1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2019 Mellanox Technologies */
3
4 #include <linux/pci.h>
5 #include "mlx5_core.h"
6 #include "pci_vsc.h"
7
8 #define MLX5_EXTRACT_C(source, offset, size) \
9 ((((u32)(source)) >> (offset)) & MLX5_ONES32(size))
10 #define MLX5_EXTRACT(src, start, len) \
11 (((len) == 32) ? (src) : MLX5_EXTRACT_C(src, start, len))
12 #define MLX5_ONES32(size) \
13 ((size) ? (0xffffffff >> (32 - (size))) : 0)
14 #define MLX5_MASK32(offset, size) \
15 (MLX5_ONES32(size) << (offset))
16 #define MLX5_MERGE_C(rsrc1, rsrc2, start, len) \
17 ((((rsrc2) << (start)) & (MLX5_MASK32((start), (len)))) | \
18 ((rsrc1) & (~MLX5_MASK32((start), (len)))))
19 #define MLX5_MERGE(rsrc1, rsrc2, start, len) \
20 (((len) == 32) ? (rsrc2) : MLX5_MERGE_C(rsrc1, rsrc2, start, len))
21 #define vsc_read(dev, offset, val) \
22 pci_read_config_dword((dev)->pdev, (dev)->vsc_addr + (offset), (val))
23 #define vsc_write(dev, offset, val) \
24 pci_write_config_dword((dev)->pdev, (dev)->vsc_addr + (offset), (val))
25 #define VSC_MAX_RETRIES 2048
26
27 enum {
28 VSC_CTRL_OFFSET = 0x4,
29 VSC_COUNTER_OFFSET = 0x8,
30 VSC_SEMAPHORE_OFFSET = 0xc,
31 VSC_ADDR_OFFSET = 0x10,
32 VSC_DATA_OFFSET = 0x14,
33
34 VSC_FLAG_BIT_OFFS = 31,
35 VSC_FLAG_BIT_LEN = 1,
36
37 VSC_SYND_BIT_OFFS = 30,
38 VSC_SYND_BIT_LEN = 1,
39
40 VSC_ADDR_BIT_OFFS = 0,
41 VSC_ADDR_BIT_LEN = 30,
42
43 VSC_SPACE_BIT_OFFS = 0,
44 VSC_SPACE_BIT_LEN = 16,
45
46 VSC_SIZE_VLD_BIT_OFFS = 28,
47 VSC_SIZE_VLD_BIT_LEN = 1,
48
49 VSC_STATUS_BIT_OFFS = 29,
50 VSC_STATUS_BIT_LEN = 3,
51 };
52
mlx5_pci_vsc_init(struct mlx5_core_dev * dev)53 void mlx5_pci_vsc_init(struct mlx5_core_dev *dev)
54 {
55 if (!mlx5_core_is_pf(dev))
56 return;
57
58 dev->vsc_addr = pci_find_capability(dev->pdev,
59 PCI_CAP_ID_VNDR);
60 if (!dev->vsc_addr)
61 mlx5_core_warn(dev, "Failed to get valid vendor specific ID\n");
62 }
63
mlx5_vsc_gw_lock(struct mlx5_core_dev * dev)64 int mlx5_vsc_gw_lock(struct mlx5_core_dev *dev)
65 {
66 u32 counter = 0;
67 int retries = 0;
68 u32 lock_val;
69 int ret;
70
71 pci_cfg_access_lock(dev->pdev);
72 do {
73 if (retries > VSC_MAX_RETRIES) {
74 ret = -EBUSY;
75 goto pci_unlock;
76 }
77 if (pci_channel_offline(dev->pdev)) {
78 ret = -EACCES;
79 goto pci_unlock;
80 }
81
82 /* Check if semaphore is already locked */
83 ret = vsc_read(dev, VSC_SEMAPHORE_OFFSET, &lock_val);
84 if (ret)
85 goto pci_unlock;
86
87 if (lock_val) {
88 retries++;
89 usleep_range(1000, 2000);
90 continue;
91 }
92
93 /* Read and write counter value, if written value is
94 * the same, semaphore was acquired successfully.
95 */
96 ret = vsc_read(dev, VSC_COUNTER_OFFSET, &counter);
97 if (ret)
98 goto pci_unlock;
99
100 ret = vsc_write(dev, VSC_SEMAPHORE_OFFSET, counter);
101 if (ret)
102 goto pci_unlock;
103
104 ret = vsc_read(dev, VSC_SEMAPHORE_OFFSET, &lock_val);
105 if (ret)
106 goto pci_unlock;
107
108 retries++;
109 } while (counter != lock_val);
110
111 return 0;
112
113 pci_unlock:
114 pci_cfg_access_unlock(dev->pdev);
115 return ret;
116 }
117
mlx5_vsc_gw_unlock(struct mlx5_core_dev * dev)118 int mlx5_vsc_gw_unlock(struct mlx5_core_dev *dev)
119 {
120 int ret;
121
122 ret = vsc_write(dev, VSC_SEMAPHORE_OFFSET, MLX5_VSC_UNLOCK);
123 pci_cfg_access_unlock(dev->pdev);
124 return ret;
125 }
126
mlx5_vsc_gw_set_space(struct mlx5_core_dev * dev,u16 space,u32 * ret_space_size)127 int mlx5_vsc_gw_set_space(struct mlx5_core_dev *dev, u16 space,
128 u32 *ret_space_size)
129 {
130 int ret;
131 u32 val = 0;
132
133 if (!mlx5_vsc_accessible(dev))
134 return -EINVAL;
135
136 if (ret_space_size)
137 *ret_space_size = 0;
138
139 /* Get a unique val */
140 ret = vsc_read(dev, VSC_CTRL_OFFSET, &val);
141 if (ret)
142 goto out;
143
144 /* Try to modify the lock */
145 val = MLX5_MERGE(val, space, VSC_SPACE_BIT_OFFS, VSC_SPACE_BIT_LEN);
146 ret = vsc_write(dev, VSC_CTRL_OFFSET, val);
147 if (ret)
148 goto out;
149
150 /* Verify lock was modified */
151 ret = vsc_read(dev, VSC_CTRL_OFFSET, &val);
152 if (ret)
153 goto out;
154
155 if (MLX5_EXTRACT(val, VSC_STATUS_BIT_OFFS, VSC_STATUS_BIT_LEN) == 0)
156 return -EINVAL;
157
158 /* Get space max address if indicated by size valid bit */
159 if (ret_space_size &&
160 MLX5_EXTRACT(val, VSC_SIZE_VLD_BIT_OFFS, VSC_SIZE_VLD_BIT_LEN)) {
161 ret = vsc_read(dev, VSC_ADDR_OFFSET, &val);
162 if (ret) {
163 mlx5_core_warn(dev, "Failed to get max space size\n");
164 goto out;
165 }
166 *ret_space_size = MLX5_EXTRACT(val, VSC_ADDR_BIT_OFFS,
167 VSC_ADDR_BIT_LEN);
168 }
169 return 0;
170
171 out:
172 return ret;
173 }
174
mlx5_vsc_wait_on_flag(struct mlx5_core_dev * dev,u8 expected_val)175 static int mlx5_vsc_wait_on_flag(struct mlx5_core_dev *dev, u8 expected_val)
176 {
177 int retries = 0;
178 u32 flag;
179 int ret;
180
181 do {
182 if (retries > VSC_MAX_RETRIES)
183 return -EBUSY;
184
185 ret = vsc_read(dev, VSC_ADDR_OFFSET, &flag);
186 if (ret)
187 return ret;
188 flag = MLX5_EXTRACT(flag, VSC_FLAG_BIT_OFFS, VSC_FLAG_BIT_LEN);
189 retries++;
190
191 if ((retries & 0xf) == 0)
192 usleep_range(1000, 2000);
193
194 } while (flag != expected_val);
195
196 return 0;
197 }
198
mlx5_vsc_gw_write(struct mlx5_core_dev * dev,unsigned int address,u32 data)199 static int mlx5_vsc_gw_write(struct mlx5_core_dev *dev, unsigned int address,
200 u32 data)
201 {
202 int ret;
203
204 if (MLX5_EXTRACT(address, VSC_SYND_BIT_OFFS,
205 VSC_FLAG_BIT_LEN + VSC_SYND_BIT_LEN))
206 return -EINVAL;
207
208 /* Set flag to 0x1 */
209 address = MLX5_MERGE(address, 1, VSC_FLAG_BIT_OFFS, 1);
210 ret = vsc_write(dev, VSC_DATA_OFFSET, data);
211 if (ret)
212 goto out;
213
214 ret = vsc_write(dev, VSC_ADDR_OFFSET, address);
215 if (ret)
216 goto out;
217
218 /* Wait for the flag to be cleared */
219 ret = mlx5_vsc_wait_on_flag(dev, 0);
220
221 out:
222 return ret;
223 }
224
mlx5_vsc_gw_read(struct mlx5_core_dev * dev,unsigned int address,u32 * data)225 static int mlx5_vsc_gw_read(struct mlx5_core_dev *dev, unsigned int address,
226 u32 *data)
227 {
228 int ret;
229
230 if (MLX5_EXTRACT(address, VSC_SYND_BIT_OFFS,
231 VSC_FLAG_BIT_LEN + VSC_SYND_BIT_LEN))
232 return -EINVAL;
233
234 ret = vsc_write(dev, VSC_ADDR_OFFSET, address);
235 if (ret)
236 goto out;
237
238 ret = mlx5_vsc_wait_on_flag(dev, 1);
239 if (ret)
240 goto out;
241
242 ret = vsc_read(dev, VSC_DATA_OFFSET, data);
243 out:
244 return ret;
245 }
246
mlx5_vsc_gw_read_fast(struct mlx5_core_dev * dev,unsigned int read_addr,unsigned int * next_read_addr,u32 * data)247 static int mlx5_vsc_gw_read_fast(struct mlx5_core_dev *dev,
248 unsigned int read_addr,
249 unsigned int *next_read_addr,
250 u32 *data)
251 {
252 int ret;
253
254 ret = mlx5_vsc_gw_read(dev, read_addr, data);
255 if (ret)
256 goto out;
257
258 ret = vsc_read(dev, VSC_ADDR_OFFSET, next_read_addr);
259 if (ret)
260 goto out;
261
262 *next_read_addr = MLX5_EXTRACT(*next_read_addr, VSC_ADDR_BIT_OFFS,
263 VSC_ADDR_BIT_LEN);
264
265 if (*next_read_addr <= read_addr)
266 ret = -EINVAL;
267 out:
268 return ret;
269 }
270
mlx5_vsc_gw_read_block_fast(struct mlx5_core_dev * dev,u32 * data,int length)271 int mlx5_vsc_gw_read_block_fast(struct mlx5_core_dev *dev, u32 *data,
272 int length)
273 {
274 unsigned int next_read_addr = 0;
275 unsigned int read_addr = 0;
276
277 while (read_addr < length) {
278 if (mlx5_vsc_gw_read_fast(dev, read_addr, &next_read_addr,
279 &data[(read_addr >> 2)]))
280 return read_addr;
281
282 read_addr = next_read_addr;
283 }
284 return length;
285 }
286
mlx5_vsc_sem_set_space(struct mlx5_core_dev * dev,u16 space,enum mlx5_vsc_state state)287 int mlx5_vsc_sem_set_space(struct mlx5_core_dev *dev, u16 space,
288 enum mlx5_vsc_state state)
289 {
290 u32 data, id = 0;
291 int ret;
292
293 ret = mlx5_vsc_gw_set_space(dev, MLX5_SEMAPHORE_SPACE_DOMAIN, NULL);
294 if (ret) {
295 mlx5_core_warn(dev, "Failed to set gw space %d\n", ret);
296 return ret;
297 }
298
299 if (state == MLX5_VSC_LOCK) {
300 /* Get a unique ID based on the counter */
301 ret = vsc_read(dev, VSC_COUNTER_OFFSET, &id);
302 if (ret)
303 return ret;
304 }
305
306 /* Try to modify lock */
307 ret = mlx5_vsc_gw_write(dev, space, id);
308 if (ret)
309 return ret;
310
311 /* Verify lock was modified */
312 ret = mlx5_vsc_gw_read(dev, space, &data);
313 if (ret)
314 return -EINVAL;
315
316 if (data != id)
317 return -EBUSY;
318
319 return 0;
320 }
321