1 // SPDX-License-Identifier: (GPL-2.0 OR MIT)
2 /* Google virtual Ethernet (gve) driver
3  *
4  * Copyright (C) 2015-2021 Google, Inc.
5  */
6 
7 #include <linux/etherdevice.h>
8 #include <linux/pci.h>
9 #include "gve.h"
10 #include "gve_adminq.h"
11 #include "gve_register.h"
12 
13 #define GVE_MAX_ADMINQ_RELEASE_CHECK	500
14 #define GVE_ADMINQ_SLEEP_LEN		20
15 #define GVE_MAX_ADMINQ_EVENT_COUNTER_CHECK	100
16 
17 #define GVE_DEVICE_OPTION_ERROR_FMT "%s option error:\n" \
18 "Expected: length=%d, feature_mask=%x.\n" \
19 "Actual: length=%d, feature_mask=%x.\n"
20 
21 #define GVE_DEVICE_OPTION_TOO_BIG_FMT "Length of %s option larger than expected. Possible older version of guest driver.\n"
22 
23 static
24 struct gve_device_option *gve_get_next_option(struct gve_device_descriptor *descriptor,
25 					      struct gve_device_option *option)
26 {
27 	void *option_end, *descriptor_end;
28 
29 	option_end = (void *)(option + 1) + be16_to_cpu(option->option_length);
30 	descriptor_end = (void *)descriptor + be16_to_cpu(descriptor->total_length);
31 
32 	return option_end > descriptor_end ? NULL : (struct gve_device_option *)option_end;
33 }
34 
35 static
36 void gve_parse_device_option(struct gve_priv *priv,
37 			     struct gve_device_descriptor *device_descriptor,
38 			     struct gve_device_option *option,
39 			     struct gve_device_option_gqi_rda **dev_op_gqi_rda,
40 			     struct gve_device_option_gqi_qpl **dev_op_gqi_qpl,
41 			     struct gve_device_option_dqo_rda **dev_op_dqo_rda,
42 			     struct gve_device_option_jumbo_frames **dev_op_jumbo_frames)
43 {
44 	u32 req_feat_mask = be32_to_cpu(option->required_features_mask);
45 	u16 option_length = be16_to_cpu(option->option_length);
46 	u16 option_id = be16_to_cpu(option->option_id);
47 
48 	/* If the length or feature mask doesn't match, continue without
49 	 * enabling the feature.
50 	 */
51 	switch (option_id) {
52 	case GVE_DEV_OPT_ID_GQI_RAW_ADDRESSING:
53 		if (option_length != GVE_DEV_OPT_LEN_GQI_RAW_ADDRESSING ||
54 		    req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RAW_ADDRESSING) {
55 			dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
56 				 "Raw Addressing",
57 				 GVE_DEV_OPT_LEN_GQI_RAW_ADDRESSING,
58 				 GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RAW_ADDRESSING,
59 				 option_length, req_feat_mask);
60 			break;
61 		}
62 
63 		dev_info(&priv->pdev->dev,
64 			 "Gqi raw addressing device option enabled.\n");
65 		priv->queue_format = GVE_GQI_RDA_FORMAT;
66 		break;
67 	case GVE_DEV_OPT_ID_GQI_RDA:
68 		if (option_length < sizeof(**dev_op_gqi_rda) ||
69 		    req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RDA) {
70 			dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
71 				 "GQI RDA", (int)sizeof(**dev_op_gqi_rda),
72 				 GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RDA,
73 				 option_length, req_feat_mask);
74 			break;
75 		}
76 
77 		if (option_length > sizeof(**dev_op_gqi_rda)) {
78 			dev_warn(&priv->pdev->dev,
79 				 GVE_DEVICE_OPTION_TOO_BIG_FMT, "GQI RDA");
80 		}
81 		*dev_op_gqi_rda = (void *)(option + 1);
82 		break;
83 	case GVE_DEV_OPT_ID_GQI_QPL:
84 		if (option_length < sizeof(**dev_op_gqi_qpl) ||
85 		    req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_GQI_QPL) {
86 			dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
87 				 "GQI QPL", (int)sizeof(**dev_op_gqi_qpl),
88 				 GVE_DEV_OPT_REQ_FEAT_MASK_GQI_QPL,
89 				 option_length, req_feat_mask);
90 			break;
91 		}
92 
93 		if (option_length > sizeof(**dev_op_gqi_qpl)) {
94 			dev_warn(&priv->pdev->dev,
95 				 GVE_DEVICE_OPTION_TOO_BIG_FMT, "GQI QPL");
96 		}
97 		*dev_op_gqi_qpl = (void *)(option + 1);
98 		break;
99 	case GVE_DEV_OPT_ID_DQO_RDA:
100 		if (option_length < sizeof(**dev_op_dqo_rda) ||
101 		    req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_DQO_RDA) {
102 			dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
103 				 "DQO RDA", (int)sizeof(**dev_op_dqo_rda),
104 				 GVE_DEV_OPT_REQ_FEAT_MASK_DQO_RDA,
105 				 option_length, req_feat_mask);
106 			break;
107 		}
108 
109 		if (option_length > sizeof(**dev_op_dqo_rda)) {
110 			dev_warn(&priv->pdev->dev,
111 				 GVE_DEVICE_OPTION_TOO_BIG_FMT, "DQO RDA");
112 		}
113 		*dev_op_dqo_rda = (void *)(option + 1);
114 		break;
115 	case GVE_DEV_OPT_ID_JUMBO_FRAMES:
116 		if (option_length < sizeof(**dev_op_jumbo_frames) ||
117 		    req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_JUMBO_FRAMES) {
118 			dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
119 				 "Jumbo Frames",
120 				 (int)sizeof(**dev_op_jumbo_frames),
121 				 GVE_DEV_OPT_REQ_FEAT_MASK_JUMBO_FRAMES,
122 				 option_length, req_feat_mask);
123 			break;
124 		}
125 
126 		if (option_length > sizeof(**dev_op_jumbo_frames)) {
127 			dev_warn(&priv->pdev->dev,
128 				 GVE_DEVICE_OPTION_TOO_BIG_FMT,
129 				 "Jumbo Frames");
130 		}
131 		*dev_op_jumbo_frames = (void *)(option + 1);
132 		break;
133 	default:
134 		/* If we don't recognize the option just continue
135 		 * without doing anything.
136 		 */
137 		dev_dbg(&priv->pdev->dev, "Unrecognized device option 0x%hx not enabled.\n",
138 			option_id);
139 	}
140 }
141 
142 /* Process all device options for a given describe device call. */
143 static int
144 gve_process_device_options(struct gve_priv *priv,
145 			   struct gve_device_descriptor *descriptor,
146 			   struct gve_device_option_gqi_rda **dev_op_gqi_rda,
147 			   struct gve_device_option_gqi_qpl **dev_op_gqi_qpl,
148 			   struct gve_device_option_dqo_rda **dev_op_dqo_rda,
149 			   struct gve_device_option_jumbo_frames **dev_op_jumbo_frames)
150 {
151 	const int num_options = be16_to_cpu(descriptor->num_device_options);
152 	struct gve_device_option *dev_opt;
153 	int i;
154 
155 	/* The options struct directly follows the device descriptor. */
156 	dev_opt = (void *)(descriptor + 1);
157 	for (i = 0; i < num_options; i++) {
158 		struct gve_device_option *next_opt;
159 
160 		next_opt = gve_get_next_option(descriptor, dev_opt);
161 		if (!next_opt) {
162 			dev_err(&priv->dev->dev,
163 				"options exceed device_descriptor's total length.\n");
164 			return -EINVAL;
165 		}
166 
167 		gve_parse_device_option(priv, descriptor, dev_opt,
168 					dev_op_gqi_rda, dev_op_gqi_qpl,
169 					dev_op_dqo_rda, dev_op_jumbo_frames);
170 		dev_opt = next_opt;
171 	}
172 
173 	return 0;
174 }
175 
176 int gve_adminq_alloc(struct device *dev, struct gve_priv *priv)
177 {
178 	priv->adminq = dma_alloc_coherent(dev, PAGE_SIZE,
179 					  &priv->adminq_bus_addr, GFP_KERNEL);
180 	if (unlikely(!priv->adminq))
181 		return -ENOMEM;
182 
183 	priv->adminq_mask = (PAGE_SIZE / sizeof(union gve_adminq_command)) - 1;
184 	priv->adminq_prod_cnt = 0;
185 	priv->adminq_cmd_fail = 0;
186 	priv->adminq_timeouts = 0;
187 	priv->adminq_describe_device_cnt = 0;
188 	priv->adminq_cfg_device_resources_cnt = 0;
189 	priv->adminq_register_page_list_cnt = 0;
190 	priv->adminq_unregister_page_list_cnt = 0;
191 	priv->adminq_create_tx_queue_cnt = 0;
192 	priv->adminq_create_rx_queue_cnt = 0;
193 	priv->adminq_destroy_tx_queue_cnt = 0;
194 	priv->adminq_destroy_rx_queue_cnt = 0;
195 	priv->adminq_dcfg_device_resources_cnt = 0;
196 	priv->adminq_set_driver_parameter_cnt = 0;
197 	priv->adminq_report_stats_cnt = 0;
198 	priv->adminq_report_link_speed_cnt = 0;
199 	priv->adminq_get_ptype_map_cnt = 0;
200 
201 	/* Setup Admin queue with the device */
202 	iowrite32be(priv->adminq_bus_addr / PAGE_SIZE,
203 		    &priv->reg_bar0->adminq_pfn);
204 
205 	gve_set_admin_queue_ok(priv);
206 	return 0;
207 }
208 
209 void gve_adminq_release(struct gve_priv *priv)
210 {
211 	int i = 0;
212 
213 	/* Tell the device the adminq is leaving */
214 	iowrite32be(0x0, &priv->reg_bar0->adminq_pfn);
215 	while (ioread32be(&priv->reg_bar0->adminq_pfn)) {
216 		/* If this is reached the device is unrecoverable and still
217 		 * holding memory. Continue looping to avoid memory corruption,
218 		 * but WARN so it is visible what is going on.
219 		 */
220 		if (i == GVE_MAX_ADMINQ_RELEASE_CHECK)
221 			WARN(1, "Unrecoverable platform error!");
222 		i++;
223 		msleep(GVE_ADMINQ_SLEEP_LEN);
224 	}
225 	gve_clear_device_rings_ok(priv);
226 	gve_clear_device_resources_ok(priv);
227 	gve_clear_admin_queue_ok(priv);
228 }
229 
230 void gve_adminq_free(struct device *dev, struct gve_priv *priv)
231 {
232 	if (!gve_get_admin_queue_ok(priv))
233 		return;
234 	gve_adminq_release(priv);
235 	dma_free_coherent(dev, PAGE_SIZE, priv->adminq, priv->adminq_bus_addr);
236 	gve_clear_admin_queue_ok(priv);
237 }
238 
239 static void gve_adminq_kick_cmd(struct gve_priv *priv, u32 prod_cnt)
240 {
241 	iowrite32be(prod_cnt, &priv->reg_bar0->adminq_doorbell);
242 }
243 
244 static bool gve_adminq_wait_for_cmd(struct gve_priv *priv, u32 prod_cnt)
245 {
246 	int i;
247 
248 	for (i = 0; i < GVE_MAX_ADMINQ_EVENT_COUNTER_CHECK; i++) {
249 		if (ioread32be(&priv->reg_bar0->adminq_event_counter)
250 		    == prod_cnt)
251 			return true;
252 		msleep(GVE_ADMINQ_SLEEP_LEN);
253 	}
254 
255 	return false;
256 }
257 
258 static int gve_adminq_parse_err(struct gve_priv *priv, u32 status)
259 {
260 	if (status != GVE_ADMINQ_COMMAND_PASSED &&
261 	    status != GVE_ADMINQ_COMMAND_UNSET) {
262 		dev_err(&priv->pdev->dev, "AQ command failed with status %d\n", status);
263 		priv->adminq_cmd_fail++;
264 	}
265 	switch (status) {
266 	case GVE_ADMINQ_COMMAND_PASSED:
267 		return 0;
268 	case GVE_ADMINQ_COMMAND_UNSET:
269 		dev_err(&priv->pdev->dev, "parse_aq_err: err and status both unset, this should not be possible.\n");
270 		return -EINVAL;
271 	case GVE_ADMINQ_COMMAND_ERROR_ABORTED:
272 	case GVE_ADMINQ_COMMAND_ERROR_CANCELLED:
273 	case GVE_ADMINQ_COMMAND_ERROR_DATALOSS:
274 	case GVE_ADMINQ_COMMAND_ERROR_FAILED_PRECONDITION:
275 	case GVE_ADMINQ_COMMAND_ERROR_UNAVAILABLE:
276 		return -EAGAIN;
277 	case GVE_ADMINQ_COMMAND_ERROR_ALREADY_EXISTS:
278 	case GVE_ADMINQ_COMMAND_ERROR_INTERNAL_ERROR:
279 	case GVE_ADMINQ_COMMAND_ERROR_INVALID_ARGUMENT:
280 	case GVE_ADMINQ_COMMAND_ERROR_NOT_FOUND:
281 	case GVE_ADMINQ_COMMAND_ERROR_OUT_OF_RANGE:
282 	case GVE_ADMINQ_COMMAND_ERROR_UNKNOWN_ERROR:
283 		return -EINVAL;
284 	case GVE_ADMINQ_COMMAND_ERROR_DEADLINE_EXCEEDED:
285 		return -ETIME;
286 	case GVE_ADMINQ_COMMAND_ERROR_PERMISSION_DENIED:
287 	case GVE_ADMINQ_COMMAND_ERROR_UNAUTHENTICATED:
288 		return -EACCES;
289 	case GVE_ADMINQ_COMMAND_ERROR_RESOURCE_EXHAUSTED:
290 		return -ENOMEM;
291 	case GVE_ADMINQ_COMMAND_ERROR_UNIMPLEMENTED:
292 		return -EOPNOTSUPP;
293 	default:
294 		dev_err(&priv->pdev->dev, "parse_aq_err: unknown status code %d\n", status);
295 		return -EINVAL;
296 	}
297 }
298 
299 /* Flushes all AQ commands currently queued and waits for them to complete.
300  * If there are failures, it will return the first error.
301  */
302 static int gve_adminq_kick_and_wait(struct gve_priv *priv)
303 {
304 	int tail, head;
305 	int i;
306 
307 	tail = ioread32be(&priv->reg_bar0->adminq_event_counter);
308 	head = priv->adminq_prod_cnt;
309 
310 	gve_adminq_kick_cmd(priv, head);
311 	if (!gve_adminq_wait_for_cmd(priv, head)) {
312 		dev_err(&priv->pdev->dev, "AQ commands timed out, need to reset AQ\n");
313 		priv->adminq_timeouts++;
314 		return -ENOTRECOVERABLE;
315 	}
316 
317 	for (i = tail; i < head; i++) {
318 		union gve_adminq_command *cmd;
319 		u32 status, err;
320 
321 		cmd = &priv->adminq[i & priv->adminq_mask];
322 		status = be32_to_cpu(READ_ONCE(cmd->status));
323 		err = gve_adminq_parse_err(priv, status);
324 		if (err)
325 			// Return the first error if we failed.
326 			return err;
327 	}
328 
329 	return 0;
330 }
331 
332 /* This function is not threadsafe - the caller is responsible for any
333  * necessary locks.
334  */
335 static int gve_adminq_issue_cmd(struct gve_priv *priv,
336 				union gve_adminq_command *cmd_orig)
337 {
338 	union gve_adminq_command *cmd;
339 	u32 opcode;
340 	u32 tail;
341 
342 	tail = ioread32be(&priv->reg_bar0->adminq_event_counter);
343 
344 	// Check if next command will overflow the buffer.
345 	if (((priv->adminq_prod_cnt + 1) & priv->adminq_mask) ==
346 	    (tail & priv->adminq_mask)) {
347 		int err;
348 
349 		// Flush existing commands to make room.
350 		err = gve_adminq_kick_and_wait(priv);
351 		if (err)
352 			return err;
353 
354 		// Retry.
355 		tail = ioread32be(&priv->reg_bar0->adminq_event_counter);
356 		if (((priv->adminq_prod_cnt + 1) & priv->adminq_mask) ==
357 		    (tail & priv->adminq_mask)) {
358 			// This should never happen. We just flushed the
359 			// command queue so there should be enough space.
360 			return -ENOMEM;
361 		}
362 	}
363 
364 	cmd = &priv->adminq[priv->adminq_prod_cnt & priv->adminq_mask];
365 	priv->adminq_prod_cnt++;
366 
367 	memcpy(cmd, cmd_orig, sizeof(*cmd_orig));
368 	opcode = be32_to_cpu(READ_ONCE(cmd->opcode));
369 
370 	switch (opcode) {
371 	case GVE_ADMINQ_DESCRIBE_DEVICE:
372 		priv->adminq_describe_device_cnt++;
373 		break;
374 	case GVE_ADMINQ_CONFIGURE_DEVICE_RESOURCES:
375 		priv->adminq_cfg_device_resources_cnt++;
376 		break;
377 	case GVE_ADMINQ_REGISTER_PAGE_LIST:
378 		priv->adminq_register_page_list_cnt++;
379 		break;
380 	case GVE_ADMINQ_UNREGISTER_PAGE_LIST:
381 		priv->adminq_unregister_page_list_cnt++;
382 		break;
383 	case GVE_ADMINQ_CREATE_TX_QUEUE:
384 		priv->adminq_create_tx_queue_cnt++;
385 		break;
386 	case GVE_ADMINQ_CREATE_RX_QUEUE:
387 		priv->adminq_create_rx_queue_cnt++;
388 		break;
389 	case GVE_ADMINQ_DESTROY_TX_QUEUE:
390 		priv->adminq_destroy_tx_queue_cnt++;
391 		break;
392 	case GVE_ADMINQ_DESTROY_RX_QUEUE:
393 		priv->adminq_destroy_rx_queue_cnt++;
394 		break;
395 	case GVE_ADMINQ_DECONFIGURE_DEVICE_RESOURCES:
396 		priv->adminq_dcfg_device_resources_cnt++;
397 		break;
398 	case GVE_ADMINQ_SET_DRIVER_PARAMETER:
399 		priv->adminq_set_driver_parameter_cnt++;
400 		break;
401 	case GVE_ADMINQ_REPORT_STATS:
402 		priv->adminq_report_stats_cnt++;
403 		break;
404 	case GVE_ADMINQ_REPORT_LINK_SPEED:
405 		priv->adminq_report_link_speed_cnt++;
406 		break;
407 	case GVE_ADMINQ_GET_PTYPE_MAP:
408 		priv->adminq_get_ptype_map_cnt++;
409 		break;
410 	case GVE_ADMINQ_VERIFY_DRIVER_COMPATIBILITY:
411 		priv->adminq_verify_driver_compatibility_cnt++;
412 		break;
413 	default:
414 		dev_err(&priv->pdev->dev, "unknown AQ command opcode %d\n", opcode);
415 	}
416 
417 	return 0;
418 }
419 
420 /* This function is not threadsafe - the caller is responsible for any
421  * necessary locks.
422  * The caller is also responsible for making sure there are no commands
423  * waiting to be executed.
424  */
425 static int gve_adminq_execute_cmd(struct gve_priv *priv,
426 				  union gve_adminq_command *cmd_orig)
427 {
428 	u32 tail, head;
429 	int err;
430 
431 	tail = ioread32be(&priv->reg_bar0->adminq_event_counter);
432 	head = priv->adminq_prod_cnt;
433 	if (tail != head)
434 		// This is not a valid path
435 		return -EINVAL;
436 
437 	err = gve_adminq_issue_cmd(priv, cmd_orig);
438 	if (err)
439 		return err;
440 
441 	return gve_adminq_kick_and_wait(priv);
442 }
443 
444 /* The device specifies that the management vector can either be the first irq
445  * or the last irq. ntfy_blk_msix_base_idx indicates the first irq assigned to
446  * the ntfy blks. It if is 0 then the management vector is last, if it is 1 then
447  * the management vector is first.
448  *
449  * gve arranges the msix vectors so that the management vector is last.
450  */
451 #define GVE_NTFY_BLK_BASE_MSIX_IDX	0
452 int gve_adminq_configure_device_resources(struct gve_priv *priv,
453 					  dma_addr_t counter_array_bus_addr,
454 					  u32 num_counters,
455 					  dma_addr_t db_array_bus_addr,
456 					  u32 num_ntfy_blks)
457 {
458 	union gve_adminq_command cmd;
459 
460 	memset(&cmd, 0, sizeof(cmd));
461 	cmd.opcode = cpu_to_be32(GVE_ADMINQ_CONFIGURE_DEVICE_RESOURCES);
462 	cmd.configure_device_resources =
463 		(struct gve_adminq_configure_device_resources) {
464 		.counter_array = cpu_to_be64(counter_array_bus_addr),
465 		.num_counters = cpu_to_be32(num_counters),
466 		.irq_db_addr = cpu_to_be64(db_array_bus_addr),
467 		.num_irq_dbs = cpu_to_be32(num_ntfy_blks),
468 		.irq_db_stride = cpu_to_be32(sizeof(*priv->irq_db_indices)),
469 		.ntfy_blk_msix_base_idx =
470 					cpu_to_be32(GVE_NTFY_BLK_BASE_MSIX_IDX),
471 		.queue_format = priv->queue_format,
472 	};
473 
474 	return gve_adminq_execute_cmd(priv, &cmd);
475 }
476 
477 int gve_adminq_deconfigure_device_resources(struct gve_priv *priv)
478 {
479 	union gve_adminq_command cmd;
480 
481 	memset(&cmd, 0, sizeof(cmd));
482 	cmd.opcode = cpu_to_be32(GVE_ADMINQ_DECONFIGURE_DEVICE_RESOURCES);
483 
484 	return gve_adminq_execute_cmd(priv, &cmd);
485 }
486 
487 static int gve_adminq_create_tx_queue(struct gve_priv *priv, u32 queue_index)
488 {
489 	struct gve_tx_ring *tx = &priv->tx[queue_index];
490 	union gve_adminq_command cmd;
491 
492 	memset(&cmd, 0, sizeof(cmd));
493 	cmd.opcode = cpu_to_be32(GVE_ADMINQ_CREATE_TX_QUEUE);
494 	cmd.create_tx_queue = (struct gve_adminq_create_tx_queue) {
495 		.queue_id = cpu_to_be32(queue_index),
496 		.queue_resources_addr =
497 			cpu_to_be64(tx->q_resources_bus),
498 		.tx_ring_addr = cpu_to_be64(tx->bus),
499 		.ntfy_id = cpu_to_be32(tx->ntfy_id),
500 	};
501 
502 	if (gve_is_gqi(priv)) {
503 		u32 qpl_id = priv->queue_format == GVE_GQI_RDA_FORMAT ?
504 			GVE_RAW_ADDRESSING_QPL_ID : tx->tx_fifo.qpl->id;
505 
506 		cmd.create_tx_queue.queue_page_list_id = cpu_to_be32(qpl_id);
507 	} else {
508 		cmd.create_tx_queue.tx_ring_size =
509 			cpu_to_be16(priv->tx_desc_cnt);
510 		cmd.create_tx_queue.tx_comp_ring_addr =
511 			cpu_to_be64(tx->complq_bus_dqo);
512 		cmd.create_tx_queue.tx_comp_ring_size =
513 			cpu_to_be16(priv->options_dqo_rda.tx_comp_ring_entries);
514 	}
515 
516 	return gve_adminq_issue_cmd(priv, &cmd);
517 }
518 
519 int gve_adminq_create_tx_queues(struct gve_priv *priv, u32 num_queues)
520 {
521 	int err;
522 	int i;
523 
524 	for (i = 0; i < num_queues; i++) {
525 		err = gve_adminq_create_tx_queue(priv, i);
526 		if (err)
527 			return err;
528 	}
529 
530 	return gve_adminq_kick_and_wait(priv);
531 }
532 
533 static int gve_adminq_create_rx_queue(struct gve_priv *priv, u32 queue_index)
534 {
535 	struct gve_rx_ring *rx = &priv->rx[queue_index];
536 	union gve_adminq_command cmd;
537 
538 	memset(&cmd, 0, sizeof(cmd));
539 	cmd.opcode = cpu_to_be32(GVE_ADMINQ_CREATE_RX_QUEUE);
540 	cmd.create_rx_queue = (struct gve_adminq_create_rx_queue) {
541 		.queue_id = cpu_to_be32(queue_index),
542 		.ntfy_id = cpu_to_be32(rx->ntfy_id),
543 		.queue_resources_addr = cpu_to_be64(rx->q_resources_bus),
544 	};
545 
546 	if (gve_is_gqi(priv)) {
547 		u32 qpl_id = priv->queue_format == GVE_GQI_RDA_FORMAT ?
548 			GVE_RAW_ADDRESSING_QPL_ID : rx->data.qpl->id;
549 
550 		cmd.create_rx_queue.rx_desc_ring_addr =
551 			cpu_to_be64(rx->desc.bus),
552 		cmd.create_rx_queue.rx_data_ring_addr =
553 			cpu_to_be64(rx->data.data_bus),
554 		cmd.create_rx_queue.index = cpu_to_be32(queue_index);
555 		cmd.create_rx_queue.queue_page_list_id = cpu_to_be32(qpl_id);
556 		cmd.create_rx_queue.packet_buffer_size = cpu_to_be16(rx->packet_buffer_size);
557 	} else {
558 		cmd.create_rx_queue.rx_ring_size =
559 			cpu_to_be16(priv->rx_desc_cnt);
560 		cmd.create_rx_queue.rx_desc_ring_addr =
561 			cpu_to_be64(rx->dqo.complq.bus);
562 		cmd.create_rx_queue.rx_data_ring_addr =
563 			cpu_to_be64(rx->dqo.bufq.bus);
564 		cmd.create_rx_queue.packet_buffer_size =
565 			cpu_to_be16(priv->data_buffer_size_dqo);
566 		cmd.create_rx_queue.rx_buff_ring_size =
567 			cpu_to_be16(priv->options_dqo_rda.rx_buff_ring_entries);
568 		cmd.create_rx_queue.enable_rsc =
569 			!!(priv->dev->features & NETIF_F_LRO);
570 	}
571 
572 	return gve_adminq_issue_cmd(priv, &cmd);
573 }
574 
575 int gve_adminq_create_rx_queues(struct gve_priv *priv, u32 num_queues)
576 {
577 	int err;
578 	int i;
579 
580 	for (i = 0; i < num_queues; i++) {
581 		err = gve_adminq_create_rx_queue(priv, i);
582 		if (err)
583 			return err;
584 	}
585 
586 	return gve_adminq_kick_and_wait(priv);
587 }
588 
589 static int gve_adminq_destroy_tx_queue(struct gve_priv *priv, u32 queue_index)
590 {
591 	union gve_adminq_command cmd;
592 	int err;
593 
594 	memset(&cmd, 0, sizeof(cmd));
595 	cmd.opcode = cpu_to_be32(GVE_ADMINQ_DESTROY_TX_QUEUE);
596 	cmd.destroy_tx_queue = (struct gve_adminq_destroy_tx_queue) {
597 		.queue_id = cpu_to_be32(queue_index),
598 	};
599 
600 	err = gve_adminq_issue_cmd(priv, &cmd);
601 	if (err)
602 		return err;
603 
604 	return 0;
605 }
606 
607 int gve_adminq_destroy_tx_queues(struct gve_priv *priv, u32 num_queues)
608 {
609 	int err;
610 	int i;
611 
612 	for (i = 0; i < num_queues; i++) {
613 		err = gve_adminq_destroy_tx_queue(priv, i);
614 		if (err)
615 			return err;
616 	}
617 
618 	return gve_adminq_kick_and_wait(priv);
619 }
620 
621 static int gve_adminq_destroy_rx_queue(struct gve_priv *priv, u32 queue_index)
622 {
623 	union gve_adminq_command cmd;
624 	int err;
625 
626 	memset(&cmd, 0, sizeof(cmd));
627 	cmd.opcode = cpu_to_be32(GVE_ADMINQ_DESTROY_RX_QUEUE);
628 	cmd.destroy_rx_queue = (struct gve_adminq_destroy_rx_queue) {
629 		.queue_id = cpu_to_be32(queue_index),
630 	};
631 
632 	err = gve_adminq_issue_cmd(priv, &cmd);
633 	if (err)
634 		return err;
635 
636 	return 0;
637 }
638 
639 int gve_adminq_destroy_rx_queues(struct gve_priv *priv, u32 num_queues)
640 {
641 	int err;
642 	int i;
643 
644 	for (i = 0; i < num_queues; i++) {
645 		err = gve_adminq_destroy_rx_queue(priv, i);
646 		if (err)
647 			return err;
648 	}
649 
650 	return gve_adminq_kick_and_wait(priv);
651 }
652 
653 static int gve_set_desc_cnt(struct gve_priv *priv,
654 			    struct gve_device_descriptor *descriptor)
655 {
656 	priv->tx_desc_cnt = be16_to_cpu(descriptor->tx_queue_entries);
657 	if (priv->tx_desc_cnt * sizeof(priv->tx->desc[0]) < PAGE_SIZE) {
658 		dev_err(&priv->pdev->dev, "Tx desc count %d too low\n",
659 			priv->tx_desc_cnt);
660 		return -EINVAL;
661 	}
662 	priv->rx_desc_cnt = be16_to_cpu(descriptor->rx_queue_entries);
663 	if (priv->rx_desc_cnt * sizeof(priv->rx->desc.desc_ring[0])
664 	    < PAGE_SIZE) {
665 		dev_err(&priv->pdev->dev, "Rx desc count %d too low\n",
666 			priv->rx_desc_cnt);
667 		return -EINVAL;
668 	}
669 	return 0;
670 }
671 
672 static int
673 gve_set_desc_cnt_dqo(struct gve_priv *priv,
674 		     const struct gve_device_descriptor *descriptor,
675 		     const struct gve_device_option_dqo_rda *dev_op_dqo_rda)
676 {
677 	priv->tx_desc_cnt = be16_to_cpu(descriptor->tx_queue_entries);
678 	priv->options_dqo_rda.tx_comp_ring_entries =
679 		be16_to_cpu(dev_op_dqo_rda->tx_comp_ring_entries);
680 	priv->rx_desc_cnt = be16_to_cpu(descriptor->rx_queue_entries);
681 	priv->options_dqo_rda.rx_buff_ring_entries =
682 		be16_to_cpu(dev_op_dqo_rda->rx_buff_ring_entries);
683 
684 	return 0;
685 }
686 
687 static void gve_enable_supported_features(struct gve_priv *priv,
688 					  u32 supported_features_mask,
689 					  const struct gve_device_option_jumbo_frames
690 						  *dev_op_jumbo_frames)
691 {
692 	/* Before control reaches this point, the page-size-capped max MTU from
693 	 * the gve_device_descriptor field has already been stored in
694 	 * priv->dev->max_mtu. We overwrite it with the true max MTU below.
695 	 */
696 	if (dev_op_jumbo_frames &&
697 	    (supported_features_mask & GVE_SUP_JUMBO_FRAMES_MASK)) {
698 		dev_info(&priv->pdev->dev,
699 			 "JUMBO FRAMES device option enabled.\n");
700 		priv->dev->max_mtu = be16_to_cpu(dev_op_jumbo_frames->max_mtu);
701 	}
702 }
703 
704 int gve_adminq_describe_device(struct gve_priv *priv)
705 {
706 	struct gve_device_option_jumbo_frames *dev_op_jumbo_frames = NULL;
707 	struct gve_device_option_gqi_rda *dev_op_gqi_rda = NULL;
708 	struct gve_device_option_gqi_qpl *dev_op_gqi_qpl = NULL;
709 	struct gve_device_option_dqo_rda *dev_op_dqo_rda = NULL;
710 	struct gve_device_descriptor *descriptor;
711 	u32 supported_features_mask = 0;
712 	union gve_adminq_command cmd;
713 	dma_addr_t descriptor_bus;
714 	int err = 0;
715 	u8 *mac;
716 	u16 mtu;
717 
718 	memset(&cmd, 0, sizeof(cmd));
719 	descriptor = dma_alloc_coherent(&priv->pdev->dev, PAGE_SIZE,
720 					&descriptor_bus, GFP_KERNEL);
721 	if (!descriptor)
722 		return -ENOMEM;
723 	cmd.opcode = cpu_to_be32(GVE_ADMINQ_DESCRIBE_DEVICE);
724 	cmd.describe_device.device_descriptor_addr =
725 						cpu_to_be64(descriptor_bus);
726 	cmd.describe_device.device_descriptor_version =
727 			cpu_to_be32(GVE_ADMINQ_DEVICE_DESCRIPTOR_VERSION);
728 	cmd.describe_device.available_length = cpu_to_be32(PAGE_SIZE);
729 
730 	err = gve_adminq_execute_cmd(priv, &cmd);
731 	if (err)
732 		goto free_device_descriptor;
733 
734 	err = gve_process_device_options(priv, descriptor, &dev_op_gqi_rda,
735 					 &dev_op_gqi_qpl, &dev_op_dqo_rda,
736 					 &dev_op_jumbo_frames);
737 	if (err)
738 		goto free_device_descriptor;
739 
740 	/* If the GQI_RAW_ADDRESSING option is not enabled and the queue format
741 	 * is not set to GqiRda, choose the queue format in a priority order:
742 	 * DqoRda, GqiRda, GqiQpl. Use GqiQpl as default.
743 	 */
744 	if (dev_op_dqo_rda) {
745 		priv->queue_format = GVE_DQO_RDA_FORMAT;
746 		dev_info(&priv->pdev->dev,
747 			 "Driver is running with DQO RDA queue format.\n");
748 		supported_features_mask =
749 			be32_to_cpu(dev_op_dqo_rda->supported_features_mask);
750 	} else if (dev_op_gqi_rda) {
751 		priv->queue_format = GVE_GQI_RDA_FORMAT;
752 		dev_info(&priv->pdev->dev,
753 			 "Driver is running with GQI RDA queue format.\n");
754 		supported_features_mask =
755 			be32_to_cpu(dev_op_gqi_rda->supported_features_mask);
756 	} else if (priv->queue_format == GVE_GQI_RDA_FORMAT) {
757 		dev_info(&priv->pdev->dev,
758 			 "Driver is running with GQI RDA queue format.\n");
759 	} else {
760 		priv->queue_format = GVE_GQI_QPL_FORMAT;
761 		if (dev_op_gqi_qpl)
762 			supported_features_mask =
763 				be32_to_cpu(dev_op_gqi_qpl->supported_features_mask);
764 		dev_info(&priv->pdev->dev,
765 			 "Driver is running with GQI QPL queue format.\n");
766 	}
767 	if (gve_is_gqi(priv)) {
768 		err = gve_set_desc_cnt(priv, descriptor);
769 	} else {
770 		/* DQO supports LRO. */
771 		priv->dev->hw_features |= NETIF_F_LRO;
772 		err = gve_set_desc_cnt_dqo(priv, descriptor, dev_op_dqo_rda);
773 	}
774 	if (err)
775 		goto free_device_descriptor;
776 
777 	priv->max_registered_pages =
778 				be64_to_cpu(descriptor->max_registered_pages);
779 	mtu = be16_to_cpu(descriptor->mtu);
780 	if (mtu < ETH_MIN_MTU) {
781 		dev_err(&priv->pdev->dev, "MTU %d below minimum MTU\n", mtu);
782 		err = -EINVAL;
783 		goto free_device_descriptor;
784 	}
785 	priv->dev->max_mtu = mtu;
786 	priv->num_event_counters = be16_to_cpu(descriptor->counters);
787 	eth_hw_addr_set(priv->dev, descriptor->mac);
788 	mac = descriptor->mac;
789 	dev_info(&priv->pdev->dev, "MAC addr: %pM\n", mac);
790 	priv->tx_pages_per_qpl = be16_to_cpu(descriptor->tx_pages_per_qpl);
791 	priv->rx_data_slot_cnt = be16_to_cpu(descriptor->rx_pages_per_qpl);
792 
793 	if (gve_is_gqi(priv) && priv->rx_data_slot_cnt < priv->rx_desc_cnt) {
794 		dev_err(&priv->pdev->dev, "rx_data_slot_cnt cannot be smaller than rx_desc_cnt, setting rx_desc_cnt down to %d.\n",
795 			priv->rx_data_slot_cnt);
796 		priv->rx_desc_cnt = priv->rx_data_slot_cnt;
797 	}
798 	priv->default_num_queues = be16_to_cpu(descriptor->default_num_queues);
799 
800 	gve_enable_supported_features(priv, supported_features_mask,
801 				      dev_op_jumbo_frames);
802 
803 free_device_descriptor:
804 	dma_free_coherent(&priv->pdev->dev, PAGE_SIZE, descriptor,
805 			  descriptor_bus);
806 	return err;
807 }
808 
809 int gve_adminq_register_page_list(struct gve_priv *priv,
810 				  struct gve_queue_page_list *qpl)
811 {
812 	struct device *hdev = &priv->pdev->dev;
813 	u32 num_entries = qpl->num_entries;
814 	u32 size = num_entries * sizeof(qpl->page_buses[0]);
815 	union gve_adminq_command cmd;
816 	dma_addr_t page_list_bus;
817 	__be64 *page_list;
818 	int err;
819 	int i;
820 
821 	memset(&cmd, 0, sizeof(cmd));
822 	page_list = dma_alloc_coherent(hdev, size, &page_list_bus, GFP_KERNEL);
823 	if (!page_list)
824 		return -ENOMEM;
825 
826 	for (i = 0; i < num_entries; i++)
827 		page_list[i] = cpu_to_be64(qpl->page_buses[i]);
828 
829 	cmd.opcode = cpu_to_be32(GVE_ADMINQ_REGISTER_PAGE_LIST);
830 	cmd.reg_page_list = (struct gve_adminq_register_page_list) {
831 		.page_list_id = cpu_to_be32(qpl->id),
832 		.num_pages = cpu_to_be32(num_entries),
833 		.page_address_list_addr = cpu_to_be64(page_list_bus),
834 	};
835 
836 	err = gve_adminq_execute_cmd(priv, &cmd);
837 	dma_free_coherent(hdev, size, page_list, page_list_bus);
838 	return err;
839 }
840 
841 int gve_adminq_unregister_page_list(struct gve_priv *priv, u32 page_list_id)
842 {
843 	union gve_adminq_command cmd;
844 
845 	memset(&cmd, 0, sizeof(cmd));
846 	cmd.opcode = cpu_to_be32(GVE_ADMINQ_UNREGISTER_PAGE_LIST);
847 	cmd.unreg_page_list = (struct gve_adminq_unregister_page_list) {
848 		.page_list_id = cpu_to_be32(page_list_id),
849 	};
850 
851 	return gve_adminq_execute_cmd(priv, &cmd);
852 }
853 
854 int gve_adminq_set_mtu(struct gve_priv *priv, u64 mtu)
855 {
856 	union gve_adminq_command cmd;
857 
858 	memset(&cmd, 0, sizeof(cmd));
859 	cmd.opcode = cpu_to_be32(GVE_ADMINQ_SET_DRIVER_PARAMETER);
860 	cmd.set_driver_param = (struct gve_adminq_set_driver_parameter) {
861 		.parameter_type = cpu_to_be32(GVE_SET_PARAM_MTU),
862 		.parameter_value = cpu_to_be64(mtu),
863 	};
864 
865 	return gve_adminq_execute_cmd(priv, &cmd);
866 }
867 
868 int gve_adminq_report_stats(struct gve_priv *priv, u64 stats_report_len,
869 			    dma_addr_t stats_report_addr, u64 interval)
870 {
871 	union gve_adminq_command cmd;
872 
873 	memset(&cmd, 0, sizeof(cmd));
874 	cmd.opcode = cpu_to_be32(GVE_ADMINQ_REPORT_STATS);
875 	cmd.report_stats = (struct gve_adminq_report_stats) {
876 		.stats_report_len = cpu_to_be64(stats_report_len),
877 		.stats_report_addr = cpu_to_be64(stats_report_addr),
878 		.interval = cpu_to_be64(interval),
879 	};
880 
881 	return gve_adminq_execute_cmd(priv, &cmd);
882 }
883 
884 int gve_adminq_verify_driver_compatibility(struct gve_priv *priv,
885 					   u64 driver_info_len,
886 					   dma_addr_t driver_info_addr)
887 {
888 	union gve_adminq_command cmd;
889 
890 	memset(&cmd, 0, sizeof(cmd));
891 	cmd.opcode = cpu_to_be32(GVE_ADMINQ_VERIFY_DRIVER_COMPATIBILITY);
892 	cmd.verify_driver_compatibility = (struct gve_adminq_verify_driver_compatibility) {
893 		.driver_info_len = cpu_to_be64(driver_info_len),
894 		.driver_info_addr = cpu_to_be64(driver_info_addr),
895 	};
896 
897 	return gve_adminq_execute_cmd(priv, &cmd);
898 }
899 
900 int gve_adminq_report_link_speed(struct gve_priv *priv)
901 {
902 	union gve_adminq_command gvnic_cmd;
903 	dma_addr_t link_speed_region_bus;
904 	__be64 *link_speed_region;
905 	int err;
906 
907 	link_speed_region =
908 		dma_alloc_coherent(&priv->pdev->dev, sizeof(*link_speed_region),
909 				   &link_speed_region_bus, GFP_KERNEL);
910 
911 	if (!link_speed_region)
912 		return -ENOMEM;
913 
914 	memset(&gvnic_cmd, 0, sizeof(gvnic_cmd));
915 	gvnic_cmd.opcode = cpu_to_be32(GVE_ADMINQ_REPORT_LINK_SPEED);
916 	gvnic_cmd.report_link_speed.link_speed_address =
917 		cpu_to_be64(link_speed_region_bus);
918 
919 	err = gve_adminq_execute_cmd(priv, &gvnic_cmd);
920 
921 	priv->link_speed = be64_to_cpu(*link_speed_region);
922 	dma_free_coherent(&priv->pdev->dev, sizeof(*link_speed_region), link_speed_region,
923 			  link_speed_region_bus);
924 	return err;
925 }
926 
927 int gve_adminq_get_ptype_map_dqo(struct gve_priv *priv,
928 				 struct gve_ptype_lut *ptype_lut)
929 {
930 	struct gve_ptype_map *ptype_map;
931 	union gve_adminq_command cmd;
932 	dma_addr_t ptype_map_bus;
933 	int err = 0;
934 	int i;
935 
936 	memset(&cmd, 0, sizeof(cmd));
937 	ptype_map = dma_alloc_coherent(&priv->pdev->dev, sizeof(*ptype_map),
938 				       &ptype_map_bus, GFP_KERNEL);
939 	if (!ptype_map)
940 		return -ENOMEM;
941 
942 	cmd.opcode = cpu_to_be32(GVE_ADMINQ_GET_PTYPE_MAP);
943 	cmd.get_ptype_map = (struct gve_adminq_get_ptype_map) {
944 		.ptype_map_len = cpu_to_be64(sizeof(*ptype_map)),
945 		.ptype_map_addr = cpu_to_be64(ptype_map_bus),
946 	};
947 
948 	err = gve_adminq_execute_cmd(priv, &cmd);
949 	if (err)
950 		goto err;
951 
952 	/* Populate ptype_lut. */
953 	for (i = 0; i < GVE_NUM_PTYPES; i++) {
954 		ptype_lut->ptypes[i].l3_type =
955 			ptype_map->ptypes[i].l3_type;
956 		ptype_lut->ptypes[i].l4_type =
957 			ptype_map->ptypes[i].l4_type;
958 	}
959 err:
960 	dma_free_coherent(&priv->pdev->dev, sizeof(*ptype_map), ptype_map,
961 			  ptype_map_bus);
962 	return err;
963 }
964