1 // SPDX-License-Identifier: (GPL-2.0 OR MIT)
2 /* Google virtual Ethernet (gve) driver
3  *
4  * Copyright (C) 2015-2021 Google, Inc.
5  */
6 
7 #include <linux/etherdevice.h>
8 #include <linux/pci.h>
9 #include "gve.h"
10 #include "gve_adminq.h"
11 #include "gve_register.h"
12 
13 #define GVE_MAX_ADMINQ_RELEASE_CHECK	500
14 #define GVE_ADMINQ_SLEEP_LEN		20
15 #define GVE_MAX_ADMINQ_EVENT_COUNTER_CHECK	100
16 
17 #define GVE_DEVICE_OPTION_ERROR_FMT "%s option error:\n" \
18 "Expected: length=%d, feature_mask=%x.\n" \
19 "Actual: length=%d, feature_mask=%x.\n"
20 
21 #define GVE_DEVICE_OPTION_TOO_BIG_FMT "Length of %s option larger than expected. Possible older version of guest driver.\n"
22 
23 static
24 struct gve_device_option *gve_get_next_option(struct gve_device_descriptor *descriptor,
25 					      struct gve_device_option *option)
26 {
27 	void *option_end, *descriptor_end;
28 
29 	option_end = (void *)(option + 1) + be16_to_cpu(option->option_length);
30 	descriptor_end = (void *)descriptor + be16_to_cpu(descriptor->total_length);
31 
32 	return option_end > descriptor_end ? NULL : (struct gve_device_option *)option_end;
33 }
34 
35 static
36 void gve_parse_device_option(struct gve_priv *priv,
37 			     struct gve_device_descriptor *device_descriptor,
38 			     struct gve_device_option *option,
39 			     struct gve_device_option_gqi_rda **dev_op_gqi_rda,
40 			     struct gve_device_option_gqi_qpl **dev_op_gqi_qpl,
41 			     struct gve_device_option_dqo_rda **dev_op_dqo_rda,
42 			     struct gve_device_option_jumbo_frames **dev_op_jumbo_frames)
43 {
44 	u32 req_feat_mask = be32_to_cpu(option->required_features_mask);
45 	u16 option_length = be16_to_cpu(option->option_length);
46 	u16 option_id = be16_to_cpu(option->option_id);
47 
48 	/* If the length or feature mask doesn't match, continue without
49 	 * enabling the feature.
50 	 */
51 	switch (option_id) {
52 	case GVE_DEV_OPT_ID_GQI_RAW_ADDRESSING:
53 		if (option_length != GVE_DEV_OPT_LEN_GQI_RAW_ADDRESSING ||
54 		    req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RAW_ADDRESSING) {
55 			dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
56 				 "Raw Addressing",
57 				 GVE_DEV_OPT_LEN_GQI_RAW_ADDRESSING,
58 				 GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RAW_ADDRESSING,
59 				 option_length, req_feat_mask);
60 			break;
61 		}
62 
63 		dev_info(&priv->pdev->dev,
64 			 "Gqi raw addressing device option enabled.\n");
65 		priv->queue_format = GVE_GQI_RDA_FORMAT;
66 		break;
67 	case GVE_DEV_OPT_ID_GQI_RDA:
68 		if (option_length < sizeof(**dev_op_gqi_rda) ||
69 		    req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RDA) {
70 			dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
71 				 "GQI RDA", (int)sizeof(**dev_op_gqi_rda),
72 				 GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RDA,
73 				 option_length, req_feat_mask);
74 			break;
75 		}
76 
77 		if (option_length > sizeof(**dev_op_gqi_rda)) {
78 			dev_warn(&priv->pdev->dev,
79 				 GVE_DEVICE_OPTION_TOO_BIG_FMT, "GQI RDA");
80 		}
81 		*dev_op_gqi_rda = (void *)(option + 1);
82 		break;
83 	case GVE_DEV_OPT_ID_GQI_QPL:
84 		if (option_length < sizeof(**dev_op_gqi_qpl) ||
85 		    req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_GQI_QPL) {
86 			dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
87 				 "GQI QPL", (int)sizeof(**dev_op_gqi_qpl),
88 				 GVE_DEV_OPT_REQ_FEAT_MASK_GQI_QPL,
89 				 option_length, req_feat_mask);
90 			break;
91 		}
92 
93 		if (option_length > sizeof(**dev_op_gqi_qpl)) {
94 			dev_warn(&priv->pdev->dev,
95 				 GVE_DEVICE_OPTION_TOO_BIG_FMT, "GQI QPL");
96 		}
97 		*dev_op_gqi_qpl = (void *)(option + 1);
98 		break;
99 	case GVE_DEV_OPT_ID_DQO_RDA:
100 		if (option_length < sizeof(**dev_op_dqo_rda) ||
101 		    req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_DQO_RDA) {
102 			dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
103 				 "DQO RDA", (int)sizeof(**dev_op_dqo_rda),
104 				 GVE_DEV_OPT_REQ_FEAT_MASK_DQO_RDA,
105 				 option_length, req_feat_mask);
106 			break;
107 		}
108 
109 		if (option_length > sizeof(**dev_op_dqo_rda)) {
110 			dev_warn(&priv->pdev->dev,
111 				 GVE_DEVICE_OPTION_TOO_BIG_FMT, "DQO RDA");
112 		}
113 		*dev_op_dqo_rda = (void *)(option + 1);
114 		break;
115 	case GVE_DEV_OPT_ID_JUMBO_FRAMES:
116 		if (option_length < sizeof(**dev_op_jumbo_frames) ||
117 		    req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_JUMBO_FRAMES) {
118 			dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
119 				 "Jumbo Frames",
120 				 (int)sizeof(**dev_op_jumbo_frames),
121 				 GVE_DEV_OPT_REQ_FEAT_MASK_JUMBO_FRAMES,
122 				 option_length, req_feat_mask);
123 			break;
124 		}
125 
126 		if (option_length > sizeof(**dev_op_jumbo_frames)) {
127 			dev_warn(&priv->pdev->dev,
128 				 GVE_DEVICE_OPTION_TOO_BIG_FMT,
129 				 "Jumbo Frames");
130 		}
131 		*dev_op_jumbo_frames = (void *)(option + 1);
132 		break;
133 	default:
134 		/* If we don't recognize the option just continue
135 		 * without doing anything.
136 		 */
137 		dev_dbg(&priv->pdev->dev, "Unrecognized device option 0x%hx not enabled.\n",
138 			option_id);
139 	}
140 }
141 
142 /* Process all device options for a given describe device call. */
143 static int
144 gve_process_device_options(struct gve_priv *priv,
145 			   struct gve_device_descriptor *descriptor,
146 			   struct gve_device_option_gqi_rda **dev_op_gqi_rda,
147 			   struct gve_device_option_gqi_qpl **dev_op_gqi_qpl,
148 			   struct gve_device_option_dqo_rda **dev_op_dqo_rda,
149 			   struct gve_device_option_jumbo_frames **dev_op_jumbo_frames)
150 {
151 	const int num_options = be16_to_cpu(descriptor->num_device_options);
152 	struct gve_device_option *dev_opt;
153 	int i;
154 
155 	/* The options struct directly follows the device descriptor. */
156 	dev_opt = (void *)(descriptor + 1);
157 	for (i = 0; i < num_options; i++) {
158 		struct gve_device_option *next_opt;
159 
160 		next_opt = gve_get_next_option(descriptor, dev_opt);
161 		if (!next_opt) {
162 			dev_err(&priv->dev->dev,
163 				"options exceed device_descriptor's total length.\n");
164 			return -EINVAL;
165 		}
166 
167 		gve_parse_device_option(priv, descriptor, dev_opt,
168 					dev_op_gqi_rda, dev_op_gqi_qpl,
169 					dev_op_dqo_rda, dev_op_jumbo_frames);
170 		dev_opt = next_opt;
171 	}
172 
173 	return 0;
174 }
175 
176 int gve_adminq_alloc(struct device *dev, struct gve_priv *priv)
177 {
178 	priv->adminq = dma_alloc_coherent(dev, PAGE_SIZE,
179 					  &priv->adminq_bus_addr, GFP_KERNEL);
180 	if (unlikely(!priv->adminq))
181 		return -ENOMEM;
182 
183 	priv->adminq_mask = (PAGE_SIZE / sizeof(union gve_adminq_command)) - 1;
184 	priv->adminq_prod_cnt = 0;
185 	priv->adminq_cmd_fail = 0;
186 	priv->adminq_timeouts = 0;
187 	priv->adminq_describe_device_cnt = 0;
188 	priv->adminq_cfg_device_resources_cnt = 0;
189 	priv->adminq_register_page_list_cnt = 0;
190 	priv->adminq_unregister_page_list_cnt = 0;
191 	priv->adminq_create_tx_queue_cnt = 0;
192 	priv->adminq_create_rx_queue_cnt = 0;
193 	priv->adminq_destroy_tx_queue_cnt = 0;
194 	priv->adminq_destroy_rx_queue_cnt = 0;
195 	priv->adminq_dcfg_device_resources_cnt = 0;
196 	priv->adminq_set_driver_parameter_cnt = 0;
197 	priv->adminq_report_stats_cnt = 0;
198 	priv->adminq_report_link_speed_cnt = 0;
199 	priv->adminq_get_ptype_map_cnt = 0;
200 
201 	/* Setup Admin queue with the device */
202 	iowrite32be(priv->adminq_bus_addr / PAGE_SIZE,
203 		    &priv->reg_bar0->adminq_pfn);
204 
205 	gve_set_admin_queue_ok(priv);
206 	return 0;
207 }
208 
209 void gve_adminq_release(struct gve_priv *priv)
210 {
211 	int i = 0;
212 
213 	/* Tell the device the adminq is leaving */
214 	iowrite32be(0x0, &priv->reg_bar0->adminq_pfn);
215 	while (ioread32be(&priv->reg_bar0->adminq_pfn)) {
216 		/* If this is reached the device is unrecoverable and still
217 		 * holding memory. Continue looping to avoid memory corruption,
218 		 * but WARN so it is visible what is going on.
219 		 */
220 		if (i == GVE_MAX_ADMINQ_RELEASE_CHECK)
221 			WARN(1, "Unrecoverable platform error!");
222 		i++;
223 		msleep(GVE_ADMINQ_SLEEP_LEN);
224 	}
225 	gve_clear_device_rings_ok(priv);
226 	gve_clear_device_resources_ok(priv);
227 	gve_clear_admin_queue_ok(priv);
228 }
229 
230 void gve_adminq_free(struct device *dev, struct gve_priv *priv)
231 {
232 	if (!gve_get_admin_queue_ok(priv))
233 		return;
234 	gve_adminq_release(priv);
235 	dma_free_coherent(dev, PAGE_SIZE, priv->adminq, priv->adminq_bus_addr);
236 	gve_clear_admin_queue_ok(priv);
237 }
238 
239 static void gve_adminq_kick_cmd(struct gve_priv *priv, u32 prod_cnt)
240 {
241 	iowrite32be(prod_cnt, &priv->reg_bar0->adminq_doorbell);
242 }
243 
244 static bool gve_adminq_wait_for_cmd(struct gve_priv *priv, u32 prod_cnt)
245 {
246 	int i;
247 
248 	for (i = 0; i < GVE_MAX_ADMINQ_EVENT_COUNTER_CHECK; i++) {
249 		if (ioread32be(&priv->reg_bar0->adminq_event_counter)
250 		    == prod_cnt)
251 			return true;
252 		msleep(GVE_ADMINQ_SLEEP_LEN);
253 	}
254 
255 	return false;
256 }
257 
258 static int gve_adminq_parse_err(struct gve_priv *priv, u32 status)
259 {
260 	if (status != GVE_ADMINQ_COMMAND_PASSED &&
261 	    status != GVE_ADMINQ_COMMAND_UNSET) {
262 		dev_err(&priv->pdev->dev, "AQ command failed with status %d\n", status);
263 		priv->adminq_cmd_fail++;
264 	}
265 	switch (status) {
266 	case GVE_ADMINQ_COMMAND_PASSED:
267 		return 0;
268 	case GVE_ADMINQ_COMMAND_UNSET:
269 		dev_err(&priv->pdev->dev, "parse_aq_err: err and status both unset, this should not be possible.\n");
270 		return -EINVAL;
271 	case GVE_ADMINQ_COMMAND_ERROR_ABORTED:
272 	case GVE_ADMINQ_COMMAND_ERROR_CANCELLED:
273 	case GVE_ADMINQ_COMMAND_ERROR_DATALOSS:
274 	case GVE_ADMINQ_COMMAND_ERROR_FAILED_PRECONDITION:
275 	case GVE_ADMINQ_COMMAND_ERROR_UNAVAILABLE:
276 		return -EAGAIN;
277 	case GVE_ADMINQ_COMMAND_ERROR_ALREADY_EXISTS:
278 	case GVE_ADMINQ_COMMAND_ERROR_INTERNAL_ERROR:
279 	case GVE_ADMINQ_COMMAND_ERROR_INVALID_ARGUMENT:
280 	case GVE_ADMINQ_COMMAND_ERROR_NOT_FOUND:
281 	case GVE_ADMINQ_COMMAND_ERROR_OUT_OF_RANGE:
282 	case GVE_ADMINQ_COMMAND_ERROR_UNKNOWN_ERROR:
283 		return -EINVAL;
284 	case GVE_ADMINQ_COMMAND_ERROR_DEADLINE_EXCEEDED:
285 		return -ETIME;
286 	case GVE_ADMINQ_COMMAND_ERROR_PERMISSION_DENIED:
287 	case GVE_ADMINQ_COMMAND_ERROR_UNAUTHENTICATED:
288 		return -EACCES;
289 	case GVE_ADMINQ_COMMAND_ERROR_RESOURCE_EXHAUSTED:
290 		return -ENOMEM;
291 	case GVE_ADMINQ_COMMAND_ERROR_UNIMPLEMENTED:
292 		return -ENOTSUPP;
293 	default:
294 		dev_err(&priv->pdev->dev, "parse_aq_err: unknown status code %d\n", status);
295 		return -EINVAL;
296 	}
297 }
298 
299 /* Flushes all AQ commands currently queued and waits for them to complete.
300  * If there are failures, it will return the first error.
301  */
302 static int gve_adminq_kick_and_wait(struct gve_priv *priv)
303 {
304 	u32 tail, head;
305 	int i;
306 
307 	tail = ioread32be(&priv->reg_bar0->adminq_event_counter);
308 	head = priv->adminq_prod_cnt;
309 
310 	gve_adminq_kick_cmd(priv, head);
311 	if (!gve_adminq_wait_for_cmd(priv, head)) {
312 		dev_err(&priv->pdev->dev, "AQ commands timed out, need to reset AQ\n");
313 		priv->adminq_timeouts++;
314 		return -ENOTRECOVERABLE;
315 	}
316 
317 	for (i = tail; i < head; i++) {
318 		union gve_adminq_command *cmd;
319 		u32 status, err;
320 
321 		cmd = &priv->adminq[i & priv->adminq_mask];
322 		status = be32_to_cpu(READ_ONCE(cmd->status));
323 		err = gve_adminq_parse_err(priv, status);
324 		if (err)
325 			// Return the first error if we failed.
326 			return err;
327 	}
328 
329 	return 0;
330 }
331 
332 /* This function is not threadsafe - the caller is responsible for any
333  * necessary locks.
334  */
335 static int gve_adminq_issue_cmd(struct gve_priv *priv,
336 				union gve_adminq_command *cmd_orig)
337 {
338 	union gve_adminq_command *cmd;
339 	u32 opcode;
340 	u32 tail;
341 
342 	tail = ioread32be(&priv->reg_bar0->adminq_event_counter);
343 
344 	// Check if next command will overflow the buffer.
345 	if (((priv->adminq_prod_cnt + 1) & priv->adminq_mask) ==
346 	    (tail & priv->adminq_mask)) {
347 		int err;
348 
349 		// Flush existing commands to make room.
350 		err = gve_adminq_kick_and_wait(priv);
351 		if (err)
352 			return err;
353 
354 		// Retry.
355 		tail = ioread32be(&priv->reg_bar0->adminq_event_counter);
356 		if (((priv->adminq_prod_cnt + 1) & priv->adminq_mask) ==
357 		    (tail & priv->adminq_mask)) {
358 			// This should never happen. We just flushed the
359 			// command queue so there should be enough space.
360 			return -ENOMEM;
361 		}
362 	}
363 
364 	cmd = &priv->adminq[priv->adminq_prod_cnt & priv->adminq_mask];
365 	priv->adminq_prod_cnt++;
366 
367 	memcpy(cmd, cmd_orig, sizeof(*cmd_orig));
368 	opcode = be32_to_cpu(READ_ONCE(cmd->opcode));
369 
370 	switch (opcode) {
371 	case GVE_ADMINQ_DESCRIBE_DEVICE:
372 		priv->adminq_describe_device_cnt++;
373 		break;
374 	case GVE_ADMINQ_CONFIGURE_DEVICE_RESOURCES:
375 		priv->adminq_cfg_device_resources_cnt++;
376 		break;
377 	case GVE_ADMINQ_REGISTER_PAGE_LIST:
378 		priv->adminq_register_page_list_cnt++;
379 		break;
380 	case GVE_ADMINQ_UNREGISTER_PAGE_LIST:
381 		priv->adminq_unregister_page_list_cnt++;
382 		break;
383 	case GVE_ADMINQ_CREATE_TX_QUEUE:
384 		priv->adminq_create_tx_queue_cnt++;
385 		break;
386 	case GVE_ADMINQ_CREATE_RX_QUEUE:
387 		priv->adminq_create_rx_queue_cnt++;
388 		break;
389 	case GVE_ADMINQ_DESTROY_TX_QUEUE:
390 		priv->adminq_destroy_tx_queue_cnt++;
391 		break;
392 	case GVE_ADMINQ_DESTROY_RX_QUEUE:
393 		priv->adminq_destroy_rx_queue_cnt++;
394 		break;
395 	case GVE_ADMINQ_DECONFIGURE_DEVICE_RESOURCES:
396 		priv->adminq_dcfg_device_resources_cnt++;
397 		break;
398 	case GVE_ADMINQ_SET_DRIVER_PARAMETER:
399 		priv->adminq_set_driver_parameter_cnt++;
400 		break;
401 	case GVE_ADMINQ_REPORT_STATS:
402 		priv->adminq_report_stats_cnt++;
403 		break;
404 	case GVE_ADMINQ_REPORT_LINK_SPEED:
405 		priv->adminq_report_link_speed_cnt++;
406 		break;
407 	case GVE_ADMINQ_GET_PTYPE_MAP:
408 		priv->adminq_get_ptype_map_cnt++;
409 		break;
410 	default:
411 		dev_err(&priv->pdev->dev, "unknown AQ command opcode %d\n", opcode);
412 	}
413 
414 	return 0;
415 }
416 
417 /* This function is not threadsafe - the caller is responsible for any
418  * necessary locks.
419  * The caller is also responsible for making sure there are no commands
420  * waiting to be executed.
421  */
422 static int gve_adminq_execute_cmd(struct gve_priv *priv,
423 				  union gve_adminq_command *cmd_orig)
424 {
425 	u32 tail, head;
426 	int err;
427 
428 	tail = ioread32be(&priv->reg_bar0->adminq_event_counter);
429 	head = priv->adminq_prod_cnt;
430 	if (tail != head)
431 		// This is not a valid path
432 		return -EINVAL;
433 
434 	err = gve_adminq_issue_cmd(priv, cmd_orig);
435 	if (err)
436 		return err;
437 
438 	return gve_adminq_kick_and_wait(priv);
439 }
440 
441 /* The device specifies that the management vector can either be the first irq
442  * or the last irq. ntfy_blk_msix_base_idx indicates the first irq assigned to
443  * the ntfy blks. It if is 0 then the management vector is last, if it is 1 then
444  * the management vector is first.
445  *
446  * gve arranges the msix vectors so that the management vector is last.
447  */
448 #define GVE_NTFY_BLK_BASE_MSIX_IDX	0
449 int gve_adminq_configure_device_resources(struct gve_priv *priv,
450 					  dma_addr_t counter_array_bus_addr,
451 					  u32 num_counters,
452 					  dma_addr_t db_array_bus_addr,
453 					  u32 num_ntfy_blks)
454 {
455 	union gve_adminq_command cmd;
456 
457 	memset(&cmd, 0, sizeof(cmd));
458 	cmd.opcode = cpu_to_be32(GVE_ADMINQ_CONFIGURE_DEVICE_RESOURCES);
459 	cmd.configure_device_resources =
460 		(struct gve_adminq_configure_device_resources) {
461 		.counter_array = cpu_to_be64(counter_array_bus_addr),
462 		.num_counters = cpu_to_be32(num_counters),
463 		.irq_db_addr = cpu_to_be64(db_array_bus_addr),
464 		.num_irq_dbs = cpu_to_be32(num_ntfy_blks),
465 		.irq_db_stride = cpu_to_be32(sizeof(priv->ntfy_blocks[0])),
466 		.ntfy_blk_msix_base_idx =
467 					cpu_to_be32(GVE_NTFY_BLK_BASE_MSIX_IDX),
468 		.queue_format = priv->queue_format,
469 	};
470 
471 	return gve_adminq_execute_cmd(priv, &cmd);
472 }
473 
474 int gve_adminq_deconfigure_device_resources(struct gve_priv *priv)
475 {
476 	union gve_adminq_command cmd;
477 
478 	memset(&cmd, 0, sizeof(cmd));
479 	cmd.opcode = cpu_to_be32(GVE_ADMINQ_DECONFIGURE_DEVICE_RESOURCES);
480 
481 	return gve_adminq_execute_cmd(priv, &cmd);
482 }
483 
484 static int gve_adminq_create_tx_queue(struct gve_priv *priv, u32 queue_index)
485 {
486 	struct gve_tx_ring *tx = &priv->tx[queue_index];
487 	union gve_adminq_command cmd;
488 
489 	memset(&cmd, 0, sizeof(cmd));
490 	cmd.opcode = cpu_to_be32(GVE_ADMINQ_CREATE_TX_QUEUE);
491 	cmd.create_tx_queue = (struct gve_adminq_create_tx_queue) {
492 		.queue_id = cpu_to_be32(queue_index),
493 		.queue_resources_addr =
494 			cpu_to_be64(tx->q_resources_bus),
495 		.tx_ring_addr = cpu_to_be64(tx->bus),
496 		.ntfy_id = cpu_to_be32(tx->ntfy_id),
497 	};
498 
499 	if (gve_is_gqi(priv)) {
500 		u32 qpl_id = priv->queue_format == GVE_GQI_RDA_FORMAT ?
501 			GVE_RAW_ADDRESSING_QPL_ID : tx->tx_fifo.qpl->id;
502 
503 		cmd.create_tx_queue.queue_page_list_id = cpu_to_be32(qpl_id);
504 	} else {
505 		cmd.create_tx_queue.tx_ring_size =
506 			cpu_to_be16(priv->tx_desc_cnt);
507 		cmd.create_tx_queue.tx_comp_ring_addr =
508 			cpu_to_be64(tx->complq_bus_dqo);
509 		cmd.create_tx_queue.tx_comp_ring_size =
510 			cpu_to_be16(priv->options_dqo_rda.tx_comp_ring_entries);
511 	}
512 
513 	return gve_adminq_issue_cmd(priv, &cmd);
514 }
515 
516 int gve_adminq_create_tx_queues(struct gve_priv *priv, u32 num_queues)
517 {
518 	int err;
519 	int i;
520 
521 	for (i = 0; i < num_queues; i++) {
522 		err = gve_adminq_create_tx_queue(priv, i);
523 		if (err)
524 			return err;
525 	}
526 
527 	return gve_adminq_kick_and_wait(priv);
528 }
529 
530 static int gve_adminq_create_rx_queue(struct gve_priv *priv, u32 queue_index)
531 {
532 	struct gve_rx_ring *rx = &priv->rx[queue_index];
533 	union gve_adminq_command cmd;
534 
535 	memset(&cmd, 0, sizeof(cmd));
536 	cmd.opcode = cpu_to_be32(GVE_ADMINQ_CREATE_RX_QUEUE);
537 	cmd.create_rx_queue = (struct gve_adminq_create_rx_queue) {
538 		.queue_id = cpu_to_be32(queue_index),
539 		.ntfy_id = cpu_to_be32(rx->ntfy_id),
540 		.queue_resources_addr = cpu_to_be64(rx->q_resources_bus),
541 	};
542 
543 	if (gve_is_gqi(priv)) {
544 		u32 qpl_id = priv->queue_format == GVE_GQI_RDA_FORMAT ?
545 			GVE_RAW_ADDRESSING_QPL_ID : rx->data.qpl->id;
546 
547 		cmd.create_rx_queue.rx_desc_ring_addr =
548 			cpu_to_be64(rx->desc.bus),
549 		cmd.create_rx_queue.rx_data_ring_addr =
550 			cpu_to_be64(rx->data.data_bus),
551 		cmd.create_rx_queue.index = cpu_to_be32(queue_index);
552 		cmd.create_rx_queue.queue_page_list_id = cpu_to_be32(qpl_id);
553 		cmd.create_rx_queue.packet_buffer_size = cpu_to_be16(rx->packet_buffer_size);
554 	} else {
555 		cmd.create_rx_queue.rx_ring_size =
556 			cpu_to_be16(priv->rx_desc_cnt);
557 		cmd.create_rx_queue.rx_desc_ring_addr =
558 			cpu_to_be64(rx->dqo.complq.bus);
559 		cmd.create_rx_queue.rx_data_ring_addr =
560 			cpu_to_be64(rx->dqo.bufq.bus);
561 		cmd.create_rx_queue.packet_buffer_size =
562 			cpu_to_be16(priv->data_buffer_size_dqo);
563 		cmd.create_rx_queue.rx_buff_ring_size =
564 			cpu_to_be16(priv->options_dqo_rda.rx_buff_ring_entries);
565 		cmd.create_rx_queue.enable_rsc =
566 			!!(priv->dev->features & NETIF_F_LRO);
567 	}
568 
569 	return gve_adminq_issue_cmd(priv, &cmd);
570 }
571 
572 int gve_adminq_create_rx_queues(struct gve_priv *priv, u32 num_queues)
573 {
574 	int err;
575 	int i;
576 
577 	for (i = 0; i < num_queues; i++) {
578 		err = gve_adminq_create_rx_queue(priv, i);
579 		if (err)
580 			return err;
581 	}
582 
583 	return gve_adminq_kick_and_wait(priv);
584 }
585 
586 static int gve_adminq_destroy_tx_queue(struct gve_priv *priv, u32 queue_index)
587 {
588 	union gve_adminq_command cmd;
589 	int err;
590 
591 	memset(&cmd, 0, sizeof(cmd));
592 	cmd.opcode = cpu_to_be32(GVE_ADMINQ_DESTROY_TX_QUEUE);
593 	cmd.destroy_tx_queue = (struct gve_adminq_destroy_tx_queue) {
594 		.queue_id = cpu_to_be32(queue_index),
595 	};
596 
597 	err = gve_adminq_issue_cmd(priv, &cmd);
598 	if (err)
599 		return err;
600 
601 	return 0;
602 }
603 
604 int gve_adminq_destroy_tx_queues(struct gve_priv *priv, u32 num_queues)
605 {
606 	int err;
607 	int i;
608 
609 	for (i = 0; i < num_queues; i++) {
610 		err = gve_adminq_destroy_tx_queue(priv, i);
611 		if (err)
612 			return err;
613 	}
614 
615 	return gve_adminq_kick_and_wait(priv);
616 }
617 
618 static int gve_adminq_destroy_rx_queue(struct gve_priv *priv, u32 queue_index)
619 {
620 	union gve_adminq_command cmd;
621 	int err;
622 
623 	memset(&cmd, 0, sizeof(cmd));
624 	cmd.opcode = cpu_to_be32(GVE_ADMINQ_DESTROY_RX_QUEUE);
625 	cmd.destroy_rx_queue = (struct gve_adminq_destroy_rx_queue) {
626 		.queue_id = cpu_to_be32(queue_index),
627 	};
628 
629 	err = gve_adminq_issue_cmd(priv, &cmd);
630 	if (err)
631 		return err;
632 
633 	return 0;
634 }
635 
636 int gve_adminq_destroy_rx_queues(struct gve_priv *priv, u32 num_queues)
637 {
638 	int err;
639 	int i;
640 
641 	for (i = 0; i < num_queues; i++) {
642 		err = gve_adminq_destroy_rx_queue(priv, i);
643 		if (err)
644 			return err;
645 	}
646 
647 	return gve_adminq_kick_and_wait(priv);
648 }
649 
650 static int gve_set_desc_cnt(struct gve_priv *priv,
651 			    struct gve_device_descriptor *descriptor)
652 {
653 	priv->tx_desc_cnt = be16_to_cpu(descriptor->tx_queue_entries);
654 	if (priv->tx_desc_cnt * sizeof(priv->tx->desc[0]) < PAGE_SIZE) {
655 		dev_err(&priv->pdev->dev, "Tx desc count %d too low\n",
656 			priv->tx_desc_cnt);
657 		return -EINVAL;
658 	}
659 	priv->rx_desc_cnt = be16_to_cpu(descriptor->rx_queue_entries);
660 	if (priv->rx_desc_cnt * sizeof(priv->rx->desc.desc_ring[0])
661 	    < PAGE_SIZE) {
662 		dev_err(&priv->pdev->dev, "Rx desc count %d too low\n",
663 			priv->rx_desc_cnt);
664 		return -EINVAL;
665 	}
666 	return 0;
667 }
668 
669 static int
670 gve_set_desc_cnt_dqo(struct gve_priv *priv,
671 		     const struct gve_device_descriptor *descriptor,
672 		     const struct gve_device_option_dqo_rda *dev_op_dqo_rda)
673 {
674 	priv->tx_desc_cnt = be16_to_cpu(descriptor->tx_queue_entries);
675 	priv->options_dqo_rda.tx_comp_ring_entries =
676 		be16_to_cpu(dev_op_dqo_rda->tx_comp_ring_entries);
677 	priv->rx_desc_cnt = be16_to_cpu(descriptor->rx_queue_entries);
678 	priv->options_dqo_rda.rx_buff_ring_entries =
679 		be16_to_cpu(dev_op_dqo_rda->rx_buff_ring_entries);
680 
681 	return 0;
682 }
683 
684 static void gve_enable_supported_features(struct gve_priv *priv,
685 					  u32 supported_features_mask,
686 					  const struct gve_device_option_jumbo_frames
687 						  *dev_op_jumbo_frames)
688 {
689 	/* Before control reaches this point, the page-size-capped max MTU from
690 	 * the gve_device_descriptor field has already been stored in
691 	 * priv->dev->max_mtu. We overwrite it with the true max MTU below.
692 	 */
693 	if (dev_op_jumbo_frames &&
694 	    (supported_features_mask & GVE_SUP_JUMBO_FRAMES_MASK)) {
695 		dev_info(&priv->pdev->dev,
696 			 "JUMBO FRAMES device option enabled.\n");
697 		priv->dev->max_mtu = be16_to_cpu(dev_op_jumbo_frames->max_mtu);
698 	}
699 }
700 
701 int gve_adminq_describe_device(struct gve_priv *priv)
702 {
703 	struct gve_device_option_jumbo_frames *dev_op_jumbo_frames = NULL;
704 	struct gve_device_option_gqi_rda *dev_op_gqi_rda = NULL;
705 	struct gve_device_option_gqi_qpl *dev_op_gqi_qpl = NULL;
706 	struct gve_device_option_dqo_rda *dev_op_dqo_rda = NULL;
707 	struct gve_device_descriptor *descriptor;
708 	u32 supported_features_mask = 0;
709 	union gve_adminq_command cmd;
710 	dma_addr_t descriptor_bus;
711 	int err = 0;
712 	u8 *mac;
713 	u16 mtu;
714 
715 	memset(&cmd, 0, sizeof(cmd));
716 	descriptor = dma_alloc_coherent(&priv->pdev->dev, PAGE_SIZE,
717 					&descriptor_bus, GFP_KERNEL);
718 	if (!descriptor)
719 		return -ENOMEM;
720 	cmd.opcode = cpu_to_be32(GVE_ADMINQ_DESCRIBE_DEVICE);
721 	cmd.describe_device.device_descriptor_addr =
722 						cpu_to_be64(descriptor_bus);
723 	cmd.describe_device.device_descriptor_version =
724 			cpu_to_be32(GVE_ADMINQ_DEVICE_DESCRIPTOR_VERSION);
725 	cmd.describe_device.available_length = cpu_to_be32(PAGE_SIZE);
726 
727 	err = gve_adminq_execute_cmd(priv, &cmd);
728 	if (err)
729 		goto free_device_descriptor;
730 
731 	err = gve_process_device_options(priv, descriptor, &dev_op_gqi_rda,
732 					 &dev_op_gqi_qpl, &dev_op_dqo_rda,
733 					 &dev_op_jumbo_frames);
734 	if (err)
735 		goto free_device_descriptor;
736 
737 	/* If the GQI_RAW_ADDRESSING option is not enabled and the queue format
738 	 * is not set to GqiRda, choose the queue format in a priority order:
739 	 * DqoRda, GqiRda, GqiQpl. Use GqiQpl as default.
740 	 */
741 	if (priv->queue_format == GVE_GQI_RDA_FORMAT) {
742 		dev_info(&priv->pdev->dev,
743 			 "Driver is running with GQI RDA queue format.\n");
744 	} else if (dev_op_dqo_rda) {
745 		priv->queue_format = GVE_DQO_RDA_FORMAT;
746 		dev_info(&priv->pdev->dev,
747 			 "Driver is running with DQO RDA queue format.\n");
748 		supported_features_mask =
749 			be32_to_cpu(dev_op_dqo_rda->supported_features_mask);
750 	} else if (dev_op_gqi_rda) {
751 		priv->queue_format = GVE_GQI_RDA_FORMAT;
752 		dev_info(&priv->pdev->dev,
753 			 "Driver is running with GQI RDA queue format.\n");
754 		supported_features_mask =
755 			be32_to_cpu(dev_op_gqi_rda->supported_features_mask);
756 	} else {
757 		priv->queue_format = GVE_GQI_QPL_FORMAT;
758 		if (dev_op_gqi_qpl)
759 			supported_features_mask =
760 				be32_to_cpu(dev_op_gqi_qpl->supported_features_mask);
761 		dev_info(&priv->pdev->dev,
762 			 "Driver is running with GQI QPL queue format.\n");
763 	}
764 	if (gve_is_gqi(priv)) {
765 		err = gve_set_desc_cnt(priv, descriptor);
766 	} else {
767 		/* DQO supports LRO. */
768 		priv->dev->hw_features |= NETIF_F_LRO;
769 		err = gve_set_desc_cnt_dqo(priv, descriptor, dev_op_dqo_rda);
770 	}
771 	if (err)
772 		goto free_device_descriptor;
773 
774 	priv->max_registered_pages =
775 				be64_to_cpu(descriptor->max_registered_pages);
776 	mtu = be16_to_cpu(descriptor->mtu);
777 	if (mtu < ETH_MIN_MTU) {
778 		dev_err(&priv->pdev->dev, "MTU %d below minimum MTU\n", mtu);
779 		err = -EINVAL;
780 		goto free_device_descriptor;
781 	}
782 	priv->dev->max_mtu = mtu;
783 	priv->num_event_counters = be16_to_cpu(descriptor->counters);
784 	eth_hw_addr_set(priv->dev, descriptor->mac);
785 	mac = descriptor->mac;
786 	dev_info(&priv->pdev->dev, "MAC addr: %pM\n", mac);
787 	priv->tx_pages_per_qpl = be16_to_cpu(descriptor->tx_pages_per_qpl);
788 	priv->rx_data_slot_cnt = be16_to_cpu(descriptor->rx_pages_per_qpl);
789 
790 	if (gve_is_gqi(priv) && priv->rx_data_slot_cnt < priv->rx_desc_cnt) {
791 		dev_err(&priv->pdev->dev, "rx_data_slot_cnt cannot be smaller than rx_desc_cnt, setting rx_desc_cnt down to %d.\n",
792 			priv->rx_data_slot_cnt);
793 		priv->rx_desc_cnt = priv->rx_data_slot_cnt;
794 	}
795 	priv->default_num_queues = be16_to_cpu(descriptor->default_num_queues);
796 
797 	gve_enable_supported_features(priv, supported_features_mask,
798 				      dev_op_jumbo_frames);
799 
800 free_device_descriptor:
801 	dma_free_coherent(&priv->pdev->dev, PAGE_SIZE, descriptor,
802 			  descriptor_bus);
803 	return err;
804 }
805 
806 int gve_adminq_register_page_list(struct gve_priv *priv,
807 				  struct gve_queue_page_list *qpl)
808 {
809 	struct device *hdev = &priv->pdev->dev;
810 	u32 num_entries = qpl->num_entries;
811 	u32 size = num_entries * sizeof(qpl->page_buses[0]);
812 	union gve_adminq_command cmd;
813 	dma_addr_t page_list_bus;
814 	__be64 *page_list;
815 	int err;
816 	int i;
817 
818 	memset(&cmd, 0, sizeof(cmd));
819 	page_list = dma_alloc_coherent(hdev, size, &page_list_bus, GFP_KERNEL);
820 	if (!page_list)
821 		return -ENOMEM;
822 
823 	for (i = 0; i < num_entries; i++)
824 		page_list[i] = cpu_to_be64(qpl->page_buses[i]);
825 
826 	cmd.opcode = cpu_to_be32(GVE_ADMINQ_REGISTER_PAGE_LIST);
827 	cmd.reg_page_list = (struct gve_adminq_register_page_list) {
828 		.page_list_id = cpu_to_be32(qpl->id),
829 		.num_pages = cpu_to_be32(num_entries),
830 		.page_address_list_addr = cpu_to_be64(page_list_bus),
831 	};
832 
833 	err = gve_adminq_execute_cmd(priv, &cmd);
834 	dma_free_coherent(hdev, size, page_list, page_list_bus);
835 	return err;
836 }
837 
838 int gve_adminq_unregister_page_list(struct gve_priv *priv, u32 page_list_id)
839 {
840 	union gve_adminq_command cmd;
841 
842 	memset(&cmd, 0, sizeof(cmd));
843 	cmd.opcode = cpu_to_be32(GVE_ADMINQ_UNREGISTER_PAGE_LIST);
844 	cmd.unreg_page_list = (struct gve_adminq_unregister_page_list) {
845 		.page_list_id = cpu_to_be32(page_list_id),
846 	};
847 
848 	return gve_adminq_execute_cmd(priv, &cmd);
849 }
850 
851 int gve_adminq_set_mtu(struct gve_priv *priv, u64 mtu)
852 {
853 	union gve_adminq_command cmd;
854 
855 	memset(&cmd, 0, sizeof(cmd));
856 	cmd.opcode = cpu_to_be32(GVE_ADMINQ_SET_DRIVER_PARAMETER);
857 	cmd.set_driver_param = (struct gve_adminq_set_driver_parameter) {
858 		.parameter_type = cpu_to_be32(GVE_SET_PARAM_MTU),
859 		.parameter_value = cpu_to_be64(mtu),
860 	};
861 
862 	return gve_adminq_execute_cmd(priv, &cmd);
863 }
864 
865 int gve_adminq_report_stats(struct gve_priv *priv, u64 stats_report_len,
866 			    dma_addr_t stats_report_addr, u64 interval)
867 {
868 	union gve_adminq_command cmd;
869 
870 	memset(&cmd, 0, sizeof(cmd));
871 	cmd.opcode = cpu_to_be32(GVE_ADMINQ_REPORT_STATS);
872 	cmd.report_stats = (struct gve_adminq_report_stats) {
873 		.stats_report_len = cpu_to_be64(stats_report_len),
874 		.stats_report_addr = cpu_to_be64(stats_report_addr),
875 		.interval = cpu_to_be64(interval),
876 	};
877 
878 	return gve_adminq_execute_cmd(priv, &cmd);
879 }
880 
881 int gve_adminq_report_link_speed(struct gve_priv *priv)
882 {
883 	union gve_adminq_command gvnic_cmd;
884 	dma_addr_t link_speed_region_bus;
885 	__be64 *link_speed_region;
886 	int err;
887 
888 	link_speed_region =
889 		dma_alloc_coherent(&priv->pdev->dev, sizeof(*link_speed_region),
890 				   &link_speed_region_bus, GFP_KERNEL);
891 
892 	if (!link_speed_region)
893 		return -ENOMEM;
894 
895 	memset(&gvnic_cmd, 0, sizeof(gvnic_cmd));
896 	gvnic_cmd.opcode = cpu_to_be32(GVE_ADMINQ_REPORT_LINK_SPEED);
897 	gvnic_cmd.report_link_speed.link_speed_address =
898 		cpu_to_be64(link_speed_region_bus);
899 
900 	err = gve_adminq_execute_cmd(priv, &gvnic_cmd);
901 
902 	priv->link_speed = be64_to_cpu(*link_speed_region);
903 	dma_free_coherent(&priv->pdev->dev, sizeof(*link_speed_region), link_speed_region,
904 			  link_speed_region_bus);
905 	return err;
906 }
907 
908 int gve_adminq_get_ptype_map_dqo(struct gve_priv *priv,
909 				 struct gve_ptype_lut *ptype_lut)
910 {
911 	struct gve_ptype_map *ptype_map;
912 	union gve_adminq_command cmd;
913 	dma_addr_t ptype_map_bus;
914 	int err = 0;
915 	int i;
916 
917 	memset(&cmd, 0, sizeof(cmd));
918 	ptype_map = dma_alloc_coherent(&priv->pdev->dev, sizeof(*ptype_map),
919 				       &ptype_map_bus, GFP_KERNEL);
920 	if (!ptype_map)
921 		return -ENOMEM;
922 
923 	cmd.opcode = cpu_to_be32(GVE_ADMINQ_GET_PTYPE_MAP);
924 	cmd.get_ptype_map = (struct gve_adminq_get_ptype_map) {
925 		.ptype_map_len = cpu_to_be64(sizeof(*ptype_map)),
926 		.ptype_map_addr = cpu_to_be64(ptype_map_bus),
927 	};
928 
929 	err = gve_adminq_execute_cmd(priv, &cmd);
930 	if (err)
931 		goto err;
932 
933 	/* Populate ptype_lut. */
934 	for (i = 0; i < GVE_NUM_PTYPES; i++) {
935 		ptype_lut->ptypes[i].l3_type =
936 			ptype_map->ptypes[i].l3_type;
937 		ptype_lut->ptypes[i].l4_type =
938 			ptype_map->ptypes[i].l4_type;
939 	}
940 err:
941 	dma_free_coherent(&priv->pdev->dev, sizeof(*ptype_map), ptype_map,
942 			  ptype_map_bus);
943 	return err;
944 }
945