1 // SPDX-License-Identifier: (GPL-2.0 OR MIT)
2 /* Google virtual Ethernet (gve) driver
3  *
4  * Copyright (C) 2015-2021 Google, Inc.
5  */
6 
7 #include <linux/etherdevice.h>
8 #include <linux/pci.h>
9 #include "gve.h"
10 #include "gve_adminq.h"
11 #include "gve_register.h"
12 
13 #define GVE_MAX_ADMINQ_RELEASE_CHECK	500
14 #define GVE_ADMINQ_SLEEP_LEN		20
15 #define GVE_MAX_ADMINQ_EVENT_COUNTER_CHECK	100
16 
17 #define GVE_DEVICE_OPTION_ERROR_FMT "%s option error:\n" \
18 "Expected: length=%d, feature_mask=%x.\n" \
19 "Actual: length=%d, feature_mask=%x.\n"
20 
21 #define GVE_DEVICE_OPTION_TOO_BIG_FMT "Length of %s option larger than expected. Possible older version of guest driver.\n"
22 
23 static
gve_get_next_option(struct gve_device_descriptor * descriptor,struct gve_device_option * option)24 struct gve_device_option *gve_get_next_option(struct gve_device_descriptor *descriptor,
25 					      struct gve_device_option *option)
26 {
27 	void *option_end, *descriptor_end;
28 
29 	option_end = (void *)(option + 1) + be16_to_cpu(option->option_length);
30 	descriptor_end = (void *)descriptor + be16_to_cpu(descriptor->total_length);
31 
32 	return option_end > descriptor_end ? NULL : (struct gve_device_option *)option_end;
33 }
34 
35 static
gve_parse_device_option(struct gve_priv * priv,struct gve_device_descriptor * device_descriptor,struct gve_device_option * option,struct gve_device_option_gqi_rda ** dev_op_gqi_rda,struct gve_device_option_gqi_qpl ** dev_op_gqi_qpl,struct gve_device_option_dqo_rda ** dev_op_dqo_rda,struct gve_device_option_jumbo_frames ** dev_op_jumbo_frames,struct gve_device_option_dqo_qpl ** dev_op_dqo_qpl)36 void gve_parse_device_option(struct gve_priv *priv,
37 			     struct gve_device_descriptor *device_descriptor,
38 			     struct gve_device_option *option,
39 			     struct gve_device_option_gqi_rda **dev_op_gqi_rda,
40 			     struct gve_device_option_gqi_qpl **dev_op_gqi_qpl,
41 			     struct gve_device_option_dqo_rda **dev_op_dqo_rda,
42 			     struct gve_device_option_jumbo_frames **dev_op_jumbo_frames,
43 			     struct gve_device_option_dqo_qpl **dev_op_dqo_qpl)
44 {
45 	u32 req_feat_mask = be32_to_cpu(option->required_features_mask);
46 	u16 option_length = be16_to_cpu(option->option_length);
47 	u16 option_id = be16_to_cpu(option->option_id);
48 
49 	/* If the length or feature mask doesn't match, continue without
50 	 * enabling the feature.
51 	 */
52 	switch (option_id) {
53 	case GVE_DEV_OPT_ID_GQI_RAW_ADDRESSING:
54 		if (option_length != GVE_DEV_OPT_LEN_GQI_RAW_ADDRESSING ||
55 		    req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RAW_ADDRESSING) {
56 			dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
57 				 "Raw Addressing",
58 				 GVE_DEV_OPT_LEN_GQI_RAW_ADDRESSING,
59 				 GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RAW_ADDRESSING,
60 				 option_length, req_feat_mask);
61 			break;
62 		}
63 
64 		dev_info(&priv->pdev->dev,
65 			 "Gqi raw addressing device option enabled.\n");
66 		priv->queue_format = GVE_GQI_RDA_FORMAT;
67 		break;
68 	case GVE_DEV_OPT_ID_GQI_RDA:
69 		if (option_length < sizeof(**dev_op_gqi_rda) ||
70 		    req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RDA) {
71 			dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
72 				 "GQI RDA", (int)sizeof(**dev_op_gqi_rda),
73 				 GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RDA,
74 				 option_length, req_feat_mask);
75 			break;
76 		}
77 
78 		if (option_length > sizeof(**dev_op_gqi_rda)) {
79 			dev_warn(&priv->pdev->dev,
80 				 GVE_DEVICE_OPTION_TOO_BIG_FMT, "GQI RDA");
81 		}
82 		*dev_op_gqi_rda = (void *)(option + 1);
83 		break;
84 	case GVE_DEV_OPT_ID_GQI_QPL:
85 		if (option_length < sizeof(**dev_op_gqi_qpl) ||
86 		    req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_GQI_QPL) {
87 			dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
88 				 "GQI QPL", (int)sizeof(**dev_op_gqi_qpl),
89 				 GVE_DEV_OPT_REQ_FEAT_MASK_GQI_QPL,
90 				 option_length, req_feat_mask);
91 			break;
92 		}
93 
94 		if (option_length > sizeof(**dev_op_gqi_qpl)) {
95 			dev_warn(&priv->pdev->dev,
96 				 GVE_DEVICE_OPTION_TOO_BIG_FMT, "GQI QPL");
97 		}
98 		*dev_op_gqi_qpl = (void *)(option + 1);
99 		break;
100 	case GVE_DEV_OPT_ID_DQO_RDA:
101 		if (option_length < sizeof(**dev_op_dqo_rda) ||
102 		    req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_DQO_RDA) {
103 			dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
104 				 "DQO RDA", (int)sizeof(**dev_op_dqo_rda),
105 				 GVE_DEV_OPT_REQ_FEAT_MASK_DQO_RDA,
106 				 option_length, req_feat_mask);
107 			break;
108 		}
109 
110 		if (option_length > sizeof(**dev_op_dqo_rda)) {
111 			dev_warn(&priv->pdev->dev,
112 				 GVE_DEVICE_OPTION_TOO_BIG_FMT, "DQO RDA");
113 		}
114 		*dev_op_dqo_rda = (void *)(option + 1);
115 		break;
116 	case GVE_DEV_OPT_ID_DQO_QPL:
117 		if (option_length < sizeof(**dev_op_dqo_qpl) ||
118 		    req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_DQO_QPL) {
119 			dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
120 				 "DQO QPL", (int)sizeof(**dev_op_dqo_qpl),
121 				 GVE_DEV_OPT_REQ_FEAT_MASK_DQO_QPL,
122 				 option_length, req_feat_mask);
123 			break;
124 		}
125 
126 		if (option_length > sizeof(**dev_op_dqo_qpl)) {
127 			dev_warn(&priv->pdev->dev,
128 				 GVE_DEVICE_OPTION_TOO_BIG_FMT, "DQO QPL");
129 		}
130 		*dev_op_dqo_qpl = (void *)(option + 1);
131 		break;
132 	case GVE_DEV_OPT_ID_JUMBO_FRAMES:
133 		if (option_length < sizeof(**dev_op_jumbo_frames) ||
134 		    req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_JUMBO_FRAMES) {
135 			dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
136 				 "Jumbo Frames",
137 				 (int)sizeof(**dev_op_jumbo_frames),
138 				 GVE_DEV_OPT_REQ_FEAT_MASK_JUMBO_FRAMES,
139 				 option_length, req_feat_mask);
140 			break;
141 		}
142 
143 		if (option_length > sizeof(**dev_op_jumbo_frames)) {
144 			dev_warn(&priv->pdev->dev,
145 				 GVE_DEVICE_OPTION_TOO_BIG_FMT,
146 				 "Jumbo Frames");
147 		}
148 		*dev_op_jumbo_frames = (void *)(option + 1);
149 		break;
150 	default:
151 		/* If we don't recognize the option just continue
152 		 * without doing anything.
153 		 */
154 		dev_dbg(&priv->pdev->dev, "Unrecognized device option 0x%hx not enabled.\n",
155 			option_id);
156 	}
157 }
158 
159 /* Process all device options for a given describe device call. */
160 static int
gve_process_device_options(struct gve_priv * priv,struct gve_device_descriptor * descriptor,struct gve_device_option_gqi_rda ** dev_op_gqi_rda,struct gve_device_option_gqi_qpl ** dev_op_gqi_qpl,struct gve_device_option_dqo_rda ** dev_op_dqo_rda,struct gve_device_option_jumbo_frames ** dev_op_jumbo_frames,struct gve_device_option_dqo_qpl ** dev_op_dqo_qpl)161 gve_process_device_options(struct gve_priv *priv,
162 			   struct gve_device_descriptor *descriptor,
163 			   struct gve_device_option_gqi_rda **dev_op_gqi_rda,
164 			   struct gve_device_option_gqi_qpl **dev_op_gqi_qpl,
165 			   struct gve_device_option_dqo_rda **dev_op_dqo_rda,
166 			   struct gve_device_option_jumbo_frames **dev_op_jumbo_frames,
167 			   struct gve_device_option_dqo_qpl **dev_op_dqo_qpl)
168 {
169 	const int num_options = be16_to_cpu(descriptor->num_device_options);
170 	struct gve_device_option *dev_opt;
171 	int i;
172 
173 	/* The options struct directly follows the device descriptor. */
174 	dev_opt = (void *)(descriptor + 1);
175 	for (i = 0; i < num_options; i++) {
176 		struct gve_device_option *next_opt;
177 
178 		next_opt = gve_get_next_option(descriptor, dev_opt);
179 		if (!next_opt) {
180 			dev_err(&priv->dev->dev,
181 				"options exceed device_descriptor's total length.\n");
182 			return -EINVAL;
183 		}
184 
185 		gve_parse_device_option(priv, descriptor, dev_opt,
186 					dev_op_gqi_rda, dev_op_gqi_qpl,
187 					dev_op_dqo_rda, dev_op_jumbo_frames,
188 					dev_op_dqo_qpl);
189 		dev_opt = next_opt;
190 	}
191 
192 	return 0;
193 }
194 
gve_adminq_alloc(struct device * dev,struct gve_priv * priv)195 int gve_adminq_alloc(struct device *dev, struct gve_priv *priv)
196 {
197 	priv->adminq = dma_alloc_coherent(dev, PAGE_SIZE,
198 					  &priv->adminq_bus_addr, GFP_KERNEL);
199 	if (unlikely(!priv->adminq))
200 		return -ENOMEM;
201 
202 	priv->adminq_mask = (PAGE_SIZE / sizeof(union gve_adminq_command)) - 1;
203 	priv->adminq_prod_cnt = 0;
204 	priv->adminq_cmd_fail = 0;
205 	priv->adminq_timeouts = 0;
206 	priv->adminq_describe_device_cnt = 0;
207 	priv->adminq_cfg_device_resources_cnt = 0;
208 	priv->adminq_register_page_list_cnt = 0;
209 	priv->adminq_unregister_page_list_cnt = 0;
210 	priv->adminq_create_tx_queue_cnt = 0;
211 	priv->adminq_create_rx_queue_cnt = 0;
212 	priv->adminq_destroy_tx_queue_cnt = 0;
213 	priv->adminq_destroy_rx_queue_cnt = 0;
214 	priv->adminq_dcfg_device_resources_cnt = 0;
215 	priv->adminq_set_driver_parameter_cnt = 0;
216 	priv->adminq_report_stats_cnt = 0;
217 	priv->adminq_report_link_speed_cnt = 0;
218 	priv->adminq_get_ptype_map_cnt = 0;
219 
220 	/* Setup Admin queue with the device */
221 	iowrite32be(priv->adminq_bus_addr / PAGE_SIZE,
222 		    &priv->reg_bar0->adminq_pfn);
223 
224 	gve_set_admin_queue_ok(priv);
225 	return 0;
226 }
227 
gve_adminq_release(struct gve_priv * priv)228 void gve_adminq_release(struct gve_priv *priv)
229 {
230 	int i = 0;
231 
232 	/* Tell the device the adminq is leaving */
233 	iowrite32be(0x0, &priv->reg_bar0->adminq_pfn);
234 	while (ioread32be(&priv->reg_bar0->adminq_pfn)) {
235 		/* If this is reached the device is unrecoverable and still
236 		 * holding memory. Continue looping to avoid memory corruption,
237 		 * but WARN so it is visible what is going on.
238 		 */
239 		if (i == GVE_MAX_ADMINQ_RELEASE_CHECK)
240 			WARN(1, "Unrecoverable platform error!");
241 		i++;
242 		msleep(GVE_ADMINQ_SLEEP_LEN);
243 	}
244 	gve_clear_device_rings_ok(priv);
245 	gve_clear_device_resources_ok(priv);
246 	gve_clear_admin_queue_ok(priv);
247 }
248 
gve_adminq_free(struct device * dev,struct gve_priv * priv)249 void gve_adminq_free(struct device *dev, struct gve_priv *priv)
250 {
251 	if (!gve_get_admin_queue_ok(priv))
252 		return;
253 	gve_adminq_release(priv);
254 	dma_free_coherent(dev, PAGE_SIZE, priv->adminq, priv->adminq_bus_addr);
255 	gve_clear_admin_queue_ok(priv);
256 }
257 
gve_adminq_kick_cmd(struct gve_priv * priv,u32 prod_cnt)258 static void gve_adminq_kick_cmd(struct gve_priv *priv, u32 prod_cnt)
259 {
260 	iowrite32be(prod_cnt, &priv->reg_bar0->adminq_doorbell);
261 }
262 
gve_adminq_wait_for_cmd(struct gve_priv * priv,u32 prod_cnt)263 static bool gve_adminq_wait_for_cmd(struct gve_priv *priv, u32 prod_cnt)
264 {
265 	int i;
266 
267 	for (i = 0; i < GVE_MAX_ADMINQ_EVENT_COUNTER_CHECK; i++) {
268 		if (ioread32be(&priv->reg_bar0->adminq_event_counter)
269 		    == prod_cnt)
270 			return true;
271 		msleep(GVE_ADMINQ_SLEEP_LEN);
272 	}
273 
274 	return false;
275 }
276 
gve_adminq_parse_err(struct gve_priv * priv,u32 status)277 static int gve_adminq_parse_err(struct gve_priv *priv, u32 status)
278 {
279 	if (status != GVE_ADMINQ_COMMAND_PASSED &&
280 	    status != GVE_ADMINQ_COMMAND_UNSET) {
281 		dev_err(&priv->pdev->dev, "AQ command failed with status %d\n", status);
282 		priv->adminq_cmd_fail++;
283 	}
284 	switch (status) {
285 	case GVE_ADMINQ_COMMAND_PASSED:
286 		return 0;
287 	case GVE_ADMINQ_COMMAND_UNSET:
288 		dev_err(&priv->pdev->dev, "parse_aq_err: err and status both unset, this should not be possible.\n");
289 		return -EINVAL;
290 	case GVE_ADMINQ_COMMAND_ERROR_ABORTED:
291 	case GVE_ADMINQ_COMMAND_ERROR_CANCELLED:
292 	case GVE_ADMINQ_COMMAND_ERROR_DATALOSS:
293 	case GVE_ADMINQ_COMMAND_ERROR_FAILED_PRECONDITION:
294 	case GVE_ADMINQ_COMMAND_ERROR_UNAVAILABLE:
295 		return -EAGAIN;
296 	case GVE_ADMINQ_COMMAND_ERROR_ALREADY_EXISTS:
297 	case GVE_ADMINQ_COMMAND_ERROR_INTERNAL_ERROR:
298 	case GVE_ADMINQ_COMMAND_ERROR_INVALID_ARGUMENT:
299 	case GVE_ADMINQ_COMMAND_ERROR_NOT_FOUND:
300 	case GVE_ADMINQ_COMMAND_ERROR_OUT_OF_RANGE:
301 	case GVE_ADMINQ_COMMAND_ERROR_UNKNOWN_ERROR:
302 		return -EINVAL;
303 	case GVE_ADMINQ_COMMAND_ERROR_DEADLINE_EXCEEDED:
304 		return -ETIME;
305 	case GVE_ADMINQ_COMMAND_ERROR_PERMISSION_DENIED:
306 	case GVE_ADMINQ_COMMAND_ERROR_UNAUTHENTICATED:
307 		return -EACCES;
308 	case GVE_ADMINQ_COMMAND_ERROR_RESOURCE_EXHAUSTED:
309 		return -ENOMEM;
310 	case GVE_ADMINQ_COMMAND_ERROR_UNIMPLEMENTED:
311 		return -EOPNOTSUPP;
312 	default:
313 		dev_err(&priv->pdev->dev, "parse_aq_err: unknown status code %d\n", status);
314 		return -EINVAL;
315 	}
316 }
317 
318 /* Flushes all AQ commands currently queued and waits for them to complete.
319  * If there are failures, it will return the first error.
320  */
gve_adminq_kick_and_wait(struct gve_priv * priv)321 static int gve_adminq_kick_and_wait(struct gve_priv *priv)
322 {
323 	int tail, head;
324 	int i;
325 
326 	tail = ioread32be(&priv->reg_bar0->adminq_event_counter);
327 	head = priv->adminq_prod_cnt;
328 
329 	gve_adminq_kick_cmd(priv, head);
330 	if (!gve_adminq_wait_for_cmd(priv, head)) {
331 		dev_err(&priv->pdev->dev, "AQ commands timed out, need to reset AQ\n");
332 		priv->adminq_timeouts++;
333 		return -ENOTRECOVERABLE;
334 	}
335 
336 	for (i = tail; i < head; i++) {
337 		union gve_adminq_command *cmd;
338 		u32 status, err;
339 
340 		cmd = &priv->adminq[i & priv->adminq_mask];
341 		status = be32_to_cpu(READ_ONCE(cmd->status));
342 		err = gve_adminq_parse_err(priv, status);
343 		if (err)
344 			// Return the first error if we failed.
345 			return err;
346 	}
347 
348 	return 0;
349 }
350 
351 /* This function is not threadsafe - the caller is responsible for any
352  * necessary locks.
353  */
gve_adminq_issue_cmd(struct gve_priv * priv,union gve_adminq_command * cmd_orig)354 static int gve_adminq_issue_cmd(struct gve_priv *priv,
355 				union gve_adminq_command *cmd_orig)
356 {
357 	union gve_adminq_command *cmd;
358 	u32 opcode;
359 	u32 tail;
360 
361 	tail = ioread32be(&priv->reg_bar0->adminq_event_counter);
362 
363 	// Check if next command will overflow the buffer.
364 	if (((priv->adminq_prod_cnt + 1) & priv->adminq_mask) ==
365 	    (tail & priv->adminq_mask)) {
366 		int err;
367 
368 		// Flush existing commands to make room.
369 		err = gve_adminq_kick_and_wait(priv);
370 		if (err)
371 			return err;
372 
373 		// Retry.
374 		tail = ioread32be(&priv->reg_bar0->adminq_event_counter);
375 		if (((priv->adminq_prod_cnt + 1) & priv->adminq_mask) ==
376 		    (tail & priv->adminq_mask)) {
377 			// This should never happen. We just flushed the
378 			// command queue so there should be enough space.
379 			return -ENOMEM;
380 		}
381 	}
382 
383 	cmd = &priv->adminq[priv->adminq_prod_cnt & priv->adminq_mask];
384 	priv->adminq_prod_cnt++;
385 
386 	memcpy(cmd, cmd_orig, sizeof(*cmd_orig));
387 	opcode = be32_to_cpu(READ_ONCE(cmd->opcode));
388 
389 	switch (opcode) {
390 	case GVE_ADMINQ_DESCRIBE_DEVICE:
391 		priv->adminq_describe_device_cnt++;
392 		break;
393 	case GVE_ADMINQ_CONFIGURE_DEVICE_RESOURCES:
394 		priv->adminq_cfg_device_resources_cnt++;
395 		break;
396 	case GVE_ADMINQ_REGISTER_PAGE_LIST:
397 		priv->adminq_register_page_list_cnt++;
398 		break;
399 	case GVE_ADMINQ_UNREGISTER_PAGE_LIST:
400 		priv->adminq_unregister_page_list_cnt++;
401 		break;
402 	case GVE_ADMINQ_CREATE_TX_QUEUE:
403 		priv->adminq_create_tx_queue_cnt++;
404 		break;
405 	case GVE_ADMINQ_CREATE_RX_QUEUE:
406 		priv->adminq_create_rx_queue_cnt++;
407 		break;
408 	case GVE_ADMINQ_DESTROY_TX_QUEUE:
409 		priv->adminq_destroy_tx_queue_cnt++;
410 		break;
411 	case GVE_ADMINQ_DESTROY_RX_QUEUE:
412 		priv->adminq_destroy_rx_queue_cnt++;
413 		break;
414 	case GVE_ADMINQ_DECONFIGURE_DEVICE_RESOURCES:
415 		priv->adminq_dcfg_device_resources_cnt++;
416 		break;
417 	case GVE_ADMINQ_SET_DRIVER_PARAMETER:
418 		priv->adminq_set_driver_parameter_cnt++;
419 		break;
420 	case GVE_ADMINQ_REPORT_STATS:
421 		priv->adminq_report_stats_cnt++;
422 		break;
423 	case GVE_ADMINQ_REPORT_LINK_SPEED:
424 		priv->adminq_report_link_speed_cnt++;
425 		break;
426 	case GVE_ADMINQ_GET_PTYPE_MAP:
427 		priv->adminq_get_ptype_map_cnt++;
428 		break;
429 	case GVE_ADMINQ_VERIFY_DRIVER_COMPATIBILITY:
430 		priv->adminq_verify_driver_compatibility_cnt++;
431 		break;
432 	default:
433 		dev_err(&priv->pdev->dev, "unknown AQ command opcode %d\n", opcode);
434 	}
435 
436 	return 0;
437 }
438 
439 /* This function is not threadsafe - the caller is responsible for any
440  * necessary locks.
441  * The caller is also responsible for making sure there are no commands
442  * waiting to be executed.
443  */
gve_adminq_execute_cmd(struct gve_priv * priv,union gve_adminq_command * cmd_orig)444 static int gve_adminq_execute_cmd(struct gve_priv *priv,
445 				  union gve_adminq_command *cmd_orig)
446 {
447 	u32 tail, head;
448 	int err;
449 
450 	tail = ioread32be(&priv->reg_bar0->adminq_event_counter);
451 	head = priv->adminq_prod_cnt;
452 	if (tail != head)
453 		// This is not a valid path
454 		return -EINVAL;
455 
456 	err = gve_adminq_issue_cmd(priv, cmd_orig);
457 	if (err)
458 		return err;
459 
460 	return gve_adminq_kick_and_wait(priv);
461 }
462 
463 /* The device specifies that the management vector can either be the first irq
464  * or the last irq. ntfy_blk_msix_base_idx indicates the first irq assigned to
465  * the ntfy blks. It if is 0 then the management vector is last, if it is 1 then
466  * the management vector is first.
467  *
468  * gve arranges the msix vectors so that the management vector is last.
469  */
470 #define GVE_NTFY_BLK_BASE_MSIX_IDX	0
gve_adminq_configure_device_resources(struct gve_priv * priv,dma_addr_t counter_array_bus_addr,u32 num_counters,dma_addr_t db_array_bus_addr,u32 num_ntfy_blks)471 int gve_adminq_configure_device_resources(struct gve_priv *priv,
472 					  dma_addr_t counter_array_bus_addr,
473 					  u32 num_counters,
474 					  dma_addr_t db_array_bus_addr,
475 					  u32 num_ntfy_blks)
476 {
477 	union gve_adminq_command cmd;
478 
479 	memset(&cmd, 0, sizeof(cmd));
480 	cmd.opcode = cpu_to_be32(GVE_ADMINQ_CONFIGURE_DEVICE_RESOURCES);
481 	cmd.configure_device_resources =
482 		(struct gve_adminq_configure_device_resources) {
483 		.counter_array = cpu_to_be64(counter_array_bus_addr),
484 		.num_counters = cpu_to_be32(num_counters),
485 		.irq_db_addr = cpu_to_be64(db_array_bus_addr),
486 		.num_irq_dbs = cpu_to_be32(num_ntfy_blks),
487 		.irq_db_stride = cpu_to_be32(sizeof(*priv->irq_db_indices)),
488 		.ntfy_blk_msix_base_idx =
489 					cpu_to_be32(GVE_NTFY_BLK_BASE_MSIX_IDX),
490 		.queue_format = priv->queue_format,
491 	};
492 
493 	return gve_adminq_execute_cmd(priv, &cmd);
494 }
495 
gve_adminq_deconfigure_device_resources(struct gve_priv * priv)496 int gve_adminq_deconfigure_device_resources(struct gve_priv *priv)
497 {
498 	union gve_adminq_command cmd;
499 
500 	memset(&cmd, 0, sizeof(cmd));
501 	cmd.opcode = cpu_to_be32(GVE_ADMINQ_DECONFIGURE_DEVICE_RESOURCES);
502 
503 	return gve_adminq_execute_cmd(priv, &cmd);
504 }
505 
gve_adminq_create_tx_queue(struct gve_priv * priv,u32 queue_index)506 static int gve_adminq_create_tx_queue(struct gve_priv *priv, u32 queue_index)
507 {
508 	struct gve_tx_ring *tx = &priv->tx[queue_index];
509 	union gve_adminq_command cmd;
510 
511 	memset(&cmd, 0, sizeof(cmd));
512 	cmd.opcode = cpu_to_be32(GVE_ADMINQ_CREATE_TX_QUEUE);
513 	cmd.create_tx_queue = (struct gve_adminq_create_tx_queue) {
514 		.queue_id = cpu_to_be32(queue_index),
515 		.queue_resources_addr =
516 			cpu_to_be64(tx->q_resources_bus),
517 		.tx_ring_addr = cpu_to_be64(tx->bus),
518 		.ntfy_id = cpu_to_be32(tx->ntfy_id),
519 	};
520 
521 	if (gve_is_gqi(priv)) {
522 		u32 qpl_id = priv->queue_format == GVE_GQI_RDA_FORMAT ?
523 			GVE_RAW_ADDRESSING_QPL_ID : tx->tx_fifo.qpl->id;
524 
525 		cmd.create_tx_queue.queue_page_list_id = cpu_to_be32(qpl_id);
526 	} else {
527 		u16 comp_ring_size;
528 		u32 qpl_id = 0;
529 
530 		if (priv->queue_format == GVE_DQO_RDA_FORMAT) {
531 			qpl_id = GVE_RAW_ADDRESSING_QPL_ID;
532 			comp_ring_size =
533 				priv->options_dqo_rda.tx_comp_ring_entries;
534 		} else {
535 			qpl_id = tx->dqo.qpl->id;
536 			comp_ring_size = priv->tx_desc_cnt;
537 		}
538 		cmd.create_tx_queue.queue_page_list_id = cpu_to_be32(qpl_id);
539 		cmd.create_tx_queue.tx_ring_size =
540 			cpu_to_be16(priv->tx_desc_cnt);
541 		cmd.create_tx_queue.tx_comp_ring_addr =
542 			cpu_to_be64(tx->complq_bus_dqo);
543 		cmd.create_tx_queue.tx_comp_ring_size =
544 			cpu_to_be16(comp_ring_size);
545 	}
546 
547 	return gve_adminq_issue_cmd(priv, &cmd);
548 }
549 
gve_adminq_create_tx_queues(struct gve_priv * priv,u32 start_id,u32 num_queues)550 int gve_adminq_create_tx_queues(struct gve_priv *priv, u32 start_id, u32 num_queues)
551 {
552 	int err;
553 	int i;
554 
555 	for (i = start_id; i < start_id + num_queues; i++) {
556 		err = gve_adminq_create_tx_queue(priv, i);
557 		if (err)
558 			return err;
559 	}
560 
561 	return gve_adminq_kick_and_wait(priv);
562 }
563 
gve_adminq_create_rx_queue(struct gve_priv * priv,u32 queue_index)564 static int gve_adminq_create_rx_queue(struct gve_priv *priv, u32 queue_index)
565 {
566 	struct gve_rx_ring *rx = &priv->rx[queue_index];
567 	union gve_adminq_command cmd;
568 
569 	memset(&cmd, 0, sizeof(cmd));
570 	cmd.opcode = cpu_to_be32(GVE_ADMINQ_CREATE_RX_QUEUE);
571 	cmd.create_rx_queue = (struct gve_adminq_create_rx_queue) {
572 		.queue_id = cpu_to_be32(queue_index),
573 		.ntfy_id = cpu_to_be32(rx->ntfy_id),
574 		.queue_resources_addr = cpu_to_be64(rx->q_resources_bus),
575 	};
576 
577 	if (gve_is_gqi(priv)) {
578 		u32 qpl_id = priv->queue_format == GVE_GQI_RDA_FORMAT ?
579 			GVE_RAW_ADDRESSING_QPL_ID : rx->data.qpl->id;
580 
581 		cmd.create_rx_queue.rx_desc_ring_addr =
582 			cpu_to_be64(rx->desc.bus),
583 		cmd.create_rx_queue.rx_data_ring_addr =
584 			cpu_to_be64(rx->data.data_bus),
585 		cmd.create_rx_queue.index = cpu_to_be32(queue_index);
586 		cmd.create_rx_queue.queue_page_list_id = cpu_to_be32(qpl_id);
587 		cmd.create_rx_queue.packet_buffer_size = cpu_to_be16(rx->packet_buffer_size);
588 	} else {
589 		u16 rx_buff_ring_entries;
590 		u32 qpl_id = 0;
591 
592 		if (priv->queue_format == GVE_DQO_RDA_FORMAT) {
593 			qpl_id = GVE_RAW_ADDRESSING_QPL_ID;
594 			rx_buff_ring_entries =
595 				priv->options_dqo_rda.rx_buff_ring_entries;
596 		} else {
597 			qpl_id = rx->dqo.qpl->id;
598 			rx_buff_ring_entries = priv->rx_desc_cnt;
599 		}
600 		cmd.create_rx_queue.queue_page_list_id = cpu_to_be32(qpl_id);
601 		cmd.create_rx_queue.rx_ring_size =
602 			cpu_to_be16(priv->rx_desc_cnt);
603 		cmd.create_rx_queue.rx_desc_ring_addr =
604 			cpu_to_be64(rx->dqo.complq.bus);
605 		cmd.create_rx_queue.rx_data_ring_addr =
606 			cpu_to_be64(rx->dqo.bufq.bus);
607 		cmd.create_rx_queue.packet_buffer_size =
608 			cpu_to_be16(priv->data_buffer_size_dqo);
609 		cmd.create_rx_queue.rx_buff_ring_size =
610 			cpu_to_be16(rx_buff_ring_entries);
611 		cmd.create_rx_queue.enable_rsc =
612 			!!(priv->dev->features & NETIF_F_LRO);
613 	}
614 
615 	return gve_adminq_issue_cmd(priv, &cmd);
616 }
617 
gve_adminq_create_rx_queues(struct gve_priv * priv,u32 num_queues)618 int gve_adminq_create_rx_queues(struct gve_priv *priv, u32 num_queues)
619 {
620 	int err;
621 	int i;
622 
623 	for (i = 0; i < num_queues; i++) {
624 		err = gve_adminq_create_rx_queue(priv, i);
625 		if (err)
626 			return err;
627 	}
628 
629 	return gve_adminq_kick_and_wait(priv);
630 }
631 
gve_adminq_destroy_tx_queue(struct gve_priv * priv,u32 queue_index)632 static int gve_adminq_destroy_tx_queue(struct gve_priv *priv, u32 queue_index)
633 {
634 	union gve_adminq_command cmd;
635 	int err;
636 
637 	memset(&cmd, 0, sizeof(cmd));
638 	cmd.opcode = cpu_to_be32(GVE_ADMINQ_DESTROY_TX_QUEUE);
639 	cmd.destroy_tx_queue = (struct gve_adminq_destroy_tx_queue) {
640 		.queue_id = cpu_to_be32(queue_index),
641 	};
642 
643 	err = gve_adminq_issue_cmd(priv, &cmd);
644 	if (err)
645 		return err;
646 
647 	return 0;
648 }
649 
gve_adminq_destroy_tx_queues(struct gve_priv * priv,u32 start_id,u32 num_queues)650 int gve_adminq_destroy_tx_queues(struct gve_priv *priv, u32 start_id, u32 num_queues)
651 {
652 	int err;
653 	int i;
654 
655 	for (i = start_id; i < start_id + num_queues; i++) {
656 		err = gve_adminq_destroy_tx_queue(priv, i);
657 		if (err)
658 			return err;
659 	}
660 
661 	return gve_adminq_kick_and_wait(priv);
662 }
663 
gve_adminq_destroy_rx_queue(struct gve_priv * priv,u32 queue_index)664 static int gve_adminq_destroy_rx_queue(struct gve_priv *priv, u32 queue_index)
665 {
666 	union gve_adminq_command cmd;
667 	int err;
668 
669 	memset(&cmd, 0, sizeof(cmd));
670 	cmd.opcode = cpu_to_be32(GVE_ADMINQ_DESTROY_RX_QUEUE);
671 	cmd.destroy_rx_queue = (struct gve_adminq_destroy_rx_queue) {
672 		.queue_id = cpu_to_be32(queue_index),
673 	};
674 
675 	err = gve_adminq_issue_cmd(priv, &cmd);
676 	if (err)
677 		return err;
678 
679 	return 0;
680 }
681 
gve_adminq_destroy_rx_queues(struct gve_priv * priv,u32 num_queues)682 int gve_adminq_destroy_rx_queues(struct gve_priv *priv, u32 num_queues)
683 {
684 	int err;
685 	int i;
686 
687 	for (i = 0; i < num_queues; i++) {
688 		err = gve_adminq_destroy_rx_queue(priv, i);
689 		if (err)
690 			return err;
691 	}
692 
693 	return gve_adminq_kick_and_wait(priv);
694 }
695 
gve_set_desc_cnt(struct gve_priv * priv,struct gve_device_descriptor * descriptor)696 static int gve_set_desc_cnt(struct gve_priv *priv,
697 			    struct gve_device_descriptor *descriptor)
698 {
699 	priv->tx_desc_cnt = be16_to_cpu(descriptor->tx_queue_entries);
700 	if (priv->tx_desc_cnt * sizeof(priv->tx->desc[0]) < PAGE_SIZE) {
701 		dev_err(&priv->pdev->dev, "Tx desc count %d too low\n",
702 			priv->tx_desc_cnt);
703 		return -EINVAL;
704 	}
705 	priv->rx_desc_cnt = be16_to_cpu(descriptor->rx_queue_entries);
706 	if (priv->rx_desc_cnt * sizeof(priv->rx->desc.desc_ring[0])
707 	    < PAGE_SIZE) {
708 		dev_err(&priv->pdev->dev, "Rx desc count %d too low\n",
709 			priv->rx_desc_cnt);
710 		return -EINVAL;
711 	}
712 	return 0;
713 }
714 
715 static int
gve_set_desc_cnt_dqo(struct gve_priv * priv,const struct gve_device_descriptor * descriptor,const struct gve_device_option_dqo_rda * dev_op_dqo_rda)716 gve_set_desc_cnt_dqo(struct gve_priv *priv,
717 		     const struct gve_device_descriptor *descriptor,
718 		     const struct gve_device_option_dqo_rda *dev_op_dqo_rda)
719 {
720 	priv->tx_desc_cnt = be16_to_cpu(descriptor->tx_queue_entries);
721 	priv->rx_desc_cnt = be16_to_cpu(descriptor->rx_queue_entries);
722 
723 	if (priv->queue_format == GVE_DQO_QPL_FORMAT)
724 		return 0;
725 
726 	priv->options_dqo_rda.tx_comp_ring_entries =
727 		be16_to_cpu(dev_op_dqo_rda->tx_comp_ring_entries);
728 	priv->options_dqo_rda.rx_buff_ring_entries =
729 		be16_to_cpu(dev_op_dqo_rda->rx_buff_ring_entries);
730 
731 	return 0;
732 }
733 
gve_enable_supported_features(struct gve_priv * priv,u32 supported_features_mask,const struct gve_device_option_jumbo_frames * dev_op_jumbo_frames,const struct gve_device_option_dqo_qpl * dev_op_dqo_qpl)734 static void gve_enable_supported_features(struct gve_priv *priv,
735 					  u32 supported_features_mask,
736 					  const struct gve_device_option_jumbo_frames
737 					  *dev_op_jumbo_frames,
738 					  const struct gve_device_option_dqo_qpl
739 					  *dev_op_dqo_qpl)
740 {
741 	/* Before control reaches this point, the page-size-capped max MTU from
742 	 * the gve_device_descriptor field has already been stored in
743 	 * priv->dev->max_mtu. We overwrite it with the true max MTU below.
744 	 */
745 	if (dev_op_jumbo_frames &&
746 	    (supported_features_mask & GVE_SUP_JUMBO_FRAMES_MASK)) {
747 		dev_info(&priv->pdev->dev,
748 			 "JUMBO FRAMES device option enabled.\n");
749 		priv->dev->max_mtu = be16_to_cpu(dev_op_jumbo_frames->max_mtu);
750 	}
751 
752 	/* Override pages for qpl for DQO-QPL */
753 	if (dev_op_dqo_qpl) {
754 		priv->tx_pages_per_qpl =
755 			be16_to_cpu(dev_op_dqo_qpl->tx_pages_per_qpl);
756 		priv->rx_pages_per_qpl =
757 			be16_to_cpu(dev_op_dqo_qpl->rx_pages_per_qpl);
758 		if (priv->tx_pages_per_qpl == 0)
759 			priv->tx_pages_per_qpl = DQO_QPL_DEFAULT_TX_PAGES;
760 		if (priv->rx_pages_per_qpl == 0)
761 			priv->rx_pages_per_qpl = DQO_QPL_DEFAULT_RX_PAGES;
762 	}
763 }
764 
gve_adminq_describe_device(struct gve_priv * priv)765 int gve_adminq_describe_device(struct gve_priv *priv)
766 {
767 	struct gve_device_option_jumbo_frames *dev_op_jumbo_frames = NULL;
768 	struct gve_device_option_gqi_rda *dev_op_gqi_rda = NULL;
769 	struct gve_device_option_gqi_qpl *dev_op_gqi_qpl = NULL;
770 	struct gve_device_option_dqo_rda *dev_op_dqo_rda = NULL;
771 	struct gve_device_option_dqo_qpl *dev_op_dqo_qpl = NULL;
772 	struct gve_device_descriptor *descriptor;
773 	u32 supported_features_mask = 0;
774 	union gve_adminq_command cmd;
775 	dma_addr_t descriptor_bus;
776 	int err = 0;
777 	u8 *mac;
778 	u16 mtu;
779 
780 	memset(&cmd, 0, sizeof(cmd));
781 	descriptor = dma_alloc_coherent(&priv->pdev->dev, PAGE_SIZE,
782 					&descriptor_bus, GFP_KERNEL);
783 	if (!descriptor)
784 		return -ENOMEM;
785 	cmd.opcode = cpu_to_be32(GVE_ADMINQ_DESCRIBE_DEVICE);
786 	cmd.describe_device.device_descriptor_addr =
787 						cpu_to_be64(descriptor_bus);
788 	cmd.describe_device.device_descriptor_version =
789 			cpu_to_be32(GVE_ADMINQ_DEVICE_DESCRIPTOR_VERSION);
790 	cmd.describe_device.available_length = cpu_to_be32(PAGE_SIZE);
791 
792 	err = gve_adminq_execute_cmd(priv, &cmd);
793 	if (err)
794 		goto free_device_descriptor;
795 
796 	err = gve_process_device_options(priv, descriptor, &dev_op_gqi_rda,
797 					 &dev_op_gqi_qpl, &dev_op_dqo_rda,
798 					 &dev_op_jumbo_frames,
799 					 &dev_op_dqo_qpl);
800 	if (err)
801 		goto free_device_descriptor;
802 
803 	/* If the GQI_RAW_ADDRESSING option is not enabled and the queue format
804 	 * is not set to GqiRda, choose the queue format in a priority order:
805 	 * DqoRda, DqoQpl, GqiRda, GqiQpl. Use GqiQpl as default.
806 	 */
807 	if (dev_op_dqo_rda) {
808 		priv->queue_format = GVE_DQO_RDA_FORMAT;
809 		dev_info(&priv->pdev->dev,
810 			 "Driver is running with DQO RDA queue format.\n");
811 		supported_features_mask =
812 			be32_to_cpu(dev_op_dqo_rda->supported_features_mask);
813 	} else if (dev_op_dqo_qpl) {
814 		priv->queue_format = GVE_DQO_QPL_FORMAT;
815 		supported_features_mask =
816 			be32_to_cpu(dev_op_dqo_qpl->supported_features_mask);
817 	}  else if (dev_op_gqi_rda) {
818 		priv->queue_format = GVE_GQI_RDA_FORMAT;
819 		dev_info(&priv->pdev->dev,
820 			 "Driver is running with GQI RDA queue format.\n");
821 		supported_features_mask =
822 			be32_to_cpu(dev_op_gqi_rda->supported_features_mask);
823 	} else if (priv->queue_format == GVE_GQI_RDA_FORMAT) {
824 		dev_info(&priv->pdev->dev,
825 			 "Driver is running with GQI RDA queue format.\n");
826 	} else {
827 		priv->queue_format = GVE_GQI_QPL_FORMAT;
828 		if (dev_op_gqi_qpl)
829 			supported_features_mask =
830 				be32_to_cpu(dev_op_gqi_qpl->supported_features_mask);
831 		dev_info(&priv->pdev->dev,
832 			 "Driver is running with GQI QPL queue format.\n");
833 	}
834 	if (gve_is_gqi(priv)) {
835 		err = gve_set_desc_cnt(priv, descriptor);
836 	} else {
837 		/* DQO supports LRO. */
838 		priv->dev->hw_features |= NETIF_F_LRO;
839 		err = gve_set_desc_cnt_dqo(priv, descriptor, dev_op_dqo_rda);
840 	}
841 	if (err)
842 		goto free_device_descriptor;
843 
844 	priv->max_registered_pages =
845 				be64_to_cpu(descriptor->max_registered_pages);
846 	mtu = be16_to_cpu(descriptor->mtu);
847 	if (mtu < ETH_MIN_MTU) {
848 		dev_err(&priv->pdev->dev, "MTU %d below minimum MTU\n", mtu);
849 		err = -EINVAL;
850 		goto free_device_descriptor;
851 	}
852 	priv->dev->max_mtu = mtu;
853 	priv->num_event_counters = be16_to_cpu(descriptor->counters);
854 	eth_hw_addr_set(priv->dev, descriptor->mac);
855 	mac = descriptor->mac;
856 	dev_info(&priv->pdev->dev, "MAC addr: %pM\n", mac);
857 	priv->tx_pages_per_qpl = be16_to_cpu(descriptor->tx_pages_per_qpl);
858 	priv->rx_data_slot_cnt = be16_to_cpu(descriptor->rx_pages_per_qpl);
859 
860 	if (gve_is_gqi(priv) && priv->rx_data_slot_cnt < priv->rx_desc_cnt) {
861 		dev_err(&priv->pdev->dev, "rx_data_slot_cnt cannot be smaller than rx_desc_cnt, setting rx_desc_cnt down to %d.\n",
862 			priv->rx_data_slot_cnt);
863 		priv->rx_desc_cnt = priv->rx_data_slot_cnt;
864 	}
865 	priv->default_num_queues = be16_to_cpu(descriptor->default_num_queues);
866 
867 	gve_enable_supported_features(priv, supported_features_mask,
868 				      dev_op_jumbo_frames, dev_op_dqo_qpl);
869 
870 free_device_descriptor:
871 	dma_free_coherent(&priv->pdev->dev, PAGE_SIZE, descriptor,
872 			  descriptor_bus);
873 	return err;
874 }
875 
gve_adminq_register_page_list(struct gve_priv * priv,struct gve_queue_page_list * qpl)876 int gve_adminq_register_page_list(struct gve_priv *priv,
877 				  struct gve_queue_page_list *qpl)
878 {
879 	struct device *hdev = &priv->pdev->dev;
880 	u32 num_entries = qpl->num_entries;
881 	u32 size = num_entries * sizeof(qpl->page_buses[0]);
882 	union gve_adminq_command cmd;
883 	dma_addr_t page_list_bus;
884 	__be64 *page_list;
885 	int err;
886 	int i;
887 
888 	memset(&cmd, 0, sizeof(cmd));
889 	page_list = dma_alloc_coherent(hdev, size, &page_list_bus, GFP_KERNEL);
890 	if (!page_list)
891 		return -ENOMEM;
892 
893 	for (i = 0; i < num_entries; i++)
894 		page_list[i] = cpu_to_be64(qpl->page_buses[i]);
895 
896 	cmd.opcode = cpu_to_be32(GVE_ADMINQ_REGISTER_PAGE_LIST);
897 	cmd.reg_page_list = (struct gve_adminq_register_page_list) {
898 		.page_list_id = cpu_to_be32(qpl->id),
899 		.num_pages = cpu_to_be32(num_entries),
900 		.page_address_list_addr = cpu_to_be64(page_list_bus),
901 	};
902 
903 	err = gve_adminq_execute_cmd(priv, &cmd);
904 	dma_free_coherent(hdev, size, page_list, page_list_bus);
905 	return err;
906 }
907 
gve_adminq_unregister_page_list(struct gve_priv * priv,u32 page_list_id)908 int gve_adminq_unregister_page_list(struct gve_priv *priv, u32 page_list_id)
909 {
910 	union gve_adminq_command cmd;
911 
912 	memset(&cmd, 0, sizeof(cmd));
913 	cmd.opcode = cpu_to_be32(GVE_ADMINQ_UNREGISTER_PAGE_LIST);
914 	cmd.unreg_page_list = (struct gve_adminq_unregister_page_list) {
915 		.page_list_id = cpu_to_be32(page_list_id),
916 	};
917 
918 	return gve_adminq_execute_cmd(priv, &cmd);
919 }
920 
gve_adminq_set_mtu(struct gve_priv * priv,u64 mtu)921 int gve_adminq_set_mtu(struct gve_priv *priv, u64 mtu)
922 {
923 	union gve_adminq_command cmd;
924 
925 	memset(&cmd, 0, sizeof(cmd));
926 	cmd.opcode = cpu_to_be32(GVE_ADMINQ_SET_DRIVER_PARAMETER);
927 	cmd.set_driver_param = (struct gve_adminq_set_driver_parameter) {
928 		.parameter_type = cpu_to_be32(GVE_SET_PARAM_MTU),
929 		.parameter_value = cpu_to_be64(mtu),
930 	};
931 
932 	return gve_adminq_execute_cmd(priv, &cmd);
933 }
934 
gve_adminq_report_stats(struct gve_priv * priv,u64 stats_report_len,dma_addr_t stats_report_addr,u64 interval)935 int gve_adminq_report_stats(struct gve_priv *priv, u64 stats_report_len,
936 			    dma_addr_t stats_report_addr, u64 interval)
937 {
938 	union gve_adminq_command cmd;
939 
940 	memset(&cmd, 0, sizeof(cmd));
941 	cmd.opcode = cpu_to_be32(GVE_ADMINQ_REPORT_STATS);
942 	cmd.report_stats = (struct gve_adminq_report_stats) {
943 		.stats_report_len = cpu_to_be64(stats_report_len),
944 		.stats_report_addr = cpu_to_be64(stats_report_addr),
945 		.interval = cpu_to_be64(interval),
946 	};
947 
948 	return gve_adminq_execute_cmd(priv, &cmd);
949 }
950 
gve_adminq_verify_driver_compatibility(struct gve_priv * priv,u64 driver_info_len,dma_addr_t driver_info_addr)951 int gve_adminq_verify_driver_compatibility(struct gve_priv *priv,
952 					   u64 driver_info_len,
953 					   dma_addr_t driver_info_addr)
954 {
955 	union gve_adminq_command cmd;
956 
957 	memset(&cmd, 0, sizeof(cmd));
958 	cmd.opcode = cpu_to_be32(GVE_ADMINQ_VERIFY_DRIVER_COMPATIBILITY);
959 	cmd.verify_driver_compatibility = (struct gve_adminq_verify_driver_compatibility) {
960 		.driver_info_len = cpu_to_be64(driver_info_len),
961 		.driver_info_addr = cpu_to_be64(driver_info_addr),
962 	};
963 
964 	return gve_adminq_execute_cmd(priv, &cmd);
965 }
966 
gve_adminq_report_link_speed(struct gve_priv * priv)967 int gve_adminq_report_link_speed(struct gve_priv *priv)
968 {
969 	union gve_adminq_command gvnic_cmd;
970 	dma_addr_t link_speed_region_bus;
971 	__be64 *link_speed_region;
972 	int err;
973 
974 	link_speed_region =
975 		dma_alloc_coherent(&priv->pdev->dev, sizeof(*link_speed_region),
976 				   &link_speed_region_bus, GFP_KERNEL);
977 
978 	if (!link_speed_region)
979 		return -ENOMEM;
980 
981 	memset(&gvnic_cmd, 0, sizeof(gvnic_cmd));
982 	gvnic_cmd.opcode = cpu_to_be32(GVE_ADMINQ_REPORT_LINK_SPEED);
983 	gvnic_cmd.report_link_speed.link_speed_address =
984 		cpu_to_be64(link_speed_region_bus);
985 
986 	err = gve_adminq_execute_cmd(priv, &gvnic_cmd);
987 
988 	priv->link_speed = be64_to_cpu(*link_speed_region);
989 	dma_free_coherent(&priv->pdev->dev, sizeof(*link_speed_region), link_speed_region,
990 			  link_speed_region_bus);
991 	return err;
992 }
993 
gve_adminq_get_ptype_map_dqo(struct gve_priv * priv,struct gve_ptype_lut * ptype_lut)994 int gve_adminq_get_ptype_map_dqo(struct gve_priv *priv,
995 				 struct gve_ptype_lut *ptype_lut)
996 {
997 	struct gve_ptype_map *ptype_map;
998 	union gve_adminq_command cmd;
999 	dma_addr_t ptype_map_bus;
1000 	int err = 0;
1001 	int i;
1002 
1003 	memset(&cmd, 0, sizeof(cmd));
1004 	ptype_map = dma_alloc_coherent(&priv->pdev->dev, sizeof(*ptype_map),
1005 				       &ptype_map_bus, GFP_KERNEL);
1006 	if (!ptype_map)
1007 		return -ENOMEM;
1008 
1009 	cmd.opcode = cpu_to_be32(GVE_ADMINQ_GET_PTYPE_MAP);
1010 	cmd.get_ptype_map = (struct gve_adminq_get_ptype_map) {
1011 		.ptype_map_len = cpu_to_be64(sizeof(*ptype_map)),
1012 		.ptype_map_addr = cpu_to_be64(ptype_map_bus),
1013 	};
1014 
1015 	err = gve_adminq_execute_cmd(priv, &cmd);
1016 	if (err)
1017 		goto err;
1018 
1019 	/* Populate ptype_lut. */
1020 	for (i = 0; i < GVE_NUM_PTYPES; i++) {
1021 		ptype_lut->ptypes[i].l3_type =
1022 			ptype_map->ptypes[i].l3_type;
1023 		ptype_lut->ptypes[i].l4_type =
1024 			ptype_map->ptypes[i].l4_type;
1025 	}
1026 err:
1027 	dma_free_coherent(&priv->pdev->dev, sizeof(*ptype_map), ptype_map,
1028 			  ptype_map_bus);
1029 	return err;
1030 }
1031