1 // SPDX-License-Identifier: (GPL-2.0 OR MIT) 2 /* Google virtual Ethernet (gve) driver 3 * 4 * Copyright (C) 2015-2021 Google, Inc. 5 */ 6 7 #include <linux/etherdevice.h> 8 #include <linux/pci.h> 9 #include "gve.h" 10 #include "gve_adminq.h" 11 #include "gve_register.h" 12 13 #define GVE_MAX_ADMINQ_RELEASE_CHECK 500 14 #define GVE_ADMINQ_SLEEP_LEN 20 15 #define GVE_MAX_ADMINQ_EVENT_COUNTER_CHECK 100 16 17 #define GVE_DEVICE_OPTION_ERROR_FMT "%s option error:\n" \ 18 "Expected: length=%d, feature_mask=%x.\n" \ 19 "Actual: length=%d, feature_mask=%x.\n" 20 21 #define GVE_DEVICE_OPTION_TOO_BIG_FMT "Length of %s option larger than expected. Possible older version of guest driver.\n" 22 23 static 24 struct gve_device_option *gve_get_next_option(struct gve_device_descriptor *descriptor, 25 struct gve_device_option *option) 26 { 27 void *option_end, *descriptor_end; 28 29 option_end = (void *)(option + 1) + be16_to_cpu(option->option_length); 30 descriptor_end = (void *)descriptor + be16_to_cpu(descriptor->total_length); 31 32 return option_end > descriptor_end ? NULL : (struct gve_device_option *)option_end; 33 } 34 35 static 36 void gve_parse_device_option(struct gve_priv *priv, 37 struct gve_device_descriptor *device_descriptor, 38 struct gve_device_option *option, 39 struct gve_device_option_gqi_rda **dev_op_gqi_rda, 40 struct gve_device_option_gqi_qpl **dev_op_gqi_qpl, 41 struct gve_device_option_dqo_rda **dev_op_dqo_rda) 42 { 43 u32 req_feat_mask = be32_to_cpu(option->required_features_mask); 44 u16 option_length = be16_to_cpu(option->option_length); 45 u16 option_id = be16_to_cpu(option->option_id); 46 47 /* If the length or feature mask doesn't match, continue without 48 * enabling the feature. 49 */ 50 switch (option_id) { 51 case GVE_DEV_OPT_ID_GQI_RAW_ADDRESSING: 52 if (option_length != GVE_DEV_OPT_LEN_GQI_RAW_ADDRESSING || 53 req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RAW_ADDRESSING) { 54 dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT, 55 "Raw Addressing", 56 GVE_DEV_OPT_LEN_GQI_RAW_ADDRESSING, 57 GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RAW_ADDRESSING, 58 option_length, req_feat_mask); 59 break; 60 } 61 62 dev_info(&priv->pdev->dev, 63 "Gqi raw addressing device option enabled.\n"); 64 priv->queue_format = GVE_GQI_RDA_FORMAT; 65 break; 66 case GVE_DEV_OPT_ID_GQI_RDA: 67 if (option_length < sizeof(**dev_op_gqi_rda) || 68 req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RDA) { 69 dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT, 70 "GQI RDA", (int)sizeof(**dev_op_gqi_rda), 71 GVE_DEV_OPT_REQ_FEAT_MASK_GQI_RDA, 72 option_length, req_feat_mask); 73 break; 74 } 75 76 if (option_length > sizeof(**dev_op_gqi_rda)) { 77 dev_warn(&priv->pdev->dev, 78 GVE_DEVICE_OPTION_TOO_BIG_FMT, "GQI RDA"); 79 } 80 *dev_op_gqi_rda = (void *)(option + 1); 81 break; 82 case GVE_DEV_OPT_ID_GQI_QPL: 83 if (option_length < sizeof(**dev_op_gqi_qpl) || 84 req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_GQI_QPL) { 85 dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT, 86 "GQI QPL", (int)sizeof(**dev_op_gqi_qpl), 87 GVE_DEV_OPT_REQ_FEAT_MASK_GQI_QPL, 88 option_length, req_feat_mask); 89 break; 90 } 91 92 if (option_length > sizeof(**dev_op_gqi_qpl)) { 93 dev_warn(&priv->pdev->dev, 94 GVE_DEVICE_OPTION_TOO_BIG_FMT, "GQI QPL"); 95 } 96 *dev_op_gqi_qpl = (void *)(option + 1); 97 break; 98 case GVE_DEV_OPT_ID_DQO_RDA: 99 if (option_length < sizeof(**dev_op_dqo_rda) || 100 req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_DQO_RDA) { 101 dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT, 102 "DQO RDA", (int)sizeof(**dev_op_dqo_rda), 103 GVE_DEV_OPT_REQ_FEAT_MASK_DQO_RDA, 104 option_length, req_feat_mask); 105 break; 106 } 107 108 if (option_length > sizeof(**dev_op_dqo_rda)) { 109 dev_warn(&priv->pdev->dev, 110 GVE_DEVICE_OPTION_TOO_BIG_FMT, "DQO RDA"); 111 } 112 *dev_op_dqo_rda = (void *)(option + 1); 113 break; 114 default: 115 /* If we don't recognize the option just continue 116 * without doing anything. 117 */ 118 dev_dbg(&priv->pdev->dev, "Unrecognized device option 0x%hx not enabled.\n", 119 option_id); 120 } 121 } 122 123 /* Process all device options for a given describe device call. */ 124 static int 125 gve_process_device_options(struct gve_priv *priv, 126 struct gve_device_descriptor *descriptor, 127 struct gve_device_option_gqi_rda **dev_op_gqi_rda, 128 struct gve_device_option_gqi_qpl **dev_op_gqi_qpl, 129 struct gve_device_option_dqo_rda **dev_op_dqo_rda) 130 { 131 const int num_options = be16_to_cpu(descriptor->num_device_options); 132 struct gve_device_option *dev_opt; 133 int i; 134 135 /* The options struct directly follows the device descriptor. */ 136 dev_opt = (void *)(descriptor + 1); 137 for (i = 0; i < num_options; i++) { 138 struct gve_device_option *next_opt; 139 140 next_opt = gve_get_next_option(descriptor, dev_opt); 141 if (!next_opt) { 142 dev_err(&priv->dev->dev, 143 "options exceed device_descriptor's total length.\n"); 144 return -EINVAL; 145 } 146 147 gve_parse_device_option(priv, descriptor, dev_opt, 148 dev_op_gqi_rda, dev_op_gqi_qpl, 149 dev_op_dqo_rda); 150 dev_opt = next_opt; 151 } 152 153 return 0; 154 } 155 156 int gve_adminq_alloc(struct device *dev, struct gve_priv *priv) 157 { 158 priv->adminq = dma_alloc_coherent(dev, PAGE_SIZE, 159 &priv->adminq_bus_addr, GFP_KERNEL); 160 if (unlikely(!priv->adminq)) 161 return -ENOMEM; 162 163 priv->adminq_mask = (PAGE_SIZE / sizeof(union gve_adminq_command)) - 1; 164 priv->adminq_prod_cnt = 0; 165 priv->adminq_cmd_fail = 0; 166 priv->adminq_timeouts = 0; 167 priv->adminq_describe_device_cnt = 0; 168 priv->adminq_cfg_device_resources_cnt = 0; 169 priv->adminq_register_page_list_cnt = 0; 170 priv->adminq_unregister_page_list_cnt = 0; 171 priv->adminq_create_tx_queue_cnt = 0; 172 priv->adminq_create_rx_queue_cnt = 0; 173 priv->adminq_destroy_tx_queue_cnt = 0; 174 priv->adminq_destroy_rx_queue_cnt = 0; 175 priv->adminq_dcfg_device_resources_cnt = 0; 176 priv->adminq_set_driver_parameter_cnt = 0; 177 priv->adminq_report_stats_cnt = 0; 178 priv->adminq_report_link_speed_cnt = 0; 179 priv->adminq_get_ptype_map_cnt = 0; 180 181 /* Setup Admin queue with the device */ 182 iowrite32be(priv->adminq_bus_addr / PAGE_SIZE, 183 &priv->reg_bar0->adminq_pfn); 184 185 gve_set_admin_queue_ok(priv); 186 return 0; 187 } 188 189 void gve_adminq_release(struct gve_priv *priv) 190 { 191 int i = 0; 192 193 /* Tell the device the adminq is leaving */ 194 iowrite32be(0x0, &priv->reg_bar0->adminq_pfn); 195 while (ioread32be(&priv->reg_bar0->adminq_pfn)) { 196 /* If this is reached the device is unrecoverable and still 197 * holding memory. Continue looping to avoid memory corruption, 198 * but WARN so it is visible what is going on. 199 */ 200 if (i == GVE_MAX_ADMINQ_RELEASE_CHECK) 201 WARN(1, "Unrecoverable platform error!"); 202 i++; 203 msleep(GVE_ADMINQ_SLEEP_LEN); 204 } 205 gve_clear_device_rings_ok(priv); 206 gve_clear_device_resources_ok(priv); 207 gve_clear_admin_queue_ok(priv); 208 } 209 210 void gve_adminq_free(struct device *dev, struct gve_priv *priv) 211 { 212 if (!gve_get_admin_queue_ok(priv)) 213 return; 214 gve_adminq_release(priv); 215 dma_free_coherent(dev, PAGE_SIZE, priv->adminq, priv->adminq_bus_addr); 216 gve_clear_admin_queue_ok(priv); 217 } 218 219 static void gve_adminq_kick_cmd(struct gve_priv *priv, u32 prod_cnt) 220 { 221 iowrite32be(prod_cnt, &priv->reg_bar0->adminq_doorbell); 222 } 223 224 static bool gve_adminq_wait_for_cmd(struct gve_priv *priv, u32 prod_cnt) 225 { 226 int i; 227 228 for (i = 0; i < GVE_MAX_ADMINQ_EVENT_COUNTER_CHECK; i++) { 229 if (ioread32be(&priv->reg_bar0->adminq_event_counter) 230 == prod_cnt) 231 return true; 232 msleep(GVE_ADMINQ_SLEEP_LEN); 233 } 234 235 return false; 236 } 237 238 static int gve_adminq_parse_err(struct gve_priv *priv, u32 status) 239 { 240 if (status != GVE_ADMINQ_COMMAND_PASSED && 241 status != GVE_ADMINQ_COMMAND_UNSET) { 242 dev_err(&priv->pdev->dev, "AQ command failed with status %d\n", status); 243 priv->adminq_cmd_fail++; 244 } 245 switch (status) { 246 case GVE_ADMINQ_COMMAND_PASSED: 247 return 0; 248 case GVE_ADMINQ_COMMAND_UNSET: 249 dev_err(&priv->pdev->dev, "parse_aq_err: err and status both unset, this should not be possible.\n"); 250 return -EINVAL; 251 case GVE_ADMINQ_COMMAND_ERROR_ABORTED: 252 case GVE_ADMINQ_COMMAND_ERROR_CANCELLED: 253 case GVE_ADMINQ_COMMAND_ERROR_DATALOSS: 254 case GVE_ADMINQ_COMMAND_ERROR_FAILED_PRECONDITION: 255 case GVE_ADMINQ_COMMAND_ERROR_UNAVAILABLE: 256 return -EAGAIN; 257 case GVE_ADMINQ_COMMAND_ERROR_ALREADY_EXISTS: 258 case GVE_ADMINQ_COMMAND_ERROR_INTERNAL_ERROR: 259 case GVE_ADMINQ_COMMAND_ERROR_INVALID_ARGUMENT: 260 case GVE_ADMINQ_COMMAND_ERROR_NOT_FOUND: 261 case GVE_ADMINQ_COMMAND_ERROR_OUT_OF_RANGE: 262 case GVE_ADMINQ_COMMAND_ERROR_UNKNOWN_ERROR: 263 return -EINVAL; 264 case GVE_ADMINQ_COMMAND_ERROR_DEADLINE_EXCEEDED: 265 return -ETIME; 266 case GVE_ADMINQ_COMMAND_ERROR_PERMISSION_DENIED: 267 case GVE_ADMINQ_COMMAND_ERROR_UNAUTHENTICATED: 268 return -EACCES; 269 case GVE_ADMINQ_COMMAND_ERROR_RESOURCE_EXHAUSTED: 270 return -ENOMEM; 271 case GVE_ADMINQ_COMMAND_ERROR_UNIMPLEMENTED: 272 return -ENOTSUPP; 273 default: 274 dev_err(&priv->pdev->dev, "parse_aq_err: unknown status code %d\n", status); 275 return -EINVAL; 276 } 277 } 278 279 /* Flushes all AQ commands currently queued and waits for them to complete. 280 * If there are failures, it will return the first error. 281 */ 282 static int gve_adminq_kick_and_wait(struct gve_priv *priv) 283 { 284 u32 tail, head; 285 int i; 286 287 tail = ioread32be(&priv->reg_bar0->adminq_event_counter); 288 head = priv->adminq_prod_cnt; 289 290 gve_adminq_kick_cmd(priv, head); 291 if (!gve_adminq_wait_for_cmd(priv, head)) { 292 dev_err(&priv->pdev->dev, "AQ commands timed out, need to reset AQ\n"); 293 priv->adminq_timeouts++; 294 return -ENOTRECOVERABLE; 295 } 296 297 for (i = tail; i < head; i++) { 298 union gve_adminq_command *cmd; 299 u32 status, err; 300 301 cmd = &priv->adminq[i & priv->adminq_mask]; 302 status = be32_to_cpu(READ_ONCE(cmd->status)); 303 err = gve_adminq_parse_err(priv, status); 304 if (err) 305 // Return the first error if we failed. 306 return err; 307 } 308 309 return 0; 310 } 311 312 /* This function is not threadsafe - the caller is responsible for any 313 * necessary locks. 314 */ 315 static int gve_adminq_issue_cmd(struct gve_priv *priv, 316 union gve_adminq_command *cmd_orig) 317 { 318 union gve_adminq_command *cmd; 319 u32 opcode; 320 u32 tail; 321 322 tail = ioread32be(&priv->reg_bar0->adminq_event_counter); 323 324 // Check if next command will overflow the buffer. 325 if (((priv->adminq_prod_cnt + 1) & priv->adminq_mask) == tail) { 326 int err; 327 328 // Flush existing commands to make room. 329 err = gve_adminq_kick_and_wait(priv); 330 if (err) 331 return err; 332 333 // Retry. 334 tail = ioread32be(&priv->reg_bar0->adminq_event_counter); 335 if (((priv->adminq_prod_cnt + 1) & priv->adminq_mask) == tail) { 336 // This should never happen. We just flushed the 337 // command queue so there should be enough space. 338 return -ENOMEM; 339 } 340 } 341 342 cmd = &priv->adminq[priv->adminq_prod_cnt & priv->adminq_mask]; 343 priv->adminq_prod_cnt++; 344 345 memcpy(cmd, cmd_orig, sizeof(*cmd_orig)); 346 opcode = be32_to_cpu(READ_ONCE(cmd->opcode)); 347 348 switch (opcode) { 349 case GVE_ADMINQ_DESCRIBE_DEVICE: 350 priv->adminq_describe_device_cnt++; 351 break; 352 case GVE_ADMINQ_CONFIGURE_DEVICE_RESOURCES: 353 priv->adminq_cfg_device_resources_cnt++; 354 break; 355 case GVE_ADMINQ_REGISTER_PAGE_LIST: 356 priv->adminq_register_page_list_cnt++; 357 break; 358 case GVE_ADMINQ_UNREGISTER_PAGE_LIST: 359 priv->adminq_unregister_page_list_cnt++; 360 break; 361 case GVE_ADMINQ_CREATE_TX_QUEUE: 362 priv->adminq_create_tx_queue_cnt++; 363 break; 364 case GVE_ADMINQ_CREATE_RX_QUEUE: 365 priv->adminq_create_rx_queue_cnt++; 366 break; 367 case GVE_ADMINQ_DESTROY_TX_QUEUE: 368 priv->adminq_destroy_tx_queue_cnt++; 369 break; 370 case GVE_ADMINQ_DESTROY_RX_QUEUE: 371 priv->adminq_destroy_rx_queue_cnt++; 372 break; 373 case GVE_ADMINQ_DECONFIGURE_DEVICE_RESOURCES: 374 priv->adminq_dcfg_device_resources_cnt++; 375 break; 376 case GVE_ADMINQ_SET_DRIVER_PARAMETER: 377 priv->adminq_set_driver_parameter_cnt++; 378 break; 379 case GVE_ADMINQ_REPORT_STATS: 380 priv->adminq_report_stats_cnt++; 381 break; 382 case GVE_ADMINQ_REPORT_LINK_SPEED: 383 priv->adminq_report_link_speed_cnt++; 384 break; 385 case GVE_ADMINQ_GET_PTYPE_MAP: 386 priv->adminq_get_ptype_map_cnt++; 387 break; 388 default: 389 dev_err(&priv->pdev->dev, "unknown AQ command opcode %d\n", opcode); 390 } 391 392 return 0; 393 } 394 395 /* This function is not threadsafe - the caller is responsible for any 396 * necessary locks. 397 * The caller is also responsible for making sure there are no commands 398 * waiting to be executed. 399 */ 400 static int gve_adminq_execute_cmd(struct gve_priv *priv, 401 union gve_adminq_command *cmd_orig) 402 { 403 u32 tail, head; 404 int err; 405 406 tail = ioread32be(&priv->reg_bar0->adminq_event_counter); 407 head = priv->adminq_prod_cnt; 408 if (tail != head) 409 // This is not a valid path 410 return -EINVAL; 411 412 err = gve_adminq_issue_cmd(priv, cmd_orig); 413 if (err) 414 return err; 415 416 return gve_adminq_kick_and_wait(priv); 417 } 418 419 /* The device specifies that the management vector can either be the first irq 420 * or the last irq. ntfy_blk_msix_base_idx indicates the first irq assigned to 421 * the ntfy blks. It if is 0 then the management vector is last, if it is 1 then 422 * the management vector is first. 423 * 424 * gve arranges the msix vectors so that the management vector is last. 425 */ 426 #define GVE_NTFY_BLK_BASE_MSIX_IDX 0 427 int gve_adminq_configure_device_resources(struct gve_priv *priv, 428 dma_addr_t counter_array_bus_addr, 429 u32 num_counters, 430 dma_addr_t db_array_bus_addr, 431 u32 num_ntfy_blks) 432 { 433 union gve_adminq_command cmd; 434 435 memset(&cmd, 0, sizeof(cmd)); 436 cmd.opcode = cpu_to_be32(GVE_ADMINQ_CONFIGURE_DEVICE_RESOURCES); 437 cmd.configure_device_resources = 438 (struct gve_adminq_configure_device_resources) { 439 .counter_array = cpu_to_be64(counter_array_bus_addr), 440 .num_counters = cpu_to_be32(num_counters), 441 .irq_db_addr = cpu_to_be64(db_array_bus_addr), 442 .num_irq_dbs = cpu_to_be32(num_ntfy_blks), 443 .irq_db_stride = cpu_to_be32(sizeof(priv->ntfy_blocks[0])), 444 .ntfy_blk_msix_base_idx = 445 cpu_to_be32(GVE_NTFY_BLK_BASE_MSIX_IDX), 446 .queue_format = priv->queue_format, 447 }; 448 449 return gve_adminq_execute_cmd(priv, &cmd); 450 } 451 452 int gve_adminq_deconfigure_device_resources(struct gve_priv *priv) 453 { 454 union gve_adminq_command cmd; 455 456 memset(&cmd, 0, sizeof(cmd)); 457 cmd.opcode = cpu_to_be32(GVE_ADMINQ_DECONFIGURE_DEVICE_RESOURCES); 458 459 return gve_adminq_execute_cmd(priv, &cmd); 460 } 461 462 static int gve_adminq_create_tx_queue(struct gve_priv *priv, u32 queue_index) 463 { 464 struct gve_tx_ring *tx = &priv->tx[queue_index]; 465 union gve_adminq_command cmd; 466 467 memset(&cmd, 0, sizeof(cmd)); 468 cmd.opcode = cpu_to_be32(GVE_ADMINQ_CREATE_TX_QUEUE); 469 cmd.create_tx_queue = (struct gve_adminq_create_tx_queue) { 470 .queue_id = cpu_to_be32(queue_index), 471 .queue_resources_addr = 472 cpu_to_be64(tx->q_resources_bus), 473 .tx_ring_addr = cpu_to_be64(tx->bus), 474 .ntfy_id = cpu_to_be32(tx->ntfy_id), 475 }; 476 477 if (gve_is_gqi(priv)) { 478 u32 qpl_id = priv->queue_format == GVE_GQI_RDA_FORMAT ? 479 GVE_RAW_ADDRESSING_QPL_ID : tx->tx_fifo.qpl->id; 480 481 cmd.create_tx_queue.queue_page_list_id = cpu_to_be32(qpl_id); 482 } else { 483 cmd.create_tx_queue.tx_ring_size = 484 cpu_to_be16(priv->tx_desc_cnt); 485 cmd.create_tx_queue.tx_comp_ring_addr = 486 cpu_to_be64(tx->complq_bus_dqo); 487 cmd.create_tx_queue.tx_comp_ring_size = 488 cpu_to_be16(priv->options_dqo_rda.tx_comp_ring_entries); 489 } 490 491 return gve_adminq_issue_cmd(priv, &cmd); 492 } 493 494 int gve_adminq_create_tx_queues(struct gve_priv *priv, u32 num_queues) 495 { 496 int err; 497 int i; 498 499 for (i = 0; i < num_queues; i++) { 500 err = gve_adminq_create_tx_queue(priv, i); 501 if (err) 502 return err; 503 } 504 505 return gve_adminq_kick_and_wait(priv); 506 } 507 508 static int gve_adminq_create_rx_queue(struct gve_priv *priv, u32 queue_index) 509 { 510 struct gve_rx_ring *rx = &priv->rx[queue_index]; 511 union gve_adminq_command cmd; 512 513 memset(&cmd, 0, sizeof(cmd)); 514 cmd.opcode = cpu_to_be32(GVE_ADMINQ_CREATE_RX_QUEUE); 515 cmd.create_rx_queue = (struct gve_adminq_create_rx_queue) { 516 .queue_id = cpu_to_be32(queue_index), 517 .ntfy_id = cpu_to_be32(rx->ntfy_id), 518 .queue_resources_addr = cpu_to_be64(rx->q_resources_bus), 519 }; 520 521 if (gve_is_gqi(priv)) { 522 u32 qpl_id = priv->queue_format == GVE_GQI_RDA_FORMAT ? 523 GVE_RAW_ADDRESSING_QPL_ID : rx->data.qpl->id; 524 525 cmd.create_rx_queue.rx_desc_ring_addr = 526 cpu_to_be64(rx->desc.bus), 527 cmd.create_rx_queue.rx_data_ring_addr = 528 cpu_to_be64(rx->data.data_bus), 529 cmd.create_rx_queue.index = cpu_to_be32(queue_index); 530 cmd.create_rx_queue.queue_page_list_id = cpu_to_be32(qpl_id); 531 } else { 532 cmd.create_rx_queue.rx_ring_size = 533 cpu_to_be16(priv->rx_desc_cnt); 534 cmd.create_rx_queue.rx_desc_ring_addr = 535 cpu_to_be64(rx->dqo.complq.bus); 536 cmd.create_rx_queue.rx_data_ring_addr = 537 cpu_to_be64(rx->dqo.bufq.bus); 538 cmd.create_rx_queue.packet_buffer_size = 539 cpu_to_be16(priv->data_buffer_size_dqo); 540 cmd.create_rx_queue.rx_buff_ring_size = 541 cpu_to_be16(priv->options_dqo_rda.rx_buff_ring_entries); 542 cmd.create_rx_queue.enable_rsc = 543 !!(priv->dev->features & NETIF_F_LRO); 544 } 545 546 return gve_adminq_issue_cmd(priv, &cmd); 547 } 548 549 int gve_adminq_create_rx_queues(struct gve_priv *priv, u32 num_queues) 550 { 551 int err; 552 int i; 553 554 for (i = 0; i < num_queues; i++) { 555 err = gve_adminq_create_rx_queue(priv, i); 556 if (err) 557 return err; 558 } 559 560 return gve_adminq_kick_and_wait(priv); 561 } 562 563 static int gve_adminq_destroy_tx_queue(struct gve_priv *priv, u32 queue_index) 564 { 565 union gve_adminq_command cmd; 566 int err; 567 568 memset(&cmd, 0, sizeof(cmd)); 569 cmd.opcode = cpu_to_be32(GVE_ADMINQ_DESTROY_TX_QUEUE); 570 cmd.destroy_tx_queue = (struct gve_adminq_destroy_tx_queue) { 571 .queue_id = cpu_to_be32(queue_index), 572 }; 573 574 err = gve_adminq_issue_cmd(priv, &cmd); 575 if (err) 576 return err; 577 578 return 0; 579 } 580 581 int gve_adminq_destroy_tx_queues(struct gve_priv *priv, u32 num_queues) 582 { 583 int err; 584 int i; 585 586 for (i = 0; i < num_queues; i++) { 587 err = gve_adminq_destroy_tx_queue(priv, i); 588 if (err) 589 return err; 590 } 591 592 return gve_adminq_kick_and_wait(priv); 593 } 594 595 static int gve_adminq_destroy_rx_queue(struct gve_priv *priv, u32 queue_index) 596 { 597 union gve_adminq_command cmd; 598 int err; 599 600 memset(&cmd, 0, sizeof(cmd)); 601 cmd.opcode = cpu_to_be32(GVE_ADMINQ_DESTROY_RX_QUEUE); 602 cmd.destroy_rx_queue = (struct gve_adminq_destroy_rx_queue) { 603 .queue_id = cpu_to_be32(queue_index), 604 }; 605 606 err = gve_adminq_issue_cmd(priv, &cmd); 607 if (err) 608 return err; 609 610 return 0; 611 } 612 613 int gve_adminq_destroy_rx_queues(struct gve_priv *priv, u32 num_queues) 614 { 615 int err; 616 int i; 617 618 for (i = 0; i < num_queues; i++) { 619 err = gve_adminq_destroy_rx_queue(priv, i); 620 if (err) 621 return err; 622 } 623 624 return gve_adminq_kick_and_wait(priv); 625 } 626 627 static int gve_set_desc_cnt(struct gve_priv *priv, 628 struct gve_device_descriptor *descriptor) 629 { 630 priv->tx_desc_cnt = be16_to_cpu(descriptor->tx_queue_entries); 631 if (priv->tx_desc_cnt * sizeof(priv->tx->desc[0]) < PAGE_SIZE) { 632 dev_err(&priv->pdev->dev, "Tx desc count %d too low\n", 633 priv->tx_desc_cnt); 634 return -EINVAL; 635 } 636 priv->rx_desc_cnt = be16_to_cpu(descriptor->rx_queue_entries); 637 if (priv->rx_desc_cnt * sizeof(priv->rx->desc.desc_ring[0]) 638 < PAGE_SIZE) { 639 dev_err(&priv->pdev->dev, "Rx desc count %d too low\n", 640 priv->rx_desc_cnt); 641 return -EINVAL; 642 } 643 return 0; 644 } 645 646 static int 647 gve_set_desc_cnt_dqo(struct gve_priv *priv, 648 const struct gve_device_descriptor *descriptor, 649 const struct gve_device_option_dqo_rda *dev_op_dqo_rda) 650 { 651 priv->tx_desc_cnt = be16_to_cpu(descriptor->tx_queue_entries); 652 priv->options_dqo_rda.tx_comp_ring_entries = 653 be16_to_cpu(dev_op_dqo_rda->tx_comp_ring_entries); 654 priv->rx_desc_cnt = be16_to_cpu(descriptor->rx_queue_entries); 655 priv->options_dqo_rda.rx_buff_ring_entries = 656 be16_to_cpu(dev_op_dqo_rda->rx_buff_ring_entries); 657 658 return 0; 659 } 660 661 int gve_adminq_describe_device(struct gve_priv *priv) 662 { 663 struct gve_device_option_gqi_rda *dev_op_gqi_rda = NULL; 664 struct gve_device_option_gqi_qpl *dev_op_gqi_qpl = NULL; 665 struct gve_device_option_dqo_rda *dev_op_dqo_rda = NULL; 666 struct gve_device_descriptor *descriptor; 667 union gve_adminq_command cmd; 668 dma_addr_t descriptor_bus; 669 int err = 0; 670 u8 *mac; 671 u16 mtu; 672 673 memset(&cmd, 0, sizeof(cmd)); 674 descriptor = dma_alloc_coherent(&priv->pdev->dev, PAGE_SIZE, 675 &descriptor_bus, GFP_KERNEL); 676 if (!descriptor) 677 return -ENOMEM; 678 cmd.opcode = cpu_to_be32(GVE_ADMINQ_DESCRIBE_DEVICE); 679 cmd.describe_device.device_descriptor_addr = 680 cpu_to_be64(descriptor_bus); 681 cmd.describe_device.device_descriptor_version = 682 cpu_to_be32(GVE_ADMINQ_DEVICE_DESCRIPTOR_VERSION); 683 cmd.describe_device.available_length = cpu_to_be32(PAGE_SIZE); 684 685 err = gve_adminq_execute_cmd(priv, &cmd); 686 if (err) 687 goto free_device_descriptor; 688 689 err = gve_process_device_options(priv, descriptor, &dev_op_gqi_rda, 690 &dev_op_gqi_qpl, &dev_op_dqo_rda); 691 if (err) 692 goto free_device_descriptor; 693 694 /* If the GQI_RAW_ADDRESSING option is not enabled and the queue format 695 * is not set to GqiRda, choose the queue format in a priority order: 696 * DqoRda, GqiRda, GqiQpl. Use GqiQpl as default. 697 */ 698 if (priv->queue_format == GVE_GQI_RDA_FORMAT) { 699 dev_info(&priv->pdev->dev, 700 "Driver is running with GQI RDA queue format.\n"); 701 } else if (dev_op_dqo_rda) { 702 priv->queue_format = GVE_DQO_RDA_FORMAT; 703 dev_info(&priv->pdev->dev, 704 "Driver is running with DQO RDA queue format.\n"); 705 } else if (dev_op_gqi_rda) { 706 priv->queue_format = GVE_GQI_RDA_FORMAT; 707 dev_info(&priv->pdev->dev, 708 "Driver is running with GQI RDA queue format.\n"); 709 } else { 710 priv->queue_format = GVE_GQI_QPL_FORMAT; 711 dev_info(&priv->pdev->dev, 712 "Driver is running with GQI QPL queue format.\n"); 713 } 714 if (gve_is_gqi(priv)) { 715 err = gve_set_desc_cnt(priv, descriptor); 716 } else { 717 /* DQO supports LRO. */ 718 priv->dev->hw_features |= NETIF_F_LRO; 719 err = gve_set_desc_cnt_dqo(priv, descriptor, dev_op_dqo_rda); 720 } 721 if (err) 722 goto free_device_descriptor; 723 724 priv->max_registered_pages = 725 be64_to_cpu(descriptor->max_registered_pages); 726 mtu = be16_to_cpu(descriptor->mtu); 727 if (mtu < ETH_MIN_MTU) { 728 dev_err(&priv->pdev->dev, "MTU %d below minimum MTU\n", mtu); 729 err = -EINVAL; 730 goto free_device_descriptor; 731 } 732 priv->dev->max_mtu = mtu; 733 priv->num_event_counters = be16_to_cpu(descriptor->counters); 734 ether_addr_copy(priv->dev->dev_addr, descriptor->mac); 735 mac = descriptor->mac; 736 dev_info(&priv->pdev->dev, "MAC addr: %pM\n", mac); 737 priv->tx_pages_per_qpl = be16_to_cpu(descriptor->tx_pages_per_qpl); 738 priv->rx_data_slot_cnt = be16_to_cpu(descriptor->rx_pages_per_qpl); 739 740 if (gve_is_gqi(priv) && priv->rx_data_slot_cnt < priv->rx_desc_cnt) { 741 dev_err(&priv->pdev->dev, "rx_data_slot_cnt cannot be smaller than rx_desc_cnt, setting rx_desc_cnt down to %d.\n", 742 priv->rx_data_slot_cnt); 743 priv->rx_desc_cnt = priv->rx_data_slot_cnt; 744 } 745 priv->default_num_queues = be16_to_cpu(descriptor->default_num_queues); 746 747 free_device_descriptor: 748 dma_free_coherent(&priv->pdev->dev, PAGE_SIZE, descriptor, 749 descriptor_bus); 750 return err; 751 } 752 753 int gve_adminq_register_page_list(struct gve_priv *priv, 754 struct gve_queue_page_list *qpl) 755 { 756 struct device *hdev = &priv->pdev->dev; 757 u32 num_entries = qpl->num_entries; 758 u32 size = num_entries * sizeof(qpl->page_buses[0]); 759 union gve_adminq_command cmd; 760 dma_addr_t page_list_bus; 761 __be64 *page_list; 762 int err; 763 int i; 764 765 memset(&cmd, 0, sizeof(cmd)); 766 page_list = dma_alloc_coherent(hdev, size, &page_list_bus, GFP_KERNEL); 767 if (!page_list) 768 return -ENOMEM; 769 770 for (i = 0; i < num_entries; i++) 771 page_list[i] = cpu_to_be64(qpl->page_buses[i]); 772 773 cmd.opcode = cpu_to_be32(GVE_ADMINQ_REGISTER_PAGE_LIST); 774 cmd.reg_page_list = (struct gve_adminq_register_page_list) { 775 .page_list_id = cpu_to_be32(qpl->id), 776 .num_pages = cpu_to_be32(num_entries), 777 .page_address_list_addr = cpu_to_be64(page_list_bus), 778 }; 779 780 err = gve_adminq_execute_cmd(priv, &cmd); 781 dma_free_coherent(hdev, size, page_list, page_list_bus); 782 return err; 783 } 784 785 int gve_adminq_unregister_page_list(struct gve_priv *priv, u32 page_list_id) 786 { 787 union gve_adminq_command cmd; 788 789 memset(&cmd, 0, sizeof(cmd)); 790 cmd.opcode = cpu_to_be32(GVE_ADMINQ_UNREGISTER_PAGE_LIST); 791 cmd.unreg_page_list = (struct gve_adminq_unregister_page_list) { 792 .page_list_id = cpu_to_be32(page_list_id), 793 }; 794 795 return gve_adminq_execute_cmd(priv, &cmd); 796 } 797 798 int gve_adminq_set_mtu(struct gve_priv *priv, u64 mtu) 799 { 800 union gve_adminq_command cmd; 801 802 memset(&cmd, 0, sizeof(cmd)); 803 cmd.opcode = cpu_to_be32(GVE_ADMINQ_SET_DRIVER_PARAMETER); 804 cmd.set_driver_param = (struct gve_adminq_set_driver_parameter) { 805 .parameter_type = cpu_to_be32(GVE_SET_PARAM_MTU), 806 .parameter_value = cpu_to_be64(mtu), 807 }; 808 809 return gve_adminq_execute_cmd(priv, &cmd); 810 } 811 812 int gve_adminq_report_stats(struct gve_priv *priv, u64 stats_report_len, 813 dma_addr_t stats_report_addr, u64 interval) 814 { 815 union gve_adminq_command cmd; 816 817 memset(&cmd, 0, sizeof(cmd)); 818 cmd.opcode = cpu_to_be32(GVE_ADMINQ_REPORT_STATS); 819 cmd.report_stats = (struct gve_adminq_report_stats) { 820 .stats_report_len = cpu_to_be64(stats_report_len), 821 .stats_report_addr = cpu_to_be64(stats_report_addr), 822 .interval = cpu_to_be64(interval), 823 }; 824 825 return gve_adminq_execute_cmd(priv, &cmd); 826 } 827 828 int gve_adminq_report_link_speed(struct gve_priv *priv) 829 { 830 union gve_adminq_command gvnic_cmd; 831 dma_addr_t link_speed_region_bus; 832 __be64 *link_speed_region; 833 int err; 834 835 link_speed_region = 836 dma_alloc_coherent(&priv->pdev->dev, sizeof(*link_speed_region), 837 &link_speed_region_bus, GFP_KERNEL); 838 839 if (!link_speed_region) 840 return -ENOMEM; 841 842 memset(&gvnic_cmd, 0, sizeof(gvnic_cmd)); 843 gvnic_cmd.opcode = cpu_to_be32(GVE_ADMINQ_REPORT_LINK_SPEED); 844 gvnic_cmd.report_link_speed.link_speed_address = 845 cpu_to_be64(link_speed_region_bus); 846 847 err = gve_adminq_execute_cmd(priv, &gvnic_cmd); 848 849 priv->link_speed = be64_to_cpu(*link_speed_region); 850 dma_free_coherent(&priv->pdev->dev, sizeof(*link_speed_region), link_speed_region, 851 link_speed_region_bus); 852 return err; 853 } 854 855 int gve_adminq_get_ptype_map_dqo(struct gve_priv *priv, 856 struct gve_ptype_lut *ptype_lut) 857 { 858 struct gve_ptype_map *ptype_map; 859 union gve_adminq_command cmd; 860 dma_addr_t ptype_map_bus; 861 int err = 0; 862 int i; 863 864 memset(&cmd, 0, sizeof(cmd)); 865 ptype_map = dma_alloc_coherent(&priv->pdev->dev, sizeof(*ptype_map), 866 &ptype_map_bus, GFP_KERNEL); 867 if (!ptype_map) 868 return -ENOMEM; 869 870 cmd.opcode = cpu_to_be32(GVE_ADMINQ_GET_PTYPE_MAP); 871 cmd.get_ptype_map = (struct gve_adminq_get_ptype_map) { 872 .ptype_map_len = cpu_to_be64(sizeof(*ptype_map)), 873 .ptype_map_addr = cpu_to_be64(ptype_map_bus), 874 }; 875 876 err = gve_adminq_execute_cmd(priv, &cmd); 877 if (err) 878 goto err; 879 880 /* Populate ptype_lut. */ 881 for (i = 0; i < GVE_NUM_PTYPES; i++) { 882 ptype_lut->ptypes[i].l3_type = 883 ptype_map->ptypes[i].l3_type; 884 ptype_lut->ptypes[i].l4_type = 885 ptype_map->ptypes[i].l4_type; 886 } 887 err: 888 dma_free_coherent(&priv->pdev->dev, sizeof(*ptype_map), ptype_map, 889 ptype_map_bus); 890 return err; 891 } 892