Merge branch 'gve-queue-api' · tjh.dev/kernel@cdc74c9

+7 -24

drivers/net/ethernet/google/gve/gve.h

··· 610 610 struct gve_priv *priv; 611 611 struct gve_tx_ring *tx; /* tx rings on this block */ 612 612 struct gve_rx_ring *rx; /* rx rings on this block */ 613 + u32 irq; 613 614 }; 614 615 615 616 /* Tracks allowed and current queue settings */ ··· 638 637 struct gve_ptype ptypes[GVE_NUM_PTYPES]; 639 638 }; 640 639 641 - /* Parameters for allocating queue page lists */ 642 - struct gve_qpls_alloc_cfg { 643 - struct gve_queue_config *tx_cfg; 644 - struct gve_queue_config *rx_cfg; 645 - 646 - u16 num_xdp_queues; 647 - bool raw_addressing; 648 - bool is_gqi; 649 - 650 - /* Allocated resources are returned here */ 651 - struct gve_queue_page_list *qpls; 652 - }; 653 - 654 640 /* Parameters for allocating resources for tx queues */ 655 641 struct gve_tx_alloc_rings_cfg { 656 642 struct gve_queue_config *qcfg; 657 - 658 - /* qpls must already be allocated */ 659 - struct gve_queue_page_list *qpls; 660 643 661 644 u16 ring_size; 662 645 u16 start_idx; ··· 656 671 /* tx config is also needed to determine QPL ids */ 657 672 struct gve_queue_config *qcfg; 658 673 struct gve_queue_config *qcfg_tx; 659 - 660 - /* qpls must already be allocated */ 661 - struct gve_queue_page_list *qpls; 662 674 663 675 u16 ring_size; 664 676 u16 packet_buffer_size; ··· 682 700 struct net_device *dev; 683 701 struct gve_tx_ring *tx; /* array of tx_cfg.num_queues */ 684 702 struct gve_rx_ring *rx; /* array of rx_cfg.num_queues */ 685 - struct gve_queue_page_list *qpls; /* array of num qpls */ 686 703 struct gve_notify_block *ntfy_blocks; /* array of num_ntfy_blks */ 687 704 struct gve_irq_db *irq_db_indices; /* array of num_ntfy_blks */ 688 705 dma_addr_t irq_db_indices_bus; ··· 1005 1024 return priv->tx_cfg.max_queues + rx_qid; 1006 1025 } 1007 1026 1008 - /* Returns the index into priv->qpls where a certain rx queue's QPL resides */ 1009 1027 static inline u32 gve_get_rx_qpl_id(const struct gve_queue_config *tx_cfg, int rx_qid) 1010 1028 { 1011 1029 return tx_cfg->max_queues + rx_qid; ··· 1015 1035 return gve_tx_qpl_id(priv, 0); 1016 1036 } 1017 1037 1018 - /* Returns the index into priv->qpls where the first rx queue's QPL resides */ 1019 1038 static inline u32 gve_rx_start_qpl_id(const struct gve_queue_config *tx_cfg) 1020 1039 { 1021 1040 return gve_get_rx_qpl_id(tx_cfg, 0); ··· 1068 1089 enum dma_data_direction, gfp_t gfp_flags); 1069 1090 void gve_free_page(struct device *dev, struct page *page, dma_addr_t dma, 1070 1091 enum dma_data_direction); 1092 + /* qpls */ 1093 + struct gve_queue_page_list *gve_alloc_queue_page_list(struct gve_priv *priv, 1094 + u32 id, int pages); 1095 + void gve_free_queue_page_list(struct gve_priv *priv, 1096 + struct gve_queue_page_list *qpl, 1097 + u32 id); 1071 1098 /* tx handling */ 1072 1099 netdev_tx_t gve_tx(struct sk_buff *skb, struct net_device *dev); 1073 1100 int gve_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames, ··· 1110 1125 void gve_schedule_reset(struct gve_priv *priv); 1111 1126 int gve_reset(struct gve_priv *priv, bool attempt_teardown); 1112 1127 void gve_get_curr_alloc_cfgs(struct gve_priv *priv, 1113 - struct gve_qpls_alloc_cfg *qpls_alloc_cfg, 1114 1128 struct gve_tx_alloc_rings_cfg *tx_alloc_cfg, 1115 1129 struct gve_rx_alloc_rings_cfg *rx_alloc_cfg); 1116 1130 int gve_adjust_config(struct gve_priv *priv, 1117 - struct gve_qpls_alloc_cfg *qpls_alloc_cfg, 1118 1131 struct gve_tx_alloc_rings_cfg *tx_alloc_cfg, 1119 1132 struct gve_rx_alloc_rings_cfg *rx_alloc_cfg); 1120 1133 int gve_adjust_queues(struct gve_priv *priv,

+52 -27

drivers/net/ethernet/google/gve/gve_adminq.c

··· 630 630 return gve_adminq_kick_and_wait(priv); 631 631 } 632 632 633 - static int gve_adminq_create_rx_queue(struct gve_priv *priv, u32 queue_index) 633 + static void gve_adminq_get_create_rx_queue_cmd(struct gve_priv *priv, 634 + union gve_adminq_command *cmd, 635 + u32 queue_index) 634 636 { 635 637 struct gve_rx_ring *rx = &priv->rx[queue_index]; 636 - union gve_adminq_command cmd; 637 638 638 - memset(&cmd, 0, sizeof(cmd)); 639 - cmd.opcode = cpu_to_be32(GVE_ADMINQ_CREATE_RX_QUEUE); 640 - cmd.create_rx_queue = (struct gve_adminq_create_rx_queue) { 639 + memset(cmd, 0, sizeof(*cmd)); 640 + cmd->opcode = cpu_to_be32(GVE_ADMINQ_CREATE_RX_QUEUE); 641 + cmd->create_rx_queue = (struct gve_adminq_create_rx_queue) { 641 642 .queue_id = cpu_to_be32(queue_index), 642 643 .ntfy_id = cpu_to_be32(rx->ntfy_id), 643 644 .queue_resources_addr = cpu_to_be64(rx->q_resources_bus), ··· 649 648 u32 qpl_id = priv->queue_format == GVE_GQI_RDA_FORMAT ? 650 649 GVE_RAW_ADDRESSING_QPL_ID : rx->data.qpl->id; 651 650 652 - cmd.create_rx_queue.rx_desc_ring_addr = 651 + cmd->create_rx_queue.rx_desc_ring_addr = 653 652 cpu_to_be64(rx->desc.bus), 654 - cmd.create_rx_queue.rx_data_ring_addr = 653 + cmd->create_rx_queue.rx_data_ring_addr = 655 654 cpu_to_be64(rx->data.data_bus), 656 - cmd.create_rx_queue.index = cpu_to_be32(queue_index); 657 - cmd.create_rx_queue.queue_page_list_id = cpu_to_be32(qpl_id); 658 - cmd.create_rx_queue.packet_buffer_size = cpu_to_be16(rx->packet_buffer_size); 655 + cmd->create_rx_queue.index = cpu_to_be32(queue_index); 656 + cmd->create_rx_queue.queue_page_list_id = cpu_to_be32(qpl_id); 657 + cmd->create_rx_queue.packet_buffer_size = cpu_to_be16(rx->packet_buffer_size); 659 658 } else { 660 659 u32 qpl_id = 0; 661 660 ··· 663 662 qpl_id = GVE_RAW_ADDRESSING_QPL_ID; 664 663 else 665 664 qpl_id = rx->dqo.qpl->id; 666 - cmd.create_rx_queue.queue_page_list_id = cpu_to_be32(qpl_id); 667 - cmd.create_rx_queue.rx_desc_ring_addr = 665 + cmd->create_rx_queue.queue_page_list_id = cpu_to_be32(qpl_id); 666 + cmd->create_rx_queue.rx_desc_ring_addr = 668 667 cpu_to_be64(rx->dqo.complq.bus); 669 - cmd.create_rx_queue.rx_data_ring_addr = 668 + cmd->create_rx_queue.rx_data_ring_addr = 670 669 cpu_to_be64(rx->dqo.bufq.bus); 671 - cmd.create_rx_queue.packet_buffer_size = 670 + cmd->create_rx_queue.packet_buffer_size = 672 671 cpu_to_be16(priv->data_buffer_size_dqo); 673 - cmd.create_rx_queue.rx_buff_ring_size = 672 + cmd->create_rx_queue.rx_buff_ring_size = 674 673 cpu_to_be16(priv->rx_desc_cnt); 675 - cmd.create_rx_queue.enable_rsc = 674 + cmd->create_rx_queue.enable_rsc = 676 675 !!(priv->dev->features & NETIF_F_LRO); 677 676 if (priv->header_split_enabled) 678 - cmd.create_rx_queue.header_buffer_size = 677 + cmd->create_rx_queue.header_buffer_size = 679 678 cpu_to_be16(priv->header_buf_size); 680 679 } 680 + } 681 681 682 + static int gve_adminq_create_rx_queue(struct gve_priv *priv, u32 queue_index) 683 + { 684 + union gve_adminq_command cmd; 685 + 686 + gve_adminq_get_create_rx_queue_cmd(priv, &cmd, queue_index); 682 687 return gve_adminq_issue_cmd(priv, &cmd); 688 + } 689 + 690 + /* Unlike gve_adminq_create_rx_queue, this actually rings the doorbell */ 691 + int gve_adminq_create_single_rx_queue(struct gve_priv *priv, u32 queue_index) 692 + { 693 + union gve_adminq_command cmd; 694 + 695 + gve_adminq_get_create_rx_queue_cmd(priv, &cmd, queue_index); 696 + return gve_adminq_execute_cmd(priv, &cmd); 683 697 } 684 698 685 699 int gve_adminq_create_rx_queues(struct gve_priv *priv, u32 num_queues) ··· 743 727 return gve_adminq_kick_and_wait(priv); 744 728 } 745 729 730 + static void gve_adminq_make_destroy_rx_queue_cmd(union gve_adminq_command *cmd, 731 + u32 queue_index) 732 + { 733 + memset(cmd, 0, sizeof(*cmd)); 734 + cmd->opcode = cpu_to_be32(GVE_ADMINQ_DESTROY_RX_QUEUE); 735 + cmd->destroy_rx_queue = (struct gve_adminq_destroy_rx_queue) { 736 + .queue_id = cpu_to_be32(queue_index), 737 + }; 738 + } 739 + 746 740 static int gve_adminq_destroy_rx_queue(struct gve_priv *priv, u32 queue_index) 747 741 { 748 742 union gve_adminq_command cmd; 749 - int err; 750 743 751 - memset(&cmd, 0, sizeof(cmd)); 752 - cmd.opcode = cpu_to_be32(GVE_ADMINQ_DESTROY_RX_QUEUE); 753 - cmd.destroy_rx_queue = (struct gve_adminq_destroy_rx_queue) { 754 - .queue_id = cpu_to_be32(queue_index), 755 - }; 744 + gve_adminq_make_destroy_rx_queue_cmd(&cmd, queue_index); 745 + return gve_adminq_issue_cmd(priv, &cmd); 746 + } 756 747 757 - err = gve_adminq_issue_cmd(priv, &cmd); 758 - if (err) 759 - return err; 748 + /* Unlike gve_adminq_destroy_rx_queue, this actually rings the doorbell */ 749 + int gve_adminq_destroy_single_rx_queue(struct gve_priv *priv, u32 queue_index) 750 + { 751 + union gve_adminq_command cmd; 760 752 761 - return 0; 753 + gve_adminq_make_destroy_rx_queue_cmd(&cmd, queue_index); 754 + return gve_adminq_execute_cmd(priv, &cmd); 762 755 } 763 756 764 757 int gve_adminq_destroy_rx_queues(struct gve_priv *priv, u32 num_queues)

+2

drivers/net/ethernet/google/gve/gve_adminq.h

··· 451 451 int gve_adminq_deconfigure_device_resources(struct gve_priv *priv); 452 452 int gve_adminq_create_tx_queues(struct gve_priv *priv, u32 start_id, u32 num_queues); 453 453 int gve_adminq_destroy_tx_queues(struct gve_priv *priv, u32 start_id, u32 num_queues); 454 + int gve_adminq_create_single_rx_queue(struct gve_priv *priv, u32 queue_index); 454 455 int gve_adminq_create_rx_queues(struct gve_priv *priv, u32 num_queues); 456 + int gve_adminq_destroy_single_rx_queue(struct gve_priv *priv, u32 queue_index); 455 457 int gve_adminq_destroy_rx_queues(struct gve_priv *priv, u32 queue_id); 456 458 int gve_adminq_register_page_list(struct gve_priv *priv, 457 459 struct gve_queue_page_list *qpl);

+37 -11

drivers/net/ethernet/google/gve/gve_ethtool.c

··· 8 8 #include "gve.h" 9 9 #include "gve_adminq.h" 10 10 #include "gve_dqo.h" 11 + #include "gve_utils.h" 11 12 12 13 static void gve_get_drvinfo(struct net_device *netdev, 13 14 struct ethtool_drvinfo *info) ··· 166 165 struct stats *report_stats; 167 166 int *rx_qid_to_stats_idx; 168 167 int *tx_qid_to_stats_idx; 168 + int num_stopped_rxqs = 0; 169 + int num_stopped_txqs = 0; 169 170 struct gve_priv *priv; 170 171 bool skip_nic_stats; 171 172 unsigned int start; ··· 184 181 sizeof(int), GFP_KERNEL); 185 182 if (!rx_qid_to_stats_idx) 186 183 return; 184 + for (ring = 0; ring < priv->rx_cfg.num_queues; ring++) { 185 + rx_qid_to_stats_idx[ring] = -1; 186 + if (!gve_rx_was_added_to_block(priv, ring)) 187 + num_stopped_rxqs++; 188 + } 187 189 tx_qid_to_stats_idx = kmalloc_array(num_tx_queues, 188 190 sizeof(int), GFP_KERNEL); 189 191 if (!tx_qid_to_stats_idx) { 190 192 kfree(rx_qid_to_stats_idx); 191 193 return; 192 194 } 195 + for (ring = 0; ring < num_tx_queues; ring++) { 196 + tx_qid_to_stats_idx[ring] = -1; 197 + if (!gve_tx_was_added_to_block(priv, ring)) 198 + num_stopped_txqs++; 199 + } 200 + 193 201 for (rx_pkts = 0, rx_bytes = 0, rx_hsplit_pkt = 0, 194 202 rx_skb_alloc_fail = 0, rx_buf_alloc_fail = 0, 195 203 rx_desc_err_dropped_pkt = 0, rx_hsplit_unsplit_pkt = 0, ··· 274 260 /* For rx cross-reporting stats, start from nic rx stats in report */ 275 261 base_stats_idx = GVE_TX_STATS_REPORT_NUM * num_tx_queues + 276 262 GVE_RX_STATS_REPORT_NUM * priv->rx_cfg.num_queues; 277 - max_stats_idx = NIC_RX_STATS_REPORT_NUM * priv->rx_cfg.num_queues + 263 + /* The boundary between driver stats and NIC stats shifts if there are 264 + * stopped queues. 265 + */ 266 + base_stats_idx += NIC_RX_STATS_REPORT_NUM * num_stopped_rxqs + 267 + NIC_TX_STATS_REPORT_NUM * num_stopped_txqs; 268 + max_stats_idx = NIC_RX_STATS_REPORT_NUM * 269 + (priv->rx_cfg.num_queues - num_stopped_rxqs) + 278 270 base_stats_idx; 279 271 /* Preprocess the stats report for rx, map queue id to start index */ 280 272 skip_nic_stats = false; ··· 293 273 /* no stats written by NIC yet */ 294 274 skip_nic_stats = true; 295 275 break; 276 + } 277 + if (queue_id < 0 || queue_id >= priv->rx_cfg.num_queues) { 278 + net_err_ratelimited("Invalid rxq id in NIC stats\n"); 279 + continue; 296 280 } 297 281 rx_qid_to_stats_idx[queue_id] = stats_idx; 298 282 } ··· 332 308 data[i++] = rx->rx_copybreak_pkt; 333 309 data[i++] = rx->rx_copied_pkt; 334 310 /* stats from NIC */ 335 - if (skip_nic_stats) { 311 + stats_idx = rx_qid_to_stats_idx[ring]; 312 + if (skip_nic_stats || stats_idx < 0) { 336 313 /* skip NIC rx stats */ 337 314 i += NIC_RX_STATS_REPORT_NUM; 338 315 } else { 339 - stats_idx = rx_qid_to_stats_idx[ring]; 340 316 for (j = 0; j < NIC_RX_STATS_REPORT_NUM; j++) { 341 317 u64 value = 342 318 be64_to_cpu(report_stats[stats_idx + j].value); ··· 362 338 363 339 /* For tx cross-reporting stats, start from nic tx stats in report */ 364 340 base_stats_idx = max_stats_idx; 365 - max_stats_idx = NIC_TX_STATS_REPORT_NUM * num_tx_queues + 341 + max_stats_idx = NIC_TX_STATS_REPORT_NUM * 342 + (num_tx_queues - num_stopped_txqs) + 366 343 max_stats_idx; 367 344 /* Preprocess the stats report for tx, map queue id to start index */ 368 345 skip_nic_stats = false; ··· 376 351 /* no stats written by NIC yet */ 377 352 skip_nic_stats = true; 378 353 break; 354 + } 355 + if (queue_id < 0 || queue_id >= num_tx_queues) { 356 + net_err_ratelimited("Invalid txq id in NIC stats\n"); 357 + continue; 379 358 } 380 359 tx_qid_to_stats_idx[queue_id] = stats_idx; 381 360 } ··· 412 383 data[i++] = gve_tx_load_event_counter(priv, tx); 413 384 data[i++] = tx->dma_mapping_error; 414 385 /* stats from NIC */ 415 - if (skip_nic_stats) { 386 + stats_idx = tx_qid_to_stats_idx[ring]; 387 + if (skip_nic_stats || stats_idx < 0) { 416 388 /* skip NIC tx stats */ 417 389 i += NIC_TX_STATS_REPORT_NUM; 418 390 } else { 419 - stats_idx = tx_qid_to_stats_idx[ring]; 420 391 for (j = 0; j < NIC_TX_STATS_REPORT_NUM; j++) { 421 392 u64 value = 422 393 be64_to_cpu(report_stats[stats_idx + j].value); ··· 538 509 { 539 510 struct gve_tx_alloc_rings_cfg tx_alloc_cfg = {0}; 540 511 struct gve_rx_alloc_rings_cfg rx_alloc_cfg = {0}; 541 - struct gve_qpls_alloc_cfg qpls_alloc_cfg = {0}; 542 512 int err; 543 513 544 514 /* get current queue configuration */ 545 - gve_get_curr_alloc_cfgs(priv, &qpls_alloc_cfg, 546 - &tx_alloc_cfg, &rx_alloc_cfg); 515 + gve_get_curr_alloc_cfgs(priv, &tx_alloc_cfg, &rx_alloc_cfg); 547 516 548 517 /* copy over the new ring_size from ethtool */ 549 518 tx_alloc_cfg.ring_size = new_tx_desc_cnt; 550 519 rx_alloc_cfg.ring_size = new_rx_desc_cnt; 551 520 552 521 if (netif_running(priv->dev)) { 553 - err = gve_adjust_config(priv, &qpls_alloc_cfg, 554 - &tx_alloc_cfg, &rx_alloc_cfg); 522 + err = gve_adjust_config(priv, &tx_alloc_cfg, &rx_alloc_cfg); 555 523 if (err) 556 524 return err; 557 525 }

+141 -259

drivers/net/ethernet/google/gve/gve_main.c

··· 9 9 #include <linux/etherdevice.h> 10 10 #include <linux/filter.h> 11 11 #include <linux/interrupt.h> 12 + #include <linux/irq.h> 12 13 #include <linux/module.h> 13 14 #include <linux/pci.h> 14 15 #include <linux/sched.h> ··· 254 253 return IRQ_HANDLED; 255 254 } 256 255 256 + static int gve_is_napi_on_home_cpu(struct gve_priv *priv, u32 irq) 257 + { 258 + int cpu_curr = smp_processor_id(); 259 + const struct cpumask *aff_mask; 260 + 261 + aff_mask = irq_get_effective_affinity_mask(irq); 262 + if (unlikely(!aff_mask)) 263 + return 1; 264 + 265 + return cpumask_test_cpu(cpu_curr, aff_mask); 266 + } 267 + 257 268 int gve_napi_poll(struct napi_struct *napi, int budget) 258 269 { 259 270 struct gve_notify_block *block; ··· 335 322 reschedule |= work_done == budget; 336 323 } 337 324 338 - if (reschedule) 339 - return budget; 325 + if (reschedule) { 326 + /* Reschedule by returning budget only if already on the correct 327 + * cpu. 328 + */ 329 + if (likely(gve_is_napi_on_home_cpu(priv, block->irq))) 330 + return budget; 331 + 332 + /* If not on the cpu with which this queue's irq has affinity 333 + * with, we avoid rescheduling napi and arm the irq instead so 334 + * that napi gets rescheduled back eventually onto the right 335 + * cpu. 336 + */ 337 + if (work_done == budget) 338 + work_done--; 339 + } 340 340 341 341 if (likely(napi_complete_done(napi, work_done))) { 342 342 /* Enable interrupts again. ··· 454 428 "Failed to receive msix vector %d\n", i); 455 429 goto abort_with_some_ntfy_blocks; 456 430 } 431 + block->irq = priv->msix_vectors[msix_idx].vector; 457 432 irq_set_affinity_hint(priv->msix_vectors[msix_idx].vector, 458 433 get_cpu_mask(i % active_cpus)); 459 434 block->irq_db_index = &priv->irq_db_indices[i].index; ··· 468 441 irq_set_affinity_hint(priv->msix_vectors[msix_idx].vector, 469 442 NULL); 470 443 free_irq(priv->msix_vectors[msix_idx].vector, block); 444 + block->irq = 0; 471 445 } 472 446 kvfree(priv->ntfy_blocks); 473 447 priv->ntfy_blocks = NULL; ··· 502 474 irq_set_affinity_hint(priv->msix_vectors[msix_idx].vector, 503 475 NULL); 504 476 free_irq(priv->msix_vectors[msix_idx].vector, block); 477 + block->irq = 0; 505 478 } 506 479 free_irq(priv->msix_vectors[priv->mgmt_msix_idx].vector, priv); 507 480 kvfree(priv->ntfy_blocks); ··· 611 582 gve_clear_device_resources_ok(priv); 612 583 } 613 584 614 - static int gve_unregister_qpl(struct gve_priv *priv, u32 i) 585 + static int gve_unregister_qpl(struct gve_priv *priv, 586 + struct gve_queue_page_list *qpl) 615 587 { 616 588 int err; 617 589 618 - err = gve_adminq_unregister_page_list(priv, priv->qpls[i].id); 590 + if (!qpl) 591 + return 0; 592 + 593 + err = gve_adminq_unregister_page_list(priv, qpl->id); 619 594 if (err) { 620 595 netif_err(priv, drv, priv->dev, 621 596 "Failed to unregister queue page list %d\n", 622 - priv->qpls[i].id); 597 + qpl->id); 623 598 return err; 624 599 } 625 600 626 - priv->num_registered_pages -= priv->qpls[i].num_entries; 601 + priv->num_registered_pages -= qpl->num_entries; 627 602 return 0; 628 603 } 629 604 630 - static int gve_register_qpl(struct gve_priv *priv, u32 i) 605 + static int gve_register_qpl(struct gve_priv *priv, 606 + struct gve_queue_page_list *qpl) 631 607 { 632 - int num_rx_qpls; 633 608 int pages; 634 609 int err; 635 610 636 - /* Rx QPLs succeed Tx QPLs in the priv->qpls array. */ 637 - num_rx_qpls = gve_num_rx_qpls(&priv->rx_cfg, gve_is_qpl(priv)); 638 - if (i >= gve_rx_start_qpl_id(&priv->tx_cfg) + num_rx_qpls) { 639 - netif_err(priv, drv, priv->dev, 640 - "Cannot register nonexisting QPL at index %d\n", i); 641 - return -EINVAL; 642 - } 611 + if (!qpl) 612 + return 0; 643 613 644 - pages = priv->qpls[i].num_entries; 614 + pages = qpl->num_entries; 645 615 646 616 if (pages + priv->num_registered_pages > priv->max_registered_pages) { 647 617 netif_err(priv, drv, priv->dev, ··· 650 622 return -EINVAL; 651 623 } 652 624 653 - err = gve_adminq_register_page_list(priv, &priv->qpls[i]); 625 + err = gve_adminq_register_page_list(priv, qpl); 654 626 if (err) { 655 627 netif_err(priv, drv, priv->dev, 656 628 "failed to register queue page list %d\n", 657 - priv->qpls[i].id); 658 - /* This failure will trigger a reset - no need to clean 659 - * up 660 - */ 629 + qpl->id); 661 630 return err; 662 631 } 663 632 664 633 priv->num_registered_pages += pages; 665 634 return 0; 635 + } 636 + 637 + static struct gve_queue_page_list *gve_tx_get_qpl(struct gve_priv *priv, int idx) 638 + { 639 + struct gve_tx_ring *tx = &priv->tx[idx]; 640 + 641 + if (gve_is_gqi(priv)) 642 + return tx->tx_fifo.qpl; 643 + else 644 + return tx->dqo.qpl; 645 + } 646 + 647 + static struct gve_queue_page_list *gve_rx_get_qpl(struct gve_priv *priv, int idx) 648 + { 649 + struct gve_rx_ring *rx = &priv->rx[idx]; 650 + 651 + if (gve_is_gqi(priv)) 652 + return rx->data.qpl; 653 + else 654 + return rx->dqo.qpl; 666 655 } 667 656 668 657 static int gve_register_xdp_qpls(struct gve_priv *priv) ··· 690 645 691 646 start_id = gve_xdp_tx_start_queue_id(priv); 692 647 for (i = start_id; i < start_id + gve_num_xdp_qpls(priv); i++) { 693 - err = gve_register_qpl(priv, i); 648 + err = gve_register_qpl(priv, gve_tx_get_qpl(priv, i)); 694 649 /* This failure will trigger a reset - no need to clean up */ 695 650 if (err) 696 651 return err; ··· 701 656 static int gve_register_qpls(struct gve_priv *priv) 702 657 { 703 658 int num_tx_qpls, num_rx_qpls; 704 - int start_id; 705 659 int err; 706 660 int i; 707 661 ··· 709 665 num_rx_qpls = gve_num_rx_qpls(&priv->rx_cfg, gve_is_qpl(priv)); 710 666 711 667 for (i = 0; i < num_tx_qpls; i++) { 712 - err = gve_register_qpl(priv, i); 668 + err = gve_register_qpl(priv, gve_tx_get_qpl(priv, i)); 713 669 if (err) 714 670 return err; 715 671 } 716 672 717 - /* there might be a gap between the tx and rx qpl ids */ 718 - start_id = gve_rx_start_qpl_id(&priv->tx_cfg); 719 673 for (i = 0; i < num_rx_qpls; i++) { 720 - err = gve_register_qpl(priv, start_id + i); 674 + err = gve_register_qpl(priv, gve_rx_get_qpl(priv, i)); 721 675 if (err) 722 676 return err; 723 677 } ··· 731 689 732 690 start_id = gve_xdp_tx_start_queue_id(priv); 733 691 for (i = start_id; i < start_id + gve_num_xdp_qpls(priv); i++) { 734 - err = gve_unregister_qpl(priv, i); 692 + err = gve_unregister_qpl(priv, gve_tx_get_qpl(priv, i)); 735 693 /* This failure will trigger a reset - no need to clean */ 736 694 if (err) 737 695 return err; ··· 742 700 static int gve_unregister_qpls(struct gve_priv *priv) 743 701 { 744 702 int num_tx_qpls, num_rx_qpls; 745 - int start_id; 746 703 int err; 747 704 int i; 748 705 ··· 750 709 num_rx_qpls = gve_num_rx_qpls(&priv->rx_cfg, gve_is_qpl(priv)); 751 710 752 711 for (i = 0; i < num_tx_qpls; i++) { 753 - err = gve_unregister_qpl(priv, i); 712 + err = gve_unregister_qpl(priv, gve_tx_get_qpl(priv, i)); 754 713 /* This failure will trigger a reset - no need to clean */ 755 714 if (err) 756 715 return err; 757 716 } 758 717 759 - start_id = gve_rx_start_qpl_id(&priv->tx_cfg); 760 718 for (i = 0; i < num_rx_qpls; i++) { 761 - err = gve_unregister_qpl(priv, start_id + i); 719 + err = gve_unregister_qpl(priv, gve_rx_get_qpl(priv, i)); 762 720 /* This failure will trigger a reset - no need to clean */ 763 721 if (err) 764 722 return err; ··· 868 828 { 869 829 cfg->qcfg = &priv->tx_cfg; 870 830 cfg->raw_addressing = !gve_is_qpl(priv); 871 - cfg->qpls = priv->qpls; 872 831 cfg->ring_size = priv->tx_desc_cnt; 873 832 cfg->start_idx = 0; 874 833 cfg->num_rings = gve_num_tx_queues(priv); ··· 924 885 return 0; 925 886 } 926 887 927 - static int gve_alloc_rings(struct gve_priv *priv, 928 - struct gve_tx_alloc_rings_cfg *tx_alloc_cfg, 929 - struct gve_rx_alloc_rings_cfg *rx_alloc_cfg) 888 + static int gve_queues_mem_alloc(struct gve_priv *priv, 889 + struct gve_tx_alloc_rings_cfg *tx_alloc_cfg, 890 + struct gve_rx_alloc_rings_cfg *rx_alloc_cfg) 930 891 { 931 892 int err; 932 893 ··· 1012 973 } 1013 974 } 1014 975 1015 - static void gve_free_rings(struct gve_priv *priv, 1016 - struct gve_tx_alloc_rings_cfg *tx_cfg, 1017 - struct gve_rx_alloc_rings_cfg *rx_cfg) 976 + static void gve_queues_mem_free(struct gve_priv *priv, 977 + struct gve_tx_alloc_rings_cfg *tx_cfg, 978 + struct gve_rx_alloc_rings_cfg *rx_cfg) 1018 979 { 1019 980 if (gve_is_gqi(priv)) { 1020 981 gve_tx_free_rings_gqi(priv, tx_cfg); ··· 1043 1004 return 0; 1044 1005 } 1045 1006 1046 - static int gve_alloc_queue_page_list(struct gve_priv *priv, 1047 - struct gve_queue_page_list *qpl, 1048 - u32 id, int pages) 1007 + struct gve_queue_page_list *gve_alloc_queue_page_list(struct gve_priv *priv, 1008 + u32 id, int pages) 1049 1009 { 1010 + struct gve_queue_page_list *qpl; 1050 1011 int err; 1051 1012 int i; 1013 + 1014 + qpl = kvzalloc(sizeof(*qpl), GFP_KERNEL); 1015 + if (!qpl) 1016 + return NULL; 1052 1017 1053 1018 qpl->id = id; 1054 1019 qpl->num_entries = 0; 1055 1020 qpl->pages = kvcalloc(pages, sizeof(*qpl->pages), GFP_KERNEL); 1056 - /* caller handles clean up */ 1057 1021 if (!qpl->pages) 1058 - return -ENOMEM; 1022 + goto abort; 1023 + 1059 1024 qpl->page_buses = kvcalloc(pages, sizeof(*qpl->page_buses), GFP_KERNEL); 1060 - /* caller handles clean up */ 1061 1025 if (!qpl->page_buses) 1062 - return -ENOMEM; 1026 + goto abort; 1063 1027 1064 1028 for (i = 0; i < pages; i++) { 1065 1029 err = gve_alloc_page(priv, &priv->pdev->dev, &qpl->pages[i], 1066 1030 &qpl->page_buses[i], 1067 1031 gve_qpl_dma_dir(priv, id), GFP_KERNEL); 1068 - /* caller handles clean up */ 1069 1032 if (err) 1070 - return -ENOMEM; 1033 + goto abort; 1071 1034 qpl->num_entries++; 1072 1035 } 1073 1036 1074 - return 0; 1037 + return qpl; 1038 + 1039 + abort: 1040 + gve_free_queue_page_list(priv, qpl, id); 1041 + return NULL; 1075 1042 } 1076 1043 1077 1044 void gve_free_page(struct device *dev, struct page *page, dma_addr_t dma, ··· 1089 1044 put_page(page); 1090 1045 } 1091 1046 1092 - static void gve_free_queue_page_list(struct gve_priv *priv, 1093 - struct gve_queue_page_list *qpl, 1094 - int id) 1047 + void gve_free_queue_page_list(struct gve_priv *priv, 1048 + struct gve_queue_page_list *qpl, 1049 + u32 id) 1095 1050 { 1096 1051 int i; 1097 1052 1098 - if (!qpl->pages) 1053 + if (!qpl) 1099 1054 return; 1055 + if (!qpl->pages) 1056 + goto free_qpl; 1100 1057 if (!qpl->page_buses) 1101 1058 goto free_pages; 1102 1059 ··· 1111 1064 free_pages: 1112 1065 kvfree(qpl->pages); 1113 1066 qpl->pages = NULL; 1114 - } 1115 - 1116 - static void gve_free_n_qpls(struct gve_priv *priv, 1117 - struct gve_queue_page_list *qpls, 1118 - int start_id, 1119 - int num_qpls) 1120 - { 1121 - int i; 1122 - 1123 - for (i = start_id; i < start_id + num_qpls; i++) 1124 - gve_free_queue_page_list(priv, &qpls[i], i); 1125 - } 1126 - 1127 - static int gve_alloc_n_qpls(struct gve_priv *priv, 1128 - struct gve_queue_page_list *qpls, 1129 - int page_count, 1130 - int start_id, 1131 - int num_qpls) 1132 - { 1133 - int err; 1134 - int i; 1135 - 1136 - for (i = start_id; i < start_id + num_qpls; i++) { 1137 - err = gve_alloc_queue_page_list(priv, &qpls[i], i, page_count); 1138 - if (err) 1139 - goto free_qpls; 1140 - } 1141 - 1142 - return 0; 1143 - 1144 - free_qpls: 1145 - /* Must include the failing QPL too for gve_alloc_queue_page_list fails 1146 - * without cleaning up. 1147 - */ 1148 - gve_free_n_qpls(priv, qpls, start_id, i - start_id + 1); 1149 - return err; 1150 - } 1151 - 1152 - static int gve_alloc_qpls(struct gve_priv *priv, struct gve_qpls_alloc_cfg *cfg, 1153 - struct gve_rx_alloc_rings_cfg *rx_alloc_cfg) 1154 - { 1155 - int max_queues = cfg->tx_cfg->max_queues + cfg->rx_cfg->max_queues; 1156 - int rx_start_id, tx_num_qpls, rx_num_qpls; 1157 - struct gve_queue_page_list *qpls; 1158 - u32 page_count; 1159 - int err; 1160 - 1161 - if (cfg->raw_addressing) 1162 - return 0; 1163 - 1164 - qpls = kvcalloc(max_queues, sizeof(*qpls), GFP_KERNEL); 1165 - if (!qpls) 1166 - return -ENOMEM; 1167 - 1168 - /* Allocate TX QPLs */ 1169 - page_count = priv->tx_pages_per_qpl; 1170 - tx_num_qpls = gve_num_tx_qpls(cfg->tx_cfg, cfg->num_xdp_queues, 1171 - gve_is_qpl(priv)); 1172 - err = gve_alloc_n_qpls(priv, qpls, page_count, 0, tx_num_qpls); 1173 - if (err) 1174 - goto free_qpl_array; 1175 - 1176 - /* Allocate RX QPLs */ 1177 - rx_start_id = gve_rx_start_qpl_id(cfg->tx_cfg); 1178 - /* For GQI_QPL number of pages allocated have 1:1 relationship with 1179 - * number of descriptors. For DQO, number of pages required are 1180 - * more than descriptors (because of out of order completions). 1181 - * Set it to twice the number of descriptors. 1182 - */ 1183 - if (cfg->is_gqi) 1184 - page_count = rx_alloc_cfg->ring_size; 1185 - else 1186 - page_count = gve_get_rx_pages_per_qpl_dqo(rx_alloc_cfg->ring_size); 1187 - rx_num_qpls = gve_num_rx_qpls(cfg->rx_cfg, gve_is_qpl(priv)); 1188 - err = gve_alloc_n_qpls(priv, qpls, page_count, rx_start_id, rx_num_qpls); 1189 - if (err) 1190 - goto free_tx_qpls; 1191 - 1192 - cfg->qpls = qpls; 1193 - return 0; 1194 - 1195 - free_tx_qpls: 1196 - gve_free_n_qpls(priv, qpls, 0, tx_num_qpls); 1197 - free_qpl_array: 1198 - kvfree(qpls); 1199 - return err; 1200 - } 1201 - 1202 - static void gve_free_qpls(struct gve_priv *priv, 1203 - struct gve_qpls_alloc_cfg *cfg) 1204 - { 1205 - int max_queues = cfg->tx_cfg->max_queues + cfg->rx_cfg->max_queues; 1206 - struct gve_queue_page_list *qpls = cfg->qpls; 1207 - int i; 1208 - 1209 - if (!qpls) 1210 - return; 1211 - 1212 - for (i = 0; i < max_queues; i++) 1213 - gve_free_queue_page_list(priv, &qpls[i], i); 1214 - 1215 - kvfree(qpls); 1216 - cfg->qpls = NULL; 1067 + free_qpl: 1068 + kvfree(qpl); 1217 1069 } 1218 1070 1219 1071 /* Use this to schedule a reset when the device is capable of continuing ··· 1216 1270 page_frag_cache_drain(&priv->rx[i].page_cache); 1217 1271 } 1218 1272 1219 - static void gve_qpls_get_curr_alloc_cfg(struct gve_priv *priv, 1220 - struct gve_qpls_alloc_cfg *cfg) 1221 - { 1222 - cfg->raw_addressing = !gve_is_qpl(priv); 1223 - cfg->is_gqi = gve_is_gqi(priv); 1224 - cfg->num_xdp_queues = priv->num_xdp_queues; 1225 - cfg->tx_cfg = &priv->tx_cfg; 1226 - cfg->rx_cfg = &priv->rx_cfg; 1227 - cfg->qpls = priv->qpls; 1228 - } 1229 - 1230 1273 static void gve_rx_get_curr_alloc_cfg(struct gve_priv *priv, 1231 1274 struct gve_rx_alloc_rings_cfg *cfg) 1232 1275 { ··· 1223 1288 cfg->qcfg_tx = &priv->tx_cfg; 1224 1289 cfg->raw_addressing = !gve_is_qpl(priv); 1225 1290 cfg->enable_header_split = priv->header_split_enabled; 1226 - cfg->qpls = priv->qpls; 1227 1291 cfg->ring_size = priv->rx_desc_cnt; 1228 1292 cfg->packet_buffer_size = gve_is_gqi(priv) ? 1229 1293 GVE_DEFAULT_RX_BUFFER_SIZE : ··· 1231 1297 } 1232 1298 1233 1299 void gve_get_curr_alloc_cfgs(struct gve_priv *priv, 1234 - struct gve_qpls_alloc_cfg *qpls_alloc_cfg, 1235 1300 struct gve_tx_alloc_rings_cfg *tx_alloc_cfg, 1236 1301 struct gve_rx_alloc_rings_cfg *rx_alloc_cfg) 1237 1302 { 1238 - gve_qpls_get_curr_alloc_cfg(priv, qpls_alloc_cfg); 1239 1303 gve_tx_get_curr_alloc_cfg(priv, tx_alloc_cfg); 1240 1304 gve_rx_get_curr_alloc_cfg(priv, rx_alloc_cfg); 1241 1305 } ··· 1265 1333 } 1266 1334 } 1267 1335 1268 - static void gve_queues_mem_free(struct gve_priv *priv, 1269 - struct gve_qpls_alloc_cfg *qpls_alloc_cfg, 1270 - struct gve_tx_alloc_rings_cfg *tx_alloc_cfg, 1271 - struct gve_rx_alloc_rings_cfg *rx_alloc_cfg) 1272 - { 1273 - gve_free_rings(priv, tx_alloc_cfg, rx_alloc_cfg); 1274 - gve_free_qpls(priv, qpls_alloc_cfg); 1275 - } 1276 - 1277 - static int gve_queues_mem_alloc(struct gve_priv *priv, 1278 - struct gve_qpls_alloc_cfg *qpls_alloc_cfg, 1279 - struct gve_tx_alloc_rings_cfg *tx_alloc_cfg, 1280 - struct gve_rx_alloc_rings_cfg *rx_alloc_cfg) 1281 - { 1282 - int err; 1283 - 1284 - err = gve_alloc_qpls(priv, qpls_alloc_cfg, rx_alloc_cfg); 1285 - if (err) { 1286 - netif_err(priv, drv, priv->dev, "Failed to alloc QPLs\n"); 1287 - return err; 1288 - } 1289 - tx_alloc_cfg->qpls = qpls_alloc_cfg->qpls; 1290 - rx_alloc_cfg->qpls = qpls_alloc_cfg->qpls; 1291 - err = gve_alloc_rings(priv, tx_alloc_cfg, rx_alloc_cfg); 1292 - if (err) { 1293 - netif_err(priv, drv, priv->dev, "Failed to alloc rings\n"); 1294 - goto free_qpls; 1295 - } 1296 - 1297 - return 0; 1298 - 1299 - free_qpls: 1300 - gve_free_qpls(priv, qpls_alloc_cfg); 1301 - return err; 1302 - } 1303 - 1304 1336 static void gve_queues_mem_remove(struct gve_priv *priv) 1305 1337 { 1306 1338 struct gve_tx_alloc_rings_cfg tx_alloc_cfg = {0}; 1307 1339 struct gve_rx_alloc_rings_cfg rx_alloc_cfg = {0}; 1308 - struct gve_qpls_alloc_cfg qpls_alloc_cfg = {0}; 1309 1340 1310 - gve_get_curr_alloc_cfgs(priv, &qpls_alloc_cfg, 1311 - &tx_alloc_cfg, &rx_alloc_cfg); 1312 - gve_queues_mem_free(priv, &qpls_alloc_cfg, 1313 - &tx_alloc_cfg, &rx_alloc_cfg); 1314 - priv->qpls = NULL; 1341 + gve_get_curr_alloc_cfgs(priv, &tx_alloc_cfg, &rx_alloc_cfg); 1342 + gve_queues_mem_free(priv, &tx_alloc_cfg, &rx_alloc_cfg); 1315 1343 priv->tx = NULL; 1316 1344 priv->rx = NULL; 1317 1345 } ··· 1280 1388 * No memory is allocated. Passed-in memory is freed on errors. 1281 1389 */ 1282 1390 static int gve_queues_start(struct gve_priv *priv, 1283 - struct gve_qpls_alloc_cfg *qpls_alloc_cfg, 1284 1391 struct gve_tx_alloc_rings_cfg *tx_alloc_cfg, 1285 1392 struct gve_rx_alloc_rings_cfg *rx_alloc_cfg) 1286 1393 { ··· 1287 1396 int err; 1288 1397 1289 1398 /* Record new resources into priv */ 1290 - priv->qpls = qpls_alloc_cfg->qpls; 1291 1399 priv->tx = tx_alloc_cfg->tx; 1292 1400 priv->rx = rx_alloc_cfg->rx; 1293 1401 ··· 1358 1468 { 1359 1469 struct gve_tx_alloc_rings_cfg tx_alloc_cfg = {0}; 1360 1470 struct gve_rx_alloc_rings_cfg rx_alloc_cfg = {0}; 1361 - struct gve_qpls_alloc_cfg qpls_alloc_cfg = {0}; 1362 1471 struct gve_priv *priv = netdev_priv(dev); 1363 1472 int err; 1364 1473 1365 - gve_get_curr_alloc_cfgs(priv, &qpls_alloc_cfg, 1366 - &tx_alloc_cfg, &rx_alloc_cfg); 1474 + gve_get_curr_alloc_cfgs(priv, &tx_alloc_cfg, &rx_alloc_cfg); 1367 1475 1368 - err = gve_queues_mem_alloc(priv, &qpls_alloc_cfg, 1369 - &tx_alloc_cfg, &rx_alloc_cfg); 1476 + err = gve_queues_mem_alloc(priv, &tx_alloc_cfg, &rx_alloc_cfg); 1370 1477 if (err) 1371 1478 return err; 1372 1479 1373 1480 /* No need to free on error: ownership of resources is lost after 1374 1481 * calling gve_queues_start. 1375 1482 */ 1376 - err = gve_queues_start(priv, &qpls_alloc_cfg, 1377 - &tx_alloc_cfg, &rx_alloc_cfg); 1483 + err = gve_queues_start(priv, &tx_alloc_cfg, &rx_alloc_cfg); 1378 1484 if (err) 1379 1485 return err; 1380 1486 ··· 1429 1543 1430 1544 static int gve_remove_xdp_queues(struct gve_priv *priv) 1431 1545 { 1432 - int qpl_start_id; 1433 1546 int err; 1434 - 1435 - qpl_start_id = gve_xdp_tx_start_queue_id(priv); 1436 1547 1437 1548 err = gve_destroy_xdp_rings(priv); 1438 1549 if (err) ··· 1442 1559 gve_unreg_xdp_info(priv); 1443 1560 gve_free_xdp_rings(priv); 1444 1561 1445 - gve_free_n_qpls(priv, priv->qpls, qpl_start_id, gve_num_xdp_qpls(priv)); 1446 1562 priv->num_xdp_queues = 0; 1447 1563 return 0; 1448 1564 } 1449 1565 1450 1566 static int gve_add_xdp_queues(struct gve_priv *priv) 1451 1567 { 1452 - int start_id; 1453 1568 int err; 1454 1569 1455 1570 priv->num_xdp_queues = priv->rx_cfg.num_queues; 1456 1571 1457 - start_id = gve_xdp_tx_start_queue_id(priv); 1458 - err = gve_alloc_n_qpls(priv, priv->qpls, priv->tx_pages_per_qpl, 1459 - start_id, gve_num_xdp_qpls(priv)); 1460 - if (err) 1461 - goto err; 1462 - 1463 1572 err = gve_alloc_xdp_rings(priv); 1464 1573 if (err) 1465 - goto free_xdp_qpls; 1574 + goto err; 1466 1575 1467 1576 err = gve_reg_xdp_info(priv, priv->dev); 1468 1577 if (err) ··· 1472 1597 1473 1598 free_xdp_rings: 1474 1599 gve_free_xdp_rings(priv); 1475 - free_xdp_qpls: 1476 - gve_free_n_qpls(priv, priv->qpls, start_id, gve_num_xdp_qpls(priv)); 1477 1600 err: 1478 1601 priv->num_xdp_queues = 0; 1479 1602 return err; ··· 1722 1849 } 1723 1850 1724 1851 int gve_adjust_config(struct gve_priv *priv, 1725 - struct gve_qpls_alloc_cfg *qpls_alloc_cfg, 1726 1852 struct gve_tx_alloc_rings_cfg *tx_alloc_cfg, 1727 1853 struct gve_rx_alloc_rings_cfg *rx_alloc_cfg) 1728 1854 { 1729 1855 int err; 1730 1856 1731 1857 /* Allocate resources for the new confiugration */ 1732 - err = gve_queues_mem_alloc(priv, qpls_alloc_cfg, 1733 - tx_alloc_cfg, rx_alloc_cfg); 1858 + err = gve_queues_mem_alloc(priv, tx_alloc_cfg, rx_alloc_cfg); 1734 1859 if (err) { 1735 1860 netif_err(priv, drv, priv->dev, 1736 1861 "Adjust config failed to alloc new queues"); ··· 1740 1869 if (err) { 1741 1870 netif_err(priv, drv, priv->dev, 1742 1871 "Adjust config failed to close old queues"); 1743 - gve_queues_mem_free(priv, qpls_alloc_cfg, 1744 - tx_alloc_cfg, rx_alloc_cfg); 1872 + gve_queues_mem_free(priv, tx_alloc_cfg, rx_alloc_cfg); 1745 1873 return err; 1746 1874 } 1747 1875 1748 1876 /* Bring the device back up again with the new resources. */ 1749 - err = gve_queues_start(priv, qpls_alloc_cfg, 1750 - tx_alloc_cfg, rx_alloc_cfg); 1877 + err = gve_queues_start(priv, tx_alloc_cfg, rx_alloc_cfg); 1751 1878 if (err) { 1752 1879 netif_err(priv, drv, priv->dev, 1753 1880 "Adjust config failed to start new queues, !!! DISABLING ALL QUEUES !!!\n"); ··· 1765 1896 { 1766 1897 struct gve_tx_alloc_rings_cfg tx_alloc_cfg = {0}; 1767 1898 struct gve_rx_alloc_rings_cfg rx_alloc_cfg = {0}; 1768 - struct gve_qpls_alloc_cfg qpls_alloc_cfg = {0}; 1769 1899 int err; 1770 1900 1771 - gve_get_curr_alloc_cfgs(priv, &qpls_alloc_cfg, 1772 - &tx_alloc_cfg, &rx_alloc_cfg); 1901 + gve_get_curr_alloc_cfgs(priv, &tx_alloc_cfg, &rx_alloc_cfg); 1773 1902 1774 1903 /* Relay the new config from ethtool */ 1775 - qpls_alloc_cfg.tx_cfg = &new_tx_config; 1776 1904 tx_alloc_cfg.qcfg = &new_tx_config; 1777 1905 rx_alloc_cfg.qcfg_tx = &new_tx_config; 1778 - qpls_alloc_cfg.rx_cfg = &new_rx_config; 1779 1906 rx_alloc_cfg.qcfg = &new_rx_config; 1780 1907 tx_alloc_cfg.num_rings = new_tx_config.num_queues; 1781 1908 1782 1909 if (netif_carrier_ok(priv->dev)) { 1783 - err = gve_adjust_config(priv, &qpls_alloc_cfg, 1784 - &tx_alloc_cfg, &rx_alloc_cfg); 1910 + err = gve_adjust_config(priv, &tx_alloc_cfg, &rx_alloc_cfg); 1785 1911 return err; 1786 1912 } 1787 1913 /* Set the config for the next up. */ ··· 1801 1937 int ntfy_idx = gve_tx_idx_to_ntfy(priv, idx); 1802 1938 struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx]; 1803 1939 1940 + if (!gve_tx_was_added_to_block(priv, idx)) 1941 + continue; 1804 1942 napi_disable(&block->napi); 1805 1943 } 1806 1944 for (idx = 0; idx < priv->rx_cfg.num_queues; idx++) { 1807 1945 int ntfy_idx = gve_rx_idx_to_ntfy(priv, idx); 1808 1946 struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx]; 1809 1947 1948 + if (!gve_rx_was_added_to_block(priv, idx)) 1949 + continue; 1810 1950 napi_disable(&block->napi); 1811 1951 } 1812 1952 ··· 1833 1965 int ntfy_idx = gve_tx_idx_to_ntfy(priv, idx); 1834 1966 struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx]; 1835 1967 1968 + if (!gve_tx_was_added_to_block(priv, idx)) 1969 + continue; 1970 + 1836 1971 napi_enable(&block->napi); 1837 1972 if (gve_is_gqi(priv)) { 1838 1973 iowrite32be(0, gve_irq_doorbell(priv, block)); ··· 1843 1972 gve_set_itr_coalesce_usecs_dqo(priv, block, 1844 1973 priv->tx_coalesce_usecs); 1845 1974 } 1975 + 1976 + /* Any descs written by the NIC before this barrier will be 1977 + * handled by the one-off napi schedule below. Whereas any 1978 + * descs after the barrier will generate interrupts. 1979 + */ 1980 + mb(); 1981 + napi_schedule(&block->napi); 1846 1982 } 1847 1983 for (idx = 0; idx < priv->rx_cfg.num_queues; idx++) { 1848 1984 int ntfy_idx = gve_rx_idx_to_ntfy(priv, idx); 1849 1985 struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx]; 1986 + 1987 + if (!gve_rx_was_added_to_block(priv, idx)) 1988 + continue; 1850 1989 1851 1990 napi_enable(&block->napi); 1852 1991 if (gve_is_gqi(priv)) { ··· 1865 1984 gve_set_itr_coalesce_usecs_dqo(priv, block, 1866 1985 priv->rx_coalesce_usecs); 1867 1986 } 1987 + 1988 + /* Any descs written by the NIC before this barrier will be 1989 + * handled by the one-off napi schedule below. Whereas any 1990 + * descs after the barrier will generate interrupts. 1991 + */ 1992 + mb(); 1993 + napi_schedule(&block->napi); 1868 1994 } 1869 1995 1870 1996 gve_set_napi_enabled(priv); ··· 1941 2053 { 1942 2054 struct gve_tx_alloc_rings_cfg tx_alloc_cfg = {0}; 1943 2055 struct gve_rx_alloc_rings_cfg rx_alloc_cfg = {0}; 1944 - struct gve_qpls_alloc_cfg qpls_alloc_cfg = {0}; 1945 2056 bool enable_hdr_split; 1946 2057 int err = 0; 1947 2058 ··· 1960 2073 if (enable_hdr_split == priv->header_split_enabled) 1961 2074 return 0; 1962 2075 1963 - gve_get_curr_alloc_cfgs(priv, &qpls_alloc_cfg, 1964 - &tx_alloc_cfg, &rx_alloc_cfg); 2076 + gve_get_curr_alloc_cfgs(priv, &tx_alloc_cfg, &rx_alloc_cfg); 1965 2077 1966 2078 rx_alloc_cfg.enable_header_split = enable_hdr_split; 1967 2079 rx_alloc_cfg.packet_buffer_size = gve_get_pkt_buf_size(priv, enable_hdr_split); 1968 2080 1969 2081 if (netif_running(priv->dev)) 1970 - err = gve_adjust_config(priv, &qpls_alloc_cfg, 1971 - &tx_alloc_cfg, &rx_alloc_cfg); 2082 + err = gve_adjust_config(priv, &tx_alloc_cfg, &rx_alloc_cfg); 1972 2083 return err; 1973 2084 } 1974 2085 ··· 1976 2091 const netdev_features_t orig_features = netdev->features; 1977 2092 struct gve_tx_alloc_rings_cfg tx_alloc_cfg = {0}; 1978 2093 struct gve_rx_alloc_rings_cfg rx_alloc_cfg = {0}; 1979 - struct gve_qpls_alloc_cfg qpls_alloc_cfg = {0}; 1980 2094 struct gve_priv *priv = netdev_priv(netdev); 1981 2095 int err; 1982 2096 1983 - gve_get_curr_alloc_cfgs(priv, &qpls_alloc_cfg, 1984 - &tx_alloc_cfg, &rx_alloc_cfg); 2097 + gve_get_curr_alloc_cfgs(priv, &tx_alloc_cfg, &rx_alloc_cfg); 1985 2098 1986 2099 if ((netdev->features & NETIF_F_LRO) != (features & NETIF_F_LRO)) { 1987 2100 netdev->features ^= NETIF_F_LRO; 1988 2101 if (netif_carrier_ok(netdev)) { 1989 - err = gve_adjust_config(priv, &qpls_alloc_cfg, 1990 - &tx_alloc_cfg, &rx_alloc_cfg); 2102 + err = gve_adjust_config(priv, &tx_alloc_cfg, &rx_alloc_cfg); 1991 2103 if (err) { 1992 2104 /* Revert the change on error. */ 1993 2105 netdev->features = orig_features;

+83 -37

drivers/net/ethernet/google/gve/gve_rx.c

··· 30 30 u32 slots = rx->mask + 1; 31 31 int i; 32 32 33 + if (!rx->data.page_info) 34 + return; 35 + 33 36 if (rx->data.raw_addressing) { 34 37 for (i = 0; i < slots; i++) 35 38 gve_rx_free_buffer(&priv->pdev->dev, &rx->data.page_info[i], ··· 41 38 for (i = 0; i < slots; i++) 42 39 page_ref_sub(rx->data.page_info[i].page, 43 40 rx->data.page_info[i].pagecnt_bias - 1); 44 - rx->data.qpl = NULL; 45 41 46 42 for (i = 0; i < rx->qpl_copy_pool_mask + 1; i++) { 47 43 page_ref_sub(rx->qpl_copy_pool[i].page, ··· 52 50 rx->data.page_info = NULL; 53 51 } 54 52 53 + static void gve_rx_ctx_clear(struct gve_rx_ctx *ctx) 54 + { 55 + ctx->skb_head = NULL; 56 + ctx->skb_tail = NULL; 57 + ctx->total_size = 0; 58 + ctx->frag_cnt = 0; 59 + ctx->drop_pkt = false; 60 + } 61 + 62 + static void gve_rx_init_ring_state_gqi(struct gve_rx_ring *rx) 63 + { 64 + rx->desc.seqno = 1; 65 + rx->cnt = 0; 66 + gve_rx_ctx_clear(&rx->ctx); 67 + } 68 + 69 + static void gve_rx_reset_ring_gqi(struct gve_priv *priv, int idx) 70 + { 71 + struct gve_rx_ring *rx = &priv->rx[idx]; 72 + const u32 slots = priv->rx_desc_cnt; 73 + size_t size; 74 + 75 + /* Reset desc ring */ 76 + if (rx->desc.desc_ring) { 77 + size = slots * sizeof(rx->desc.desc_ring[0]); 78 + memset(rx->desc.desc_ring, 0, size); 79 + } 80 + 81 + /* Reset q_resources */ 82 + if (rx->q_resources) 83 + memset(rx->q_resources, 0, sizeof(*rx->q_resources)); 84 + 85 + gve_rx_init_ring_state_gqi(rx); 86 + } 87 + 55 88 void gve_rx_stop_ring_gqi(struct gve_priv *priv, int idx) 56 89 { 57 90 int ntfy_idx = gve_rx_idx_to_ntfy(priv, idx); ··· 96 59 97 60 gve_remove_napi(priv, ntfy_idx); 98 61 gve_rx_remove_from_block(priv, idx); 62 + gve_rx_reset_ring_gqi(priv, idx); 99 63 } 100 64 101 65 static void gve_rx_free_ring_gqi(struct gve_priv *priv, struct gve_rx_ring *rx, ··· 106 68 u32 slots = rx->mask + 1; 107 69 int idx = rx->q_num; 108 70 size_t bytes; 71 + u32 qpl_id; 109 72 110 - bytes = sizeof(struct gve_rx_desc) * cfg->ring_size; 111 - dma_free_coherent(dev, bytes, rx->desc.desc_ring, rx->desc.bus); 112 - rx->desc.desc_ring = NULL; 73 + if (rx->desc.desc_ring) { 74 + bytes = sizeof(struct gve_rx_desc) * cfg->ring_size; 75 + dma_free_coherent(dev, bytes, rx->desc.desc_ring, rx->desc.bus); 76 + rx->desc.desc_ring = NULL; 77 + } 113 78 114 - dma_free_coherent(dev, sizeof(*rx->q_resources), 115 - rx->q_resources, rx->q_resources_bus); 116 - rx->q_resources = NULL; 79 + if (rx->q_resources) { 80 + dma_free_coherent(dev, sizeof(*rx->q_resources), 81 + rx->q_resources, rx->q_resources_bus); 82 + rx->q_resources = NULL; 83 + } 117 84 118 85 gve_rx_unfill_pages(priv, rx, cfg); 119 86 120 - bytes = sizeof(*rx->data.data_ring) * slots; 121 - dma_free_coherent(dev, bytes, rx->data.data_ring, 122 - rx->data.data_bus); 123 - rx->data.data_ring = NULL; 87 + if (rx->data.data_ring) { 88 + bytes = sizeof(*rx->data.data_ring) * slots; 89 + dma_free_coherent(dev, bytes, rx->data.data_ring, 90 + rx->data.data_bus); 91 + rx->data.data_ring = NULL; 92 + } 124 93 125 94 kvfree(rx->qpl_copy_pool); 126 95 rx->qpl_copy_pool = NULL; 96 + 97 + if (rx->data.qpl) { 98 + qpl_id = gve_get_rx_qpl_id(cfg->qcfg_tx, idx); 99 + gve_free_queue_page_list(priv, rx->data.qpl, qpl_id); 100 + rx->data.qpl = NULL; 101 + } 127 102 128 103 netif_dbg(priv, drv, priv->dev, "freed rx ring %d\n", idx); 129 104 } ··· 194 143 if (!rx->data.page_info) 195 144 return -ENOMEM; 196 145 197 - if (!rx->data.raw_addressing) { 198 - u32 qpl_id = gve_get_rx_qpl_id(cfg->qcfg_tx, rx->q_num); 199 - 200 - rx->data.qpl = &cfg->qpls[qpl_id]; 201 - } 202 - 203 146 for (i = 0; i < slots; i++) { 204 147 if (!rx->data.raw_addressing) { 205 148 struct page *page = rx->data.qpl->pages[i]; ··· 246 201 page_ref_sub(rx->data.page_info[i].page, 247 202 rx->data.page_info[i].pagecnt_bias - 1); 248 203 249 - rx->data.qpl = NULL; 250 - 251 204 return err; 252 205 253 206 alloc_err_rda: ··· 254 211 &rx->data.page_info[i], 255 212 &rx->data.data_ring[i]); 256 213 return err; 257 - } 258 - 259 - static void gve_rx_ctx_clear(struct gve_rx_ctx *ctx) 260 - { 261 - ctx->skb_head = NULL; 262 - ctx->skb_tail = NULL; 263 - ctx->total_size = 0; 264 - ctx->frag_cnt = 0; 265 - ctx->drop_pkt = false; 266 214 } 267 215 268 216 void gve_rx_start_ring_gqi(struct gve_priv *priv, int idx) ··· 272 238 struct device *hdev = &priv->pdev->dev; 273 239 u32 slots = cfg->ring_size; 274 240 int filled_pages; 241 + int qpl_page_cnt; 242 + u32 qpl_id = 0; 275 243 size_t bytes; 276 244 int err; 277 245 ··· 306 270 goto abort_with_slots; 307 271 } 308 272 273 + if (!rx->data.raw_addressing) { 274 + qpl_id = gve_get_rx_qpl_id(cfg->qcfg_tx, rx->q_num); 275 + qpl_page_cnt = cfg->ring_size; 276 + 277 + rx->data.qpl = gve_alloc_queue_page_list(priv, qpl_id, 278 + qpl_page_cnt); 279 + if (!rx->data.qpl) { 280 + err = -ENOMEM; 281 + goto abort_with_copy_pool; 282 + } 283 + } 284 + 309 285 filled_pages = gve_rx_prefill_pages(rx, cfg); 310 286 if (filled_pages < 0) { 311 287 err = -ENOMEM; 312 - goto abort_with_copy_pool; 288 + goto abort_with_qpl; 313 289 } 314 290 rx->fill_cnt = filled_pages; 315 291 /* Ensure data ring slots (packet buffers) are visible. */ ··· 348 300 err = -ENOMEM; 349 301 goto abort_with_q_resources; 350 302 } 351 - rx->cnt = 0; 352 303 rx->db_threshold = slots / 2; 353 - rx->desc.seqno = 1; 304 + gve_rx_init_ring_state_gqi(rx); 354 305 355 306 rx->packet_buffer_size = GVE_DEFAULT_RX_BUFFER_SIZE; 356 307 gve_rx_ctx_clear(&rx->ctx); ··· 362 315 rx->q_resources = NULL; 363 316 abort_filled: 364 317 gve_rx_unfill_pages(priv, rx, cfg); 318 + abort_with_qpl: 319 + if (!rx->data.raw_addressing) { 320 + gve_free_queue_page_list(priv, rx->data.qpl, qpl_id); 321 + rx->data.qpl = NULL; 322 + } 365 323 abort_with_copy_pool: 366 324 kvfree(rx->qpl_copy_pool); 367 325 rx->qpl_copy_pool = NULL; ··· 384 332 struct gve_rx_ring *rx; 385 333 int err = 0; 386 334 int i, j; 387 - 388 - if (!cfg->raw_addressing && !cfg->qpls) { 389 - netif_err(priv, drv, priv->dev, 390 - "Cannot alloc QPL ring before allocing QPLs\n"); 391 - return -EINVAL; 392 - } 393 335 394 336 rx = kvcalloc(cfg->qcfg->max_queues, sizeof(struct gve_rx_ring), 395 337 GFP_KERNEL);

+97 -28

drivers/net/ethernet/google/gve/gve_rx_dqo.c

··· 211 211 } 212 212 } 213 213 214 + static void gve_rx_init_ring_state_dqo(struct gve_rx_ring *rx, 215 + const u32 buffer_queue_slots, 216 + const u32 completion_queue_slots) 217 + { 218 + int i; 219 + 220 + /* Set buffer queue state */ 221 + rx->dqo.bufq.mask = buffer_queue_slots - 1; 222 + rx->dqo.bufq.head = 0; 223 + rx->dqo.bufq.tail = 0; 224 + 225 + /* Set completion queue state */ 226 + rx->dqo.complq.num_free_slots = completion_queue_slots; 227 + rx->dqo.complq.mask = completion_queue_slots - 1; 228 + rx->dqo.complq.cur_gen_bit = 0; 229 + rx->dqo.complq.head = 0; 230 + 231 + /* Set RX SKB context */ 232 + rx->ctx.skb_head = NULL; 233 + rx->ctx.skb_tail = NULL; 234 + 235 + /* Set up linked list of buffer IDs */ 236 + if (rx->dqo.buf_states) { 237 + for (i = 0; i < rx->dqo.num_buf_states - 1; i++) 238 + rx->dqo.buf_states[i].next = i + 1; 239 + rx->dqo.buf_states[rx->dqo.num_buf_states - 1].next = -1; 240 + } 241 + 242 + rx->dqo.free_buf_states = 0; 243 + rx->dqo.recycled_buf_states.head = -1; 244 + rx->dqo.recycled_buf_states.tail = -1; 245 + rx->dqo.used_buf_states.head = -1; 246 + rx->dqo.used_buf_states.tail = -1; 247 + } 248 + 249 + static void gve_rx_reset_ring_dqo(struct gve_priv *priv, int idx) 250 + { 251 + struct gve_rx_ring *rx = &priv->rx[idx]; 252 + size_t size; 253 + int i; 254 + 255 + const u32 buffer_queue_slots = priv->rx_desc_cnt; 256 + const u32 completion_queue_slots = priv->rx_desc_cnt; 257 + 258 + /* Reset buffer queue */ 259 + if (rx->dqo.bufq.desc_ring) { 260 + size = sizeof(rx->dqo.bufq.desc_ring[0]) * 261 + buffer_queue_slots; 262 + memset(rx->dqo.bufq.desc_ring, 0, size); 263 + } 264 + 265 + /* Reset completion queue */ 266 + if (rx->dqo.complq.desc_ring) { 267 + size = sizeof(rx->dqo.complq.desc_ring[0]) * 268 + completion_queue_slots; 269 + memset(rx->dqo.complq.desc_ring, 0, size); 270 + } 271 + 272 + /* Reset q_resources */ 273 + if (rx->q_resources) 274 + memset(rx->q_resources, 0, sizeof(*rx->q_resources)); 275 + 276 + /* Reset buf states */ 277 + if (rx->dqo.buf_states) { 278 + for (i = 0; i < rx->dqo.num_buf_states; i++) { 279 + struct gve_rx_buf_state_dqo *bs = &rx->dqo.buf_states[i]; 280 + 281 + if (bs->page_info.page) 282 + gve_free_page_dqo(priv, bs, !rx->dqo.qpl); 283 + } 284 + } 285 + 286 + gve_rx_init_ring_state_dqo(rx, buffer_queue_slots, 287 + completion_queue_slots); 288 + } 289 + 214 290 void gve_rx_stop_ring_dqo(struct gve_priv *priv, int idx) 215 291 { 216 292 int ntfy_idx = gve_rx_idx_to_ntfy(priv, idx); ··· 296 220 297 221 gve_remove_napi(priv, ntfy_idx); 298 222 gve_rx_remove_from_block(priv, idx); 223 + gve_rx_reset_ring_dqo(priv, idx); 299 224 } 300 225 301 226 static void gve_rx_free_ring_dqo(struct gve_priv *priv, struct gve_rx_ring *rx, ··· 307 230 size_t buffer_queue_slots; 308 231 int idx = rx->q_num; 309 232 size_t size; 233 + u32 qpl_id; 310 234 int i; 311 235 312 236 completion_queue_slots = rx->dqo.complq.mask + 1; ··· 326 248 gve_free_page_dqo(priv, bs, !rx->dqo.qpl); 327 249 } 328 250 329 - rx->dqo.qpl = NULL; 251 + if (rx->dqo.qpl) { 252 + qpl_id = gve_get_rx_qpl_id(cfg->qcfg_tx, rx->q_num); 253 + gve_free_queue_page_list(priv, rx->dqo.qpl, qpl_id); 254 + rx->dqo.qpl = NULL; 255 + } 330 256 331 257 if (rx->dqo.bufq.desc_ring) { 332 258 size = sizeof(rx->dqo.bufq.desc_ring[0]) * buffer_queue_slots; ··· 355 273 netif_dbg(priv, drv, priv->dev, "freed rx ring %d\n", idx); 356 274 } 357 275 358 - static int gve_rx_alloc_hdr_bufs(struct gve_priv *priv, struct gve_rx_ring *rx) 276 + static int gve_rx_alloc_hdr_bufs(struct gve_priv *priv, struct gve_rx_ring *rx, 277 + const u32 buf_count) 359 278 { 360 279 struct device *hdev = &priv->pdev->dev; 361 - int buf_count = rx->dqo.bufq.mask + 1; 362 280 363 281 rx->dqo.hdr_bufs.data = dma_alloc_coherent(hdev, priv->header_buf_size * buf_count, 364 282 &rx->dqo.hdr_bufs.addr, GFP_KERNEL); ··· 382 300 int idx) 383 301 { 384 302 struct device *hdev = &priv->pdev->dev; 303 + int qpl_page_cnt; 385 304 size_t size; 386 - int i; 305 + u32 qpl_id; 387 306 388 307 const u32 buffer_queue_slots = cfg->ring_size; 389 308 const u32 completion_queue_slots = cfg->ring_size; ··· 394 311 memset(rx, 0, sizeof(*rx)); 395 312 rx->gve = priv; 396 313 rx->q_num = idx; 397 - rx->dqo.bufq.mask = buffer_queue_slots - 1; 398 - rx->dqo.complq.num_free_slots = completion_queue_slots; 399 - rx->dqo.complq.mask = completion_queue_slots - 1; 400 - rx->ctx.skb_head = NULL; 401 - rx->ctx.skb_tail = NULL; 402 314 403 315 rx->dqo.num_buf_states = cfg->raw_addressing ? 404 316 min_t(s16, S16_MAX, buffer_queue_slots * 4) : ··· 406 328 407 329 /* Allocate header buffers for header-split */ 408 330 if (cfg->enable_header_split) 409 - if (gve_rx_alloc_hdr_bufs(priv, rx)) 331 + if (gve_rx_alloc_hdr_bufs(priv, rx, buffer_queue_slots)) 410 332 goto err; 411 - 412 - /* Set up linked list of buffer IDs */ 413 - for (i = 0; i < rx->dqo.num_buf_states - 1; i++) 414 - rx->dqo.buf_states[i].next = i + 1; 415 - 416 - rx->dqo.buf_states[rx->dqo.num_buf_states - 1].next = -1; 417 - rx->dqo.recycled_buf_states.head = -1; 418 - rx->dqo.recycled_buf_states.tail = -1; 419 - rx->dqo.used_buf_states.head = -1; 420 - rx->dqo.used_buf_states.tail = -1; 421 333 422 334 /* Allocate RX completion queue */ 423 335 size = sizeof(rx->dqo.complq.desc_ring[0]) * ··· 425 357 goto err; 426 358 427 359 if (!cfg->raw_addressing) { 428 - u32 qpl_id = gve_get_rx_qpl_id(cfg->qcfg_tx, rx->q_num); 360 + qpl_id = gve_get_rx_qpl_id(cfg->qcfg_tx, rx->q_num); 361 + qpl_page_cnt = gve_get_rx_pages_per_qpl_dqo(cfg->ring_size); 429 362 430 - rx->dqo.qpl = &cfg->qpls[qpl_id]; 363 + rx->dqo.qpl = gve_alloc_queue_page_list(priv, qpl_id, 364 + qpl_page_cnt); 365 + if (!rx->dqo.qpl) 366 + goto err; 431 367 rx->dqo.next_qpl_page_idx = 0; 432 368 } 433 369 ··· 439 367 &rx->q_resources_bus, GFP_KERNEL); 440 368 if (!rx->q_resources) 441 369 goto err; 370 + 371 + gve_rx_init_ring_state_dqo(rx, buffer_queue_slots, 372 + completion_queue_slots); 442 373 443 374 return 0; 444 375 ··· 464 389 struct gve_rx_ring *rx; 465 390 int err; 466 391 int i; 467 - 468 - if (!cfg->raw_addressing && !cfg->qpls) { 469 - netif_err(priv, drv, priv->dev, 470 - "Cannot alloc QPL ring before allocing QPLs\n"); 471 - return -EINVAL; 472 - } 473 392 474 393 rx = kvcalloc(cfg->qcfg->max_queues, sizeof(struct gve_rx_ring), 475 394 GFP_KERNEL);

+22 -11

drivers/net/ethernet/google/gve/gve_tx.c

··· 216 216 struct device *hdev = &priv->pdev->dev; 217 217 int idx = tx->q_num; 218 218 size_t bytes; 219 + u32 qpl_id; 219 220 u32 slots; 220 221 221 222 slots = tx->mask + 1; ··· 224 223 tx->q_resources, tx->q_resources_bus); 225 224 tx->q_resources = NULL; 226 225 227 - if (!tx->raw_addressing) { 228 - gve_tx_fifo_release(priv, &tx->tx_fifo); 226 + if (tx->tx_fifo.qpl) { 227 + if (tx->tx_fifo.base) 228 + gve_tx_fifo_release(priv, &tx->tx_fifo); 229 + 230 + qpl_id = gve_tx_qpl_id(priv, tx->q_num); 231 + gve_free_queue_page_list(priv, tx->tx_fifo.qpl, qpl_id); 229 232 tx->tx_fifo.qpl = NULL; 230 233 } 231 234 ··· 260 255 int idx) 261 256 { 262 257 struct device *hdev = &priv->pdev->dev; 258 + int qpl_page_cnt; 259 + u32 qpl_id = 0; 263 260 size_t bytes; 264 261 265 262 /* Make sure everything is zeroed to start */ ··· 286 279 tx->raw_addressing = cfg->raw_addressing; 287 280 tx->dev = hdev; 288 281 if (!tx->raw_addressing) { 289 - u32 qpl_id = gve_tx_qpl_id(priv, tx->q_num); 282 + qpl_id = gve_tx_qpl_id(priv, tx->q_num); 283 + qpl_page_cnt = priv->tx_pages_per_qpl; 290 284 291 - tx->tx_fifo.qpl = &cfg->qpls[qpl_id]; 285 + tx->tx_fifo.qpl = gve_alloc_queue_page_list(priv, qpl_id, 286 + qpl_page_cnt); 287 + if (!tx->tx_fifo.qpl) 288 + goto abort_with_desc; 289 + 292 290 /* map Tx FIFO */ 293 291 if (gve_tx_fifo_init(priv, &tx->tx_fifo)) 294 - goto abort_with_desc; 292 + goto abort_with_qpl; 295 293 } 296 294 297 295 tx->q_resources = ··· 312 300 abort_with_fifo: 313 301 if (!tx->raw_addressing) 314 302 gve_tx_fifo_release(priv, &tx->tx_fifo); 303 + abort_with_qpl: 304 + if (!tx->raw_addressing) { 305 + gve_free_queue_page_list(priv, tx->tx_fifo.qpl, qpl_id); 306 + tx->tx_fifo.qpl = NULL; 307 + } 315 308 abort_with_desc: 316 309 dma_free_coherent(hdev, bytes, tx->desc, tx->bus); 317 310 tx->desc = NULL; ··· 332 315 struct gve_tx_ring *tx = cfg->tx; 333 316 int err = 0; 334 317 int i, j; 335 - 336 - if (!cfg->raw_addressing && !cfg->qpls) { 337 - netif_err(priv, drv, priv->dev, 338 - "Cannot alloc QPL ring before allocing QPLs\n"); 339 - return -EINVAL; 340 - } 341 318 342 319 if (cfg->start_idx + cfg->num_rings > cfg->qcfg->max_queues) { 343 320 netif_err(priv, drv, priv->dev,

+14 -9

drivers/net/ethernet/google/gve/gve_tx_dqo.c

··· 209 209 struct device *hdev = &priv->pdev->dev; 210 210 int idx = tx->q_num; 211 211 size_t bytes; 212 + u32 qpl_id; 212 213 213 214 if (tx->q_resources) { 214 215 dma_free_coherent(hdev, sizeof(*tx->q_resources), ··· 237 236 kvfree(tx->dqo.tx_qpl_buf_next); 238 237 tx->dqo.tx_qpl_buf_next = NULL; 239 238 240 - tx->dqo.qpl = NULL; 239 + if (tx->dqo.qpl) { 240 + qpl_id = gve_tx_qpl_id(priv, tx->q_num); 241 + gve_free_queue_page_list(priv, tx->dqo.qpl, qpl_id); 242 + tx->dqo.qpl = NULL; 243 + } 241 244 242 245 netif_dbg(priv, drv, priv->dev, "freed tx queue %d\n", idx); 243 246 } ··· 287 282 { 288 283 struct device *hdev = &priv->pdev->dev; 289 284 int num_pending_packets; 285 + int qpl_page_cnt; 290 286 size_t bytes; 287 + u32 qpl_id; 291 288 int i; 292 289 293 290 memset(tx, 0, sizeof(*tx)); ··· 356 349 goto err; 357 350 358 351 if (!cfg->raw_addressing) { 359 - u32 qpl_id = gve_tx_qpl_id(priv, tx->q_num); 352 + qpl_id = gve_tx_qpl_id(priv, tx->q_num); 353 + qpl_page_cnt = priv->tx_pages_per_qpl; 360 354 361 - tx->dqo.qpl = &cfg->qpls[qpl_id]; 355 + tx->dqo.qpl = gve_alloc_queue_page_list(priv, qpl_id, 356 + qpl_page_cnt); 357 + if (!tx->dqo.qpl) 358 + goto err; 362 359 363 360 if (gve_tx_qpl_buf_init(tx)) 364 361 goto err; ··· 381 370 struct gve_tx_ring *tx = cfg->tx; 382 371 int err = 0; 383 372 int i, j; 384 - 385 - if (!cfg->raw_addressing && !cfg->qpls) { 386 - netif_err(priv, drv, priv->dev, 387 - "Cannot alloc QPL ring before allocing QPLs\n"); 388 - return -EINVAL; 389 - } 390 373 391 374 if (cfg->start_idx + cfg->num_rings > cfg->qcfg->max_queues) { 392 375 netif_err(priv, drv, priv->dev,

+3

include/linux/netdevice.h

··· 1957 1957 * @sysfs_rx_queue_group: Space for optional per-rx queue attributes 1958 1958 * @rtnl_link_ops: Rtnl_link_ops 1959 1959 * @stat_ops: Optional ops for queue-aware statistics 1960 + * @queue_mgmt_ops: Optional ops for queue management 1960 1961 * 1961 1962 * @gso_max_size: Maximum size of generic segmentation offload 1962 1963 * @tso_max_size: Device (as in HW) limit on the max TSO request size ··· 2340 2339 const struct rtnl_link_ops *rtnl_link_ops; 2341 2340 2342 2341 const struct netdev_stat_ops *stat_ops; 2342 + 2343 + const struct netdev_queue_mgmt_ops *queue_mgmt_ops; 2343 2344 2344 2345 /* for setting kernel sock attribute on TCP connection setup */ 2345 2346 #define GSO_MAX_SEGS 65535u

+31

include/net/netdev_queues.h

··· 88 88 }; 89 89 90 90 /** 91 + * struct netdev_queue_mgmt_ops - netdev ops for queue management 92 + * 93 + * @ndo_queue_mem_size: Size of the struct that describes a queue's memory. 94 + * 95 + * @ndo_queue_mem_alloc: Allocate memory for an RX queue at the specified index. 96 + * The new memory is written at the specified address. 97 + * 98 + * @ndo_queue_mem_free: Free memory from an RX queue. 99 + * 100 + * @ndo_queue_start: Start an RX queue with the specified memory and at the 101 + * specified index. 102 + * 103 + * @ndo_queue_stop: Stop the RX queue at the specified index. The stopped 104 + * queue's memory is written at the specified address. 105 + */ 106 + struct netdev_queue_mgmt_ops { 107 + size_t ndo_queue_mem_size; 108 + int (*ndo_queue_mem_alloc)(struct net_device *dev, 109 + void *per_queue_mem, 110 + int idx); 111 + void (*ndo_queue_mem_free)(struct net_device *dev, 112 + void *per_queue_mem); 113 + int (*ndo_queue_start)(struct net_device *dev, 114 + void *per_queue_mem, 115 + int idx); 116 + int (*ndo_queue_stop)(struct net_device *dev, 117 + void *per_queue_mem, 118 + int idx); 119 + }; 120 + 121 + /** 91 122 * DOC: Lockless queue stopping / waking helpers. 92 123 * 93 124 * The netif_txq_maybe_stop() and __netif_txq_completed_wake()

Configure Feed

Configure Feed