Merge branch 'mlxsw-preparations-for-improving-performance'

+224 -220

2 changed files

expand all

drivers

net

ethernet

mellanox

mlxsw

pci.c

pci_hw.h

+222 -218

drivers/net/ethernet/mellanox/mlxsw/pci.c

··· 8 8 #include <linux/device.h> 9 9 #include <linux/pci.h> 10 10 #include <linux/interrupt.h> 11 - #include <linux/wait.h> 12 11 #include <linux/types.h> 13 12 #include <linux/skbuff.h> 14 13 #include <linux/if_vlan.h> ··· 34 35 }; 35 36 36 37 #define MLXSW_PCI_QUEUE_TYPE_COUNT 4 38 + 39 + enum mlxsw_pci_cq_type { 40 + MLXSW_PCI_CQ_SDQ, 41 + MLXSW_PCI_CQ_RDQ, 42 + }; 37 43 38 44 static const u16 mlxsw_pci_doorbell_type_offset[] = { 39 45 MLXSW_PCI_DOORBELL_SDQ_OFFSET, /* for type MLXSW_PCI_QUEUE_TYPE_SDQ */ ··· 84 80 enum mlxsw_pci_queue_type type; 85 81 struct tasklet_struct tasklet; /* queue processing tasklet */ 86 82 struct mlxsw_pci *pci; 87 - union { 88 - struct { 89 - u32 comp_sdq_count; 90 - u32 comp_rdq_count; 91 - enum mlxsw_pci_cqe_v v; 92 - } cq; 93 - struct { 94 - u32 ev_cmd_count; 95 - u32 ev_comp_count; 96 - u32 ev_other_count; 97 - } eq; 98 - } u; 83 + struct { 84 + enum mlxsw_pci_cqe_v v; 85 + struct mlxsw_pci_queue *dq; 86 + } cq; 99 87 }; 100 88 101 89 struct mlxsw_pci_queue_type_group { ··· 116 120 struct mlxsw_pci_mem_item out_mbox; 117 121 struct mlxsw_pci_mem_item in_mbox; 118 122 struct mutex lock; /* Lock access to command registers */ 119 - bool nopoll; 120 - wait_queue_head_t wait; 121 - bool wait_done; 122 123 struct { 123 124 u8 status; 124 125 u64 out_param; ··· 124 131 struct mlxsw_bus_info bus_info; 125 132 const struct pci_device_id *id; 126 133 enum mlxsw_pci_cqe_v max_cqe_ver; /* Maximal supported CQE version */ 127 - u8 num_sdq_cqs; /* Number of CQs used for SDQs */ 134 + u8 num_cqs; /* Number of CQs */ 135 + u8 num_sdqs; /* Number of SDQs */ 128 136 bool skip_reset; 129 137 }; 130 138 ··· 181 187 return &mlxsw_pci->queues[q_type]; 182 188 } 183 189 184 - static u8 __mlxsw_pci_queue_count(struct mlxsw_pci *mlxsw_pci, 185 - enum mlxsw_pci_queue_type q_type) 186 - { 187 - struct mlxsw_pci_queue_type_group *queue_group; 188 - 189 - queue_group = mlxsw_pci_queue_type_group_get(mlxsw_pci, q_type); 190 - return queue_group->count; 191 - } 192 - 193 - static u8 mlxsw_pci_sdq_count(struct mlxsw_pci *mlxsw_pci) 194 - { 195 - return __mlxsw_pci_queue_count(mlxsw_pci, MLXSW_PCI_QUEUE_TYPE_SDQ); 196 - } 197 - 198 - static u8 mlxsw_pci_cq_count(struct mlxsw_pci *mlxsw_pci) 199 - { 200 - return __mlxsw_pci_queue_count(mlxsw_pci, MLXSW_PCI_QUEUE_TYPE_CQ); 201 - } 202 - 203 190 static struct mlxsw_pci_queue * 204 191 __mlxsw_pci_queue_get(struct mlxsw_pci *mlxsw_pci, 205 192 enum mlxsw_pci_queue_type q_type, u8 q_num) ··· 195 220 MLXSW_PCI_QUEUE_TYPE_SDQ, q_num); 196 221 } 197 222 198 - static struct mlxsw_pci_queue *mlxsw_pci_rdq_get(struct mlxsw_pci *mlxsw_pci, 199 - u8 q_num) 200 - { 201 - return __mlxsw_pci_queue_get(mlxsw_pci, 202 - MLXSW_PCI_QUEUE_TYPE_RDQ, q_num); 203 - } 204 - 205 223 static struct mlxsw_pci_queue *mlxsw_pci_cq_get(struct mlxsw_pci *mlxsw_pci, 206 224 u8 q_num) 207 225 { 208 226 return __mlxsw_pci_queue_get(mlxsw_pci, MLXSW_PCI_QUEUE_TYPE_CQ, q_num); 209 227 } 210 228 211 - static struct mlxsw_pci_queue *mlxsw_pci_eq_get(struct mlxsw_pci *mlxsw_pci, 212 - u8 q_num) 229 + static struct mlxsw_pci_queue *mlxsw_pci_eq_get(struct mlxsw_pci *mlxsw_pci) 213 230 { 214 - return __mlxsw_pci_queue_get(mlxsw_pci, MLXSW_PCI_QUEUE_TYPE_EQ, q_num); 231 + /* There is only one EQ at index 0. */ 232 + return __mlxsw_pci_queue_get(mlxsw_pci, MLXSW_PCI_QUEUE_TYPE_EQ, 0); 215 233 } 216 234 217 235 static void __mlxsw_pci_queue_doorbell_set(struct mlxsw_pci *mlxsw_pci, ··· 259 291 static int mlxsw_pci_sdq_init(struct mlxsw_pci *mlxsw_pci, char *mbox, 260 292 struct mlxsw_pci_queue *q) 261 293 { 294 + struct mlxsw_pci_queue *cq; 262 295 int tclass; 296 + u8 cq_num; 263 297 int lp; 264 298 int i; 265 299 int err; ··· 274 304 MLXSW_CMD_MBOX_SW2HW_DQ_SDQ_LP_WQE; 275 305 276 306 /* Set CQ of same number of this SDQ. */ 277 - mlxsw_cmd_mbox_sw2hw_dq_cq_set(mbox, q->num); 307 + cq_num = q->num; 308 + mlxsw_cmd_mbox_sw2hw_dq_cq_set(mbox, cq_num); 278 309 mlxsw_cmd_mbox_sw2hw_dq_sdq_lp_set(mbox, lp); 279 310 mlxsw_cmd_mbox_sw2hw_dq_sdq_tclass_set(mbox, tclass); 280 311 mlxsw_cmd_mbox_sw2hw_dq_log2_dq_sz_set(mbox, 3); /* 8 pages */ ··· 288 317 err = mlxsw_cmd_sw2hw_sdq(mlxsw_pci->core, mbox, q->num); 289 318 if (err) 290 319 return err; 320 + 321 + cq = mlxsw_pci_cq_get(mlxsw_pci, cq_num); 322 + cq->cq.dq = q; 291 323 mlxsw_pci_queue_doorbell_producer_ring(mlxsw_pci, q); 292 324 return 0; 293 325 } ··· 373 399 struct mlxsw_pci_queue *q) 374 400 { 375 401 struct mlxsw_pci_queue_elem_info *elem_info; 376 - u8 sdq_count = mlxsw_pci_sdq_count(mlxsw_pci); 402 + u8 sdq_count = mlxsw_pci->num_sdqs; 403 + struct mlxsw_pci_queue *cq; 404 + u8 cq_num; 377 405 int i; 378 406 int err; 379 407 ··· 385 409 /* Set CQ of same number of this RDQ with base 386 410 * above SDQ count as the lower ones are assigned to SDQs. 387 411 */ 388 - mlxsw_cmd_mbox_sw2hw_dq_cq_set(mbox, sdq_count + q->num); 412 + cq_num = sdq_count + q->num; 413 + mlxsw_cmd_mbox_sw2hw_dq_cq_set(mbox, cq_num); 389 414 mlxsw_cmd_mbox_sw2hw_dq_log2_dq_sz_set(mbox, 3); /* 8 pages */ 390 415 for (i = 0; i < MLXSW_PCI_AQ_PAGES; i++) { 391 416 dma_addr_t mapaddr = __mlxsw_pci_queue_page_get(q, i); ··· 397 420 err = mlxsw_cmd_sw2hw_rdq(mlxsw_pci->core, mbox, q->num); 398 421 if (err) 399 422 return err; 423 + 424 + cq = mlxsw_pci_cq_get(mlxsw_pci, cq_num); 425 + cq->cq.dq = q; 400 426 401 427 mlxsw_pci_queue_doorbell_producer_ring(mlxsw_pci, q); 402 428 ··· 421 441 elem_info = mlxsw_pci_queue_elem_info_get(q, i); 422 442 mlxsw_pci_rdq_skb_free(mlxsw_pci, elem_info); 423 443 } 444 + cq->cq.dq = NULL; 424 445 mlxsw_cmd_hw2sw_rdq(mlxsw_pci->core, q->num); 425 446 426 447 return err; ··· 443 462 static void mlxsw_pci_cq_pre_init(struct mlxsw_pci *mlxsw_pci, 444 463 struct mlxsw_pci_queue *q) 445 464 { 446 - q->u.cq.v = mlxsw_pci->max_cqe_ver; 465 + q->cq.v = mlxsw_pci->max_cqe_ver; 447 466 448 - if (q->u.cq.v == MLXSW_PCI_CQE_V2 && 449 - q->num < mlxsw_pci->num_sdq_cqs && 467 + if (q->cq.v == MLXSW_PCI_CQE_V2 && 468 + q->num < mlxsw_pci->num_sdqs && 450 469 !mlxsw_core_sdq_supports_cqe_v2(mlxsw_pci->core)) 451 - q->u.cq.v = MLXSW_PCI_CQE_V1; 452 - } 453 - 454 - static int mlxsw_pci_cq_init(struct mlxsw_pci *mlxsw_pci, char *mbox, 455 - struct mlxsw_pci_queue *q) 456 - { 457 - int i; 458 - int err; 459 - 460 - q->consumer_counter = 0; 461 - 462 - for (i = 0; i < q->count; i++) { 463 - char *elem = mlxsw_pci_queue_elem_get(q, i); 464 - 465 - mlxsw_pci_cqe_owner_set(q->u.cq.v, elem, 1); 466 - } 467 - 468 - if (q->u.cq.v == MLXSW_PCI_CQE_V1) 469 - mlxsw_cmd_mbox_sw2hw_cq_cqe_ver_set(mbox, 470 - MLXSW_CMD_MBOX_SW2HW_CQ_CQE_VER_1); 471 - else if (q->u.cq.v == MLXSW_PCI_CQE_V2) 472 - mlxsw_cmd_mbox_sw2hw_cq_cqe_ver_set(mbox, 473 - MLXSW_CMD_MBOX_SW2HW_CQ_CQE_VER_2); 474 - 475 - mlxsw_cmd_mbox_sw2hw_cq_c_eqn_set(mbox, MLXSW_PCI_EQ_COMP_NUM); 476 - mlxsw_cmd_mbox_sw2hw_cq_st_set(mbox, 0); 477 - mlxsw_cmd_mbox_sw2hw_cq_log_cq_size_set(mbox, ilog2(q->count)); 478 - for (i = 0; i < MLXSW_PCI_AQ_PAGES; i++) { 479 - dma_addr_t mapaddr = __mlxsw_pci_queue_page_get(q, i); 480 - 481 - mlxsw_cmd_mbox_sw2hw_cq_pa_set(mbox, i, mapaddr); 482 - } 483 - err = mlxsw_cmd_sw2hw_cq(mlxsw_pci->core, mbox, q->num); 484 - if (err) 485 - return err; 486 - mlxsw_pci_queue_doorbell_consumer_ring(mlxsw_pci, q); 487 - mlxsw_pci_queue_doorbell_arm_consumer_ring(mlxsw_pci, q); 488 - return 0; 489 - } 490 - 491 - static void mlxsw_pci_cq_fini(struct mlxsw_pci *mlxsw_pci, 492 - struct mlxsw_pci_queue *q) 493 - { 494 - mlxsw_cmd_hw2sw_cq(mlxsw_pci->core, q->num); 470 + q->cq.v = MLXSW_PCI_CQE_V1; 495 471 } 496 472 497 473 static unsigned int mlxsw_pci_read32_off(struct mlxsw_pci *mlxsw_pci, ··· 644 706 645 707 elem_info = mlxsw_pci_queue_elem_info_consumer_get(q); 646 708 elem = elem_info->elem; 647 - owner_bit = mlxsw_pci_cqe_owner_get(q->u.cq.v, elem); 709 + owner_bit = mlxsw_pci_cqe_owner_get(q->cq.v, elem); 648 710 if (mlxsw_pci_elem_hw_owned(q, owner_bit)) 649 711 return NULL; 650 712 q->consumer_counter++; ··· 652 714 return elem; 653 715 } 654 716 655 - static void mlxsw_pci_cq_tasklet(struct tasklet_struct *t) 717 + static void mlxsw_pci_cq_rx_tasklet(struct tasklet_struct *t) 656 718 { 657 719 struct mlxsw_pci_queue *q = from_tasklet(q, t, tasklet); 720 + struct mlxsw_pci_queue *rdq = q->cq.dq; 658 721 struct mlxsw_pci *mlxsw_pci = q->pci; 659 - char *cqe; 660 - int items = 0; 661 722 int credits = q->count >> 1; 723 + int items = 0; 724 + char *cqe; 662 725 663 726 while ((cqe = mlxsw_pci_cq_sw_cqe_get(q))) { 664 727 u16 wqe_counter = mlxsw_pci_cqe_wqe_counter_get(cqe); 665 - u8 sendq = mlxsw_pci_cqe_sr_get(q->u.cq.v, cqe); 666 - u8 dqn = mlxsw_pci_cqe_dqn_get(q->u.cq.v, cqe); 728 + u8 sendq = mlxsw_pci_cqe_sr_get(q->cq.v, cqe); 729 + u8 dqn = mlxsw_pci_cqe_dqn_get(q->cq.v, cqe); 667 730 char ncqe[MLXSW_PCI_CQE_SIZE_MAX]; 731 + 732 + if (unlikely(sendq)) { 733 + WARN_ON_ONCE(1); 734 + continue; 735 + } 736 + 737 + if (unlikely(dqn != rdq->num)) { 738 + WARN_ON_ONCE(1); 739 + continue; 740 + } 668 741 669 742 memcpy(ncqe, cqe, q->elem_size); 670 743 mlxsw_pci_queue_doorbell_consumer_ring(mlxsw_pci, q); 671 744 672 - if (sendq) { 673 - struct mlxsw_pci_queue *sdq; 745 + mlxsw_pci_cqe_rdq_handle(mlxsw_pci, rdq, 746 + wqe_counter, q->cq.v, ncqe); 674 747 675 - sdq = mlxsw_pci_sdq_get(mlxsw_pci, dqn); 676 - mlxsw_pci_cqe_sdq_handle(mlxsw_pci, sdq, 677 - wqe_counter, q->u.cq.v, ncqe); 678 - q->u.cq.comp_sdq_count++; 679 - } else { 680 - struct mlxsw_pci_queue *rdq; 681 - 682 - rdq = mlxsw_pci_rdq_get(mlxsw_pci, dqn); 683 - mlxsw_pci_cqe_rdq_handle(mlxsw_pci, rdq, 684 - wqe_counter, q->u.cq.v, ncqe); 685 - q->u.cq.comp_rdq_count++; 686 - } 687 748 if (++items == credits) 688 749 break; 689 750 } 690 - if (items) 691 - mlxsw_pci_queue_doorbell_arm_consumer_ring(mlxsw_pci, q); 751 + 752 + mlxsw_pci_queue_doorbell_arm_consumer_ring(mlxsw_pci, q); 692 753 } 693 754 694 - static u16 mlxsw_pci_cq_elem_count(const struct mlxsw_pci_queue *q) 755 + static void mlxsw_pci_cq_tx_tasklet(struct tasklet_struct *t) 695 756 { 696 - return q->u.cq.v == MLXSW_PCI_CQE_V2 ? MLXSW_PCI_CQE2_COUNT : 697 - MLXSW_PCI_CQE01_COUNT; 757 + struct mlxsw_pci_queue *q = from_tasklet(q, t, tasklet); 758 + struct mlxsw_pci_queue *sdq = q->cq.dq; 759 + struct mlxsw_pci *mlxsw_pci = q->pci; 760 + int credits = q->count >> 1; 761 + int items = 0; 762 + char *cqe; 763 + 764 + while ((cqe = mlxsw_pci_cq_sw_cqe_get(q))) { 765 + u16 wqe_counter = mlxsw_pci_cqe_wqe_counter_get(cqe); 766 + u8 sendq = mlxsw_pci_cqe_sr_get(q->cq.v, cqe); 767 + u8 dqn = mlxsw_pci_cqe_dqn_get(q->cq.v, cqe); 768 + char ncqe[MLXSW_PCI_CQE_SIZE_MAX]; 769 + 770 + if (unlikely(!sendq)) { 771 + WARN_ON_ONCE(1); 772 + continue; 773 + } 774 + 775 + if (unlikely(dqn != sdq->num)) { 776 + WARN_ON_ONCE(1); 777 + continue; 778 + } 779 + 780 + memcpy(ncqe, cqe, q->elem_size); 781 + mlxsw_pci_queue_doorbell_consumer_ring(mlxsw_pci, q); 782 + 783 + mlxsw_pci_cqe_sdq_handle(mlxsw_pci, sdq, 784 + wqe_counter, q->cq.v, ncqe); 785 + 786 + if (++items == credits) 787 + break; 788 + } 789 + 790 + mlxsw_pci_queue_doorbell_arm_consumer_ring(mlxsw_pci, q); 698 791 } 699 792 700 - static u8 mlxsw_pci_cq_elem_size(const struct mlxsw_pci_queue *q) 793 + static enum mlxsw_pci_cq_type 794 + mlxsw_pci_cq_type(const struct mlxsw_pci *mlxsw_pci, 795 + const struct mlxsw_pci_queue *q) 701 796 { 702 - return q->u.cq.v == MLXSW_PCI_CQE_V2 ? MLXSW_PCI_CQE2_SIZE : 703 - MLXSW_PCI_CQE01_SIZE; 797 + /* Each CQ is mapped to one DQ. The first 'num_sdqs' queues are used 798 + * for SDQs and the rest are used for RDQs. 799 + */ 800 + if (q->num < mlxsw_pci->num_sdqs) 801 + return MLXSW_PCI_CQ_SDQ; 802 + 803 + return MLXSW_PCI_CQ_RDQ; 704 804 } 705 805 706 - static int mlxsw_pci_eq_init(struct mlxsw_pci *mlxsw_pci, char *mbox, 806 + static void mlxsw_pci_cq_tasklet_setup(struct mlxsw_pci_queue *q, 807 + enum mlxsw_pci_cq_type cq_type) 808 + { 809 + switch (cq_type) { 810 + case MLXSW_PCI_CQ_SDQ: 811 + tasklet_setup(&q->tasklet, mlxsw_pci_cq_tx_tasklet); 812 + break; 813 + case MLXSW_PCI_CQ_RDQ: 814 + tasklet_setup(&q->tasklet, mlxsw_pci_cq_rx_tasklet); 815 + break; 816 + } 817 + } 818 + 819 + static int mlxsw_pci_cq_init(struct mlxsw_pci *mlxsw_pci, char *mbox, 707 820 struct mlxsw_pci_queue *q) 708 821 { 709 822 int i; ··· 765 776 for (i = 0; i < q->count; i++) { 766 777 char *elem = mlxsw_pci_queue_elem_get(q, i); 767 778 768 - mlxsw_pci_eqe_owner_set(elem, 1); 779 + mlxsw_pci_cqe_owner_set(q->cq.v, elem, 1); 769 780 } 770 781 771 - mlxsw_cmd_mbox_sw2hw_eq_int_msix_set(mbox, 1); /* MSI-X used */ 772 - mlxsw_cmd_mbox_sw2hw_eq_st_set(mbox, 1); /* armed */ 773 - mlxsw_cmd_mbox_sw2hw_eq_log_eq_size_set(mbox, ilog2(q->count)); 782 + if (q->cq.v == MLXSW_PCI_CQE_V1) 783 + mlxsw_cmd_mbox_sw2hw_cq_cqe_ver_set(mbox, 784 + MLXSW_CMD_MBOX_SW2HW_CQ_CQE_VER_1); 785 + else if (q->cq.v == MLXSW_PCI_CQE_V2) 786 + mlxsw_cmd_mbox_sw2hw_cq_cqe_ver_set(mbox, 787 + MLXSW_CMD_MBOX_SW2HW_CQ_CQE_VER_2); 788 + 789 + mlxsw_cmd_mbox_sw2hw_cq_c_eqn_set(mbox, MLXSW_PCI_EQ_COMP_NUM); 790 + mlxsw_cmd_mbox_sw2hw_cq_st_set(mbox, 0); 791 + mlxsw_cmd_mbox_sw2hw_cq_log_cq_size_set(mbox, ilog2(q->count)); 774 792 for (i = 0; i < MLXSW_PCI_AQ_PAGES; i++) { 775 793 dma_addr_t mapaddr = __mlxsw_pci_queue_page_get(q, i); 776 794 777 - mlxsw_cmd_mbox_sw2hw_eq_pa_set(mbox, i, mapaddr); 795 + mlxsw_cmd_mbox_sw2hw_cq_pa_set(mbox, i, mapaddr); 778 796 } 779 - err = mlxsw_cmd_sw2hw_eq(mlxsw_pci->core, mbox, q->num); 797 + err = mlxsw_cmd_sw2hw_cq(mlxsw_pci->core, mbox, q->num); 780 798 if (err) 781 799 return err; 800 + mlxsw_pci_cq_tasklet_setup(q, mlxsw_pci_cq_type(mlxsw_pci, q)); 782 801 mlxsw_pci_queue_doorbell_consumer_ring(mlxsw_pci, q); 783 802 mlxsw_pci_queue_doorbell_arm_consumer_ring(mlxsw_pci, q); 784 803 return 0; 785 804 } 786 805 787 - static void mlxsw_pci_eq_fini(struct mlxsw_pci *mlxsw_pci, 806 + static void mlxsw_pci_cq_fini(struct mlxsw_pci *mlxsw_pci, 788 807 struct mlxsw_pci_queue *q) 789 808 { 790 - mlxsw_cmd_hw2sw_eq(mlxsw_pci->core, q->num); 809 + mlxsw_cmd_hw2sw_cq(mlxsw_pci->core, q->num); 791 810 } 792 811 793 - static void mlxsw_pci_eq_cmd_event(struct mlxsw_pci *mlxsw_pci, char *eqe) 812 + static u16 mlxsw_pci_cq_elem_count(const struct mlxsw_pci_queue *q) 794 813 { 795 - mlxsw_pci->cmd.comp.status = mlxsw_pci_eqe_cmd_status_get(eqe); 796 - mlxsw_pci->cmd.comp.out_param = 797 - ((u64) mlxsw_pci_eqe_cmd_out_param_h_get(eqe)) << 32 | 798 - mlxsw_pci_eqe_cmd_out_param_l_get(eqe); 799 - mlxsw_pci->cmd.wait_done = true; 800 - wake_up(&mlxsw_pci->cmd.wait); 814 + return q->cq.v == MLXSW_PCI_CQE_V2 ? MLXSW_PCI_CQE2_COUNT : 815 + MLXSW_PCI_CQE01_COUNT; 816 + } 817 + 818 + static u8 mlxsw_pci_cq_elem_size(const struct mlxsw_pci_queue *q) 819 + { 820 + return q->cq.v == MLXSW_PCI_CQE_V2 ? MLXSW_PCI_CQE2_SIZE : 821 + MLXSW_PCI_CQE01_SIZE; 801 822 } 802 823 803 824 static char *mlxsw_pci_eq_sw_eqe_get(struct mlxsw_pci_queue *q) ··· 828 829 829 830 static void mlxsw_pci_eq_tasklet(struct tasklet_struct *t) 830 831 { 832 + unsigned long active_cqns[BITS_TO_LONGS(MLXSW_PCI_CQS_MAX)]; 831 833 struct mlxsw_pci_queue *q = from_tasklet(q, t, tasklet); 832 834 struct mlxsw_pci *mlxsw_pci = q->pci; 833 - u8 cq_count = mlxsw_pci_cq_count(mlxsw_pci); 834 - unsigned long active_cqns[BITS_TO_LONGS(MLXSW_PCI_CQS_MAX)]; 835 - char *eqe; 836 - u8 cqn; 837 - bool cq_handle = false; 838 - int items = 0; 839 835 int credits = q->count >> 1; 836 + u8 cqn, cq_count; 837 + int items = 0; 838 + char *eqe; 840 839 841 840 memset(&active_cqns, 0, sizeof(active_cqns)); 842 841 843 842 while ((eqe = mlxsw_pci_eq_sw_eqe_get(q))) { 843 + cqn = mlxsw_pci_eqe_cqn_get(eqe); 844 + set_bit(cqn, active_cqns); 844 845 845 - /* Command interface completion events are always received on 846 - * queue MLXSW_PCI_EQ_ASYNC_NUM (EQ0) and completion events 847 - * are mapped to queue MLXSW_PCI_EQ_COMP_NUM (EQ1). 848 - */ 849 - switch (q->num) { 850 - case MLXSW_PCI_EQ_ASYNC_NUM: 851 - mlxsw_pci_eq_cmd_event(mlxsw_pci, eqe); 852 - q->u.eq.ev_cmd_count++; 853 - break; 854 - case MLXSW_PCI_EQ_COMP_NUM: 855 - cqn = mlxsw_pci_eqe_cqn_get(eqe); 856 - set_bit(cqn, active_cqns); 857 - cq_handle = true; 858 - q->u.eq.ev_comp_count++; 859 - break; 860 - default: 861 - q->u.eq.ev_other_count++; 862 - } 863 846 if (++items == credits) 864 847 break; 865 848 } 866 - if (items) { 867 - mlxsw_pci_queue_doorbell_consumer_ring(mlxsw_pci, q); 868 - mlxsw_pci_queue_doorbell_arm_consumer_ring(mlxsw_pci, q); 869 - } 870 849 871 - if (!cq_handle) 850 + if (!items) 872 851 return; 852 + 853 + mlxsw_pci_queue_doorbell_consumer_ring(mlxsw_pci, q); 854 + mlxsw_pci_queue_doorbell_arm_consumer_ring(mlxsw_pci, q); 855 + 856 + cq_count = mlxsw_pci->num_cqs; 873 857 for_each_set_bit(cqn, active_cqns, cq_count) { 874 858 q = mlxsw_pci_cq_get(mlxsw_pci, cqn); 875 859 mlxsw_pci_queue_tasklet_schedule(q); 876 860 } 861 + } 862 + 863 + static int mlxsw_pci_eq_init(struct mlxsw_pci *mlxsw_pci, char *mbox, 864 + struct mlxsw_pci_queue *q) 865 + { 866 + int i; 867 + int err; 868 + 869 + /* We expect to initialize only one EQ, which gets num=0 as it is 870 + * located at index zero. We use the EQ as EQ1, so set the number for 871 + * future use. 872 + */ 873 + WARN_ON_ONCE(q->num); 874 + q->num = MLXSW_PCI_EQ_COMP_NUM; 875 + 876 + q->consumer_counter = 0; 877 + 878 + for (i = 0; i < q->count; i++) { 879 + char *elem = mlxsw_pci_queue_elem_get(q, i); 880 + 881 + mlxsw_pci_eqe_owner_set(elem, 1); 882 + } 883 + 884 + mlxsw_cmd_mbox_sw2hw_eq_int_msix_set(mbox, 1); /* MSI-X used */ 885 + mlxsw_cmd_mbox_sw2hw_eq_st_set(mbox, 1); /* armed */ 886 + mlxsw_cmd_mbox_sw2hw_eq_log_eq_size_set(mbox, ilog2(q->count)); 887 + for (i = 0; i < MLXSW_PCI_AQ_PAGES; i++) { 888 + dma_addr_t mapaddr = __mlxsw_pci_queue_page_get(q, i); 889 + 890 + mlxsw_cmd_mbox_sw2hw_eq_pa_set(mbox, i, mapaddr); 891 + } 892 + err = mlxsw_cmd_sw2hw_eq(mlxsw_pci->core, mbox, q->num); 893 + if (err) 894 + return err; 895 + tasklet_setup(&q->tasklet, mlxsw_pci_eq_tasklet); 896 + mlxsw_pci_queue_doorbell_consumer_ring(mlxsw_pci, q); 897 + mlxsw_pci_queue_doorbell_arm_consumer_ring(mlxsw_pci, q); 898 + return 0; 899 + } 900 + 901 + static void mlxsw_pci_eq_fini(struct mlxsw_pci *mlxsw_pci, 902 + struct mlxsw_pci_queue *q) 903 + { 904 + mlxsw_cmd_hw2sw_eq(mlxsw_pci->core, q->num); 877 905 } 878 906 879 907 struct mlxsw_pci_queue_ops { ··· 912 886 struct mlxsw_pci_queue *q); 913 887 void (*fini)(struct mlxsw_pci *mlxsw_pci, 914 888 struct mlxsw_pci_queue *q); 915 - void (*tasklet)(struct tasklet_struct *t); 916 889 u16 (*elem_count_f)(const struct mlxsw_pci_queue *q); 917 890 u8 (*elem_size_f)(const struct mlxsw_pci_queue *q); 918 891 u16 elem_count; ··· 939 914 .pre_init = mlxsw_pci_cq_pre_init, 940 915 .init = mlxsw_pci_cq_init, 941 916 .fini = mlxsw_pci_cq_fini, 942 - .tasklet = mlxsw_pci_cq_tasklet, 943 917 .elem_count_f = mlxsw_pci_cq_elem_count, 944 918 .elem_size_f = mlxsw_pci_cq_elem_size 945 919 }; ··· 947 923 .type = MLXSW_PCI_QUEUE_TYPE_EQ, 948 924 .init = mlxsw_pci_eq_init, 949 925 .fini = mlxsw_pci_eq_fini, 950 - .tasklet = mlxsw_pci_eq_tasklet, 951 926 .elem_count = MLXSW_PCI_EQE_COUNT, 952 927 .elem_size = MLXSW_PCI_EQE_SIZE 953 928 }; ··· 970 947 q_ops->elem_size; 971 948 q->type = q_ops->type; 972 949 q->pci = mlxsw_pci; 973 - 974 - if (q_ops->tasklet) 975 - tasklet_setup(&q->tasklet, q_ops->tasklet); 976 950 977 951 mem_item->size = MLXSW_PCI_AQ_SIZE; 978 952 mem_item->buf = dma_alloc_coherent(&mlxsw_pci->pdev->dev, ··· 1094 1074 1095 1075 if (num_sdqs + num_rdqs > num_cqs || 1096 1076 num_sdqs < MLXSW_PCI_SDQS_MIN || 1097 - num_cqs > MLXSW_PCI_CQS_MAX || num_eqs != MLXSW_PCI_EQS_COUNT) { 1077 + num_cqs > MLXSW_PCI_CQS_MAX || num_eqs != MLXSW_PCI_EQS_MAX) { 1098 1078 dev_err(&pdev->dev, "Unsupported number of queues\n"); 1099 1079 return -EINVAL; 1100 1080 } ··· 1109 1089 return -EINVAL; 1110 1090 } 1111 1091 1112 - mlxsw_pci->num_sdq_cqs = num_sdqs; 1092 + mlxsw_pci->num_cqs = num_cqs; 1093 + mlxsw_pci->num_sdqs = num_sdqs; 1113 1094 1114 1095 err = mlxsw_pci_queue_group_init(mlxsw_pci, mbox, &mlxsw_pci_eq_ops, 1115 - num_eqs); 1096 + MLXSW_PCI_EQS_COUNT); 1116 1097 if (err) { 1117 1098 dev_err(&pdev->dev, "Failed to initialize event queues\n"); 1118 1099 return err; ··· 1140 1119 goto err_rdqs_init; 1141 1120 } 1142 1121 1143 - /* We have to poll in command interface until queues are initialized */ 1144 - mlxsw_pci->cmd.nopoll = true; 1145 1122 return 0; 1146 1123 1147 1124 err_rdqs_init: ··· 1153 1134 1154 1135 static void mlxsw_pci_aqs_fini(struct mlxsw_pci *mlxsw_pci) 1155 1136 { 1156 - mlxsw_pci->cmd.nopoll = false; 1157 1137 mlxsw_pci_queue_group_fini(mlxsw_pci, &mlxsw_pci_rdq_ops); 1158 1138 mlxsw_pci_queue_group_fini(mlxsw_pci, &mlxsw_pci_sdq_ops); 1159 1139 mlxsw_pci_queue_group_fini(mlxsw_pci, &mlxsw_pci_cq_ops); ··· 1450 1432 { 1451 1433 struct mlxsw_pci *mlxsw_pci = dev_id; 1452 1434 struct mlxsw_pci_queue *q; 1453 - int i; 1454 1435 1455 - for (i = 0; i < MLXSW_PCI_EQS_COUNT; i++) { 1456 - q = mlxsw_pci_eq_get(mlxsw_pci, i); 1457 - mlxsw_pci_queue_tasklet_schedule(q); 1458 - } 1436 + q = mlxsw_pci_eq_get(mlxsw_pci); 1437 + mlxsw_pci_queue_tasklet_schedule(q); 1459 1438 return IRQ_HANDLED; 1460 1439 } 1461 1440 ··· 1778 1763 mlxsw_pci_sdq_pick(struct mlxsw_pci *mlxsw_pci, 1779 1764 const struct mlxsw_tx_info *tx_info) 1780 1765 { 1781 - u8 ctl_sdq_count = mlxsw_pci_sdq_count(mlxsw_pci) - 1; 1766 + u8 ctl_sdq_count = mlxsw_pci->num_sdqs - 1; 1782 1767 u8 sdqn; 1783 1768 1784 1769 if (tx_info->is_emad) { ··· 1877 1862 { 1878 1863 struct mlxsw_pci *mlxsw_pci = bus_priv; 1879 1864 dma_addr_t in_mapaddr = 0, out_mapaddr = 0; 1880 - bool evreq = mlxsw_pci->cmd.nopoll; 1881 1865 unsigned long timeout = msecs_to_jiffies(MLXSW_PCI_CIR_TIMEOUT_MSECS); 1882 - bool *p_wait_done = &mlxsw_pci->cmd.wait_done; 1866 + unsigned long end; 1867 + bool wait_done; 1883 1868 int err; 1884 1869 1885 1870 *p_status = MLXSW_CMD_STATUS_OK; ··· 1903 1888 mlxsw_pci_write32(mlxsw_pci, CIR_IN_MODIFIER, in_mod); 1904 1889 mlxsw_pci_write32(mlxsw_pci, CIR_TOKEN, 0); 1905 1890 1906 - *p_wait_done = false; 1891 + wait_done = false; 1907 1892 1908 1893 wmb(); /* all needs to be written before we write control register */ 1909 1894 mlxsw_pci_write32(mlxsw_pci, CIR_CTRL, 1910 1895 MLXSW_PCI_CIR_CTRL_GO_BIT | 1911 - (evreq ? MLXSW_PCI_CIR_CTRL_EVREQ_BIT : 0) | 1912 1896 (opcode_mod << MLXSW_PCI_CIR_CTRL_OPCODE_MOD_SHIFT) | 1913 1897 opcode); 1914 1898 1915 - if (!evreq) { 1916 - unsigned long end; 1899 + end = jiffies + timeout; 1900 + do { 1901 + u32 ctrl = mlxsw_pci_read32(mlxsw_pci, CIR_CTRL); 1917 1902 1918 - end = jiffies + timeout; 1919 - do { 1920 - u32 ctrl = mlxsw_pci_read32(mlxsw_pci, CIR_CTRL); 1921 - 1922 - if (!(ctrl & MLXSW_PCI_CIR_CTRL_GO_BIT)) { 1923 - *p_wait_done = true; 1924 - *p_status = ctrl >> MLXSW_PCI_CIR_CTRL_STATUS_SHIFT; 1925 - break; 1926 - } 1927 - cond_resched(); 1928 - } while (time_before(jiffies, end)); 1929 - } else { 1930 - wait_event_timeout(mlxsw_pci->cmd.wait, *p_wait_done, timeout); 1931 - *p_status = mlxsw_pci->cmd.comp.status; 1932 - } 1903 + if (!(ctrl & MLXSW_PCI_CIR_CTRL_GO_BIT)) { 1904 + wait_done = true; 1905 + *p_status = ctrl >> MLXSW_PCI_CIR_CTRL_STATUS_SHIFT; 1906 + break; 1907 + } 1908 + cond_resched(); 1909 + } while (time_before(jiffies, end)); 1933 1910 1934 1911 err = 0; 1935 - if (*p_wait_done) { 1912 + if (wait_done) { 1936 1913 if (*p_status) 1937 1914 err = -EIO; 1938 1915 } else { ··· 1938 1931 */ 1939 1932 __be32 tmp; 1940 1933 1941 - if (!evreq) { 1942 - tmp = cpu_to_be32(mlxsw_pci_read32(mlxsw_pci, 1943 - CIR_OUT_PARAM_HI)); 1944 - memcpy(out_mbox, &tmp, sizeof(tmp)); 1945 - tmp = cpu_to_be32(mlxsw_pci_read32(mlxsw_pci, 1946 - CIR_OUT_PARAM_LO)); 1947 - memcpy(out_mbox + sizeof(tmp), &tmp, sizeof(tmp)); 1948 - } 1934 + tmp = cpu_to_be32(mlxsw_pci_read32(mlxsw_pci, 1935 + CIR_OUT_PARAM_HI)); 1936 + memcpy(out_mbox, &tmp, sizeof(tmp)); 1937 + tmp = cpu_to_be32(mlxsw_pci_read32(mlxsw_pci, 1938 + CIR_OUT_PARAM_LO)); 1939 + memcpy(out_mbox + sizeof(tmp), &tmp, sizeof(tmp)); 1949 1940 } else if (!err && out_mbox) { 1950 1941 memcpy(out_mbox, mlxsw_pci->cmd.out_mbox.buf, out_mbox_size); 1951 1942 } ··· 2022 2017 int err; 2023 2018 2024 2019 mutex_init(&mlxsw_pci->cmd.lock); 2025 - init_waitqueue_head(&mlxsw_pci->cmd.wait); 2026 2020 2027 2021 err = mlxsw_pci_mbox_alloc(mlxsw_pci, &mlxsw_pci->cmd.in_mbox); 2028 2022 if (err)

+2 -2

drivers/net/ethernet/mellanox/mlxsw/pci_hw.h

··· 42 42 ((offset) + (type_offset) + (num) * 4) 43 43 44 44 #define MLXSW_PCI_CQS_MAX 96 45 - #define MLXSW_PCI_EQS_COUNT 2 46 - #define MLXSW_PCI_EQ_ASYNC_NUM 0 45 + #define MLXSW_PCI_EQS_MAX 2 46 + #define MLXSW_PCI_EQS_COUNT 1 47 47 #define MLXSW_PCI_EQ_COMP_NUM 1 48 48 49 49 #define MLXSW_PCI_SDQS_MIN 2 /* EMAD and control traffic */

Configure Feed

Configure Feed