Merge branch '200GbE' of git://git.kernel.org/pub/scm/linux/kernel/git/tnguy/next-queue

+2

drivers/net/ethernet/intel/idpf/idpf_dev.c

··· 97 97 intr->dyn_ctl = idpf_get_reg_addr(adapter, 98 98 reg_vals[vec_id].dyn_ctl_reg); 99 99 intr->dyn_ctl_intena_m = PF_GLINT_DYN_CTL_INTENA_M; 100 + intr->dyn_ctl_intena_msk_m = PF_GLINT_DYN_CTL_INTENA_MSK_M; 100 101 intr->dyn_ctl_itridx_s = PF_GLINT_DYN_CTL_ITR_INDX_S; 101 102 intr->dyn_ctl_intrvl_s = PF_GLINT_DYN_CTL_INTERVAL_S; 103 + intr->dyn_ctl_wb_on_itr_m = PF_GLINT_DYN_CTL_WB_ON_ITR_M; 102 104 103 105 spacing = IDPF_ITR_IDX_SPACING(reg_vals[vec_id].itrn_index_spacing, 104 106 IDPF_PF_ITR_IDX_SPACING);

+50 -60

drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c

··· 2 2 /* Copyright (C) 2023 Intel Corporation */ 3 3 4 4 #include <net/libeth/rx.h> 5 + #include <net/libeth/tx.h> 5 6 6 7 #include "idpf.h" 7 8 ··· 225 224 /* record length, and DMA address */ 226 225 dma_unmap_len_set(tx_buf, len, size); 227 226 dma_unmap_addr_set(tx_buf, dma, dma); 227 + tx_buf->type = LIBETH_SQE_FRAG; 228 228 229 229 /* align size to end of page */ 230 230 max_data += -dma & (IDPF_TX_MAX_READ_REQ_SIZE - 1); ··· 239 237 offsets, 240 238 max_data, 241 239 td_tag); 242 - tx_desc++; 243 - i++; 244 - 245 - if (i == tx_q->desc_count) { 240 + if (unlikely(++i == tx_q->desc_count)) { 241 + tx_buf = &tx_q->tx_buf[0]; 246 242 tx_desc = &tx_q->base_tx[0]; 247 243 i = 0; 244 + } else { 245 + tx_buf++; 246 + tx_desc++; 248 247 } 248 + 249 + tx_buf->type = LIBETH_SQE_EMPTY; 249 250 250 251 dma += max_data; 251 252 size -= max_data; ··· 262 257 263 258 tx_desc->qw1 = idpf_tx_singleq_build_ctob(td_cmd, offsets, 264 259 size, td_tag); 265 - tx_desc++; 266 - i++; 267 260 268 - if (i == tx_q->desc_count) { 261 + if (unlikely(++i == tx_q->desc_count)) { 262 + tx_buf = &tx_q->tx_buf[0]; 269 263 tx_desc = &tx_q->base_tx[0]; 270 264 i = 0; 265 + } else { 266 + tx_buf++; 267 + tx_desc++; 271 268 } 272 269 273 270 size = skb_frag_size(frag); ··· 277 270 278 271 dma = skb_frag_dma_map(tx_q->dev, frag, 0, size, 279 272 DMA_TO_DEVICE); 280 - 281 - tx_buf = &tx_q->tx_buf[i]; 282 273 } 283 274 284 275 skb_tx_timestamp(first->skb); ··· 287 282 tx_desc->qw1 = idpf_tx_singleq_build_ctob(td_cmd, offsets, 288 283 size, td_tag); 289 284 285 + first->type = LIBETH_SQE_SKB; 286 + first->rs_idx = i; 287 + 290 288 IDPF_SINGLEQ_BUMP_RING_IDX(tx_q, i); 291 289 292 - /* set next_to_watch value indicating a packet is present */ 293 - first->next_to_watch = tx_desc; 294 - 295 290 nq = netdev_get_tx_queue(tx_q->netdev, tx_q->idx); 296 - netdev_tx_sent_queue(nq, first->bytecount); 291 + netdev_tx_sent_queue(nq, first->bytes); 297 292 298 293 idpf_tx_buf_hw_update(tx_q, i, netdev_xmit_more()); 299 294 } ··· 311 306 struct idpf_base_tx_ctx_desc *ctx_desc; 312 307 int ntu = txq->next_to_use; 313 308 314 - memset(&txq->tx_buf[ntu], 0, sizeof(struct idpf_tx_buf)); 315 - txq->tx_buf[ntu].ctx_entry = true; 309 + txq->tx_buf[ntu].type = LIBETH_SQE_CTX; 316 310 317 311 ctx_desc = &txq->base_ctx[ntu]; 318 312 ··· 375 371 IDPF_TX_DESCS_FOR_CTX)) { 376 372 idpf_tx_buf_hw_update(tx_q, tx_q->next_to_use, false); 377 373 374 + u64_stats_update_begin(&tx_q->stats_sync); 375 + u64_stats_inc(&tx_q->q_stats.q_busy); 376 + u64_stats_update_end(&tx_q->stats_sync); 377 + 378 378 return NETDEV_TX_BUSY; 379 379 } 380 380 ··· 404 396 first->skb = skb; 405 397 406 398 if (tso) { 407 - first->gso_segs = offload.tso_segs; 408 - first->bytecount = skb->len + ((first->gso_segs - 1) * offload.tso_hdr_len); 399 + first->packets = offload.tso_segs; 400 + first->bytes = skb->len + ((first->packets - 1) * offload.tso_hdr_len); 409 401 } else { 410 - first->bytecount = max_t(unsigned int, skb->len, ETH_ZLEN); 411 - first->gso_segs = 1; 402 + first->bytes = max_t(unsigned int, skb->len, ETH_ZLEN); 403 + first->packets = 1; 412 404 } 413 405 idpf_tx_singleq_map(tx_q, first, &offload); 414 406 ··· 428 420 static bool idpf_tx_singleq_clean(struct idpf_tx_queue *tx_q, int napi_budget, 429 421 int *cleaned) 430 422 { 431 - unsigned int total_bytes = 0, total_pkts = 0; 423 + struct libeth_sq_napi_stats ss = { }; 432 424 struct idpf_base_tx_desc *tx_desc; 433 425 u32 budget = tx_q->clean_budget; 434 426 s16 ntc = tx_q->next_to_clean; 427 + struct libeth_cq_pp cp = { 428 + .dev = tx_q->dev, 429 + .ss = &ss, 430 + .napi = napi_budget, 431 + }; 435 432 struct idpf_netdev_priv *np; 436 433 struct idpf_tx_buf *tx_buf; 437 434 struct netdev_queue *nq; ··· 454 441 * such. We can skip this descriptor since there is no buffer 455 442 * to clean. 456 443 */ 457 - if (tx_buf->ctx_entry) { 458 - /* Clear this flag here to avoid stale flag values when 459 - * this buffer is used for actual data in the future. 460 - * There are cases where the tx_buf struct / the flags 461 - * field will not be cleared before being reused. 462 - */ 463 - tx_buf->ctx_entry = false; 444 + if (unlikely(tx_buf->type <= LIBETH_SQE_CTX)) { 445 + tx_buf->type = LIBETH_SQE_EMPTY; 464 446 goto fetch_next_txq_desc; 465 447 } 466 448 467 - /* if next_to_watch is not set then no work pending */ 468 - eop_desc = (struct idpf_base_tx_desc *)tx_buf->next_to_watch; 469 - if (!eop_desc) 449 + if (unlikely(tx_buf->type != LIBETH_SQE_SKB)) 470 450 break; 471 451 472 - /* prevent any other reads prior to eop_desc */ 452 + /* prevent any other reads prior to type */ 473 453 smp_rmb(); 454 + 455 + eop_desc = &tx_q->base_tx[tx_buf->rs_idx]; 474 456 475 457 /* if the descriptor isn't done, no work yet to do */ 476 458 if (!(eop_desc->qw1 & 477 459 cpu_to_le64(IDPF_TX_DESC_DTYPE_DESC_DONE))) 478 460 break; 479 461 480 - /* clear next_to_watch to prevent false hangs */ 481 - tx_buf->next_to_watch = NULL; 482 - 483 462 /* update the statistics for this packet */ 484 - total_bytes += tx_buf->bytecount; 485 - total_pkts += tx_buf->gso_segs; 486 - 487 - napi_consume_skb(tx_buf->skb, napi_budget); 488 - 489 - /* unmap skb header data */ 490 - dma_unmap_single(tx_q->dev, 491 - dma_unmap_addr(tx_buf, dma), 492 - dma_unmap_len(tx_buf, len), 493 - DMA_TO_DEVICE); 494 - 495 - /* clear tx_buf data */ 496 - tx_buf->skb = NULL; 497 - dma_unmap_len_set(tx_buf, len, 0); 463 + libeth_tx_complete(tx_buf, &cp); 498 464 499 465 /* unmap remaining buffers */ 500 466 while (tx_desc != eop_desc) { ··· 487 495 } 488 496 489 497 /* unmap any remaining paged data */ 490 - if (dma_unmap_len(tx_buf, len)) { 491 - dma_unmap_page(tx_q->dev, 492 - dma_unmap_addr(tx_buf, dma), 493 - dma_unmap_len(tx_buf, len), 494 - DMA_TO_DEVICE); 495 - dma_unmap_len_set(tx_buf, len, 0); 496 - } 498 + libeth_tx_complete(tx_buf, &cp); 497 499 } 498 500 499 501 /* update budget only if we did something */ ··· 507 521 ntc += tx_q->desc_count; 508 522 tx_q->next_to_clean = ntc; 509 523 510 - *cleaned += total_pkts; 524 + *cleaned += ss.packets; 511 525 512 526 u64_stats_update_begin(&tx_q->stats_sync); 513 - u64_stats_add(&tx_q->q_stats.packets, total_pkts); 514 - u64_stats_add(&tx_q->q_stats.bytes, total_bytes); 527 + u64_stats_add(&tx_q->q_stats.packets, ss.packets); 528 + u64_stats_add(&tx_q->q_stats.bytes, ss.bytes); 515 529 u64_stats_update_end(&tx_q->stats_sync); 516 530 517 531 np = netdev_priv(tx_q->netdev); ··· 519 533 520 534 dont_wake = np->state != __IDPF_VPORT_UP || 521 535 !netif_carrier_ok(tx_q->netdev); 522 - __netif_txq_completed_wake(nq, total_pkts, total_bytes, 536 + __netif_txq_completed_wake(nq, ss.packets, ss.bytes, 523 537 IDPF_DESC_UNUSED(tx_q), IDPF_TX_WAKE_THRESH, 524 538 dont_wake); 525 539 ··· 1120 1134 &work_done); 1121 1135 1122 1136 /* If work not completed, return budget and polling will return */ 1123 - if (!clean_complete) 1137 + if (!clean_complete) { 1138 + idpf_vport_intr_set_wb_on_itr(q_vector); 1124 1139 return budget; 1140 + } 1125 1141 1126 1142 work_done = min_t(int, work_done, budget - 1); 1127 1143 ··· 1132 1144 */ 1133 1145 if (likely(napi_complete_done(napi, work_done))) 1134 1146 idpf_vport_intr_update_itr_ena_irq(q_vector); 1147 + else 1148 + idpf_vport_intr_set_wb_on_itr(q_vector); 1135 1149 1136 1150 return work_done; 1137 1151 }

+184 -213

drivers/net/ethernet/intel/idpf/idpf_txrx.c

··· 2 2 /* Copyright (C) 2023 Intel Corporation */ 3 3 4 4 #include <net/libeth/rx.h> 5 + #include <net/libeth/tx.h> 5 6 6 7 #include "idpf.h" 7 8 #include "idpf_virtchnl.h" 9 + 10 + struct idpf_tx_stash { 11 + struct hlist_node hlist; 12 + struct libeth_sqe buf; 13 + }; 14 + 15 + #define idpf_tx_buf_compl_tag(buf) (*(u32 *)&(buf)->priv) 16 + LIBETH_SQE_CHECK_PRIV(u32); 8 17 9 18 static bool idpf_chk_linearize(struct sk_buff *skb, unsigned int max_bufs, 10 19 unsigned int count); ··· 70 61 } 71 62 72 63 /** 73 - * idpf_tx_buf_rel - Release a Tx buffer 74 - * @tx_q: the queue that owns the buffer 75 - * @tx_buf: the buffer to free 76 - */ 77 - static void idpf_tx_buf_rel(struct idpf_tx_queue *tx_q, 78 - struct idpf_tx_buf *tx_buf) 79 - { 80 - if (tx_buf->skb) { 81 - if (dma_unmap_len(tx_buf, len)) 82 - dma_unmap_single(tx_q->dev, 83 - dma_unmap_addr(tx_buf, dma), 84 - dma_unmap_len(tx_buf, len), 85 - DMA_TO_DEVICE); 86 - dev_kfree_skb_any(tx_buf->skb); 87 - } else if (dma_unmap_len(tx_buf, len)) { 88 - dma_unmap_page(tx_q->dev, 89 - dma_unmap_addr(tx_buf, dma), 90 - dma_unmap_len(tx_buf, len), 91 - DMA_TO_DEVICE); 92 - } 93 - 94 - tx_buf->next_to_watch = NULL; 95 - tx_buf->skb = NULL; 96 - tx_buf->compl_tag = IDPF_SPLITQ_TX_INVAL_COMPL_TAG; 97 - dma_unmap_len_set(tx_buf, len, 0); 98 - } 99 - 100 - /** 101 64 * idpf_tx_buf_rel_all - Free any empty Tx buffers 102 65 * @txq: queue to be cleaned 103 66 */ 104 67 static void idpf_tx_buf_rel_all(struct idpf_tx_queue *txq) 105 68 { 69 + struct libeth_sq_napi_stats ss = { }; 106 70 struct idpf_buf_lifo *buf_stack; 107 - u16 i; 71 + struct idpf_tx_stash *stash; 72 + struct libeth_cq_pp cp = { 73 + .dev = txq->dev, 74 + .ss = &ss, 75 + }; 76 + struct hlist_node *tmp; 77 + u32 i, tag; 108 78 109 79 /* Buffers already cleared, nothing to do */ 110 80 if (!txq->tx_buf) ··· 91 103 92 104 /* Free all the Tx buffer sk_buffs */ 93 105 for (i = 0; i < txq->desc_count; i++) 94 - idpf_tx_buf_rel(txq, &txq->tx_buf[i]); 106 + libeth_tx_complete(&txq->tx_buf[i], &cp); 95 107 96 108 kfree(txq->tx_buf); 97 109 txq->tx_buf = NULL; ··· 102 114 buf_stack = &txq->stash->buf_stack; 103 115 if (!buf_stack->bufs) 104 116 return; 117 + 118 + /* 119 + * If a Tx timeout occurred, there are potentially still bufs in the 120 + * hash table, free them here. 121 + */ 122 + hash_for_each_safe(txq->stash->sched_buf_hash, tag, tmp, stash, 123 + hlist) { 124 + if (!stash) 125 + continue; 126 + 127 + libeth_tx_complete(&stash->buf, &cp); 128 + hash_del(&stash->hlist); 129 + idpf_buf_lifo_push(buf_stack, stash); 130 + } 105 131 106 132 for (i = 0; i < buf_stack->size; i++) 107 133 kfree(buf_stack->bufs[i]); ··· 133 131 static void idpf_tx_desc_rel(struct idpf_tx_queue *txq) 134 132 { 135 133 idpf_tx_buf_rel_all(txq); 134 + netdev_tx_reset_subqueue(txq->netdev, txq->idx); 136 135 137 136 if (!txq->desc_ring) 138 137 return; ··· 205 202 tx_q->tx_buf = kzalloc(buf_size, GFP_KERNEL); 206 203 if (!tx_q->tx_buf) 207 204 return -ENOMEM; 208 - 209 - /* Initialize tx_bufs with invalid completion tags */ 210 - for (i = 0; i < tx_q->desc_count; i++) 211 - tx_q->tx_buf[i].compl_tag = IDPF_SPLITQ_TX_INVAL_COMPL_TAG; 212 205 213 206 if (!idpf_queue_has(FLOW_SCH_EN, tx_q)) 214 207 return 0; ··· 1655 1656 } 1656 1657 1657 1658 /** 1658 - * idpf_tx_splitq_clean_hdr - Clean TX buffer resources for header portion of 1659 - * packet 1660 - * @tx_q: tx queue to clean buffer from 1661 - * @tx_buf: buffer to be cleaned 1662 - * @cleaned: pointer to stats struct to track cleaned packets/bytes 1663 - * @napi_budget: Used to determine if we are in netpoll 1664 - */ 1665 - static void idpf_tx_splitq_clean_hdr(struct idpf_tx_queue *tx_q, 1666 - struct idpf_tx_buf *tx_buf, 1667 - struct idpf_cleaned_stats *cleaned, 1668 - int napi_budget) 1669 - { 1670 - napi_consume_skb(tx_buf->skb, napi_budget); 1671 - 1672 - if (dma_unmap_len(tx_buf, len)) { 1673 - dma_unmap_single(tx_q->dev, 1674 - dma_unmap_addr(tx_buf, dma), 1675 - dma_unmap_len(tx_buf, len), 1676 - DMA_TO_DEVICE); 1677 - 1678 - dma_unmap_len_set(tx_buf, len, 0); 1679 - } 1680 - 1681 - /* clear tx_buf data */ 1682 - tx_buf->skb = NULL; 1683 - 1684 - cleaned->bytes += tx_buf->bytecount; 1685 - cleaned->packets += tx_buf->gso_segs; 1686 - } 1687 - 1688 - /** 1689 1659 * idpf_tx_clean_stashed_bufs - clean bufs that were stored for 1690 1660 * out of order completions 1691 1661 * @txq: queue to clean ··· 1664 1696 */ 1665 1697 static void idpf_tx_clean_stashed_bufs(struct idpf_tx_queue *txq, 1666 1698 u16 compl_tag, 1667 - struct idpf_cleaned_stats *cleaned, 1699 + struct libeth_sq_napi_stats *cleaned, 1668 1700 int budget) 1669 1701 { 1670 1702 struct idpf_tx_stash *stash; 1671 1703 struct hlist_node *tmp_buf; 1704 + struct libeth_cq_pp cp = { 1705 + .dev = txq->dev, 1706 + .ss = cleaned, 1707 + .napi = budget, 1708 + }; 1672 1709 1673 1710 /* Buffer completion */ 1674 1711 hash_for_each_possible_safe(txq->stash->sched_buf_hash, stash, tmp_buf, 1675 1712 hlist, compl_tag) { 1676 - if (unlikely(stash->buf.compl_tag != (int)compl_tag)) 1713 + if (unlikely(idpf_tx_buf_compl_tag(&stash->buf) != compl_tag)) 1677 1714 continue; 1678 1715 1679 - if (stash->buf.skb) { 1680 - idpf_tx_splitq_clean_hdr(txq, &stash->buf, cleaned, 1681 - budget); 1682 - } else if (dma_unmap_len(&stash->buf, len)) { 1683 - dma_unmap_page(txq->dev, 1684 - dma_unmap_addr(&stash->buf, dma), 1685 - dma_unmap_len(&stash->buf, len), 1686 - DMA_TO_DEVICE); 1687 - dma_unmap_len_set(&stash->buf, len, 0); 1688 - } 1716 + hash_del(&stash->hlist); 1717 + libeth_tx_complete(&stash->buf, &cp); 1689 1718 1690 1719 /* Push shadow buf back onto stack */ 1691 1720 idpf_buf_lifo_push(&txq->stash->buf_stack, stash); 1692 - 1693 - hash_del(&stash->hlist); 1694 1721 } 1695 1722 } 1696 1723 ··· 1700 1737 { 1701 1738 struct idpf_tx_stash *stash; 1702 1739 1703 - if (unlikely(!dma_unmap_addr(tx_buf, dma) && 1704 - !dma_unmap_len(tx_buf, len))) 1740 + if (unlikely(tx_buf->type <= LIBETH_SQE_CTX)) 1705 1741 return 0; 1706 1742 1707 1743 stash = idpf_buf_lifo_pop(&txq->stash->buf_stack); ··· 1713 1751 1714 1752 /* Store buffer params in shadow buffer */ 1715 1753 stash->buf.skb = tx_buf->skb; 1716 - stash->buf.bytecount = tx_buf->bytecount; 1717 - stash->buf.gso_segs = tx_buf->gso_segs; 1754 + stash->buf.bytes = tx_buf->bytes; 1755 + stash->buf.packets = tx_buf->packets; 1756 + stash->buf.type = tx_buf->type; 1757 + stash->buf.nr_frags = tx_buf->nr_frags; 1718 1758 dma_unmap_addr_set(&stash->buf, dma, dma_unmap_addr(tx_buf, dma)); 1719 1759 dma_unmap_len_set(&stash->buf, len, dma_unmap_len(tx_buf, len)); 1720 - stash->buf.compl_tag = tx_buf->compl_tag; 1760 + idpf_tx_buf_compl_tag(&stash->buf) = idpf_tx_buf_compl_tag(tx_buf); 1721 1761 1722 1762 /* Add buffer to buf_hash table to be freed later */ 1723 1763 hash_add(txq->stash->sched_buf_hash, &stash->hlist, 1724 - stash->buf.compl_tag); 1764 + idpf_tx_buf_compl_tag(&stash->buf)); 1725 1765 1726 - memset(tx_buf, 0, sizeof(struct idpf_tx_buf)); 1727 - 1728 - /* Reinitialize buf_id portion of tag */ 1729 - tx_buf->compl_tag = IDPF_SPLITQ_TX_INVAL_COMPL_TAG; 1766 + tx_buf->type = LIBETH_SQE_EMPTY; 1730 1767 1731 1768 return 0; 1732 1769 } 1733 1770 1734 1771 #define idpf_tx_splitq_clean_bump_ntc(txq, ntc, desc, buf) \ 1735 1772 do { \ 1736 - (ntc)++; \ 1737 - if (unlikely(!(ntc))) { \ 1738 - ntc -= (txq)->desc_count; \ 1773 + if (unlikely(++(ntc) == (txq)->desc_count)) { \ 1774 + ntc = 0; \ 1739 1775 buf = (txq)->tx_buf; \ 1740 1776 desc = &(txq)->flex_tx[0]; \ 1741 1777 } else { \ ··· 1757 1797 * Separate packet completion events will be reported on the completion queue, 1758 1798 * and the buffers will be cleaned separately. The stats are not updated from 1759 1799 * this function when using flow-based scheduling. 1800 + * 1801 + * Furthermore, in flow scheduling mode, check to make sure there are enough 1802 + * reserve buffers to stash the packet. If there are not, return early, which 1803 + * will leave next_to_clean pointing to the packet that failed to be stashed. 1804 + * 1805 + * Return: false in the scenario above, true otherwise. 1760 1806 */ 1761 - static void idpf_tx_splitq_clean(struct idpf_tx_queue *tx_q, u16 end, 1807 + static bool idpf_tx_splitq_clean(struct idpf_tx_queue *tx_q, u16 end, 1762 1808 int napi_budget, 1763 - struct idpf_cleaned_stats *cleaned, 1809 + struct libeth_sq_napi_stats *cleaned, 1764 1810 bool descs_only) 1765 1811 { 1766 1812 union idpf_tx_flex_desc *next_pending_desc = NULL; 1767 1813 union idpf_tx_flex_desc *tx_desc; 1768 - s16 ntc = tx_q->next_to_clean; 1814 + u32 ntc = tx_q->next_to_clean; 1815 + struct libeth_cq_pp cp = { 1816 + .dev = tx_q->dev, 1817 + .ss = cleaned, 1818 + .napi = napi_budget, 1819 + }; 1769 1820 struct idpf_tx_buf *tx_buf; 1821 + bool clean_complete = true; 1770 1822 1771 1823 tx_desc = &tx_q->flex_tx[ntc]; 1772 1824 next_pending_desc = &tx_q->flex_tx[end]; 1773 1825 tx_buf = &tx_q->tx_buf[ntc]; 1774 - ntc -= tx_q->desc_count; 1775 1826 1776 1827 while (tx_desc != next_pending_desc) { 1777 - union idpf_tx_flex_desc *eop_desc; 1828 + u32 eop_idx; 1778 1829 1779 1830 /* If this entry in the ring was used as a context descriptor, 1780 - * it's corresponding entry in the buffer ring will have an 1781 - * invalid completion tag since no buffer was used. We can 1782 - * skip this descriptor since there is no buffer to clean. 1831 + * it's corresponding entry in the buffer ring is reserved. We 1832 + * can skip this descriptor since there is no buffer to clean. 1783 1833 */ 1784 - if (unlikely(tx_buf->compl_tag == IDPF_SPLITQ_TX_INVAL_COMPL_TAG)) 1834 + if (tx_buf->type <= LIBETH_SQE_CTX) 1785 1835 goto fetch_next_txq_desc; 1786 1836 1787 - eop_desc = (union idpf_tx_flex_desc *)tx_buf->next_to_watch; 1837 + if (unlikely(tx_buf->type != LIBETH_SQE_SKB)) 1838 + break; 1788 1839 1789 - /* clear next_to_watch to prevent false hangs */ 1790 - tx_buf->next_to_watch = NULL; 1840 + eop_idx = tx_buf->rs_idx; 1791 1841 1792 1842 if (descs_only) { 1793 - if (idpf_stash_flow_sch_buffers(tx_q, tx_buf)) 1843 + if (IDPF_TX_BUF_RSV_UNUSED(tx_q) < tx_buf->nr_frags) { 1844 + clean_complete = false; 1794 1845 goto tx_splitq_clean_out; 1846 + } 1795 1847 1796 - while (tx_desc != eop_desc) { 1848 + idpf_stash_flow_sch_buffers(tx_q, tx_buf); 1849 + 1850 + while (ntc != eop_idx) { 1797 1851 idpf_tx_splitq_clean_bump_ntc(tx_q, ntc, 1798 1852 tx_desc, tx_buf); 1799 - 1800 - if (dma_unmap_len(tx_buf, len)) { 1801 - if (idpf_stash_flow_sch_buffers(tx_q, 1802 - tx_buf)) 1803 - goto tx_splitq_clean_out; 1804 - } 1853 + idpf_stash_flow_sch_buffers(tx_q, tx_buf); 1805 1854 } 1806 1855 } else { 1807 - idpf_tx_splitq_clean_hdr(tx_q, tx_buf, cleaned, 1808 - napi_budget); 1856 + libeth_tx_complete(tx_buf, &cp); 1809 1857 1810 1858 /* unmap remaining buffers */ 1811 - while (tx_desc != eop_desc) { 1859 + while (ntc != eop_idx) { 1812 1860 idpf_tx_splitq_clean_bump_ntc(tx_q, ntc, 1813 1861 tx_desc, tx_buf); 1814 1862 1815 1863 /* unmap any remaining paged data */ 1816 - if (dma_unmap_len(tx_buf, len)) { 1817 - dma_unmap_page(tx_q->dev, 1818 - dma_unmap_addr(tx_buf, dma), 1819 - dma_unmap_len(tx_buf, len), 1820 - DMA_TO_DEVICE); 1821 - dma_unmap_len_set(tx_buf, len, 0); 1822 - } 1864 + libeth_tx_complete(tx_buf, &cp); 1823 1865 } 1824 1866 } 1825 1867 ··· 1830 1868 } 1831 1869 1832 1870 tx_splitq_clean_out: 1833 - ntc += tx_q->desc_count; 1834 1871 tx_q->next_to_clean = ntc; 1872 + 1873 + return clean_complete; 1835 1874 } 1836 1875 1837 1876 #define idpf_tx_clean_buf_ring_bump_ntc(txq, ntc, buf) \ ··· 1858 1895 * this completion tag. 1859 1896 */ 1860 1897 static bool idpf_tx_clean_buf_ring(struct idpf_tx_queue *txq, u16 compl_tag, 1861 - struct idpf_cleaned_stats *cleaned, 1898 + struct libeth_sq_napi_stats *cleaned, 1862 1899 int budget) 1863 1900 { 1864 1901 u16 idx = compl_tag & txq->compl_tag_bufid_m; 1865 1902 struct idpf_tx_buf *tx_buf = NULL; 1866 - u16 ntc = txq->next_to_clean; 1867 - u16 num_descs_cleaned = 0; 1868 - u16 orig_idx = idx; 1903 + struct libeth_cq_pp cp = { 1904 + .dev = txq->dev, 1905 + .ss = cleaned, 1906 + .napi = budget, 1907 + }; 1908 + u16 ntc, orig_idx = idx; 1869 1909 1870 1910 tx_buf = &txq->tx_buf[idx]; 1871 1911 1872 - while (tx_buf->compl_tag == (int)compl_tag) { 1873 - if (tx_buf->skb) { 1874 - idpf_tx_splitq_clean_hdr(txq, tx_buf, cleaned, budget); 1875 - } else if (dma_unmap_len(tx_buf, len)) { 1876 - dma_unmap_page(txq->dev, 1877 - dma_unmap_addr(tx_buf, dma), 1878 - dma_unmap_len(tx_buf, len), 1879 - DMA_TO_DEVICE); 1880 - dma_unmap_len_set(tx_buf, len, 0); 1881 - } 1912 + if (unlikely(tx_buf->type <= LIBETH_SQE_CTX || 1913 + idpf_tx_buf_compl_tag(tx_buf) != compl_tag)) 1914 + return false; 1882 1915 1883 - memset(tx_buf, 0, sizeof(struct idpf_tx_buf)); 1884 - tx_buf->compl_tag = IDPF_SPLITQ_TX_INVAL_COMPL_TAG; 1916 + if (tx_buf->type == LIBETH_SQE_SKB) 1917 + libeth_tx_complete(tx_buf, &cp); 1885 1918 1886 - num_descs_cleaned++; 1919 + idpf_tx_clean_buf_ring_bump_ntc(txq, idx, tx_buf); 1920 + 1921 + while (idpf_tx_buf_compl_tag(tx_buf) == compl_tag) { 1922 + libeth_tx_complete(tx_buf, &cp); 1887 1923 idpf_tx_clean_buf_ring_bump_ntc(txq, idx, tx_buf); 1888 1924 } 1889 1925 1890 - /* If we didn't clean anything on the ring for this completion, there's 1891 - * nothing more to do. 1926 + /* 1927 + * It's possible the packet we just cleaned was an out of order 1928 + * completion, which means we can stash the buffers starting from 1929 + * the original next_to_clean and reuse the descriptors. We need 1930 + * to compare the descriptor ring next_to_clean packet's "first" buffer 1931 + * to the "first" buffer of the packet we just cleaned to determine if 1932 + * this is the case. Howevever, next_to_clean can point to either a 1933 + * reserved buffer that corresponds to a context descriptor used for the 1934 + * next_to_clean packet (TSO packet) or the "first" buffer (single 1935 + * packet). The orig_idx from the packet we just cleaned will always 1936 + * point to the "first" buffer. If next_to_clean points to a reserved 1937 + * buffer, let's bump ntc once and start the comparison from there. 1892 1938 */ 1893 - if (unlikely(!num_descs_cleaned)) 1894 - return false; 1895 - 1896 - /* Otherwise, if we did clean a packet on the ring directly, it's safe 1897 - * to assume that the descriptors starting from the original 1898 - * next_to_clean up until the previously cleaned packet can be reused. 1899 - * Therefore, we will go back in the ring and stash any buffers still 1900 - * in the ring into the hash table to be cleaned later. 1901 - */ 1939 + ntc = txq->next_to_clean; 1902 1940 tx_buf = &txq->tx_buf[ntc]; 1903 - while (tx_buf != &txq->tx_buf[orig_idx]) { 1904 - idpf_stash_flow_sch_buffers(txq, tx_buf); 1905 - idpf_tx_clean_buf_ring_bump_ntc(txq, ntc, tx_buf); 1906 - } 1907 1941 1908 - /* Finally, update next_to_clean to reflect the work that was just done 1909 - * on the ring, if any. If the packet was only cleaned from the hash 1910 - * table, the ring will not be impacted, therefore we should not touch 1911 - * next_to_clean. The updated idx is used here 1942 + if (tx_buf->type == LIBETH_SQE_CTX) 1943 + idpf_tx_clean_buf_ring_bump_ntc(txq, ntc, tx_buf); 1944 + 1945 + /* 1946 + * If ntc still points to a different "first" buffer, clean the 1947 + * descriptor ring and stash all of the buffers for later cleaning. If 1948 + * we cannot stash all of the buffers, next_to_clean will point to the 1949 + * "first" buffer of the packet that could not be stashed and cleaning 1950 + * will start there next time. 1951 + */ 1952 + if (unlikely(tx_buf != &txq->tx_buf[orig_idx] && 1953 + !idpf_tx_splitq_clean(txq, orig_idx, budget, cleaned, 1954 + true))) 1955 + return true; 1956 + 1957 + /* 1958 + * Otherwise, update next_to_clean to reflect the cleaning that was 1959 + * done above. 1912 1960 */ 1913 1961 txq->next_to_clean = idx; 1914 1962 ··· 1939 1965 */ 1940 1966 static void idpf_tx_handle_rs_completion(struct idpf_tx_queue *txq, 1941 1967 struct idpf_splitq_tx_compl_desc *desc, 1942 - struct idpf_cleaned_stats *cleaned, 1968 + struct libeth_sq_napi_stats *cleaned, 1943 1969 int budget) 1944 1970 { 1945 1971 u16 compl_tag; ··· 1947 1973 if (!idpf_queue_has(FLOW_SCH_EN, txq)) { 1948 1974 u16 head = le16_to_cpu(desc->q_head_compl_tag.q_head); 1949 1975 1950 - return idpf_tx_splitq_clean(txq, head, budget, cleaned, false); 1976 + idpf_tx_splitq_clean(txq, head, budget, cleaned, false); 1977 + return; 1951 1978 } 1952 1979 1953 1980 compl_tag = le16_to_cpu(desc->q_head_compl_tag.compl_tag); ··· 1983 2008 ntc -= complq->desc_count; 1984 2009 1985 2010 do { 1986 - struct idpf_cleaned_stats cleaned_stats = { }; 2011 + struct libeth_sq_napi_stats cleaned_stats = { }; 1987 2012 struct idpf_tx_queue *tx_q; 1988 2013 int rel_tx_qid; 1989 2014 u16 hw_head; ··· 2133 2158 } 2134 2159 2135 2160 /** 2136 - * idpf_tx_maybe_stop_common - 1st level check for common Tx stop conditions 2137 - * @tx_q: the queue to be checked 2138 - * @size: number of descriptors we want to assure is available 2139 - * 2140 - * Returns 0 if stop is not needed 2141 - */ 2142 - int idpf_tx_maybe_stop_common(struct idpf_tx_queue *tx_q, unsigned int size) 2143 - { 2144 - struct netdev_queue *nq; 2145 - 2146 - if (likely(IDPF_DESC_UNUSED(tx_q) >= size)) 2147 - return 0; 2148 - 2149 - u64_stats_update_begin(&tx_q->stats_sync); 2150 - u64_stats_inc(&tx_q->q_stats.q_busy); 2151 - u64_stats_update_end(&tx_q->stats_sync); 2152 - 2153 - nq = netdev_get_tx_queue(tx_q->netdev, tx_q->idx); 2154 - 2155 - return netif_txq_maybe_stop(nq, IDPF_DESC_UNUSED(tx_q), size, size); 2156 - } 2157 - 2158 - /** 2159 2161 * idpf_tx_maybe_stop_splitq - 1st level check for Tx splitq stop conditions 2160 2162 * @tx_q: the queue to be checked 2161 2163 * @descs_needed: number of descriptors required for this packet ··· 2143 2191 unsigned int descs_needed) 2144 2192 { 2145 2193 if (idpf_tx_maybe_stop_common(tx_q, descs_needed)) 2146 - goto splitq_stop; 2194 + goto out; 2147 2195 2148 2196 /* If there are too many outstanding completions expected on the 2149 2197 * completion queue, stop the TX queue to give the device some time to ··· 2162 2210 return 0; 2163 2211 2164 2212 splitq_stop: 2213 + netif_stop_subqueue(tx_q->netdev, tx_q->idx); 2214 + 2215 + out: 2165 2216 u64_stats_update_begin(&tx_q->stats_sync); 2166 2217 u64_stats_inc(&tx_q->q_stats.q_busy); 2167 2218 u64_stats_update_end(&tx_q->stats_sync); 2168 - netif_stop_subqueue(tx_q->netdev, tx_q->idx); 2169 2219 2170 2220 return -EBUSY; 2171 2221 } ··· 2190 2236 nq = netdev_get_tx_queue(tx_q->netdev, tx_q->idx); 2191 2237 tx_q->next_to_use = val; 2192 2238 2193 - idpf_tx_maybe_stop_common(tx_q, IDPF_TX_DESC_NEEDED); 2239 + if (idpf_tx_maybe_stop_common(tx_q, IDPF_TX_DESC_NEEDED)) { 2240 + u64_stats_update_begin(&tx_q->stats_sync); 2241 + u64_stats_inc(&tx_q->q_stats.q_busy); 2242 + u64_stats_update_end(&tx_q->stats_sync); 2243 + } 2194 2244 2195 2245 /* Force memory writes to complete before letting h/w 2196 2246 * know there are new descriptors to fetch. (Only ··· 2265 2307 void idpf_tx_dma_map_error(struct idpf_tx_queue *txq, struct sk_buff *skb, 2266 2308 struct idpf_tx_buf *first, u16 idx) 2267 2309 { 2310 + struct libeth_sq_napi_stats ss = { }; 2311 + struct libeth_cq_pp cp = { 2312 + .dev = txq->dev, 2313 + .ss = &ss, 2314 + }; 2315 + 2268 2316 u64_stats_update_begin(&txq->stats_sync); 2269 2317 u64_stats_inc(&txq->q_stats.dma_map_errs); 2270 2318 u64_stats_update_end(&txq->stats_sync); ··· 2280 2316 struct idpf_tx_buf *tx_buf; 2281 2317 2282 2318 tx_buf = &txq->tx_buf[idx]; 2283 - idpf_tx_buf_rel(txq, tx_buf); 2319 + libeth_tx_complete(tx_buf, &cp); 2284 2320 if (tx_buf == first) 2285 2321 break; 2286 2322 if (idx == 0) ··· 2359 2395 dma = dma_map_single(tx_q->dev, skb->data, size, DMA_TO_DEVICE); 2360 2396 2361 2397 tx_buf = first; 2398 + first->nr_frags = 0; 2362 2399 2363 2400 params->compl_tag = 2364 2401 (tx_q->compl_tag_cur_gen << tx_q->compl_tag_gen_s) | i; ··· 2370 2405 if (dma_mapping_error(tx_q->dev, dma)) 2371 2406 return idpf_tx_dma_map_error(tx_q, skb, first, i); 2372 2407 2373 - tx_buf->compl_tag = params->compl_tag; 2408 + first->nr_frags++; 2409 + idpf_tx_buf_compl_tag(tx_buf) = params->compl_tag; 2410 + tx_buf->type = LIBETH_SQE_FRAG; 2374 2411 2375 2412 /* record length, and DMA address */ 2376 2413 dma_unmap_len_set(tx_buf, len, size); ··· 2426 2459 idpf_tx_splitq_build_desc(tx_desc, params, td_cmd, 2427 2460 max_data); 2428 2461 2429 - tx_desc++; 2430 - i++; 2431 - 2432 - if (i == tx_q->desc_count) { 2462 + if (unlikely(++i == tx_q->desc_count)) { 2463 + tx_buf = tx_q->tx_buf; 2433 2464 tx_desc = &tx_q->flex_tx[0]; 2434 2465 i = 0; 2435 2466 tx_q->compl_tag_cur_gen = 2436 2467 IDPF_TX_ADJ_COMPL_TAG_GEN(tx_q); 2468 + } else { 2469 + tx_buf++; 2470 + tx_desc++; 2437 2471 } 2438 2472 2439 2473 /* Since this packet has a buffer that is going to span ··· 2447 2479 * simply pass over these holes and finish cleaning the 2448 2480 * rest of the packet. 2449 2481 */ 2450 - memset(&tx_q->tx_buf[i], 0, sizeof(struct idpf_tx_buf)); 2451 - tx_q->tx_buf[i].compl_tag = params->compl_tag; 2482 + tx_buf->type = LIBETH_SQE_EMPTY; 2452 2483 2453 2484 /* Adjust the DMA offset and the remaining size of the 2454 2485 * fragment. On the first iteration of this loop, ··· 2471 2504 break; 2472 2505 2473 2506 idpf_tx_splitq_build_desc(tx_desc, params, td_cmd, size); 2474 - tx_desc++; 2475 - i++; 2476 2507 2477 - if (i == tx_q->desc_count) { 2508 + if (unlikely(++i == tx_q->desc_count)) { 2509 + tx_buf = tx_q->tx_buf; 2478 2510 tx_desc = &tx_q->flex_tx[0]; 2479 2511 i = 0; 2480 2512 tx_q->compl_tag_cur_gen = IDPF_TX_ADJ_COMPL_TAG_GEN(tx_q); 2513 + } else { 2514 + tx_buf++; 2515 + tx_desc++; 2481 2516 } 2482 2517 2483 2518 size = skb_frag_size(frag); ··· 2487 2518 2488 2519 dma = skb_frag_dma_map(tx_q->dev, frag, 0, size, 2489 2520 DMA_TO_DEVICE); 2490 - 2491 - tx_buf = &tx_q->tx_buf[i]; 2492 2521 } 2493 2522 2494 2523 /* record SW timestamp if HW timestamp is not available */ 2495 2524 skb_tx_timestamp(skb); 2496 2525 2526 + first->type = LIBETH_SQE_SKB; 2527 + 2497 2528 /* write last descriptor with RS and EOP bits */ 2529 + first->rs_idx = i; 2498 2530 td_cmd |= params->eop_cmd; 2499 2531 idpf_tx_splitq_build_desc(tx_desc, params, td_cmd, size); 2500 2532 i = idpf_tx_splitq_bump_ntu(tx_q, i); 2501 - 2502 - /* set next_to_watch value indicating a packet is present */ 2503 - first->next_to_watch = tx_desc; 2504 2533 2505 2534 tx_q->txq_grp->num_completions_pending++; 2506 2535 2507 2536 /* record bytecount for BQL */ 2508 2537 nq = netdev_get_tx_queue(tx_q->netdev, tx_q->idx); 2509 - netdev_tx_sent_queue(nq, first->bytecount); 2538 + netdev_tx_sent_queue(nq, first->bytes); 2510 2539 2511 2540 idpf_tx_buf_hw_update(tx_q, i, netdev_xmit_more()); 2512 2541 } ··· 2704 2737 struct idpf_flex_tx_ctx_desc *desc; 2705 2738 int i = txq->next_to_use; 2706 2739 2707 - memset(&txq->tx_buf[i], 0, sizeof(struct idpf_tx_buf)); 2708 - txq->tx_buf[i].compl_tag = IDPF_SPLITQ_TX_INVAL_COMPL_TAG; 2740 + txq->tx_buf[i].type = LIBETH_SQE_CTX; 2709 2741 2710 2742 /* grab the next descriptor */ 2711 2743 desc = &txq->flex_ctx[i]; ··· 2788 2822 first->skb = skb; 2789 2823 2790 2824 if (tso) { 2791 - first->gso_segs = tx_params.offload.tso_segs; 2792 - first->bytecount = skb->len + 2793 - ((first->gso_segs - 1) * tx_params.offload.tso_hdr_len); 2825 + first->packets = tx_params.offload.tso_segs; 2826 + first->bytes = skb->len + 2827 + ((first->packets - 1) * tx_params.offload.tso_hdr_len); 2794 2828 } else { 2795 - first->gso_segs = 1; 2796 - first->bytecount = max_t(unsigned int, skb->len, ETH_ZLEN); 2829 + first->packets = 1; 2830 + first->bytes = max_t(unsigned int, skb->len, ETH_ZLEN); 2797 2831 } 2798 2832 2799 2833 if (idpf_queue_has(FLOW_SCH_EN, tx_q)) { ··· 3715 3749 /* net_dim() updates ITR out-of-band using a work item */ 3716 3750 idpf_net_dim(q_vector); 3717 3751 3752 + q_vector->wb_on_itr = false; 3718 3753 intval = idpf_vport_intr_buildreg_itr(q_vector, 3719 3754 IDPF_NO_ITR_UPDATE_IDX, 0); 3720 3755 ··· 4018 4051 clean_complete &= idpf_tx_splitq_clean_all(q_vector, budget, &work_done); 4019 4052 4020 4053 /* If work not completed, return budget and polling will return */ 4021 - if (!clean_complete) 4054 + if (!clean_complete) { 4055 + idpf_vport_intr_set_wb_on_itr(q_vector); 4022 4056 return budget; 4057 + } 4023 4058 4024 4059 work_done = min_t(int, work_done, budget - 1); 4025 4060 ··· 4030 4061 */ 4031 4062 if (likely(napi_complete_done(napi, work_done))) 4032 4063 idpf_vport_intr_update_itr_ena_irq(q_vector); 4064 + else 4065 + idpf_vport_intr_set_wb_on_itr(q_vector); 4033 4066 4034 4067 /* Switch to poll mode in the tear-down path after sending disable 4035 4068 * queues virtchnl message, as the interrupts will be disabled after

+39 -53

drivers/net/ethernet/intel/idpf/idpf_txrx.h

··· 127 127 */ 128 128 #define IDPF_TX_COMPLQ_PENDING(txq) \ 129 129 (((txq)->num_completions_pending >= (txq)->complq->num_completions ? \ 130 - 0 : U64_MAX) + \ 130 + 0 : U32_MAX) + \ 131 131 (txq)->num_completions_pending - (txq)->complq->num_completions) 132 132 133 133 #define IDPF_TX_SPLITQ_COMPL_TAG_WIDTH 16 134 - #define IDPF_SPLITQ_TX_INVAL_COMPL_TAG -1 135 134 /* Adjust the generation for the completion tag and wrap if necessary */ 136 135 #define IDPF_TX_ADJ_COMPL_TAG_GEN(txq) \ 137 136 ((++(txq)->compl_tag_cur_gen) >= (txq)->compl_tag_gen_max ? \ ··· 148 149 struct idpf_flex_tx_sched_desc flow; /* flow based scheduling */ 149 150 }; 150 151 151 - /** 152 - * struct idpf_tx_buf 153 - * @next_to_watch: Next descriptor to clean 154 - * @skb: Pointer to the skb 155 - * @dma: DMA address 156 - * @len: DMA length 157 - * @bytecount: Number of bytes 158 - * @gso_segs: Number of GSO segments 159 - * @compl_tag: Splitq only, unique identifier for a buffer. Used to compare 160 - * with completion tag returned in buffer completion event. 161 - * Because the completion tag is expected to be the same in all 162 - * data descriptors for a given packet, and a single packet can 163 - * span multiple buffers, we need this field to track all 164 - * buffers associated with this completion tag independently of 165 - * the buf_id. The tag consists of a N bit buf_id and M upper 166 - * order "generation bits". See compl_tag_bufid_m and 167 - * compl_tag_gen_s in struct idpf_queue. We'll use a value of -1 168 - * to indicate the tag is not valid. 169 - * @ctx_entry: Singleq only. Used to indicate the corresponding entry 170 - * in the descriptor ring was used for a context descriptor and 171 - * this buffer entry should be skipped. 172 - */ 173 - struct idpf_tx_buf { 174 - void *next_to_watch; 175 - struct sk_buff *skb; 176 - DEFINE_DMA_UNMAP_ADDR(dma); 177 - DEFINE_DMA_UNMAP_LEN(len); 178 - unsigned int bytecount; 179 - unsigned short gso_segs; 180 - 181 - union { 182 - int compl_tag; 183 - 184 - bool ctx_entry; 185 - }; 186 - }; 187 - 188 - struct idpf_tx_stash { 189 - struct hlist_node hlist; 190 - struct idpf_tx_buf buf; 191 - }; 152 + #define idpf_tx_buf libeth_sqe 192 153 193 154 /** 194 155 * struct idpf_buf_lifo - LIFO for managing OOO completions ··· 349 390 * struct idpf_intr_reg 350 391 * @dyn_ctl: Dynamic control interrupt register 351 392 * @dyn_ctl_intena_m: Mask for dyn_ctl interrupt enable 393 + * @dyn_ctl_intena_msk_m: Mask for dyn_ctl interrupt enable mask 352 394 * @dyn_ctl_itridx_s: Register bit offset for ITR index 353 395 * @dyn_ctl_itridx_m: Mask for ITR index 354 396 * @dyn_ctl_intrvl_s: Register bit offset for ITR interval 397 + * @dyn_ctl_wb_on_itr_m: Mask for WB on ITR feature 355 398 * @rx_itr: RX ITR register 356 399 * @tx_itr: TX ITR register 357 400 * @icr_ena: Interrupt cause register offset ··· 362 401 struct idpf_intr_reg { 363 402 void __iomem *dyn_ctl; 364 403 u32 dyn_ctl_intena_m; 404 + u32 dyn_ctl_intena_msk_m; 365 405 u32 dyn_ctl_itridx_s; 366 406 u32 dyn_ctl_itridx_m; 367 407 u32 dyn_ctl_intrvl_s; 408 + u32 dyn_ctl_wb_on_itr_m; 368 409 void __iomem *rx_itr; 369 410 void __iomem *tx_itr; 370 411 void __iomem *icr_ena; ··· 387 424 * @intr_reg: See struct idpf_intr_reg 388 425 * @napi: napi handler 389 426 * @total_events: Number of interrupts processed 427 + * @wb_on_itr: whether WB on ITR is enabled 390 428 * @tx_dim: Data for TX net_dim algorithm 391 429 * @tx_itr_value: TX interrupt throttling rate 392 430 * @tx_intr_mode: Dynamic ITR or not ··· 418 454 __cacheline_group_begin_aligned(read_write); 419 455 struct napi_struct napi; 420 456 u16 total_events; 457 + bool wb_on_itr; 421 458 422 459 struct dim tx_dim; 423 460 u16 tx_itr_value; ··· 437 472 cpumask_var_t affinity_mask; 438 473 __cacheline_group_end_aligned(cold); 439 474 }; 440 - libeth_cacheline_set_assert(struct idpf_q_vector, 104, 475 + libeth_cacheline_set_assert(struct idpf_q_vector, 112, 441 476 424 + 2 * sizeof(struct dim), 442 477 8 + sizeof(cpumask_var_t)); 443 478 ··· 459 494 u64_stats_t q_busy; 460 495 u64_stats_t skb_drops; 461 496 u64_stats_t dma_map_errs; 462 - }; 463 - 464 - struct idpf_cleaned_stats { 465 - u32 packets; 466 - u32 bytes; 467 497 }; 468 498 469 499 #define IDPF_ITR_DYNAMIC 1 ··· 648 688 649 689 void *desc_ring; 650 690 }; 651 - struct idpf_tx_buf *tx_buf; 691 + struct libeth_sqe *tx_buf; 652 692 struct idpf_txq_group *txq_grp; 653 693 struct device *dev; 654 694 void __iomem *tail; ··· 791 831 u32 next_to_use; 792 832 u32 next_to_clean; 793 833 794 - u32 num_completions; 834 + aligned_u64 num_completions; 795 835 __cacheline_group_end_aligned(read_write); 796 836 797 837 __cacheline_group_begin_aligned(cold); ··· 923 963 924 964 struct idpf_compl_queue *complq; 925 965 926 - u32 num_completions_pending; 966 + aligned_u64 num_completions_pending; 927 967 }; 928 968 929 969 static inline int idpf_q_vector_to_mem(const struct idpf_q_vector *q_vector) ··· 993 1033 idpf_tx_splitq_build_flow_desc(desc, params, td_cmd, size); 994 1034 } 995 1035 1036 + /** 1037 + * idpf_vport_intr_set_wb_on_itr - enable descriptor writeback on disabled interrupts 1038 + * @q_vector: pointer to queue vector struct 1039 + */ 1040 + static inline void idpf_vport_intr_set_wb_on_itr(struct idpf_q_vector *q_vector) 1041 + { 1042 + struct idpf_intr_reg *reg; 1043 + 1044 + if (q_vector->wb_on_itr) 1045 + return; 1046 + 1047 + q_vector->wb_on_itr = true; 1048 + reg = &q_vector->intr_reg; 1049 + 1050 + writel(reg->dyn_ctl_wb_on_itr_m | reg->dyn_ctl_intena_msk_m | 1051 + (IDPF_NO_ITR_UPDATE_IDX << reg->dyn_ctl_itridx_s), 1052 + reg->dyn_ctl); 1053 + } 1054 + 996 1055 int idpf_vport_singleq_napi_poll(struct napi_struct *napi, int budget); 997 1056 void idpf_vport_init_num_qs(struct idpf_vport *vport, 998 1057 struct virtchnl2_create_vport *vport_msg); ··· 1043 1064 struct idpf_tx_buf *first, u16 ring_idx); 1044 1065 unsigned int idpf_tx_desc_count_required(struct idpf_tx_queue *txq, 1045 1066 struct sk_buff *skb); 1046 - int idpf_tx_maybe_stop_common(struct idpf_tx_queue *tx_q, unsigned int size); 1047 1067 void idpf_tx_timeout(struct net_device *netdev, unsigned int txqueue); 1048 1068 netdev_tx_t idpf_tx_singleq_frame(struct sk_buff *skb, 1049 1069 struct idpf_tx_queue *tx_q); ··· 1050 1072 bool idpf_rx_singleq_buf_hw_alloc_all(struct idpf_rx_queue *rxq, 1051 1073 u16 cleaned_count); 1052 1074 int idpf_tso(struct sk_buff *skb, struct idpf_tx_offload_params *off); 1075 + 1076 + static inline bool idpf_tx_maybe_stop_common(struct idpf_tx_queue *tx_q, 1077 + u32 needed) 1078 + { 1079 + return !netif_subqueue_maybe_stop(tx_q->netdev, tx_q->idx, 1080 + IDPF_DESC_UNUSED(tx_q), 1081 + needed, needed); 1082 + } 1053 1083 1054 1084 #endif /* !_IDPF_TXRX_H_ */

+2

drivers/net/ethernet/intel/idpf/idpf_vf_dev.c

··· 97 97 intr->dyn_ctl = idpf_get_reg_addr(adapter, 98 98 reg_vals[vec_id].dyn_ctl_reg); 99 99 intr->dyn_ctl_intena_m = VF_INT_DYN_CTLN_INTENA_M; 100 + intr->dyn_ctl_intena_msk_m = VF_INT_DYN_CTLN_INTENA_MSK_M; 100 101 intr->dyn_ctl_itridx_s = VF_INT_DYN_CTLN_ITR_INDX_S; 102 + intr->dyn_ctl_wb_on_itr_m = VF_INT_DYN_CTLN_WB_ON_ITR_M; 101 103 102 104 spacing = IDPF_ITR_IDX_SPACING(reg_vals[vec_id].itrn_index_spacing, 103 105 IDPF_VF_ITR_IDX_SPACING);

+12 -1

include/linux/netdevice.h

··· 3567 3567 } 3568 3568 3569 3569 /** 3570 + * netdev_tx_reset_subqueue - reset the BQL stats and state of a netdev queue 3571 + * @dev: network device 3572 + * @qid: stack index of the queue to reset 3573 + */ 3574 + static inline void netdev_tx_reset_subqueue(const struct net_device *dev, 3575 + u32 qid) 3576 + { 3577 + netdev_tx_reset_queue(netdev_get_tx_queue(dev, qid)); 3578 + } 3579 + 3580 + /** 3570 3581 * netdev_reset_queue - reset the packets and bytes count of a network device 3571 3582 * @dev_queue: network device 3572 3583 * ··· 3586 3575 */ 3587 3576 static inline void netdev_reset_queue(struct net_device *dev_queue) 3588 3577 { 3589 - netdev_tx_reset_queue(netdev_get_tx_queue(dev_queue, 0)); 3578 + netdev_tx_reset_subqueue(dev_queue, 0); 3590 3579 } 3591 3580 3592 3581 /**

+129

include/net/libeth/tx.h

··· 1 + /* SPDX-License-Identifier: GPL-2.0-only */ 2 + /* Copyright (C) 2024 Intel Corporation */ 3 + 4 + #ifndef __LIBETH_TX_H 5 + #define __LIBETH_TX_H 6 + 7 + #include <linux/skbuff.h> 8 + 9 + #include <net/libeth/types.h> 10 + 11 + /* Tx buffer completion */ 12 + 13 + /** 14 + * enum libeth_sqe_type - type of &libeth_sqe to act on Tx completion 15 + * @LIBETH_SQE_EMPTY: unused/empty, no action required 16 + * @LIBETH_SQE_CTX: context descriptor with empty SQE, no action required 17 + * @LIBETH_SQE_SLAB: kmalloc-allocated buffer, unmap and kfree() 18 + * @LIBETH_SQE_FRAG: mapped skb frag, only unmap DMA 19 + * @LIBETH_SQE_SKB: &sk_buff, unmap and napi_consume_skb(), update stats 20 + */ 21 + enum libeth_sqe_type { 22 + LIBETH_SQE_EMPTY = 0U, 23 + LIBETH_SQE_CTX, 24 + LIBETH_SQE_SLAB, 25 + LIBETH_SQE_FRAG, 26 + LIBETH_SQE_SKB, 27 + }; 28 + 29 + /** 30 + * struct libeth_sqe - represents a Send Queue Element / Tx buffer 31 + * @type: type of the buffer, see the enum above 32 + * @rs_idx: index of the last buffer from the batch this one was sent in 33 + * @raw: slab buffer to free via kfree() 34 + * @skb: &sk_buff to consume 35 + * @dma: DMA address to unmap 36 + * @len: length of the mapped region to unmap 37 + * @nr_frags: number of frags in the frame this buffer belongs to 38 + * @packets: number of physical packets sent for this frame 39 + * @bytes: number of physical bytes sent for this frame 40 + * @priv: driver-private scratchpad 41 + */ 42 + struct libeth_sqe { 43 + enum libeth_sqe_type type:32; 44 + u32 rs_idx; 45 + 46 + union { 47 + void *raw; 48 + struct sk_buff *skb; 49 + }; 50 + 51 + DEFINE_DMA_UNMAP_ADDR(dma); 52 + DEFINE_DMA_UNMAP_LEN(len); 53 + 54 + u32 nr_frags; 55 + u32 packets; 56 + u32 bytes; 57 + 58 + unsigned long priv; 59 + } __aligned_largest; 60 + 61 + /** 62 + * LIBETH_SQE_CHECK_PRIV - check the driver's private SQE data 63 + * @p: type or name of the object the driver wants to fit into &libeth_sqe 64 + * 65 + * Make sure the driver's private data fits into libeth_sqe::priv. To be used 66 + * right after its declaration. 67 + */ 68 + #define LIBETH_SQE_CHECK_PRIV(p) \ 69 + static_assert(sizeof(p) <= sizeof_field(struct libeth_sqe, priv)) 70 + 71 + /** 72 + * struct libeth_cq_pp - completion queue poll params 73 + * @dev: &device to perform DMA unmapping 74 + * @ss: onstack NAPI stats to fill 75 + * @napi: whether it's called from the NAPI context 76 + * 77 + * libeth uses this structure to access objects needed for performing full 78 + * Tx complete operation without passing lots of arguments and change the 79 + * prototypes each time a new one is added. 80 + */ 81 + struct libeth_cq_pp { 82 + struct device *dev; 83 + struct libeth_sq_napi_stats *ss; 84 + 85 + bool napi; 86 + }; 87 + 88 + /** 89 + * libeth_tx_complete - perform Tx completion for one SQE 90 + * @sqe: SQE to complete 91 + * @cp: poll params 92 + * 93 + * Do Tx complete for all the types of buffers, incl. freeing, unmapping, 94 + * updating the stats etc. 95 + */ 96 + static inline void libeth_tx_complete(struct libeth_sqe *sqe, 97 + const struct libeth_cq_pp *cp) 98 + { 99 + switch (sqe->type) { 100 + case LIBETH_SQE_EMPTY: 101 + return; 102 + case LIBETH_SQE_SKB: 103 + case LIBETH_SQE_FRAG: 104 + case LIBETH_SQE_SLAB: 105 + dma_unmap_page(cp->dev, dma_unmap_addr(sqe, dma), 106 + dma_unmap_len(sqe, len), DMA_TO_DEVICE); 107 + break; 108 + default: 109 + break; 110 + } 111 + 112 + switch (sqe->type) { 113 + case LIBETH_SQE_SKB: 114 + cp->ss->packets += sqe->packets; 115 + cp->ss->bytes += sqe->bytes; 116 + 117 + napi_consume_skb(sqe->skb, cp->napi); 118 + break; 119 + case LIBETH_SQE_SLAB: 120 + kfree(sqe->raw); 121 + break; 122 + default: 123 + break; 124 + } 125 + 126 + sqe->type = LIBETH_SQE_EMPTY; 127 + } 128 + 129 + #endif /* __LIBETH_TX_H */

+25

include/net/libeth/types.h

··· 1 + /* SPDX-License-Identifier: GPL-2.0-only */ 2 + /* Copyright (C) 2024 Intel Corporation */ 3 + 4 + #ifndef __LIBETH_TYPES_H 5 + #define __LIBETH_TYPES_H 6 + 7 + #include <linux/types.h> 8 + 9 + /** 10 + * struct libeth_sq_napi_stats - "hot" counters to update in Tx completion loop 11 + * @packets: completed frames counter 12 + * @bytes: sum of bytes of completed frames above 13 + * @raw: alias to access all the fields as an array 14 + */ 15 + struct libeth_sq_napi_stats { 16 + union { 17 + struct { 18 + u32 packets; 19 + u32 bytes; 20 + }; 21 + DECLARE_FLEX_ARRAY(u32, raw); 22 + }; 23 + }; 24 + 25 + #endif /* __LIBETH_TYPES_H */

Configure Feed

Configure Feed