Merge branch '200GbE' of git://git.kernel.org/pub/scm/linux/kernel/git/tnguy/next-queue

+1

drivers/net/ethernet/intel/idpf/Makefile

··· 23 23 idpf-$(CONFIG_PTP_1588_CLOCK) += idpf_virtchnl_ptp.o 24 24 25 25 idpf-y += xdp.o 26 + idpf-y += xsk.o

+7

drivers/net/ethernet/intel/idpf/idpf.h

··· 995 995 mutex_unlock(&np->adapter->vport_ctrl_lock); 996 996 } 997 997 998 + static inline bool idpf_vport_ctrl_is_locked(struct net_device *netdev) 999 + { 1000 + struct idpf_netdev_priv *np = netdev_priv(netdev); 1001 + 1002 + return mutex_is_locked(&np->adapter->vport_ctrl_lock); 1003 + } 1004 + 998 1005 void idpf_statistics_task(struct work_struct *work); 999 1006 void idpf_init_task(struct work_struct *work); 1000 1007 void idpf_service_task(struct work_struct *work);

+4 -4

drivers/net/ethernet/intel/idpf/idpf_ethtool.c

··· 1245 1245 * 1246 1246 * returns pointer to rx vector 1247 1247 */ 1248 - static struct idpf_q_vector *idpf_find_rxq_vec(const struct idpf_vport *vport, 1249 - int q_num) 1248 + struct idpf_q_vector *idpf_find_rxq_vec(const struct idpf_vport *vport, 1249 + u32 q_num) 1250 1250 { 1251 1251 int q_grp, q_idx; 1252 1252 ··· 1266 1266 * 1267 1267 * returns pointer to tx vector 1268 1268 */ 1269 - static struct idpf_q_vector *idpf_find_txq_vec(const struct idpf_vport *vport, 1270 - int q_num) 1269 + struct idpf_q_vector *idpf_find_txq_vec(const struct idpf_vport *vport, 1270 + u32 q_num) 1271 1271 { 1272 1272 int q_grp; 1273 1273

+6 -4

drivers/net/ethernet/intel/idpf/idpf_lib.c

··· 5 5 #include "idpf_virtchnl.h" 6 6 #include "idpf_ptp.h" 7 7 #include "xdp.h" 8 + #include "xsk.h" 8 9 9 10 static const struct net_device_ops idpf_netdev_ops; 10 11 ··· 1425 1424 goto queues_rel; 1426 1425 } 1427 1426 1428 - err = idpf_rx_bufs_init_all(vport); 1427 + err = idpf_queue_reg_init(vport); 1429 1428 if (err) { 1430 - dev_err(&adapter->pdev->dev, "Failed to initialize RX buffers for vport %u: %d\n", 1429 + dev_err(&adapter->pdev->dev, "Failed to initialize queue registers for vport %u: %d\n", 1431 1430 vport->vport_id, err); 1432 1431 goto queues_rel; 1433 1432 } 1434 1433 1435 - err = idpf_queue_reg_init(vport); 1434 + err = idpf_rx_bufs_init_all(vport); 1436 1435 if (err) { 1437 - dev_err(&adapter->pdev->dev, "Failed to initialize queue registers for vport %u: %d\n", 1436 + dev_err(&adapter->pdev->dev, "Failed to initialize RX buffers for vport %u: %d\n", 1438 1437 vport->vport_id, err); 1439 1438 goto queues_rel; 1440 1439 } ··· 2619 2618 .ndo_hwtstamp_set = idpf_hwtstamp_set, 2620 2619 .ndo_bpf = idpf_xdp, 2621 2620 .ndo_xdp_xmit = idpf_xdp_xmit, 2621 + .ndo_xsk_wakeup = idpf_xsk_wakeup, 2622 2622 };

+427 -24

drivers/net/ethernet/intel/idpf/idpf_txrx.c

··· 5 5 #include "idpf_ptp.h" 6 6 #include "idpf_virtchnl.h" 7 7 #include "xdp.h" 8 + #include "xsk.h" 8 9 9 10 #define idpf_tx_buf_next(buf) (*(u32 *)&(buf)->priv) 10 11 LIBETH_SQE_CHECK_PRIV(u32); ··· 54 53 } 55 54 } 56 55 57 - /** 58 - * idpf_tx_buf_rel_all - Free any empty Tx buffers 59 - * @txq: queue to be cleaned 60 - */ 61 - static void idpf_tx_buf_rel_all(struct idpf_tx_queue *txq) 56 + static void idpf_tx_buf_clean(struct idpf_tx_queue *txq) 62 57 { 63 58 struct libeth_sq_napi_stats ss = { }; 64 59 struct xdp_frame_bulk bq; ··· 63 66 .bq = &bq, 64 67 .ss = &ss, 65 68 }; 66 - u32 i; 67 - 68 - /* Buffers already cleared, nothing to do */ 69 - if (!txq->tx_buf) 70 - return; 71 69 72 70 xdp_frame_bulk_init(&bq); 73 71 74 72 /* Free all the Tx buffer sk_buffs */ 75 - for (i = 0; i < txq->buf_pool_size; i++) 73 + for (u32 i = 0; i < txq->buf_pool_size; i++) 76 74 libeth_tx_complete_any(&txq->tx_buf[i], &cp); 77 75 78 76 xdp_flush_frame_bulk(&bq); 77 + } 78 + 79 + /** 80 + * idpf_tx_buf_rel_all - Free any empty Tx buffers 81 + * @txq: queue to be cleaned 82 + */ 83 + static void idpf_tx_buf_rel_all(struct idpf_tx_queue *txq) 84 + { 85 + /* Buffers already cleared, nothing to do */ 86 + if (!txq->tx_buf) 87 + return; 88 + 89 + if (idpf_queue_has(XSK, txq)) 90 + idpf_xsksq_clean(txq); 91 + else 92 + idpf_tx_buf_clean(txq); 79 93 80 94 kfree(txq->tx_buf); 81 95 txq->tx_buf = NULL; ··· 110 102 if (!xdp) 111 103 netdev_tx_reset_subqueue(txq->netdev, txq->idx); 112 104 105 + idpf_xsk_clear_queue(txq, VIRTCHNL2_QUEUE_TYPE_TX); 106 + 113 107 if (!txq->desc_ring) 114 108 return; 115 109 ··· 132 122 */ 133 123 static void idpf_compl_desc_rel(struct idpf_compl_queue *complq) 134 124 { 125 + idpf_xsk_clear_queue(complq, VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION); 126 + 135 127 if (!complq->comp) 136 128 return; 137 129 ··· 226 214 tx_q->next_to_clean = 0; 227 215 idpf_queue_set(GEN_CHK, tx_q); 228 216 217 + idpf_xsk_setup_queue(vport, tx_q, VIRTCHNL2_QUEUE_TYPE_TX); 218 + 229 219 if (!idpf_queue_has(FLOW_SCH_EN, tx_q)) 230 220 return 0; 231 221 ··· 286 272 complq->next_to_use = 0; 287 273 complq->next_to_clean = 0; 288 274 idpf_queue_set(GEN_CHK, complq); 275 + 276 + idpf_xsk_setup_queue(vport, complq, 277 + VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION); 289 278 290 279 return 0; 291 280 } ··· 389 372 if (!bufq->buf) 390 373 return; 391 374 375 + if (idpf_queue_has(XSK, bufq)) { 376 + idpf_xskfq_rel(bufq); 377 + return; 378 + } 379 + 392 380 /* Free all the bufs allocated and given to hw on Rx queue */ 393 381 for (u32 i = 0; i < bufq->desc_count; i++) 394 382 idpf_rx_page_rel(&bufq->buf[i]); ··· 442 420 if (!rxq) 443 421 return; 444 422 445 - libeth_xdp_return_stash(&rxq->xdp); 423 + if (!idpf_queue_has(XSK, rxq)) 424 + libeth_xdp_return_stash(&rxq->xdp); 446 425 447 426 if (!idpf_is_queue_model_split(model)) 448 427 idpf_rx_buf_rel_all(rxq); 428 + 429 + idpf_xsk_clear_queue(rxq, VIRTCHNL2_QUEUE_TYPE_RX); 449 430 450 431 rxq->next_to_alloc = 0; 451 432 rxq->next_to_clean = 0; ··· 472 447 return; 473 448 474 449 idpf_rx_buf_rel_bufq(bufq); 450 + idpf_xsk_clear_queue(bufq, VIRTCHNL2_QUEUE_TYPE_RX_BUFFER); 475 451 476 452 bufq->next_to_alloc = 0; 477 453 bufq->next_to_clean = 0; ··· 760 734 }; 761 735 int ret; 762 736 737 + if (idpf_queue_has(XSK, bufq)) 738 + return idpf_xskfq_init(bufq); 739 + 763 740 ret = libeth_rx_fq_create(&fq, &bufq->q_vector->napi); 764 741 if (ret) 765 742 return ret; ··· 858 829 rxq->next_to_use = 0; 859 830 idpf_queue_set(GEN_CHK, rxq); 860 831 832 + idpf_xsk_setup_queue(vport, rxq, VIRTCHNL2_QUEUE_TYPE_RX); 833 + 861 834 return 0; 862 835 } 863 836 ··· 885 854 bufq->next_to_alloc = 0; 886 855 bufq->next_to_clean = 0; 887 856 bufq->next_to_use = 0; 888 - 889 857 idpf_queue_set(GEN_CHK, bufq); 858 + 859 + idpf_xsk_setup_queue(vport, bufq, VIRTCHNL2_QUEUE_TYPE_RX_BUFFER); 890 860 891 861 return 0; 892 862 } ··· 952 920 idpf_rx_desc_rel_all(vport); 953 921 954 922 return err; 923 + } 924 + 925 + static int idpf_init_queue_set(const struct idpf_queue_set *qs) 926 + { 927 + const struct idpf_vport *vport = qs->vport; 928 + bool splitq; 929 + int err; 930 + 931 + splitq = idpf_is_queue_model_split(vport->rxq_model); 932 + 933 + for (u32 i = 0; i < qs->num; i++) { 934 + const struct idpf_queue_ptr *q = &qs->qs[i]; 935 + struct idpf_buf_queue *bufq; 936 + 937 + switch (q->type) { 938 + case VIRTCHNL2_QUEUE_TYPE_RX: 939 + err = idpf_rx_desc_alloc(vport, q->rxq); 940 + if (err) 941 + break; 942 + 943 + err = idpf_xdp_rxq_info_init(q->rxq); 944 + if (err) 945 + break; 946 + 947 + if (!splitq) 948 + err = idpf_rx_bufs_init_singleq(q->rxq); 949 + 950 + break; 951 + case VIRTCHNL2_QUEUE_TYPE_RX_BUFFER: 952 + bufq = q->bufq; 953 + 954 + err = idpf_bufq_desc_alloc(vport, bufq); 955 + if (err) 956 + break; 957 + 958 + for (u32 j = 0; j < bufq->q_vector->num_bufq; j++) { 959 + struct idpf_buf_queue * const *bufqs; 960 + enum libeth_fqe_type type; 961 + u32 ts; 962 + 963 + bufqs = bufq->q_vector->bufq; 964 + if (bufqs[j] != bufq) 965 + continue; 966 + 967 + if (j) { 968 + type = LIBETH_FQE_SHORT; 969 + ts = bufqs[j - 1]->truesize >> 1; 970 + } else { 971 + type = LIBETH_FQE_MTU; 972 + ts = 0; 973 + } 974 + 975 + bufq->truesize = ts; 976 + 977 + err = idpf_rx_bufs_init(bufq, type); 978 + break; 979 + } 980 + 981 + break; 982 + case VIRTCHNL2_QUEUE_TYPE_TX: 983 + err = idpf_tx_desc_alloc(vport, q->txq); 984 + break; 985 + case VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION: 986 + err = idpf_compl_desc_alloc(vport, q->complq); 987 + break; 988 + default: 989 + continue; 990 + } 991 + 992 + if (err) 993 + return err; 994 + } 995 + 996 + return 0; 997 + } 998 + 999 + static void idpf_clean_queue_set(const struct idpf_queue_set *qs) 1000 + { 1001 + const struct idpf_vport *vport = qs->vport; 1002 + struct device *dev = vport->netdev->dev.parent; 1003 + 1004 + for (u32 i = 0; i < qs->num; i++) { 1005 + const struct idpf_queue_ptr *q = &qs->qs[i]; 1006 + 1007 + switch (q->type) { 1008 + case VIRTCHNL2_QUEUE_TYPE_RX: 1009 + idpf_xdp_rxq_info_deinit(q->rxq, vport->rxq_model); 1010 + idpf_rx_desc_rel(q->rxq, dev, vport->rxq_model); 1011 + break; 1012 + case VIRTCHNL2_QUEUE_TYPE_RX_BUFFER: 1013 + idpf_rx_desc_rel_bufq(q->bufq, dev); 1014 + break; 1015 + case VIRTCHNL2_QUEUE_TYPE_TX: 1016 + idpf_tx_desc_rel(q->txq); 1017 + 1018 + if (idpf_queue_has(XDP, q->txq)) { 1019 + q->txq->pending = 0; 1020 + q->txq->xdp_tx = 0; 1021 + } else { 1022 + q->txq->txq_grp->num_completions_pending = 0; 1023 + } 1024 + 1025 + writel(q->txq->next_to_use, q->txq->tail); 1026 + break; 1027 + case VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION: 1028 + idpf_compl_desc_rel(q->complq); 1029 + q->complq->num_completions = 0; 1030 + break; 1031 + default: 1032 + break; 1033 + } 1034 + } 1035 + } 1036 + 1037 + static void idpf_qvec_ena_irq(struct idpf_q_vector *qv) 1038 + { 1039 + if (qv->num_txq) { 1040 + u32 itr; 1041 + 1042 + if (IDPF_ITR_IS_DYNAMIC(qv->tx_intr_mode)) 1043 + itr = qv->vport->tx_itr_profile[qv->tx_dim.profile_ix]; 1044 + else 1045 + itr = qv->tx_itr_value; 1046 + 1047 + idpf_vport_intr_write_itr(qv, itr, true); 1048 + } 1049 + 1050 + if (qv->num_rxq) { 1051 + u32 itr; 1052 + 1053 + if (IDPF_ITR_IS_DYNAMIC(qv->rx_intr_mode)) 1054 + itr = qv->vport->rx_itr_profile[qv->rx_dim.profile_ix]; 1055 + else 1056 + itr = qv->rx_itr_value; 1057 + 1058 + idpf_vport_intr_write_itr(qv, itr, false); 1059 + } 1060 + 1061 + if (qv->num_txq || qv->num_rxq) 1062 + idpf_vport_intr_update_itr_ena_irq(qv); 1063 + } 1064 + 1065 + /** 1066 + * idpf_vector_to_queue_set - create a queue set associated with the given 1067 + * queue vector 1068 + * @qv: queue vector corresponding to the queue pair 1069 + * 1070 + * Returns a pointer to a dynamically allocated array of pointers to all 1071 + * queues associated with a given queue vector (@qv). 1072 + * Please note that the caller is responsible to free the memory allocated 1073 + * by this function using kfree(). 1074 + * 1075 + * Return: &idpf_queue_set on success, %NULL in case of error. 1076 + */ 1077 + static struct idpf_queue_set * 1078 + idpf_vector_to_queue_set(struct idpf_q_vector *qv) 1079 + { 1080 + bool xdp = qv->vport->xdp_txq_offset && !qv->num_xsksq; 1081 + struct idpf_vport *vport = qv->vport; 1082 + struct idpf_queue_set *qs; 1083 + u32 num; 1084 + 1085 + num = qv->num_rxq + qv->num_bufq + qv->num_txq + qv->num_complq; 1086 + num += xdp ? qv->num_rxq * 2 : qv->num_xsksq * 2; 1087 + if (!num) 1088 + return NULL; 1089 + 1090 + qs = idpf_alloc_queue_set(vport, num); 1091 + if (!qs) 1092 + return NULL; 1093 + 1094 + num = 0; 1095 + 1096 + for (u32 i = 0; i < qv->num_bufq; i++) { 1097 + qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_RX_BUFFER; 1098 + qs->qs[num++].bufq = qv->bufq[i]; 1099 + } 1100 + 1101 + for (u32 i = 0; i < qv->num_rxq; i++) { 1102 + qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_RX; 1103 + qs->qs[num++].rxq = qv->rx[i]; 1104 + } 1105 + 1106 + for (u32 i = 0; i < qv->num_txq; i++) { 1107 + qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_TX; 1108 + qs->qs[num++].txq = qv->tx[i]; 1109 + } 1110 + 1111 + for (u32 i = 0; i < qv->num_complq; i++) { 1112 + qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION; 1113 + qs->qs[num++].complq = qv->complq[i]; 1114 + } 1115 + 1116 + if (!vport->xdp_txq_offset) 1117 + goto finalize; 1118 + 1119 + if (xdp) { 1120 + for (u32 i = 0; i < qv->num_rxq; i++) { 1121 + u32 idx = vport->xdp_txq_offset + qv->rx[i]->idx; 1122 + 1123 + qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_TX; 1124 + qs->qs[num++].txq = vport->txqs[idx]; 1125 + 1126 + qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION; 1127 + qs->qs[num++].complq = vport->txqs[idx]->complq; 1128 + } 1129 + } else { 1130 + for (u32 i = 0; i < qv->num_xsksq; i++) { 1131 + qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_TX; 1132 + qs->qs[num++].txq = qv->xsksq[i]; 1133 + 1134 + qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION; 1135 + qs->qs[num++].complq = qv->xsksq[i]->complq; 1136 + } 1137 + } 1138 + 1139 + finalize: 1140 + if (num != qs->num) { 1141 + kfree(qs); 1142 + return NULL; 1143 + } 1144 + 1145 + return qs; 1146 + } 1147 + 1148 + static int idpf_qp_enable(const struct idpf_queue_set *qs, u32 qid) 1149 + { 1150 + struct idpf_vport *vport = qs->vport; 1151 + struct idpf_q_vector *q_vector; 1152 + int err; 1153 + 1154 + q_vector = idpf_find_rxq_vec(vport, qid); 1155 + 1156 + err = idpf_init_queue_set(qs); 1157 + if (err) { 1158 + netdev_err(vport->netdev, "Could not initialize queues in pair %u: %pe\n", 1159 + qid, ERR_PTR(err)); 1160 + return err; 1161 + } 1162 + 1163 + if (!vport->xdp_txq_offset) 1164 + goto config; 1165 + 1166 + q_vector->xsksq = kcalloc(DIV_ROUND_UP(vport->num_rxq_grp, 1167 + vport->num_q_vectors), 1168 + sizeof(*q_vector->xsksq), GFP_KERNEL); 1169 + if (!q_vector->xsksq) 1170 + return -ENOMEM; 1171 + 1172 + for (u32 i = 0; i < qs->num; i++) { 1173 + const struct idpf_queue_ptr *q = &qs->qs[i]; 1174 + 1175 + if (q->type != VIRTCHNL2_QUEUE_TYPE_TX) 1176 + continue; 1177 + 1178 + if (!idpf_queue_has(XSK, q->txq)) 1179 + continue; 1180 + 1181 + idpf_xsk_init_wakeup(q_vector); 1182 + 1183 + q->txq->q_vector = q_vector; 1184 + q_vector->xsksq[q_vector->num_xsksq++] = q->txq; 1185 + } 1186 + 1187 + config: 1188 + err = idpf_send_config_queue_set_msg(qs); 1189 + if (err) { 1190 + netdev_err(vport->netdev, "Could not configure queues in pair %u: %pe\n", 1191 + qid, ERR_PTR(err)); 1192 + return err; 1193 + } 1194 + 1195 + err = idpf_send_enable_queue_set_msg(qs); 1196 + if (err) { 1197 + netdev_err(vport->netdev, "Could not enable queues in pair %u: %pe\n", 1198 + qid, ERR_PTR(err)); 1199 + return err; 1200 + } 1201 + 1202 + napi_enable(&q_vector->napi); 1203 + idpf_qvec_ena_irq(q_vector); 1204 + 1205 + netif_start_subqueue(vport->netdev, qid); 1206 + 1207 + return 0; 1208 + } 1209 + 1210 + static int idpf_qp_disable(const struct idpf_queue_set *qs, u32 qid) 1211 + { 1212 + struct idpf_vport *vport = qs->vport; 1213 + struct idpf_q_vector *q_vector; 1214 + int err; 1215 + 1216 + q_vector = idpf_find_rxq_vec(vport, qid); 1217 + netif_stop_subqueue(vport->netdev, qid); 1218 + 1219 + writel(0, q_vector->intr_reg.dyn_ctl); 1220 + napi_disable(&q_vector->napi); 1221 + 1222 + err = idpf_send_disable_queue_set_msg(qs); 1223 + if (err) { 1224 + netdev_err(vport->netdev, "Could not disable queues in pair %u: %pe\n", 1225 + qid, ERR_PTR(err)); 1226 + return err; 1227 + } 1228 + 1229 + idpf_clean_queue_set(qs); 1230 + 1231 + kfree(q_vector->xsksq); 1232 + q_vector->num_xsksq = 0; 1233 + 1234 + return 0; 1235 + } 1236 + 1237 + /** 1238 + * idpf_qp_switch - enable or disable queues associated with queue pair 1239 + * @vport: vport to switch the pair for 1240 + * @qid: index of the queue pair to switch 1241 + * @en: whether to enable or disable the pair 1242 + * 1243 + * Return: 0 on success, -errno on failure. 1244 + */ 1245 + int idpf_qp_switch(struct idpf_vport *vport, u32 qid, bool en) 1246 + { 1247 + struct idpf_q_vector *q_vector = idpf_find_rxq_vec(vport, qid); 1248 + struct idpf_queue_set *qs __free(kfree) = NULL; 1249 + 1250 + if (idpf_find_txq_vec(vport, qid) != q_vector) 1251 + return -EINVAL; 1252 + 1253 + qs = idpf_vector_to_queue_set(q_vector); 1254 + if (!qs) 1255 + return -ENOMEM; 1256 + 1257 + return en ? idpf_qp_enable(qs, qid) : idpf_qp_disable(qs, qid); 955 1258 } 956 1259 957 1260 /** ··· 1732 1365 q->tx_min_pkt_len = idpf_get_min_tx_pkt_len(adapter); 1733 1366 q->netdev = vport->netdev; 1734 1367 q->txq_grp = tx_qgrp; 1368 + q->rel_q_id = j; 1735 1369 1736 1370 if (!split) { 1737 1371 q->clean_budget = vport->compln_clean_budget; ··· 3398 3030 return 0; 3399 3031 } 3400 3032 3401 - static bool idpf_rx_process_skb_fields(struct sk_buff *skb, 3402 - const struct libeth_xdp_buff *xdp, 3403 - struct libeth_rq_napi_stats *rs) 3033 + bool idpf_rx_process_skb_fields(struct sk_buff *skb, 3034 + const struct libeth_xdp_buff *xdp, 3035 + struct libeth_rq_napi_stats *rs) 3404 3036 { 3405 3037 struct idpf_rx_queue *rxq; 3406 3038 ··· 3758 3390 struct idpf_q_vector *q_vector = (struct idpf_q_vector *)data; 3759 3391 3760 3392 q_vector->total_events++; 3761 - napi_schedule(&q_vector->napi); 3393 + napi_schedule_irqoff(&q_vector->napi); 3762 3394 3763 3395 return IRQ_HANDLED; 3764 3396 } ··· 3799 3431 for (u32 v_idx = 0; v_idx < vport->num_q_vectors; v_idx++) { 3800 3432 struct idpf_q_vector *q_vector = &vport->q_vectors[v_idx]; 3801 3433 3434 + kfree(q_vector->xsksq); 3435 + q_vector->xsksq = NULL; 3802 3436 kfree(q_vector->complq); 3803 3437 q_vector->complq = NULL; 3804 3438 kfree(q_vector->bufq); ··· 4259 3889 struct idpf_rx_queue *rxq = q_vec->rx[i]; 4260 3890 int pkts_cleaned_per_q; 4261 3891 4262 - pkts_cleaned_per_q = idpf_rx_splitq_clean(rxq, budget_per_q); 3892 + pkts_cleaned_per_q = idpf_queue_has(XSK, rxq) ? 3893 + idpf_xskrq_poll(rxq, budget_per_q) : 3894 + idpf_rx_splitq_clean(rxq, budget_per_q); 4263 3895 /* if we clean as many as budgeted, we must not be done */ 4264 3896 if (pkts_cleaned_per_q >= budget_per_q) 4265 3897 clean_complete = false; ··· 4271 3899 4272 3900 nid = numa_mem_id(); 4273 3901 4274 - for (i = 0; i < q_vec->num_bufq; i++) 4275 - idpf_rx_clean_refillq_all(q_vec->bufq[i], nid); 3902 + for (i = 0; i < q_vec->num_bufq; i++) { 3903 + if (!idpf_queue_has(XSK, q_vec->bufq[i])) 3904 + idpf_rx_clean_refillq_all(q_vec->bufq[i], nid); 3905 + } 4276 3906 4277 3907 return clean_complete; 4278 3908 } ··· 4288 3914 { 4289 3915 struct idpf_q_vector *q_vector = 4290 3916 container_of(napi, struct idpf_q_vector, napi); 4291 - bool clean_complete; 3917 + bool clean_complete = true; 4292 3918 int work_done = 0; 4293 3919 4294 3920 /* Handle case where we are called by netpoll with a budget of 0 */ ··· 4298 3924 return 0; 4299 3925 } 4300 3926 4301 - clean_complete = idpf_rx_splitq_clean_all(q_vector, budget, &work_done); 4302 - clean_complete &= idpf_tx_splitq_clean_all(q_vector, budget, &work_done); 3927 + for (u32 i = 0; i < q_vector->num_xsksq; i++) 3928 + clean_complete &= idpf_xsk_xmit(q_vector->xsksq[i]); 3929 + 3930 + clean_complete &= idpf_tx_splitq_clean_all(q_vector, budget, 3931 + &work_done); 3932 + clean_complete &= idpf_rx_splitq_clean_all(q_vector, budget, 3933 + &work_done); 4303 3934 4304 3935 /* If work not completed, return budget and polling will return */ 4305 3936 if (!clean_complete) { ··· 4317 3938 /* Exit the polling mode, but don't re-enable interrupts if stack might 4318 3939 * poll us due to busy-polling 4319 3940 */ 4320 - if (likely(napi_complete_done(napi, work_done))) 3941 + if (napi_complete_done(napi, work_done)) 4321 3942 idpf_vport_intr_update_itr_ena_irq(q_vector); 4322 3943 else 4323 3944 idpf_vport_intr_set_wb_on_itr(q_vector); ··· 4409 4030 } 4410 4031 4411 4032 qv_idx++; 4033 + } 4034 + 4035 + for (i = 0; i < vport->num_xdp_txq; i++) { 4036 + struct idpf_tx_queue *xdpsq; 4037 + struct idpf_q_vector *qv; 4038 + 4039 + xdpsq = vport->txqs[vport->xdp_txq_offset + i]; 4040 + if (!idpf_queue_has(XSK, xdpsq)) 4041 + continue; 4042 + 4043 + qv = idpf_find_rxq_vec(vport, i); 4044 + idpf_xsk_init_wakeup(qv); 4045 + 4046 + xdpsq->q_vector = qv; 4047 + qv->xsksq[qv->num_xsksq++] = xdpsq; 4412 4048 } 4413 4049 } 4414 4050 ··· 4561 4167 sizeof(*q_vector->complq), 4562 4168 GFP_KERNEL); 4563 4169 if (!q_vector->complq) 4170 + goto error; 4171 + 4172 + if (!vport->xdp_txq_offset) 4173 + continue; 4174 + 4175 + q_vector->xsksq = kcalloc(rxqs_per_vector, 4176 + sizeof(*q_vector->xsksq), 4177 + GFP_KERNEL); 4178 + if (!q_vector->xsksq) 4564 4179 goto error; 4565 4180 } 4566 4181

+62 -10

drivers/net/ethernet/intel/idpf/idpf_txrx.h

··· 141 141 #define IDPF_TX_FLAGS_TUNNEL BIT(3) 142 142 #define IDPF_TX_FLAGS_TSYN BIT(4) 143 143 144 + struct libeth_rq_napi_stats; 145 + 144 146 union idpf_tx_flex_desc { 145 147 struct idpf_flex_tx_desc q; /* queue based scheduling */ 146 148 struct idpf_flex_tx_sched_desc flow; /* flow based scheduling */ ··· 287 285 * queue 288 286 * @__IDPF_Q_NOIRQ: queue is polling-driven and has no interrupt 289 287 * @__IDPF_Q_XDP: this is an XDP queue 288 + * @__IDPF_Q_XSK: the queue has an XSk pool installed 290 289 * @__IDPF_Q_FLAGS_NBITS: Must be last 291 290 */ 292 291 enum idpf_queue_flags_t { ··· 300 297 __IDPF_Q_PTP, 301 298 __IDPF_Q_NOIRQ, 302 299 __IDPF_Q_XDP, 300 + __IDPF_Q_XSK, 303 301 304 302 __IDPF_Q_FLAGS_NBITS, 305 303 }; ··· 367 363 * @num_txq: Number of TX queues 368 364 * @num_bufq: Number of buffer queues 369 365 * @num_complq: number of completion queues 366 + * @num_xsksq: number of XSk send queues 370 367 * @rx: Array of RX queues to service 371 368 * @tx: Array of TX queues to service 372 369 * @bufq: Array of buffer queues to service 373 370 * @complq: array of completion queues 371 + * @xsksq: array of XSk send queues 374 372 * @intr_reg: See struct idpf_intr_reg 375 - * @napi: napi handler 373 + * @csd: XSk wakeup CSD 376 374 * @total_events: Number of interrupts processed 377 375 * @wb_on_itr: whether WB on ITR is enabled 376 + * @napi: napi handler 378 377 * @tx_dim: Data for TX net_dim algorithm 379 378 * @tx_itr_value: TX interrupt throttling rate 380 379 * @tx_intr_mode: Dynamic ITR or not ··· 396 389 u16 num_txq; 397 390 u16 num_bufq; 398 391 u16 num_complq; 392 + u16 num_xsksq; 399 393 struct idpf_rx_queue **rx; 400 394 struct idpf_tx_queue **tx; 401 395 struct idpf_buf_queue **bufq; 402 396 struct idpf_compl_queue **complq; 397 + struct idpf_tx_queue **xsksq; 403 398 404 399 struct idpf_intr_reg intr_reg; 405 400 __cacheline_group_end_aligned(read_mostly); 406 401 407 402 __cacheline_group_begin_aligned(read_write); 408 - struct napi_struct napi; 403 + call_single_data_t csd; 404 + 409 405 u16 total_events; 410 406 bool wb_on_itr; 407 + 408 + struct napi_struct napi; 411 409 412 410 struct dim tx_dim; 413 411 u16 tx_itr_value; ··· 430 418 431 419 __cacheline_group_end_aligned(cold); 432 420 }; 433 - libeth_cacheline_set_assert(struct idpf_q_vector, 120, 434 - 24 + sizeof(struct napi_struct) + 421 + libeth_cacheline_set_assert(struct idpf_q_vector, 136, 422 + 56 + sizeof(struct napi_struct) + 435 423 2 * sizeof(struct dim), 436 424 8); 437 425 ··· 497 485 * @next_to_clean: Next descriptor to clean 498 486 * @next_to_alloc: RX buffer to allocate at 499 487 * @xdp: XDP buffer with the current frame 488 + * @xsk: current XDP buffer in XSk mode 489 + * @pool: XSk pool if installed 500 490 * @cached_phc_time: Cached PHC time for the Rx queue 501 491 * @stats_sync: See struct u64_stats_sync 502 492 * @q_stats: See union idpf_rx_queue_stats ··· 554 540 u32 next_to_clean; 555 541 u32 next_to_alloc; 556 542 557 - struct libeth_xdp_buff_stash xdp; 543 + union { 544 + struct libeth_xdp_buff_stash xdp; 545 + struct { 546 + struct libeth_xdp_buff *xsk; 547 + struct xsk_buff_pool *pool; 548 + }; 549 + }; 558 550 u64 cached_phc_time; 559 551 560 552 struct u64_stats_sync stats_sync; ··· 598 578 * @txq_grp: See struct idpf_txq_group 599 579 * @complq: corresponding completion queue in XDP mode 600 580 * @dev: Device back pointer for DMA mapping 581 + * @pool: corresponding XSk pool if installed 601 582 * @tail: Tail offset. Used for both queue models single and split 602 583 * @flags: See enum idpf_queue_flags_t 603 584 * @idx: For TX queue, it is used as index to map between TX queue group and ··· 635 614 * @dma: Physical address of ring 636 615 * @q_vector: Backreference to associated vector 637 616 * @buf_pool_size: Total number of idpf_tx_buf 617 + * @rel_q_id: relative virtchnl queue index 638 618 */ 639 619 struct idpf_tx_queue { 640 620 __cacheline_group_begin_aligned(read_mostly); ··· 652 630 struct idpf_txq_group *txq_grp; 653 631 struct idpf_compl_queue *complq; 654 632 }; 655 - struct device *dev; 633 + union { 634 + struct device *dev; 635 + struct xsk_buff_pool *pool; 636 + }; 656 637 void __iomem *tail; 657 638 658 639 DECLARE_BITMAP(flags, __IDPF_Q_FLAGS_NBITS); ··· 709 684 dma_addr_t dma; 710 685 711 686 struct idpf_q_vector *q_vector; 687 + 712 688 u32 buf_pool_size; 689 + u32 rel_q_id; 713 690 __cacheline_group_end_aligned(cold); 714 691 }; 715 692 libeth_cacheline_set_assert(struct idpf_tx_queue, 64, ··· 725 698 /** 726 699 * struct idpf_buf_queue - software structure representing a buffer queue 727 700 * @split_buf: buffer descriptor array 728 - * @hdr_buf: &libeth_fqe for header buffers 729 - * @hdr_pp: &page_pool for header buffers 730 701 * @buf: &libeth_fqe for data buffers 731 702 * @pp: &page_pool for data buffers 703 + * @xsk_buf: &xdp_buff for XSk Rx buffers 704 + * @pool: &xsk_buff_pool on XSk queues 705 + * @hdr_buf: &libeth_fqe for header buffers 706 + * @hdr_pp: &page_pool for header buffers 732 707 * @tail: Tail offset 733 708 * @flags: See enum idpf_queue_flags_t 734 709 * @desc_count: Number of descriptors 710 + * @thresh: refill threshold in XSk 735 711 * @next_to_use: Next descriptor to use 736 712 * @next_to_clean: Next descriptor to clean 737 713 * @next_to_alloc: RX buffer to allocate at 714 + * @pending: number of buffers to refill (Xsk) 738 715 * @hdr_truesize: truesize for buffer headers 739 716 * @truesize: truesize for data buffers 740 717 * @q_id: Queue id ··· 752 721 struct idpf_buf_queue { 753 722 __cacheline_group_begin_aligned(read_mostly); 754 723 struct virtchnl2_splitq_rx_buf_desc *split_buf; 724 + union { 725 + struct { 726 + struct libeth_fqe *buf; 727 + struct page_pool *pp; 728 + }; 729 + struct { 730 + struct libeth_xdp_buff **xsk_buf; 731 + struct xsk_buff_pool *pool; 732 + }; 733 + }; 755 734 struct libeth_fqe *hdr_buf; 756 735 struct page_pool *hdr_pp; 757 - struct libeth_fqe *buf; 758 - struct page_pool *pp; 759 736 void __iomem *tail; 760 737 761 738 DECLARE_BITMAP(flags, __IDPF_Q_FLAGS_NBITS); 762 739 u32 desc_count; 740 + 741 + u32 thresh; 763 742 __cacheline_group_end_aligned(read_mostly); 764 743 765 744 __cacheline_group_begin_aligned(read_write); ··· 777 736 u32 next_to_clean; 778 737 u32 next_to_alloc; 779 738 739 + u32 pending; 780 740 u32 hdr_truesize; 781 741 u32 truesize; 782 742 __cacheline_group_end_aligned(read_write); ··· 1089 1047 int idpf_init_rss(struct idpf_vport *vport); 1090 1048 void idpf_deinit_rss(struct idpf_vport *vport); 1091 1049 int idpf_rx_bufs_init_all(struct idpf_vport *vport); 1050 + 1051 + struct idpf_q_vector *idpf_find_rxq_vec(const struct idpf_vport *vport, 1052 + u32 q_num); 1053 + struct idpf_q_vector *idpf_find_txq_vec(const struct idpf_vport *vport, 1054 + u32 q_num); 1055 + int idpf_qp_switch(struct idpf_vport *vport, u32 qid, bool en); 1056 + 1092 1057 void idpf_tx_buf_hw_update(struct idpf_tx_queue *tx_q, u32 val, 1093 1058 bool xmit_more); 1094 1059 unsigned int idpf_size_to_txd_count(unsigned int size); ··· 1108 1059 netdev_tx_t idpf_tx_start(struct sk_buff *skb, struct net_device *netdev); 1109 1060 bool idpf_rx_singleq_buf_hw_alloc_all(struct idpf_rx_queue *rxq, 1110 1061 u16 cleaned_count); 1062 + bool idpf_rx_process_skb_fields(struct sk_buff *skb, 1063 + const struct libeth_xdp_buff *xdp, 1064 + struct libeth_rq_napi_stats *rs); 1111 1065 int idpf_tso(struct sk_buff *skb, struct idpf_tx_offload_params *off); 1112 1066 1113 1067 void idpf_wait_for_sw_marker_completion(const struct idpf_tx_queue *txq);

+780 -472

drivers/net/ethernet/intel/idpf/idpf_virtchnl.c

··· 716 716 return err; 717 717 } 718 718 719 + struct idpf_chunked_msg_params { 720 + u32 (*prepare_msg)(const struct idpf_vport *vport, 721 + void *buf, const void *pos, 722 + u32 num); 723 + 724 + const void *chunks; 725 + u32 num_chunks; 726 + 727 + u32 chunk_sz; 728 + u32 config_sz; 729 + 730 + u32 vc_op; 731 + }; 732 + 733 + struct idpf_queue_set *idpf_alloc_queue_set(struct idpf_vport *vport, u32 num) 734 + { 735 + struct idpf_queue_set *qp; 736 + 737 + qp = kzalloc(struct_size(qp, qs, num), GFP_KERNEL); 738 + if (!qp) 739 + return NULL; 740 + 741 + qp->vport = vport; 742 + qp->num = num; 743 + 744 + return qp; 745 + } 746 + 747 + /** 748 + * idpf_send_chunked_msg - send VC message consisting of chunks 749 + * @vport: virtual port data structure 750 + * @params: message params 751 + * 752 + * Helper function for preparing a message describing queues to be enabled 753 + * or disabled. 754 + * 755 + * Return: the total size of the prepared message. 756 + */ 757 + static int idpf_send_chunked_msg(struct idpf_vport *vport, 758 + const struct idpf_chunked_msg_params *params) 759 + { 760 + struct idpf_vc_xn_params xn_params = { 761 + .vc_op = params->vc_op, 762 + .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 763 + }; 764 + const void *pos = params->chunks; 765 + u32 num_chunks, num_msgs, buf_sz; 766 + void *buf __free(kfree) = NULL; 767 + u32 totqs = params->num_chunks; 768 + 769 + num_chunks = min(IDPF_NUM_CHUNKS_PER_MSG(params->config_sz, 770 + params->chunk_sz), totqs); 771 + num_msgs = DIV_ROUND_UP(totqs, num_chunks); 772 + 773 + buf_sz = params->config_sz + num_chunks * params->chunk_sz; 774 + buf = kzalloc(buf_sz, GFP_KERNEL); 775 + if (!buf) 776 + return -ENOMEM; 777 + 778 + xn_params.send_buf.iov_base = buf; 779 + 780 + for (u32 i = 0; i < num_msgs; i++) { 781 + ssize_t reply_sz; 782 + 783 + memset(buf, 0, buf_sz); 784 + xn_params.send_buf.iov_len = buf_sz; 785 + 786 + if (params->prepare_msg(vport, buf, pos, num_chunks) != buf_sz) 787 + return -EINVAL; 788 + 789 + reply_sz = idpf_vc_xn_exec(vport->adapter, &xn_params); 790 + if (reply_sz < 0) 791 + return reply_sz; 792 + 793 + pos += num_chunks * params->chunk_sz; 794 + totqs -= num_chunks; 795 + 796 + num_chunks = min(num_chunks, totqs); 797 + buf_sz = params->config_sz + num_chunks * params->chunk_sz; 798 + } 799 + 800 + return 0; 801 + } 802 + 803 + /** 804 + * idpf_wait_for_marker_event_set - wait for software marker response for 805 + * selected Tx queues 806 + * @qs: set of the Tx queues 807 + * 808 + * Return: 0 success, -errno on failure. 809 + */ 810 + static int idpf_wait_for_marker_event_set(const struct idpf_queue_set *qs) 811 + { 812 + struct idpf_tx_queue *txq; 813 + bool markers_rcvd = true; 814 + 815 + for (u32 i = 0; i < qs->num; i++) { 816 + switch (qs->qs[i].type) { 817 + case VIRTCHNL2_QUEUE_TYPE_TX: 818 + txq = qs->qs[i].txq; 819 + 820 + idpf_queue_set(SW_MARKER, txq); 821 + idpf_wait_for_sw_marker_completion(txq); 822 + markers_rcvd &= !idpf_queue_has(SW_MARKER, txq); 823 + break; 824 + default: 825 + break; 826 + } 827 + } 828 + 829 + if (!markers_rcvd) { 830 + netdev_warn(qs->vport->netdev, 831 + "Failed to receive marker packets\n"); 832 + return -ETIMEDOUT; 833 + } 834 + 835 + return 0; 836 + } 837 + 719 838 /** 720 839 * idpf_wait_for_marker_event - wait for software marker response 721 840 * @vport: virtual port data structure 722 841 * 723 - * Returns 0 success, negative on failure. 842 + * Return: 0 success, negative on failure. 724 843 **/ 725 844 static int idpf_wait_for_marker_event(struct idpf_vport *vport) 726 845 { 727 - bool markers_rcvd = true; 846 + struct idpf_queue_set *qs __free(kfree) = NULL; 728 847 729 - for (u32 i = 0; i < vport->num_txq; i++) { 730 - struct idpf_tx_queue *txq = vport->txqs[i]; 848 + qs = idpf_alloc_queue_set(vport, vport->num_txq); 849 + if (!qs) 850 + return -ENOMEM; 731 851 732 - idpf_queue_set(SW_MARKER, txq); 733 - idpf_wait_for_sw_marker_completion(txq); 734 - markers_rcvd &= !idpf_queue_has(SW_MARKER, txq); 852 + for (u32 i = 0; i < qs->num; i++) { 853 + qs->qs[i].type = VIRTCHNL2_QUEUE_TYPE_TX; 854 + qs->qs[i].txq = vport->txqs[i]; 735 855 } 736 856 737 - if (markers_rcvd) 738 - return 0; 739 - 740 - dev_warn(&vport->adapter->pdev->dev, "Failed to receive marker packets\n"); 741 - 742 - return -ETIMEDOUT; 857 + return idpf_wait_for_marker_event_set(qs); 743 858 } 744 859 745 860 /** ··· 1686 1571 } 1687 1572 1688 1573 /** 1689 - * idpf_send_config_tx_queues_msg - Send virtchnl config tx queues message 1574 + * idpf_fill_txq_config_chunk - fill chunk describing the Tx queue 1575 + * @vport: virtual port data structure 1576 + * @q: Tx queue to be inserted into VC chunk 1577 + * @qi: pointer to the buffer containing the VC chunk 1578 + */ 1579 + static void idpf_fill_txq_config_chunk(const struct idpf_vport *vport, 1580 + const struct idpf_tx_queue *q, 1581 + struct virtchnl2_txq_info *qi) 1582 + { 1583 + u32 val; 1584 + 1585 + qi->queue_id = cpu_to_le32(q->q_id); 1586 + qi->model = cpu_to_le16(vport->txq_model); 1587 + qi->type = cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_TX); 1588 + qi->ring_len = cpu_to_le16(q->desc_count); 1589 + qi->dma_ring_addr = cpu_to_le64(q->dma); 1590 + qi->relative_queue_id = cpu_to_le16(q->rel_q_id); 1591 + 1592 + if (!idpf_is_queue_model_split(vport->txq_model)) { 1593 + qi->sched_mode = cpu_to_le16(VIRTCHNL2_TXQ_SCHED_MODE_QUEUE); 1594 + return; 1595 + } 1596 + 1597 + if (idpf_queue_has(XDP, q)) 1598 + val = q->complq->q_id; 1599 + else 1600 + val = q->txq_grp->complq->q_id; 1601 + 1602 + qi->tx_compl_queue_id = cpu_to_le16(val); 1603 + 1604 + if (idpf_queue_has(FLOW_SCH_EN, q)) 1605 + val = VIRTCHNL2_TXQ_SCHED_MODE_FLOW; 1606 + else 1607 + val = VIRTCHNL2_TXQ_SCHED_MODE_QUEUE; 1608 + 1609 + qi->sched_mode = cpu_to_le16(val); 1610 + } 1611 + 1612 + /** 1613 + * idpf_fill_complq_config_chunk - fill chunk describing the completion queue 1614 + * @vport: virtual port data structure 1615 + * @q: completion queue to be inserted into VC chunk 1616 + * @qi: pointer to the buffer containing the VC chunk 1617 + */ 1618 + static void idpf_fill_complq_config_chunk(const struct idpf_vport *vport, 1619 + const struct idpf_compl_queue *q, 1620 + struct virtchnl2_txq_info *qi) 1621 + { 1622 + u32 val; 1623 + 1624 + qi->queue_id = cpu_to_le32(q->q_id); 1625 + qi->model = cpu_to_le16(vport->txq_model); 1626 + qi->type = cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION); 1627 + qi->ring_len = cpu_to_le16(q->desc_count); 1628 + qi->dma_ring_addr = cpu_to_le64(q->dma); 1629 + 1630 + if (idpf_queue_has(FLOW_SCH_EN, q)) 1631 + val = VIRTCHNL2_TXQ_SCHED_MODE_FLOW; 1632 + else 1633 + val = VIRTCHNL2_TXQ_SCHED_MODE_QUEUE; 1634 + 1635 + qi->sched_mode = cpu_to_le16(val); 1636 + } 1637 + 1638 + /** 1639 + * idpf_prepare_cfg_txqs_msg - prepare message to configure selected Tx queues 1640 + * @vport: virtual port data structure 1641 + * @buf: buffer containing the message 1642 + * @pos: pointer to the first chunk describing the tx queue 1643 + * @num_chunks: number of chunks in the message 1644 + * 1645 + * Helper function for preparing the message describing configuration of 1646 + * Tx queues. 1647 + * 1648 + * Return: the total size of the prepared message. 1649 + */ 1650 + static u32 idpf_prepare_cfg_txqs_msg(const struct idpf_vport *vport, 1651 + void *buf, const void *pos, 1652 + u32 num_chunks) 1653 + { 1654 + struct virtchnl2_config_tx_queues *ctq = buf; 1655 + 1656 + ctq->vport_id = cpu_to_le32(vport->vport_id); 1657 + ctq->num_qinfo = cpu_to_le16(num_chunks); 1658 + memcpy(ctq->qinfo, pos, num_chunks * sizeof(*ctq->qinfo)); 1659 + 1660 + return struct_size(ctq, qinfo, num_chunks); 1661 + } 1662 + 1663 + /** 1664 + * idpf_send_config_tx_queue_set_msg - send virtchnl config Tx queues 1665 + * message for selected queues 1666 + * @qs: set of the Tx queues to configure 1667 + * 1668 + * Send config queues virtchnl message for queues contained in the @qs array. 1669 + * The @qs array can contain Tx queues (or completion queues) only. 1670 + * 1671 + * Return: 0 on success, -errno on failure. 1672 + */ 1673 + static int idpf_send_config_tx_queue_set_msg(const struct idpf_queue_set *qs) 1674 + { 1675 + struct virtchnl2_txq_info *qi __free(kfree) = NULL; 1676 + struct idpf_chunked_msg_params params = { 1677 + .vc_op = VIRTCHNL2_OP_CONFIG_TX_QUEUES, 1678 + .prepare_msg = idpf_prepare_cfg_txqs_msg, 1679 + .config_sz = sizeof(struct virtchnl2_config_tx_queues), 1680 + .chunk_sz = sizeof(*qi), 1681 + }; 1682 + 1683 + qi = kcalloc(qs->num, sizeof(*qi), GFP_KERNEL); 1684 + if (!qi) 1685 + return -ENOMEM; 1686 + 1687 + params.chunks = qi; 1688 + 1689 + for (u32 i = 0; i < qs->num; i++) { 1690 + if (qs->qs[i].type == VIRTCHNL2_QUEUE_TYPE_TX) 1691 + idpf_fill_txq_config_chunk(qs->vport, qs->qs[i].txq, 1692 + &qi[params.num_chunks++]); 1693 + else if (qs->qs[i].type == VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION) 1694 + idpf_fill_complq_config_chunk(qs->vport, 1695 + qs->qs[i].complq, 1696 + &qi[params.num_chunks++]); 1697 + } 1698 + 1699 + return idpf_send_chunked_msg(qs->vport, &params); 1700 + } 1701 + 1702 + /** 1703 + * idpf_send_config_tx_queues_msg - send virtchnl config Tx queues message 1690 1704 * @vport: virtual port data structure 1691 1705 * 1692 - * Send config tx queues virtchnl message. Returns 0 on success, negative on 1693 - * failure. 1706 + * Return: 0 on success, -errno on failure. 1694 1707 */ 1695 1708 static int idpf_send_config_tx_queues_msg(struct idpf_vport *vport) 1696 1709 { 1697 - struct virtchnl2_config_tx_queues *ctq __free(kfree) = NULL; 1698 - struct virtchnl2_txq_info *qi __free(kfree) = NULL; 1699 - struct idpf_vc_xn_params xn_params = {}; 1700 - u32 config_sz, chunk_sz, buf_sz; 1701 - int totqs, num_msgs, num_chunks; 1702 - ssize_t reply_sz; 1703 - int i, k = 0; 1710 + struct idpf_queue_set *qs __free(kfree) = NULL; 1711 + u32 totqs = vport->num_txq + vport->num_complq; 1712 + u32 k = 0; 1704 1713 1705 - totqs = vport->num_txq + vport->num_complq; 1706 - qi = kcalloc(totqs, sizeof(struct virtchnl2_txq_info), GFP_KERNEL); 1707 - if (!qi) 1714 + qs = idpf_alloc_queue_set(vport, totqs); 1715 + if (!qs) 1708 1716 return -ENOMEM; 1709 1717 1710 1718 /* Populate the queue info buffer with all queue context info */ 1711 - for (i = 0; i < vport->num_txq_grp; i++) { 1712 - struct idpf_txq_group *tx_qgrp = &vport->txq_grps[i]; 1713 - int j, sched_mode; 1719 + for (u32 i = 0; i < vport->num_txq_grp; i++) { 1720 + const struct idpf_txq_group *tx_qgrp = &vport->txq_grps[i]; 1714 1721 1715 - for (j = 0; j < tx_qgrp->num_txq; j++, k++) { 1716 - qi[k].queue_id = 1717 - cpu_to_le32(tx_qgrp->txqs[j]->q_id); 1718 - qi[k].model = 1719 - cpu_to_le16(vport->txq_model); 1720 - qi[k].type = 1721 - cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_TX); 1722 - qi[k].ring_len = 1723 - cpu_to_le16(tx_qgrp->txqs[j]->desc_count); 1724 - qi[k].dma_ring_addr = 1725 - cpu_to_le64(tx_qgrp->txqs[j]->dma); 1726 - if (idpf_is_queue_model_split(vport->txq_model)) { 1727 - struct idpf_tx_queue *q = tx_qgrp->txqs[j]; 1728 - 1729 - qi[k].tx_compl_queue_id = 1730 - cpu_to_le16(tx_qgrp->complq->q_id); 1731 - qi[k].relative_queue_id = cpu_to_le16(j); 1732 - 1733 - if (idpf_queue_has(FLOW_SCH_EN, q)) 1734 - qi[k].sched_mode = 1735 - cpu_to_le16(VIRTCHNL2_TXQ_SCHED_MODE_FLOW); 1736 - else 1737 - qi[k].sched_mode = 1738 - cpu_to_le16(VIRTCHNL2_TXQ_SCHED_MODE_QUEUE); 1739 - } else { 1740 - qi[k].sched_mode = 1741 - cpu_to_le16(VIRTCHNL2_TXQ_SCHED_MODE_QUEUE); 1742 - } 1722 + for (u32 j = 0; j < tx_qgrp->num_txq; j++) { 1723 + qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_TX; 1724 + qs->qs[k++].txq = tx_qgrp->txqs[j]; 1743 1725 } 1744 1726 1745 - if (!idpf_is_queue_model_split(vport->txq_model)) 1746 - continue; 1747 - 1748 - qi[k].queue_id = cpu_to_le32(tx_qgrp->complq->q_id); 1749 - qi[k].model = cpu_to_le16(vport->txq_model); 1750 - qi[k].type = cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION); 1751 - qi[k].ring_len = cpu_to_le16(tx_qgrp->complq->desc_count); 1752 - qi[k].dma_ring_addr = cpu_to_le64(tx_qgrp->complq->dma); 1753 - 1754 - if (idpf_queue_has(FLOW_SCH_EN, tx_qgrp->complq)) 1755 - sched_mode = VIRTCHNL2_TXQ_SCHED_MODE_FLOW; 1756 - else 1757 - sched_mode = VIRTCHNL2_TXQ_SCHED_MODE_QUEUE; 1758 - qi[k].sched_mode = cpu_to_le16(sched_mode); 1759 - 1760 - k++; 1727 + if (idpf_is_queue_model_split(vport->txq_model)) { 1728 + qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION; 1729 + qs->qs[k++].complq = tx_qgrp->complq; 1730 + } 1761 1731 } 1762 1732 1763 1733 /* Make sure accounting agrees */ 1764 1734 if (k != totqs) 1765 1735 return -EINVAL; 1766 1736 1767 - /* Chunk up the queue contexts into multiple messages to avoid 1768 - * sending a control queue message buffer that is too large 1769 - */ 1770 - config_sz = sizeof(struct virtchnl2_config_tx_queues); 1771 - chunk_sz = sizeof(struct virtchnl2_txq_info); 1772 - 1773 - num_chunks = min_t(u32, IDPF_NUM_CHUNKS_PER_MSG(config_sz, chunk_sz), 1774 - totqs); 1775 - num_msgs = DIV_ROUND_UP(totqs, num_chunks); 1776 - 1777 - buf_sz = struct_size(ctq, qinfo, num_chunks); 1778 - ctq = kzalloc(buf_sz, GFP_KERNEL); 1779 - if (!ctq) 1780 - return -ENOMEM; 1781 - 1782 - xn_params.vc_op = VIRTCHNL2_OP_CONFIG_TX_QUEUES; 1783 - xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC; 1784 - 1785 - for (i = 0, k = 0; i < num_msgs; i++) { 1786 - memset(ctq, 0, buf_sz); 1787 - ctq->vport_id = cpu_to_le32(vport->vport_id); 1788 - ctq->num_qinfo = cpu_to_le16(num_chunks); 1789 - memcpy(ctq->qinfo, &qi[k], chunk_sz * num_chunks); 1790 - 1791 - xn_params.send_buf.iov_base = ctq; 1792 - xn_params.send_buf.iov_len = buf_sz; 1793 - reply_sz = idpf_vc_xn_exec(vport->adapter, &xn_params); 1794 - if (reply_sz < 0) 1795 - return reply_sz; 1796 - 1797 - k += num_chunks; 1798 - totqs -= num_chunks; 1799 - num_chunks = min(num_chunks, totqs); 1800 - /* Recalculate buffer size */ 1801 - buf_sz = struct_size(ctq, qinfo, num_chunks); 1802 - } 1803 - 1804 - return 0; 1737 + return idpf_send_config_tx_queue_set_msg(qs); 1805 1738 } 1806 1739 1807 1740 /** 1808 - * idpf_send_config_rx_queues_msg - Send virtchnl config rx queues message 1741 + * idpf_fill_rxq_config_chunk - fill chunk describing the Rx queue 1742 + * @vport: virtual port data structure 1743 + * @q: Rx queue to be inserted into VC chunk 1744 + * @qi: pointer to the buffer containing the VC chunk 1745 + */ 1746 + static void idpf_fill_rxq_config_chunk(const struct idpf_vport *vport, 1747 + struct idpf_rx_queue *q, 1748 + struct virtchnl2_rxq_info *qi) 1749 + { 1750 + const struct idpf_bufq_set *sets; 1751 + 1752 + qi->queue_id = cpu_to_le32(q->q_id); 1753 + qi->model = cpu_to_le16(vport->rxq_model); 1754 + qi->type = cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_RX); 1755 + qi->ring_len = cpu_to_le16(q->desc_count); 1756 + qi->dma_ring_addr = cpu_to_le64(q->dma); 1757 + qi->max_pkt_size = cpu_to_le32(q->rx_max_pkt_size); 1758 + qi->rx_buffer_low_watermark = cpu_to_le16(q->rx_buffer_low_watermark); 1759 + qi->qflags = cpu_to_le16(VIRTCHNL2_RX_DESC_SIZE_32BYTE); 1760 + if (idpf_is_feature_ena(vport, NETIF_F_GRO_HW)) 1761 + qi->qflags |= cpu_to_le16(VIRTCHNL2_RXQ_RSC); 1762 + 1763 + if (!idpf_is_queue_model_split(vport->rxq_model)) { 1764 + qi->data_buffer_size = cpu_to_le32(q->rx_buf_size); 1765 + qi->desc_ids = cpu_to_le64(q->rxdids); 1766 + 1767 + return; 1768 + } 1769 + 1770 + sets = q->bufq_sets; 1771 + 1772 + /* 1773 + * In splitq mode, RxQ buffer size should be set to that of the first 1774 + * buffer queue associated with this RxQ. 1775 + */ 1776 + q->rx_buf_size = sets[0].bufq.rx_buf_size; 1777 + qi->data_buffer_size = cpu_to_le32(q->rx_buf_size); 1778 + 1779 + qi->rx_bufq1_id = cpu_to_le16(sets[0].bufq.q_id); 1780 + if (vport->num_bufqs_per_qgrp > IDPF_SINGLE_BUFQ_PER_RXQ_GRP) { 1781 + qi->bufq2_ena = IDPF_BUFQ2_ENA; 1782 + qi->rx_bufq2_id = cpu_to_le16(sets[1].bufq.q_id); 1783 + } 1784 + 1785 + q->rx_hbuf_size = sets[0].bufq.rx_hbuf_size; 1786 + 1787 + if (idpf_queue_has(HSPLIT_EN, q)) { 1788 + qi->qflags |= cpu_to_le16(VIRTCHNL2_RXQ_HDR_SPLIT); 1789 + qi->hdr_buffer_size = cpu_to_le16(q->rx_hbuf_size); 1790 + } 1791 + 1792 + qi->desc_ids = cpu_to_le64(VIRTCHNL2_RXDID_2_FLEX_SPLITQ_M); 1793 + } 1794 + 1795 + /** 1796 + * idpf_fill_bufq_config_chunk - fill chunk describing the buffer queue 1797 + * @vport: virtual port data structure 1798 + * @q: buffer queue to be inserted into VC chunk 1799 + * @qi: pointer to the buffer containing the VC chunk 1800 + */ 1801 + static void idpf_fill_bufq_config_chunk(const struct idpf_vport *vport, 1802 + const struct idpf_buf_queue *q, 1803 + struct virtchnl2_rxq_info *qi) 1804 + { 1805 + qi->queue_id = cpu_to_le32(q->q_id); 1806 + qi->model = cpu_to_le16(vport->rxq_model); 1807 + qi->type = cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_RX_BUFFER); 1808 + qi->ring_len = cpu_to_le16(q->desc_count); 1809 + qi->dma_ring_addr = cpu_to_le64(q->dma); 1810 + qi->data_buffer_size = cpu_to_le32(q->rx_buf_size); 1811 + qi->rx_buffer_low_watermark = cpu_to_le16(q->rx_buffer_low_watermark); 1812 + qi->desc_ids = cpu_to_le64(VIRTCHNL2_RXDID_2_FLEX_SPLITQ_M); 1813 + qi->buffer_notif_stride = IDPF_RX_BUF_STRIDE; 1814 + if (idpf_is_feature_ena(vport, NETIF_F_GRO_HW)) 1815 + qi->qflags = cpu_to_le16(VIRTCHNL2_RXQ_RSC); 1816 + 1817 + if (idpf_queue_has(HSPLIT_EN, q)) { 1818 + qi->qflags |= cpu_to_le16(VIRTCHNL2_RXQ_HDR_SPLIT); 1819 + qi->hdr_buffer_size = cpu_to_le16(q->rx_hbuf_size); 1820 + } 1821 + } 1822 + 1823 + /** 1824 + * idpf_prepare_cfg_rxqs_msg - prepare message to configure selected Rx queues 1825 + * @vport: virtual port data structure 1826 + * @buf: buffer containing the message 1827 + * @pos: pointer to the first chunk describing the rx queue 1828 + * @num_chunks: number of chunks in the message 1829 + * 1830 + * Helper function for preparing the message describing configuration of 1831 + * Rx queues. 1832 + * 1833 + * Return: the total size of the prepared message. 1834 + */ 1835 + static u32 idpf_prepare_cfg_rxqs_msg(const struct idpf_vport *vport, 1836 + void *buf, const void *pos, 1837 + u32 num_chunks) 1838 + { 1839 + struct virtchnl2_config_rx_queues *crq = buf; 1840 + 1841 + crq->vport_id = cpu_to_le32(vport->vport_id); 1842 + crq->num_qinfo = cpu_to_le16(num_chunks); 1843 + memcpy(crq->qinfo, pos, num_chunks * sizeof(*crq->qinfo)); 1844 + 1845 + return struct_size(crq, qinfo, num_chunks); 1846 + } 1847 + 1848 + /** 1849 + * idpf_send_config_rx_queue_set_msg - send virtchnl config Rx queues message 1850 + * for selected queues. 1851 + * @qs: set of the Rx queues to configure 1852 + * 1853 + * Send config queues virtchnl message for queues contained in the @qs array. 1854 + * The @qs array can contain Rx queues (or buffer queues) only. 1855 + * 1856 + * Return: 0 on success, -errno on failure. 1857 + */ 1858 + static int idpf_send_config_rx_queue_set_msg(const struct idpf_queue_set *qs) 1859 + { 1860 + struct virtchnl2_rxq_info *qi __free(kfree) = NULL; 1861 + struct idpf_chunked_msg_params params = { 1862 + .vc_op = VIRTCHNL2_OP_CONFIG_RX_QUEUES, 1863 + .prepare_msg = idpf_prepare_cfg_rxqs_msg, 1864 + .config_sz = sizeof(struct virtchnl2_config_rx_queues), 1865 + .chunk_sz = sizeof(*qi), 1866 + }; 1867 + 1868 + qi = kcalloc(qs->num, sizeof(*qi), GFP_KERNEL); 1869 + if (!qi) 1870 + return -ENOMEM; 1871 + 1872 + params.chunks = qi; 1873 + 1874 + for (u32 i = 0; i < qs->num; i++) { 1875 + if (qs->qs[i].type == VIRTCHNL2_QUEUE_TYPE_RX) 1876 + idpf_fill_rxq_config_chunk(qs->vport, qs->qs[i].rxq, 1877 + &qi[params.num_chunks++]); 1878 + else if (qs->qs[i].type == VIRTCHNL2_QUEUE_TYPE_RX_BUFFER) 1879 + idpf_fill_bufq_config_chunk(qs->vport, qs->qs[i].bufq, 1880 + &qi[params.num_chunks++]); 1881 + } 1882 + 1883 + return idpf_send_chunked_msg(qs->vport, &params); 1884 + } 1885 + 1886 + /** 1887 + * idpf_send_config_rx_queues_msg - send virtchnl config Rx queues message 1809 1888 * @vport: virtual port data structure 1810 1889 * 1811 - * Send config rx queues virtchnl message. Returns 0 on success, negative on 1812 - * failure. 1890 + * Return: 0 on success, -errno on failure. 1813 1891 */ 1814 1892 static int idpf_send_config_rx_queues_msg(struct idpf_vport *vport) 1815 1893 { 1816 - struct virtchnl2_config_rx_queues *crq __free(kfree) = NULL; 1817 - struct virtchnl2_rxq_info *qi __free(kfree) = NULL; 1818 - struct idpf_vc_xn_params xn_params = {}; 1819 - u32 config_sz, chunk_sz, buf_sz; 1820 - int totqs, num_msgs, num_chunks; 1821 - ssize_t reply_sz; 1822 - int i, k = 0; 1894 + bool splitq = idpf_is_queue_model_split(vport->rxq_model); 1895 + struct idpf_queue_set *qs __free(kfree) = NULL; 1896 + u32 totqs = vport->num_rxq + vport->num_bufq; 1897 + u32 k = 0; 1823 1898 1824 - totqs = vport->num_rxq + vport->num_bufq; 1825 - qi = kcalloc(totqs, sizeof(struct virtchnl2_rxq_info), GFP_KERNEL); 1826 - if (!qi) 1899 + qs = idpf_alloc_queue_set(vport, totqs); 1900 + if (!qs) 1827 1901 return -ENOMEM; 1828 1902 1829 1903 /* Populate the queue info buffer with all queue context info */ 1830 - for (i = 0; i < vport->num_rxq_grp; i++) { 1831 - struct idpf_rxq_group *rx_qgrp = &vport->rxq_grps[i]; 1832 - u16 num_rxq; 1833 - int j; 1904 + for (u32 i = 0; i < vport->num_rxq_grp; i++) { 1905 + const struct idpf_rxq_group *rx_qgrp = &vport->rxq_grps[i]; 1906 + u32 num_rxq; 1834 1907 1835 - if (!idpf_is_queue_model_split(vport->rxq_model)) 1836 - goto setup_rxqs; 1837 - 1838 - for (j = 0; j < vport->num_bufqs_per_qgrp; j++, k++) { 1839 - struct idpf_buf_queue *bufq = 1840 - &rx_qgrp->splitq.bufq_sets[j].bufq; 1841 - 1842 - qi[k].queue_id = cpu_to_le32(bufq->q_id); 1843 - qi[k].model = cpu_to_le16(vport->rxq_model); 1844 - qi[k].type = 1845 - cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_RX_BUFFER); 1846 - qi[k].desc_ids = cpu_to_le64(VIRTCHNL2_RXDID_2_FLEX_SPLITQ_M); 1847 - qi[k].ring_len = cpu_to_le16(bufq->desc_count); 1848 - qi[k].dma_ring_addr = cpu_to_le64(bufq->dma); 1849 - qi[k].data_buffer_size = cpu_to_le32(bufq->rx_buf_size); 1850 - qi[k].buffer_notif_stride = IDPF_RX_BUF_STRIDE; 1851 - qi[k].rx_buffer_low_watermark = 1852 - cpu_to_le16(bufq->rx_buffer_low_watermark); 1853 - if (idpf_is_feature_ena(vport, NETIF_F_GRO_HW)) 1854 - qi[k].qflags |= cpu_to_le16(VIRTCHNL2_RXQ_RSC); 1908 + if (!splitq) { 1909 + num_rxq = rx_qgrp->singleq.num_rxq; 1910 + goto rxq; 1855 1911 } 1856 1912 1857 - setup_rxqs: 1858 - if (idpf_is_queue_model_split(vport->rxq_model)) 1859 - num_rxq = rx_qgrp->splitq.num_rxq_sets; 1860 - else 1861 - num_rxq = rx_qgrp->singleq.num_rxq; 1913 + for (u32 j = 0; j < vport->num_bufqs_per_qgrp; j++) { 1914 + qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_RX_BUFFER; 1915 + qs->qs[k++].bufq = &rx_qgrp->splitq.bufq_sets[j].bufq; 1916 + } 1862 1917 1863 - for (j = 0; j < num_rxq; j++, k++) { 1864 - const struct idpf_bufq_set *sets; 1865 - struct idpf_rx_queue *rxq; 1866 - u32 rxdids; 1918 + num_rxq = rx_qgrp->splitq.num_rxq_sets; 1867 1919 1868 - if (!idpf_is_queue_model_split(vport->rxq_model)) { 1869 - rxq = rx_qgrp->singleq.rxqs[j]; 1870 - rxdids = rxq->rxdids; 1920 + rxq: 1921 + for (u32 j = 0; j < num_rxq; j++) { 1922 + qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_RX; 1871 1923 1872 - goto common_qi_fields; 1873 - } 1874 - 1875 - rxq = &rx_qgrp->splitq.rxq_sets[j]->rxq; 1876 - sets = rxq->bufq_sets; 1877 - 1878 - /* In splitq mode, RXQ buffer size should be 1879 - * set to that of the first buffer queue 1880 - * associated with this RXQ. 1881 - */ 1882 - rxq->rx_buf_size = sets[0].bufq.rx_buf_size; 1883 - 1884 - qi[k].rx_bufq1_id = cpu_to_le16(sets[0].bufq.q_id); 1885 - if (vport->num_bufqs_per_qgrp > IDPF_SINGLE_BUFQ_PER_RXQ_GRP) { 1886 - qi[k].bufq2_ena = IDPF_BUFQ2_ENA; 1887 - qi[k].rx_bufq2_id = 1888 - cpu_to_le16(sets[1].bufq.q_id); 1889 - } 1890 - qi[k].rx_buffer_low_watermark = 1891 - cpu_to_le16(rxq->rx_buffer_low_watermark); 1892 - if (idpf_is_feature_ena(vport, NETIF_F_GRO_HW)) 1893 - qi[k].qflags |= cpu_to_le16(VIRTCHNL2_RXQ_RSC); 1894 - 1895 - rxq->rx_hbuf_size = sets[0].bufq.rx_hbuf_size; 1896 - 1897 - if (idpf_queue_has(HSPLIT_EN, rxq)) { 1898 - qi[k].qflags |= 1899 - cpu_to_le16(VIRTCHNL2_RXQ_HDR_SPLIT); 1900 - qi[k].hdr_buffer_size = 1901 - cpu_to_le16(rxq->rx_hbuf_size); 1902 - } 1903 - 1904 - rxdids = VIRTCHNL2_RXDID_2_FLEX_SPLITQ_M; 1905 - 1906 - common_qi_fields: 1907 - qi[k].queue_id = cpu_to_le32(rxq->q_id); 1908 - qi[k].model = cpu_to_le16(vport->rxq_model); 1909 - qi[k].type = cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_RX); 1910 - qi[k].ring_len = cpu_to_le16(rxq->desc_count); 1911 - qi[k].dma_ring_addr = cpu_to_le64(rxq->dma); 1912 - qi[k].max_pkt_size = cpu_to_le32(rxq->rx_max_pkt_size); 1913 - qi[k].data_buffer_size = cpu_to_le32(rxq->rx_buf_size); 1914 - qi[k].qflags |= 1915 - cpu_to_le16(VIRTCHNL2_RX_DESC_SIZE_32BYTE); 1916 - qi[k].desc_ids = cpu_to_le64(rxdids); 1924 + if (splitq) 1925 + qs->qs[k++].rxq = 1926 + &rx_qgrp->splitq.rxq_sets[j]->rxq; 1927 + else 1928 + qs->qs[k++].rxq = rx_qgrp->singleq.rxqs[j]; 1917 1929 } 1918 1930 } 1919 1931 ··· 2048 1806 if (k != totqs) 2049 1807 return -EINVAL; 2050 1808 2051 - /* Chunk up the queue contexts into multiple messages to avoid 2052 - * sending a control queue message buffer that is too large 2053 - */ 2054 - config_sz = sizeof(struct virtchnl2_config_rx_queues); 2055 - chunk_sz = sizeof(struct virtchnl2_rxq_info); 2056 - 2057 - num_chunks = min_t(u32, IDPF_NUM_CHUNKS_PER_MSG(config_sz, chunk_sz), 2058 - totqs); 2059 - num_msgs = DIV_ROUND_UP(totqs, num_chunks); 2060 - 2061 - buf_sz = struct_size(crq, qinfo, num_chunks); 2062 - crq = kzalloc(buf_sz, GFP_KERNEL); 2063 - if (!crq) 2064 - return -ENOMEM; 2065 - 2066 - xn_params.vc_op = VIRTCHNL2_OP_CONFIG_RX_QUEUES; 2067 - xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC; 2068 - 2069 - for (i = 0, k = 0; i < num_msgs; i++) { 2070 - memset(crq, 0, buf_sz); 2071 - crq->vport_id = cpu_to_le32(vport->vport_id); 2072 - crq->num_qinfo = cpu_to_le16(num_chunks); 2073 - memcpy(crq->qinfo, &qi[k], chunk_sz * num_chunks); 2074 - 2075 - xn_params.send_buf.iov_base = crq; 2076 - xn_params.send_buf.iov_len = buf_sz; 2077 - reply_sz = idpf_vc_xn_exec(vport->adapter, &xn_params); 2078 - if (reply_sz < 0) 2079 - return reply_sz; 2080 - 2081 - k += num_chunks; 2082 - totqs -= num_chunks; 2083 - num_chunks = min(num_chunks, totqs); 2084 - /* Recalculate buffer size */ 2085 - buf_sz = struct_size(crq, qinfo, num_chunks); 2086 - } 2087 - 2088 - return 0; 1809 + return idpf_send_config_rx_queue_set_msg(qs); 2089 1810 } 2090 1811 2091 1812 /** 2092 - * idpf_send_ena_dis_queues_msg - Send virtchnl enable or disable 2093 - * queues message 1813 + * idpf_prepare_ena_dis_qs_msg - prepare message to enable/disable selected 1814 + * queues 2094 1815 * @vport: virtual port data structure 2095 - * @ena: if true enable, false disable 1816 + * @buf: buffer containing the message 1817 + * @pos: pointer to the first chunk describing the queue 1818 + * @num_chunks: number of chunks in the message 2096 1819 * 2097 - * Send enable or disable queues virtchnl message. Returns 0 on success, 2098 - * negative on failure. 1820 + * Helper function for preparing the message describing queues to be enabled 1821 + * or disabled. 1822 + * 1823 + * Return: the total size of the prepared message. 2099 1824 */ 2100 - static int idpf_send_ena_dis_queues_msg(struct idpf_vport *vport, bool ena) 1825 + static u32 idpf_prepare_ena_dis_qs_msg(const struct idpf_vport *vport, 1826 + void *buf, const void *pos, 1827 + u32 num_chunks) 2101 1828 { 2102 - struct virtchnl2_del_ena_dis_queues *eq __free(kfree) = NULL; 2103 - struct virtchnl2_queue_chunk *qc __free(kfree) = NULL; 2104 - u32 num_msgs, num_chunks, num_txq, num_rxq, num_q; 2105 - struct idpf_vc_xn_params xn_params = { 2106 - .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 2107 - }; 2108 - struct virtchnl2_queue_chunks *qcs; 2109 - u32 config_sz, chunk_sz, buf_sz; 2110 - ssize_t reply_sz; 2111 - int i, j, k = 0; 1829 + struct virtchnl2_del_ena_dis_queues *eq = buf; 2112 1830 2113 - num_txq = vport->num_txq + vport->num_complq; 2114 - num_rxq = vport->num_rxq + vport->num_bufq; 2115 - num_q = num_txq + num_rxq; 2116 - buf_sz = sizeof(struct virtchnl2_queue_chunk) * num_q; 2117 - qc = kzalloc(buf_sz, GFP_KERNEL); 1831 + eq->vport_id = cpu_to_le32(vport->vport_id); 1832 + eq->chunks.num_chunks = cpu_to_le16(num_chunks); 1833 + memcpy(eq->chunks.chunks, pos, 1834 + num_chunks * sizeof(*eq->chunks.chunks)); 1835 + 1836 + return struct_size(eq, chunks.chunks, num_chunks); 1837 + } 1838 + 1839 + /** 1840 + * idpf_send_ena_dis_queue_set_msg - send virtchnl enable or disable queues 1841 + * message for selected queues 1842 + * @qs: set of the queues to enable or disable 1843 + * @en: whether to enable or disable queues 1844 + * 1845 + * Send enable or disable queues virtchnl message for queues contained 1846 + * in the @qs array. 1847 + * The @qs array can contain pointers to both Rx and Tx queues. 1848 + * 1849 + * Return: 0 on success, -errno on failure. 1850 + */ 1851 + static int idpf_send_ena_dis_queue_set_msg(const struct idpf_queue_set *qs, 1852 + bool en) 1853 + { 1854 + struct virtchnl2_queue_chunk *qc __free(kfree) = NULL; 1855 + struct idpf_chunked_msg_params params = { 1856 + .vc_op = en ? VIRTCHNL2_OP_ENABLE_QUEUES : 1857 + VIRTCHNL2_OP_DISABLE_QUEUES, 1858 + .prepare_msg = idpf_prepare_ena_dis_qs_msg, 1859 + .config_sz = sizeof(struct virtchnl2_del_ena_dis_queues), 1860 + .chunk_sz = sizeof(*qc), 1861 + .num_chunks = qs->num, 1862 + }; 1863 + 1864 + qc = kcalloc(qs->num, sizeof(*qc), GFP_KERNEL); 2118 1865 if (!qc) 2119 1866 return -ENOMEM; 2120 1867 2121 - for (i = 0; i < vport->num_txq_grp; i++) { 2122 - struct idpf_txq_group *tx_qgrp = &vport->txq_grps[i]; 1868 + params.chunks = qc; 2123 1869 2124 - for (j = 0; j < tx_qgrp->num_txq; j++, k++) { 2125 - qc[k].type = cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_TX); 2126 - qc[k].start_queue_id = cpu_to_le32(tx_qgrp->txqs[j]->q_id); 2127 - qc[k].num_queues = cpu_to_le32(IDPF_NUMQ_PER_CHUNK); 1870 + for (u32 i = 0; i < qs->num; i++) { 1871 + const struct idpf_queue_ptr *q = &qs->qs[i]; 1872 + u32 qid; 1873 + 1874 + qc[i].type = cpu_to_le32(q->type); 1875 + qc[i].num_queues = cpu_to_le32(IDPF_NUMQ_PER_CHUNK); 1876 + 1877 + switch (q->type) { 1878 + case VIRTCHNL2_QUEUE_TYPE_RX: 1879 + qid = q->rxq->q_id; 1880 + break; 1881 + case VIRTCHNL2_QUEUE_TYPE_TX: 1882 + qid = q->txq->q_id; 1883 + break; 1884 + case VIRTCHNL2_QUEUE_TYPE_RX_BUFFER: 1885 + qid = q->bufq->q_id; 1886 + break; 1887 + case VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION: 1888 + qid = q->complq->q_id; 1889 + break; 1890 + default: 1891 + return -EINVAL; 2128 1892 } 2129 - } 2130 - if (vport->num_txq != k) 2131 - return -EINVAL; 2132 1893 2133 - if (!idpf_is_queue_model_split(vport->txq_model)) 2134 - goto setup_rx; 2135 - 2136 - for (i = 0; i < vport->num_txq_grp; i++, k++) { 2137 - struct idpf_txq_group *tx_qgrp = &vport->txq_grps[i]; 2138 - 2139 - qc[k].type = cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION); 2140 - qc[k].start_queue_id = cpu_to_le32(tx_qgrp->complq->q_id); 2141 - qc[k].num_queues = cpu_to_le32(IDPF_NUMQ_PER_CHUNK); 2142 - } 2143 - if (vport->num_complq != (k - vport->num_txq)) 2144 - return -EINVAL; 2145 - 2146 - setup_rx: 2147 - for (i = 0; i < vport->num_rxq_grp; i++) { 2148 - struct idpf_rxq_group *rx_qgrp = &vport->rxq_grps[i]; 2149 - 2150 - if (idpf_is_queue_model_split(vport->rxq_model)) 2151 - num_rxq = rx_qgrp->splitq.num_rxq_sets; 2152 - else 2153 - num_rxq = rx_qgrp->singleq.num_rxq; 2154 - 2155 - for (j = 0; j < num_rxq; j++, k++) { 2156 - if (idpf_is_queue_model_split(vport->rxq_model)) { 2157 - qc[k].start_queue_id = 2158 - cpu_to_le32(rx_qgrp->splitq.rxq_sets[j]->rxq.q_id); 2159 - qc[k].type = 2160 - cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_RX); 2161 - } else { 2162 - qc[k].start_queue_id = 2163 - cpu_to_le32(rx_qgrp->singleq.rxqs[j]->q_id); 2164 - qc[k].type = 2165 - cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_RX); 2166 - } 2167 - qc[k].num_queues = cpu_to_le32(IDPF_NUMQ_PER_CHUNK); 2168 - } 2169 - } 2170 - if (vport->num_rxq != k - (vport->num_txq + vport->num_complq)) 2171 - return -EINVAL; 2172 - 2173 - if (!idpf_is_queue_model_split(vport->rxq_model)) 2174 - goto send_msg; 2175 - 2176 - for (i = 0; i < vport->num_rxq_grp; i++) { 2177 - struct idpf_rxq_group *rx_qgrp = &vport->rxq_grps[i]; 2178 - 2179 - for (j = 0; j < vport->num_bufqs_per_qgrp; j++, k++) { 2180 - const struct idpf_buf_queue *q; 2181 - 2182 - q = &rx_qgrp->splitq.bufq_sets[j].bufq; 2183 - qc[k].type = 2184 - cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_RX_BUFFER); 2185 - qc[k].start_queue_id = cpu_to_le32(q->q_id); 2186 - qc[k].num_queues = cpu_to_le32(IDPF_NUMQ_PER_CHUNK); 2187 - } 2188 - } 2189 - if (vport->num_bufq != k - (vport->num_txq + 2190 - vport->num_complq + 2191 - vport->num_rxq)) 2192 - return -EINVAL; 2193 - 2194 - send_msg: 2195 - /* Chunk up the queue info into multiple messages */ 2196 - config_sz = sizeof(struct virtchnl2_del_ena_dis_queues); 2197 - chunk_sz = sizeof(struct virtchnl2_queue_chunk); 2198 - 2199 - num_chunks = min_t(u32, IDPF_NUM_CHUNKS_PER_MSG(config_sz, chunk_sz), 2200 - num_q); 2201 - num_msgs = DIV_ROUND_UP(num_q, num_chunks); 2202 - 2203 - buf_sz = struct_size(eq, chunks.chunks, num_chunks); 2204 - eq = kzalloc(buf_sz, GFP_KERNEL); 2205 - if (!eq) 2206 - return -ENOMEM; 2207 - 2208 - if (ena) 2209 - xn_params.vc_op = VIRTCHNL2_OP_ENABLE_QUEUES; 2210 - else 2211 - xn_params.vc_op = VIRTCHNL2_OP_DISABLE_QUEUES; 2212 - 2213 - for (i = 0, k = 0; i < num_msgs; i++) { 2214 - memset(eq, 0, buf_sz); 2215 - eq->vport_id = cpu_to_le32(vport->vport_id); 2216 - eq->chunks.num_chunks = cpu_to_le16(num_chunks); 2217 - qcs = &eq->chunks; 2218 - memcpy(qcs->chunks, &qc[k], chunk_sz * num_chunks); 2219 - 2220 - xn_params.send_buf.iov_base = eq; 2221 - xn_params.send_buf.iov_len = buf_sz; 2222 - reply_sz = idpf_vc_xn_exec(vport->adapter, &xn_params); 2223 - if (reply_sz < 0) 2224 - return reply_sz; 2225 - 2226 - k += num_chunks; 2227 - num_q -= num_chunks; 2228 - num_chunks = min(num_chunks, num_q); 2229 - /* Recalculate buffer size */ 2230 - buf_sz = struct_size(eq, chunks.chunks, num_chunks); 1894 + qc[i].start_queue_id = cpu_to_le32(qid); 2231 1895 } 2232 1896 2233 - return 0; 1897 + return idpf_send_chunked_msg(qs->vport, &params); 2234 1898 } 2235 1899 2236 1900 /** 2237 - * idpf_send_map_unmap_queue_vector_msg - Send virtchnl map or unmap queue 2238 - * vector message 1901 + * idpf_send_ena_dis_queues_msg - send virtchnl enable or disable queues 1902 + * message 2239 1903 * @vport: virtual port data structure 2240 - * @map: true for map and false for unmap 1904 + * @en: whether to enable or disable queues 2241 1905 * 2242 - * Send map or unmap queue vector virtchnl message. Returns 0 on success, 2243 - * negative on failure. 1906 + * Return: 0 on success, -errno on failure. 2244 1907 */ 2245 - int idpf_send_map_unmap_queue_vector_msg(struct idpf_vport *vport, bool map) 1908 + static int idpf_send_ena_dis_queues_msg(struct idpf_vport *vport, bool en) 2246 1909 { 2247 - struct virtchnl2_queue_vector_maps *vqvm __free(kfree) = NULL; 2248 - struct virtchnl2_queue_vector *vqv __free(kfree) = NULL; 2249 - struct idpf_vc_xn_params xn_params = { 2250 - .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, 2251 - }; 2252 - u32 config_sz, chunk_sz, buf_sz; 2253 - u32 num_msgs, num_chunks, num_q; 2254 - ssize_t reply_sz; 2255 - int i, j, k = 0; 1910 + struct idpf_queue_set *qs __free(kfree) = NULL; 1911 + u32 num_txq, num_q, k = 0; 1912 + bool split; 2256 1913 2257 - num_q = vport->num_txq + vport->num_rxq; 1914 + num_txq = vport->num_txq + vport->num_complq; 1915 + num_q = num_txq + vport->num_rxq + vport->num_bufq; 2258 1916 2259 - buf_sz = sizeof(struct virtchnl2_queue_vector) * num_q; 2260 - vqv = kzalloc(buf_sz, GFP_KERNEL); 2261 - if (!vqv) 1917 + qs = idpf_alloc_queue_set(vport, num_q); 1918 + if (!qs) 2262 1919 return -ENOMEM; 2263 1920 2264 - for (i = 0; i < vport->num_txq_grp; i++) { 2265 - struct idpf_txq_group *tx_qgrp = &vport->txq_grps[i]; 1921 + split = idpf_is_queue_model_split(vport->txq_model); 2266 1922 2267 - for (j = 0; j < tx_qgrp->num_txq; j++, k++) { 2268 - const struct idpf_tx_queue *txq = tx_qgrp->txqs[j]; 2269 - const struct idpf_q_vector *vec; 2270 - u32 v_idx, tx_itr_idx; 1923 + for (u32 i = 0; i < vport->num_txq_grp; i++) { 1924 + const struct idpf_txq_group *tx_qgrp = &vport->txq_grps[i]; 2271 1925 2272 - vqv[k].queue_type = 2273 - cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_TX); 2274 - vqv[k].queue_id = cpu_to_le32(txq->q_id); 2275 - 2276 - if (idpf_queue_has(NOIRQ, txq)) 2277 - vec = NULL; 2278 - else if (idpf_queue_has(XDP, txq)) 2279 - vec = txq->complq->q_vector; 2280 - else if (idpf_is_queue_model_split(vport->txq_model)) 2281 - vec = txq->txq_grp->complq->q_vector; 2282 - else 2283 - vec = txq->q_vector; 2284 - 2285 - if (vec) { 2286 - v_idx = vec->v_idx; 2287 - tx_itr_idx = vec->tx_itr_idx; 2288 - } else { 2289 - v_idx = vport->noirq_v_idx; 2290 - tx_itr_idx = VIRTCHNL2_ITR_IDX_1; 2291 - } 2292 - 2293 - vqv[k].vector_id = cpu_to_le16(v_idx); 2294 - vqv[k].itr_idx = cpu_to_le32(tx_itr_idx); 1926 + for (u32 j = 0; j < tx_qgrp->num_txq; j++) { 1927 + qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_TX; 1928 + qs->qs[k++].txq = tx_qgrp->txqs[j]; 2295 1929 } 1930 + 1931 + if (!split) 1932 + continue; 1933 + 1934 + qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION; 1935 + qs->qs[k++].complq = tx_qgrp->complq; 2296 1936 } 2297 1937 2298 - for (i = 0; i < vport->num_rxq_grp; i++) { 2299 - struct idpf_rxq_group *rx_qgrp = &vport->rxq_grps[i]; 2300 - u16 num_rxq; 1938 + if (k != num_txq) 1939 + return -EINVAL; 2301 1940 2302 - if (idpf_is_queue_model_split(vport->rxq_model)) 1941 + split = idpf_is_queue_model_split(vport->rxq_model); 1942 + 1943 + for (u32 i = 0; i < vport->num_rxq_grp; i++) { 1944 + const struct idpf_rxq_group *rx_qgrp = &vport->rxq_grps[i]; 1945 + u32 num_rxq; 1946 + 1947 + if (split) 2303 1948 num_rxq = rx_qgrp->splitq.num_rxq_sets; 2304 1949 else 2305 1950 num_rxq = rx_qgrp->singleq.num_rxq; 2306 1951 2307 - for (j = 0; j < num_rxq; j++, k++) { 2308 - struct idpf_rx_queue *rxq; 2309 - u32 v_idx, rx_itr_idx; 1952 + for (u32 j = 0; j < num_rxq; j++) { 1953 + qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_RX; 2310 1954 2311 - if (idpf_is_queue_model_split(vport->rxq_model)) 2312 - rxq = &rx_qgrp->splitq.rxq_sets[j]->rxq; 1955 + if (split) 1956 + qs->qs[k++].rxq = 1957 + &rx_qgrp->splitq.rxq_sets[j]->rxq; 2313 1958 else 2314 - rxq = rx_qgrp->singleq.rxqs[j]; 1959 + qs->qs[k++].rxq = rx_qgrp->singleq.rxqs[j]; 1960 + } 2315 1961 2316 - vqv[k].queue_type = 2317 - cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_RX); 2318 - vqv[k].queue_id = cpu_to_le32(rxq->q_id); 1962 + if (!split) 1963 + continue; 2319 1964 2320 - if (idpf_queue_has(NOIRQ, rxq)) { 2321 - v_idx = vport->noirq_v_idx; 2322 - rx_itr_idx = VIRTCHNL2_ITR_IDX_0; 2323 - } else { 2324 - v_idx = rxq->q_vector->v_idx; 2325 - rx_itr_idx = rxq->q_vector->rx_itr_idx; 2326 - } 2327 - 2328 - vqv[k].vector_id = cpu_to_le16(v_idx); 2329 - vqv[k].itr_idx = cpu_to_le32(rx_itr_idx); 1965 + for (u32 j = 0; j < vport->num_bufqs_per_qgrp; j++) { 1966 + qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_RX_BUFFER; 1967 + qs->qs[k++].bufq = &rx_qgrp->splitq.bufq_sets[j].bufq; 2330 1968 } 2331 1969 } 2332 1970 2333 1971 if (k != num_q) 2334 1972 return -EINVAL; 2335 1973 2336 - /* Chunk up the vector info into multiple messages */ 2337 - config_sz = sizeof(struct virtchnl2_queue_vector_maps); 2338 - chunk_sz = sizeof(struct virtchnl2_queue_vector); 1974 + return idpf_send_ena_dis_queue_set_msg(qs, en); 1975 + } 2339 1976 2340 - num_chunks = min_t(u32, IDPF_NUM_CHUNKS_PER_MSG(config_sz, chunk_sz), 2341 - num_q); 2342 - num_msgs = DIV_ROUND_UP(num_q, num_chunks); 1977 + /** 1978 + * idpf_prep_map_unmap_queue_set_vector_msg - prepare message to map or unmap 1979 + * queue set to the interrupt vector 1980 + * @vport: virtual port data structure 1981 + * @buf: buffer containing the message 1982 + * @pos: pointer to the first chunk describing the vector mapping 1983 + * @num_chunks: number of chunks in the message 1984 + * 1985 + * Helper function for preparing the message describing mapping queues to 1986 + * q_vectors. 1987 + * 1988 + * Return: the total size of the prepared message. 1989 + */ 1990 + static u32 1991 + idpf_prep_map_unmap_queue_set_vector_msg(const struct idpf_vport *vport, 1992 + void *buf, const void *pos, 1993 + u32 num_chunks) 1994 + { 1995 + struct virtchnl2_queue_vector_maps *vqvm = buf; 2343 1996 2344 - buf_sz = struct_size(vqvm, qv_maps, num_chunks); 2345 - vqvm = kzalloc(buf_sz, GFP_KERNEL); 2346 - if (!vqvm) 1997 + vqvm->vport_id = cpu_to_le32(vport->vport_id); 1998 + vqvm->num_qv_maps = cpu_to_le16(num_chunks); 1999 + memcpy(vqvm->qv_maps, pos, num_chunks * sizeof(*vqvm->qv_maps)); 2000 + 2001 + return struct_size(vqvm, qv_maps, num_chunks); 2002 + } 2003 + 2004 + /** 2005 + * idpf_send_map_unmap_queue_set_vector_msg - send virtchnl map or unmap 2006 + * queue set vector message 2007 + * @qs: set of the queues to map or unmap 2008 + * @map: true for map and false for unmap 2009 + * 2010 + * Return: 0 on success, -errno on failure. 2011 + */ 2012 + static int 2013 + idpf_send_map_unmap_queue_set_vector_msg(const struct idpf_queue_set *qs, 2014 + bool map) 2015 + { 2016 + struct virtchnl2_queue_vector *vqv __free(kfree) = NULL; 2017 + struct idpf_chunked_msg_params params = { 2018 + .vc_op = map ? VIRTCHNL2_OP_MAP_QUEUE_VECTOR : 2019 + VIRTCHNL2_OP_UNMAP_QUEUE_VECTOR, 2020 + .prepare_msg = idpf_prep_map_unmap_queue_set_vector_msg, 2021 + .config_sz = sizeof(struct virtchnl2_queue_vector_maps), 2022 + .chunk_sz = sizeof(*vqv), 2023 + .num_chunks = qs->num, 2024 + }; 2025 + bool split; 2026 + 2027 + vqv = kcalloc(qs->num, sizeof(*vqv), GFP_KERNEL); 2028 + if (!vqv) 2347 2029 return -ENOMEM; 2348 2030 2349 - if (map) 2350 - xn_params.vc_op = VIRTCHNL2_OP_MAP_QUEUE_VECTOR; 2351 - else 2352 - xn_params.vc_op = VIRTCHNL2_OP_UNMAP_QUEUE_VECTOR; 2031 + params.chunks = vqv; 2353 2032 2354 - for (i = 0, k = 0; i < num_msgs; i++) { 2355 - memset(vqvm, 0, buf_sz); 2356 - xn_params.send_buf.iov_base = vqvm; 2357 - xn_params.send_buf.iov_len = buf_sz; 2358 - vqvm->vport_id = cpu_to_le32(vport->vport_id); 2359 - vqvm->num_qv_maps = cpu_to_le16(num_chunks); 2360 - memcpy(vqvm->qv_maps, &vqv[k], chunk_sz * num_chunks); 2033 + split = idpf_is_queue_model_split(qs->vport->txq_model); 2361 2034 2362 - reply_sz = idpf_vc_xn_exec(vport->adapter, &xn_params); 2363 - if (reply_sz < 0) 2364 - return reply_sz; 2035 + for (u32 i = 0; i < qs->num; i++) { 2036 + const struct idpf_queue_ptr *q = &qs->qs[i]; 2037 + const struct idpf_q_vector *vec; 2038 + u32 qid, v_idx, itr_idx; 2365 2039 2366 - k += num_chunks; 2367 - num_q -= num_chunks; 2368 - num_chunks = min(num_chunks, num_q); 2369 - /* Recalculate buffer size */ 2370 - buf_sz = struct_size(vqvm, qv_maps, num_chunks); 2040 + vqv[i].queue_type = cpu_to_le32(q->type); 2041 + 2042 + switch (q->type) { 2043 + case VIRTCHNL2_QUEUE_TYPE_RX: 2044 + qid = q->rxq->q_id; 2045 + 2046 + if (idpf_queue_has(NOIRQ, q->rxq)) 2047 + vec = NULL; 2048 + else 2049 + vec = q->rxq->q_vector; 2050 + 2051 + if (vec) { 2052 + v_idx = vec->v_idx; 2053 + itr_idx = vec->rx_itr_idx; 2054 + } else { 2055 + v_idx = qs->vport->noirq_v_idx; 2056 + itr_idx = VIRTCHNL2_ITR_IDX_0; 2057 + } 2058 + break; 2059 + case VIRTCHNL2_QUEUE_TYPE_TX: 2060 + qid = q->txq->q_id; 2061 + 2062 + if (idpf_queue_has(NOIRQ, q->txq)) 2063 + vec = NULL; 2064 + else if (idpf_queue_has(XDP, q->txq)) 2065 + vec = q->txq->complq->q_vector; 2066 + else if (split) 2067 + vec = q->txq->txq_grp->complq->q_vector; 2068 + else 2069 + vec = q->txq->q_vector; 2070 + 2071 + if (vec) { 2072 + v_idx = vec->v_idx; 2073 + itr_idx = vec->tx_itr_idx; 2074 + } else { 2075 + v_idx = qs->vport->noirq_v_idx; 2076 + itr_idx = VIRTCHNL2_ITR_IDX_1; 2077 + } 2078 + break; 2079 + default: 2080 + return -EINVAL; 2081 + } 2082 + 2083 + vqv[i].queue_id = cpu_to_le32(qid); 2084 + vqv[i].vector_id = cpu_to_le16(v_idx); 2085 + vqv[i].itr_idx = cpu_to_le32(itr_idx); 2371 2086 } 2372 2087 2373 - return 0; 2088 + return idpf_send_chunked_msg(qs->vport, &params); 2089 + } 2090 + 2091 + /** 2092 + * idpf_send_map_unmap_queue_vector_msg - send virtchnl map or unmap queue 2093 + * vector message 2094 + * @vport: virtual port data structure 2095 + * @map: true for map and false for unmap 2096 + * 2097 + * Return: 0 on success, -errno on failure. 2098 + */ 2099 + int idpf_send_map_unmap_queue_vector_msg(struct idpf_vport *vport, bool map) 2100 + { 2101 + struct idpf_queue_set *qs __free(kfree) = NULL; 2102 + u32 num_q = vport->num_txq + vport->num_rxq; 2103 + u32 k = 0; 2104 + 2105 + qs = idpf_alloc_queue_set(vport, num_q); 2106 + if (!qs) 2107 + return -ENOMEM; 2108 + 2109 + for (u32 i = 0; i < vport->num_txq_grp; i++) { 2110 + const struct idpf_txq_group *tx_qgrp = &vport->txq_grps[i]; 2111 + 2112 + for (u32 j = 0; j < tx_qgrp->num_txq; j++) { 2113 + qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_TX; 2114 + qs->qs[k++].txq = tx_qgrp->txqs[j]; 2115 + } 2116 + } 2117 + 2118 + if (k != vport->num_txq) 2119 + return -EINVAL; 2120 + 2121 + for (u32 i = 0; i < vport->num_rxq_grp; i++) { 2122 + const struct idpf_rxq_group *rx_qgrp = &vport->rxq_grps[i]; 2123 + u32 num_rxq; 2124 + 2125 + if (idpf_is_queue_model_split(vport->rxq_model)) 2126 + num_rxq = rx_qgrp->splitq.num_rxq_sets; 2127 + else 2128 + num_rxq = rx_qgrp->singleq.num_rxq; 2129 + 2130 + for (u32 j = 0; j < num_rxq; j++) { 2131 + qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_RX; 2132 + 2133 + if (idpf_is_queue_model_split(vport->rxq_model)) 2134 + qs->qs[k++].rxq = 2135 + &rx_qgrp->splitq.rxq_sets[j]->rxq; 2136 + else 2137 + qs->qs[k++].rxq = rx_qgrp->singleq.rxqs[j]; 2138 + } 2139 + } 2140 + 2141 + if (k != num_q) 2142 + return -EINVAL; 2143 + 2144 + return idpf_send_map_unmap_queue_set_vector_msg(qs, map); 2145 + } 2146 + 2147 + /** 2148 + * idpf_send_enable_queue_set_msg - send enable queues virtchnl message for 2149 + * selected queues 2150 + * @qs: set of the queues 2151 + * 2152 + * Send enable queues virtchnl message for queues contained in the @qs array. 2153 + * 2154 + * Return: 0 on success, -errno on failure. 2155 + */ 2156 + int idpf_send_enable_queue_set_msg(const struct idpf_queue_set *qs) 2157 + { 2158 + return idpf_send_ena_dis_queue_set_msg(qs, true); 2159 + } 2160 + 2161 + /** 2162 + * idpf_send_disable_queue_set_msg - send disable queues virtchnl message for 2163 + * selected queues 2164 + * @qs: set of the queues 2165 + * 2166 + * Return: 0 on success, -errno on failure. 2167 + */ 2168 + int idpf_send_disable_queue_set_msg(const struct idpf_queue_set *qs) 2169 + { 2170 + int err; 2171 + 2172 + err = idpf_send_ena_dis_queue_set_msg(qs, false); 2173 + if (err) 2174 + return err; 2175 + 2176 + return idpf_wait_for_marker_event_set(qs); 2177 + } 2178 + 2179 + /** 2180 + * idpf_send_config_queue_set_msg - send virtchnl config queues message for 2181 + * selected queues 2182 + * @qs: set of the queues 2183 + * 2184 + * Send config queues virtchnl message for queues contained in the @qs array. 2185 + * The @qs array can contain both Rx or Tx queues. 2186 + * 2187 + * Return: 0 on success, -errno on failure. 2188 + */ 2189 + int idpf_send_config_queue_set_msg(const struct idpf_queue_set *qs) 2190 + { 2191 + int err; 2192 + 2193 + err = idpf_send_config_tx_queue_set_msg(qs); 2194 + if (err) 2195 + return err; 2196 + 2197 + return idpf_send_config_rx_queue_set_msg(qs); 2374 2198 } 2375 2199 2376 2200 /**

+29 -3

drivers/net/ethernet/intel/idpf/idpf_virtchnl.h

··· 4 4 #ifndef _IDPF_VIRTCHNL_H_ 5 5 #define _IDPF_VIRTCHNL_H_ 6 6 7 + #include "virtchnl2.h" 8 + 7 9 #define IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC (60 * 1000) 8 10 #define IDPF_VC_XN_IDX_M GENMASK(7, 0) 9 11 #define IDPF_VC_XN_SALT_M GENMASK(15, 8) ··· 116 114 int idpf_send_mb_msg(struct idpf_adapter *adapter, u32 op, 117 115 u16 msg_size, u8 *msg, u16 cookie); 118 116 117 + struct idpf_queue_ptr { 118 + enum virtchnl2_queue_type type; 119 + union { 120 + struct idpf_rx_queue *rxq; 121 + struct idpf_tx_queue *txq; 122 + struct idpf_buf_queue *bufq; 123 + struct idpf_compl_queue *complq; 124 + }; 125 + }; 126 + 127 + struct idpf_queue_set { 128 + struct idpf_vport *vport; 129 + 130 + u32 num; 131 + struct idpf_queue_ptr qs[] __counted_by(num); 132 + }; 133 + 134 + struct idpf_queue_set *idpf_alloc_queue_set(struct idpf_vport *vport, u32 num); 135 + 136 + int idpf_send_enable_queue_set_msg(const struct idpf_queue_set *qs); 137 + int idpf_send_disable_queue_set_msg(const struct idpf_queue_set *qs); 138 + int idpf_send_config_queue_set_msg(const struct idpf_queue_set *qs); 139 + 140 + int idpf_send_disable_queues_msg(struct idpf_vport *vport); 141 + int idpf_send_config_queues_msg(struct idpf_vport *vport); 142 + int idpf_send_enable_queues_msg(struct idpf_vport *vport); 143 + 119 144 void idpf_vport_init(struct idpf_vport *vport, struct idpf_vport_max_q *max_q); 120 145 u32 idpf_get_vport_id(struct idpf_vport *vport); 121 146 int idpf_send_create_vport_msg(struct idpf_adapter *adapter, ··· 159 130 int idpf_send_add_queues_msg(const struct idpf_vport *vport, u16 num_tx_q, 160 131 u16 num_complq, u16 num_rx_q, u16 num_rx_bufq); 161 132 int idpf_send_delete_queues_msg(struct idpf_vport *vport); 162 - int idpf_send_enable_queues_msg(struct idpf_vport *vport); 163 - int idpf_send_disable_queues_msg(struct idpf_vport *vport); 164 - int idpf_send_config_queues_msg(struct idpf_vport *vport); 165 133 166 134 int idpf_vport_alloc_vec_indexes(struct idpf_vport *vport); 167 135 int idpf_get_vec_ids(struct idpf_adapter *adapter,

+38 -6

drivers/net/ethernet/intel/idpf/xdp.c

··· 4 4 #include "idpf.h" 5 5 #include "idpf_virtchnl.h" 6 6 #include "xdp.h" 7 + #include "xsk.h" 7 8 8 9 static int idpf_rxq_for_each(const struct idpf_vport *vport, 9 10 int (*fn)(struct idpf_rx_queue *rxq, void *arg), ··· 46 45 { 47 46 const struct idpf_vport *vport = rxq->q_vector->vport; 48 47 bool split = idpf_is_queue_model_split(vport->rxq_model); 49 - const struct page_pool *pp; 50 48 int err; 51 49 52 50 err = __xdp_rxq_info_reg(&rxq->xdp_rxq, vport->netdev, rxq->idx, ··· 54 54 if (err) 55 55 return err; 56 56 57 - pp = split ? rxq->bufq_sets[0].bufq.pp : rxq->pp; 58 - xdp_rxq_info_attach_page_pool(&rxq->xdp_rxq, pp); 57 + if (idpf_queue_has(XSK, rxq)) { 58 + err = xdp_rxq_info_reg_mem_model(&rxq->xdp_rxq, 59 + MEM_TYPE_XSK_BUFF_POOL, 60 + rxq->pool); 61 + if (err) 62 + goto unreg; 63 + } else { 64 + const struct page_pool *pp; 65 + 66 + pp = split ? rxq->bufq_sets[0].bufq.pp : rxq->pp; 67 + xdp_rxq_info_attach_page_pool(&rxq->xdp_rxq, pp); 68 + } 59 69 60 70 if (!split) 61 71 return 0; ··· 74 64 rxq->num_xdp_txq = vport->num_xdp_txq; 75 65 76 66 return 0; 67 + 68 + unreg: 69 + xdp_rxq_info_unreg(&rxq->xdp_rxq); 70 + 71 + return err; 72 + } 73 + 74 + int idpf_xdp_rxq_info_init(struct idpf_rx_queue *rxq) 75 + { 76 + return __idpf_xdp_rxq_info_init(rxq, NULL); 77 77 } 78 78 79 79 int idpf_xdp_rxq_info_init_all(const struct idpf_vport *vport) ··· 98 78 rxq->num_xdp_txq = 0; 99 79 } 100 80 101 - xdp_rxq_info_detach_mem_model(&rxq->xdp_rxq); 81 + if (!idpf_queue_has(XSK, rxq)) 82 + xdp_rxq_info_detach_mem_model(&rxq->xdp_rxq); 83 + 102 84 xdp_rxq_info_unreg(&rxq->xdp_rxq); 103 85 104 86 return 0; 87 + } 88 + 89 + void idpf_xdp_rxq_info_deinit(struct idpf_rx_queue *rxq, u32 model) 90 + { 91 + __idpf_xdp_rxq_info_deinit(rxq, (void *)(size_t)model); 105 92 } 106 93 107 94 void idpf_xdp_rxq_info_deinit_all(const struct idpf_vport *vport) ··· 241 214 return upper_16_bits(val); 242 215 } 243 216 244 - static u32 idpf_xdpsq_poll(struct idpf_tx_queue *xdpsq, u32 budget) 217 + u32 idpf_xdpsq_poll(struct idpf_tx_queue *xdpsq, u32 budget) 245 218 { 246 219 struct idpf_compl_queue *cq = xdpsq->complq; 247 220 u32 tx_ntc = xdpsq->next_to_clean; ··· 400 373 if (!idpf_is_queue_model_split(vport->rxq_model)) 401 374 return; 402 375 403 - libeth_xdp_set_features_noredir(vport->netdev, &idpf_xdpmo); 376 + libeth_xdp_set_features_noredir(vport->netdev, &idpf_xdpmo, 377 + idpf_get_max_tx_bufs(vport->adapter), 378 + libeth_xsktmo); 404 379 } 405 380 406 381 static int idpf_xdp_setup_prog(struct idpf_vport *vport, ··· 470 441 switch (xdp->command) { 471 442 case XDP_SETUP_PROG: 472 443 ret = idpf_xdp_setup_prog(vport, xdp); 444 + break; 445 + case XDP_SETUP_XSK_POOL: 446 + ret = idpf_xsk_pool_setup(vport, xdp); 473 447 break; 474 448 default: 475 449 notsupp:

+3

drivers/net/ethernet/intel/idpf/xdp.h

··· 8 8 9 9 #include "idpf_txrx.h" 10 10 11 + int idpf_xdp_rxq_info_init(struct idpf_rx_queue *rxq); 11 12 int idpf_xdp_rxq_info_init_all(const struct idpf_vport *vport); 13 + void idpf_xdp_rxq_info_deinit(struct idpf_rx_queue *rxq, u32 model); 12 14 void idpf_xdp_rxq_info_deinit_all(const struct idpf_vport *vport); 13 15 void idpf_xdp_copy_prog_to_rqs(const struct idpf_vport *vport, 14 16 struct bpf_prog *xdp_prog); ··· 18 16 int idpf_xdpsqs_get(const struct idpf_vport *vport); 19 17 void idpf_xdpsqs_put(const struct idpf_vport *vport); 20 18 19 + u32 idpf_xdpsq_poll(struct idpf_tx_queue *xdpsq, u32 budget); 21 20 bool idpf_xdp_tx_flush_bulk(struct libeth_xdp_tx_bulk *bq, u32 flags); 22 21 23 22 /**

+633

drivers/net/ethernet/intel/idpf/xsk.c

··· 1 + // SPDX-License-Identifier: GPL-2.0-only 2 + /* Copyright (C) 2025 Intel Corporation */ 3 + 4 + #include <net/libeth/xsk.h> 5 + 6 + #include "idpf.h" 7 + #include "xdp.h" 8 + #include "xsk.h" 9 + 10 + static void idpf_xsk_tx_timer(struct work_struct *work); 11 + 12 + static void idpf_xsk_setup_rxq(const struct idpf_vport *vport, 13 + struct idpf_rx_queue *rxq) 14 + { 15 + struct xsk_buff_pool *pool; 16 + 17 + pool = xsk_get_pool_from_qid(vport->netdev, rxq->idx); 18 + if (!pool || !pool->dev || !xsk_buff_can_alloc(pool, 1)) 19 + return; 20 + 21 + rxq->pool = pool; 22 + 23 + idpf_queue_set(XSK, rxq); 24 + } 25 + 26 + static void idpf_xsk_setup_bufq(const struct idpf_vport *vport, 27 + struct idpf_buf_queue *bufq) 28 + { 29 + struct xsk_buff_pool *pool; 30 + u32 qid = U32_MAX; 31 + 32 + for (u32 i = 0; i < vport->num_rxq_grp; i++) { 33 + const struct idpf_rxq_group *grp = &vport->rxq_grps[i]; 34 + 35 + for (u32 j = 0; j < vport->num_bufqs_per_qgrp; j++) { 36 + if (&grp->splitq.bufq_sets[j].bufq == bufq) { 37 + qid = grp->splitq.rxq_sets[0]->rxq.idx; 38 + goto setup; 39 + } 40 + } 41 + } 42 + 43 + setup: 44 + pool = xsk_get_pool_from_qid(vport->netdev, qid); 45 + if (!pool || !pool->dev || !xsk_buff_can_alloc(pool, 1)) 46 + return; 47 + 48 + bufq->pool = pool; 49 + 50 + idpf_queue_set(XSK, bufq); 51 + } 52 + 53 + static void idpf_xsk_setup_txq(const struct idpf_vport *vport, 54 + struct idpf_tx_queue *txq) 55 + { 56 + struct xsk_buff_pool *pool; 57 + u32 qid; 58 + 59 + idpf_queue_clear(XSK, txq); 60 + 61 + if (!idpf_queue_has(XDP, txq)) 62 + return; 63 + 64 + qid = txq->idx - vport->xdp_txq_offset; 65 + 66 + pool = xsk_get_pool_from_qid(vport->netdev, qid); 67 + if (!pool || !pool->dev) 68 + return; 69 + 70 + txq->pool = pool; 71 + libeth_xdpsq_init_timer(txq->timer, txq, &txq->xdp_lock, 72 + idpf_xsk_tx_timer); 73 + 74 + idpf_queue_assign(NOIRQ, txq, xsk_uses_need_wakeup(pool)); 75 + idpf_queue_set(XSK, txq); 76 + } 77 + 78 + static void idpf_xsk_setup_complq(const struct idpf_vport *vport, 79 + struct idpf_compl_queue *complq) 80 + { 81 + const struct xsk_buff_pool *pool; 82 + u32 qid; 83 + 84 + idpf_queue_clear(XSK, complq); 85 + 86 + if (!idpf_queue_has(XDP, complq)) 87 + return; 88 + 89 + qid = complq->txq_grp->txqs[0]->idx - vport->xdp_txq_offset; 90 + 91 + pool = xsk_get_pool_from_qid(vport->netdev, qid); 92 + if (!pool || !pool->dev) 93 + return; 94 + 95 + idpf_queue_set(XSK, complq); 96 + } 97 + 98 + void idpf_xsk_setup_queue(const struct idpf_vport *vport, void *q, 99 + enum virtchnl2_queue_type type) 100 + { 101 + if (!idpf_xdp_enabled(vport)) 102 + return; 103 + 104 + switch (type) { 105 + case VIRTCHNL2_QUEUE_TYPE_RX: 106 + idpf_xsk_setup_rxq(vport, q); 107 + break; 108 + case VIRTCHNL2_QUEUE_TYPE_RX_BUFFER: 109 + idpf_xsk_setup_bufq(vport, q); 110 + break; 111 + case VIRTCHNL2_QUEUE_TYPE_TX: 112 + idpf_xsk_setup_txq(vport, q); 113 + break; 114 + case VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION: 115 + idpf_xsk_setup_complq(vport, q); 116 + break; 117 + default: 118 + break; 119 + } 120 + } 121 + 122 + void idpf_xsk_clear_queue(void *q, enum virtchnl2_queue_type type) 123 + { 124 + struct idpf_compl_queue *complq; 125 + struct idpf_buf_queue *bufq; 126 + struct idpf_rx_queue *rxq; 127 + struct idpf_tx_queue *txq; 128 + 129 + switch (type) { 130 + case VIRTCHNL2_QUEUE_TYPE_RX: 131 + rxq = q; 132 + if (!idpf_queue_has_clear(XSK, rxq)) 133 + return; 134 + 135 + rxq->pool = NULL; 136 + break; 137 + case VIRTCHNL2_QUEUE_TYPE_RX_BUFFER: 138 + bufq = q; 139 + if (!idpf_queue_has_clear(XSK, bufq)) 140 + return; 141 + 142 + bufq->pool = NULL; 143 + break; 144 + case VIRTCHNL2_QUEUE_TYPE_TX: 145 + txq = q; 146 + if (!idpf_queue_has_clear(XSK, txq)) 147 + return; 148 + 149 + idpf_queue_set(NOIRQ, txq); 150 + txq->dev = txq->netdev->dev.parent; 151 + break; 152 + case VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION: 153 + complq = q; 154 + idpf_queue_clear(XSK, complq); 155 + break; 156 + default: 157 + break; 158 + } 159 + } 160 + 161 + void idpf_xsk_init_wakeup(struct idpf_q_vector *qv) 162 + { 163 + libeth_xsk_init_wakeup(&qv->csd, &qv->napi); 164 + } 165 + 166 + void idpf_xsksq_clean(struct idpf_tx_queue *xdpsq) 167 + { 168 + struct libeth_xdpsq_napi_stats ss = { }; 169 + u32 ntc = xdpsq->next_to_clean; 170 + struct xdp_frame_bulk bq; 171 + struct libeth_cq_pp cp = { 172 + .dev = xdpsq->pool->dev, 173 + .bq = &bq, 174 + .xss = &ss, 175 + }; 176 + u32 xsk_frames = 0; 177 + 178 + xdp_frame_bulk_init(&bq); 179 + 180 + while (ntc != xdpsq->next_to_use) { 181 + struct libeth_sqe *sqe = &xdpsq->tx_buf[ntc]; 182 + 183 + if (sqe->type) 184 + libeth_xdp_complete_tx(sqe, &cp); 185 + else 186 + xsk_frames++; 187 + 188 + if (unlikely(++ntc == xdpsq->desc_count)) 189 + ntc = 0; 190 + } 191 + 192 + xdp_flush_frame_bulk(&bq); 193 + 194 + if (xsk_frames) 195 + xsk_tx_completed(xdpsq->pool, xsk_frames); 196 + } 197 + 198 + static noinline u32 idpf_xsksq_complete_slow(struct idpf_tx_queue *xdpsq, 199 + u32 done) 200 + { 201 + struct libeth_xdpsq_napi_stats ss = { }; 202 + u32 ntc = xdpsq->next_to_clean; 203 + u32 cnt = xdpsq->desc_count; 204 + struct xdp_frame_bulk bq; 205 + struct libeth_cq_pp cp = { 206 + .dev = xdpsq->pool->dev, 207 + .bq = &bq, 208 + .xss = &ss, 209 + .napi = true, 210 + }; 211 + u32 xsk_frames = 0; 212 + 213 + xdp_frame_bulk_init(&bq); 214 + 215 + for (u32 i = 0; likely(i < done); i++) { 216 + struct libeth_sqe *sqe = &xdpsq->tx_buf[ntc]; 217 + 218 + if (sqe->type) 219 + libeth_xdp_complete_tx(sqe, &cp); 220 + else 221 + xsk_frames++; 222 + 223 + if (unlikely(++ntc == cnt)) 224 + ntc = 0; 225 + } 226 + 227 + xdp_flush_frame_bulk(&bq); 228 + 229 + xdpsq->next_to_clean = ntc; 230 + xdpsq->xdp_tx -= cp.xdp_tx; 231 + 232 + return xsk_frames; 233 + } 234 + 235 + static __always_inline u32 idpf_xsksq_complete(void *_xdpsq, u32 budget) 236 + { 237 + struct idpf_tx_queue *xdpsq = _xdpsq; 238 + u32 tx_ntc = xdpsq->next_to_clean; 239 + u32 tx_cnt = xdpsq->desc_count; 240 + u32 done_frames; 241 + u32 xsk_frames; 242 + 243 + done_frames = idpf_xdpsq_poll(xdpsq, budget); 244 + if (unlikely(!done_frames)) 245 + return 0; 246 + 247 + if (likely(!xdpsq->xdp_tx)) { 248 + tx_ntc += done_frames; 249 + if (tx_ntc >= tx_cnt) 250 + tx_ntc -= tx_cnt; 251 + 252 + xdpsq->next_to_clean = tx_ntc; 253 + xsk_frames = done_frames; 254 + 255 + goto finalize; 256 + } 257 + 258 + xsk_frames = idpf_xsksq_complete_slow(xdpsq, done_frames); 259 + if (xsk_frames) 260 + finalize: 261 + xsk_tx_completed(xdpsq->pool, xsk_frames); 262 + 263 + xdpsq->pending -= done_frames; 264 + 265 + return done_frames; 266 + } 267 + 268 + static u32 idpf_xsk_tx_prep(void *_xdpsq, struct libeth_xdpsq *sq) 269 + { 270 + struct idpf_tx_queue *xdpsq = _xdpsq; 271 + u32 free; 272 + 273 + libeth_xdpsq_lock(&xdpsq->xdp_lock); 274 + 275 + free = xdpsq->desc_count - xdpsq->pending; 276 + if (free < xdpsq->thresh) 277 + free += idpf_xsksq_complete(xdpsq, xdpsq->thresh); 278 + 279 + *sq = (struct libeth_xdpsq){ 280 + .pool = xdpsq->pool, 281 + .sqes = xdpsq->tx_buf, 282 + .descs = xdpsq->desc_ring, 283 + .count = xdpsq->desc_count, 284 + .lock = &xdpsq->xdp_lock, 285 + .ntu = &xdpsq->next_to_use, 286 + .pending = &xdpsq->pending, 287 + .xdp_tx = &xdpsq->xdp_tx, 288 + }; 289 + 290 + return free; 291 + } 292 + 293 + static u32 idpf_xsk_xmit_prep(void *_xdpsq, struct libeth_xdpsq *sq) 294 + { 295 + struct idpf_tx_queue *xdpsq = _xdpsq; 296 + 297 + *sq = (struct libeth_xdpsq){ 298 + .pool = xdpsq->pool, 299 + .sqes = xdpsq->tx_buf, 300 + .descs = xdpsq->desc_ring, 301 + .count = xdpsq->desc_count, 302 + .lock = &xdpsq->xdp_lock, 303 + .ntu = &xdpsq->next_to_use, 304 + .pending = &xdpsq->pending, 305 + }; 306 + 307 + /* 308 + * The queue is cleaned, the budget is already known, optimize out 309 + * the second min() by passing the type limit. 310 + */ 311 + return U32_MAX; 312 + } 313 + 314 + bool idpf_xsk_xmit(struct idpf_tx_queue *xsksq) 315 + { 316 + u32 free; 317 + 318 + libeth_xdpsq_lock(&xsksq->xdp_lock); 319 + 320 + free = xsksq->desc_count - xsksq->pending; 321 + if (free < xsksq->thresh) 322 + free += idpf_xsksq_complete(xsksq, xsksq->thresh); 323 + 324 + return libeth_xsk_xmit_do_bulk(xsksq->pool, xsksq, 325 + min(free - 1, xsksq->thresh), 326 + libeth_xsktmo, idpf_xsk_xmit_prep, 327 + idpf_xdp_tx_xmit, idpf_xdp_tx_finalize); 328 + } 329 + 330 + LIBETH_XDP_DEFINE_START(); 331 + LIBETH_XDP_DEFINE_TIMER(static idpf_xsk_tx_timer, idpf_xsksq_complete); 332 + LIBETH_XSK_DEFINE_FLUSH_TX(static idpf_xsk_tx_flush_bulk, idpf_xsk_tx_prep, 333 + idpf_xdp_tx_xmit); 334 + LIBETH_XSK_DEFINE_RUN(static idpf_xsk_run_pass, idpf_xsk_run_prog, 335 + idpf_xsk_tx_flush_bulk, idpf_rx_process_skb_fields); 336 + LIBETH_XSK_DEFINE_FINALIZE(static idpf_xsk_finalize_rx, idpf_xsk_tx_flush_bulk, 337 + idpf_xdp_tx_finalize); 338 + LIBETH_XDP_DEFINE_END(); 339 + 340 + static void idpf_xskfqe_init(const struct libeth_xskfq_fp *fq, u32 i) 341 + { 342 + struct virtchnl2_splitq_rx_buf_desc *desc = fq->descs; 343 + 344 + desc = &desc[i]; 345 + #ifdef __LIBETH_WORD_ACCESS 346 + *(u64 *)&desc->qword0 = i; 347 + #else 348 + desc->qword0.buf_id = cpu_to_le16(i); 349 + #endif 350 + desc->pkt_addr = cpu_to_le64(libeth_xsk_buff_xdp_get_dma(fq->fqes[i])); 351 + } 352 + 353 + static bool idpf_xskfq_refill_thresh(struct idpf_buf_queue *bufq, u32 count) 354 + { 355 + struct libeth_xskfq_fp fq = { 356 + .pool = bufq->pool, 357 + .fqes = bufq->xsk_buf, 358 + .descs = bufq->split_buf, 359 + .ntu = bufq->next_to_use, 360 + .count = bufq->desc_count, 361 + }; 362 + u32 done; 363 + 364 + done = libeth_xskfqe_alloc(&fq, count, idpf_xskfqe_init); 365 + writel(fq.ntu, bufq->tail); 366 + 367 + bufq->next_to_use = fq.ntu; 368 + bufq->pending -= done; 369 + 370 + return done == count; 371 + } 372 + 373 + static bool idpf_xskfq_refill(struct idpf_buf_queue *bufq) 374 + { 375 + u32 count, rx_thresh = bufq->thresh; 376 + 377 + count = ALIGN_DOWN(bufq->pending - 1, rx_thresh); 378 + 379 + for (u32 i = 0; i < count; i += rx_thresh) { 380 + if (unlikely(!idpf_xskfq_refill_thresh(bufq, rx_thresh))) 381 + return false; 382 + } 383 + 384 + return true; 385 + } 386 + 387 + int idpf_xskfq_init(struct idpf_buf_queue *bufq) 388 + { 389 + struct libeth_xskfq fq = { 390 + .pool = bufq->pool, 391 + .count = bufq->desc_count, 392 + .nid = idpf_q_vector_to_mem(bufq->q_vector), 393 + }; 394 + int ret; 395 + 396 + ret = libeth_xskfq_create(&fq); 397 + if (ret) 398 + return ret; 399 + 400 + bufq->xsk_buf = fq.fqes; 401 + bufq->pending = fq.pending; 402 + bufq->thresh = fq.thresh; 403 + bufq->rx_buf_size = fq.buf_len; 404 + 405 + if (!idpf_xskfq_refill(bufq)) 406 + netdev_err(bufq->pool->netdev, 407 + "failed to allocate XSk buffers for qid %d\n", 408 + bufq->pool->queue_id); 409 + 410 + bufq->next_to_alloc = bufq->next_to_use; 411 + 412 + idpf_queue_clear(HSPLIT_EN, bufq); 413 + bufq->rx_hbuf_size = 0; 414 + 415 + return 0; 416 + } 417 + 418 + void idpf_xskfq_rel(struct idpf_buf_queue *bufq) 419 + { 420 + struct libeth_xskfq fq = { 421 + .fqes = bufq->xsk_buf, 422 + }; 423 + 424 + libeth_xskfq_destroy(&fq); 425 + 426 + bufq->rx_buf_size = fq.buf_len; 427 + bufq->thresh = fq.thresh; 428 + bufq->pending = fq.pending; 429 + } 430 + 431 + struct idpf_xskfq_refill_set { 432 + struct { 433 + struct idpf_buf_queue *q; 434 + u32 buf_id; 435 + u32 pending; 436 + } bufqs[IDPF_MAX_BUFQS_PER_RXQ_GRP]; 437 + }; 438 + 439 + static bool idpf_xskfq_refill_set(const struct idpf_xskfq_refill_set *set) 440 + { 441 + bool ret = true; 442 + 443 + for (u32 i = 0; i < ARRAY_SIZE(set->bufqs); i++) { 444 + struct idpf_buf_queue *bufq = set->bufqs[i].q; 445 + u32 ntc; 446 + 447 + if (!bufq) 448 + continue; 449 + 450 + ntc = set->bufqs[i].buf_id; 451 + if (unlikely(++ntc == bufq->desc_count)) 452 + ntc = 0; 453 + 454 + bufq->next_to_clean = ntc; 455 + bufq->pending += set->bufqs[i].pending; 456 + 457 + if (bufq->pending > bufq->thresh) 458 + ret &= idpf_xskfq_refill(bufq); 459 + } 460 + 461 + return ret; 462 + } 463 + 464 + int idpf_xskrq_poll(struct idpf_rx_queue *rxq, u32 budget) 465 + { 466 + struct idpf_xskfq_refill_set set = { }; 467 + struct libeth_rq_napi_stats rs = { }; 468 + bool wake, gen, fail = false; 469 + u32 ntc = rxq->next_to_clean; 470 + struct libeth_xdp_buff *xdp; 471 + LIBETH_XDP_ONSTACK_BULK(bq); 472 + u32 cnt = rxq->desc_count; 473 + 474 + wake = xsk_uses_need_wakeup(rxq->pool); 475 + if (wake) 476 + xsk_clear_rx_need_wakeup(rxq->pool); 477 + 478 + gen = idpf_queue_has(GEN_CHK, rxq); 479 + 480 + libeth_xsk_tx_init_bulk(&bq, rxq->xdp_prog, rxq->xdp_rxq.dev, 481 + rxq->xdpsqs, rxq->num_xdp_txq); 482 + xdp = rxq->xsk; 483 + 484 + while (likely(rs.packets < budget)) { 485 + const struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc; 486 + struct idpf_xdp_rx_desc desc __uninitialized; 487 + struct idpf_buf_queue *bufq; 488 + u32 bufq_id, buf_id; 489 + 490 + rx_desc = &rxq->rx[ntc].flex_adv_nic_3_wb; 491 + 492 + idpf_xdp_get_qw0(&desc, rx_desc); 493 + if (idpf_xdp_rx_gen(&desc) != gen) 494 + break; 495 + 496 + dma_rmb(); 497 + 498 + bufq_id = idpf_xdp_rx_bufq(&desc); 499 + bufq = set.bufqs[bufq_id].q; 500 + if (!bufq) { 501 + bufq = &rxq->bufq_sets[bufq_id].bufq; 502 + set.bufqs[bufq_id].q = bufq; 503 + } 504 + 505 + idpf_xdp_get_qw1(&desc, rx_desc); 506 + buf_id = idpf_xdp_rx_buf(&desc); 507 + 508 + set.bufqs[bufq_id].buf_id = buf_id; 509 + set.bufqs[bufq_id].pending++; 510 + 511 + xdp = libeth_xsk_process_buff(xdp, bufq->xsk_buf[buf_id], 512 + idpf_xdp_rx_len(&desc)); 513 + 514 + if (unlikely(++ntc == cnt)) { 515 + ntc = 0; 516 + gen = !gen; 517 + idpf_queue_change(GEN_CHK, rxq); 518 + } 519 + 520 + if (!idpf_xdp_rx_eop(&desc) || unlikely(!xdp)) 521 + continue; 522 + 523 + fail = !idpf_xsk_run_pass(xdp, &bq, rxq->napi, &rs, rx_desc); 524 + xdp = NULL; 525 + 526 + if (fail) 527 + break; 528 + } 529 + 530 + idpf_xsk_finalize_rx(&bq); 531 + 532 + rxq->next_to_clean = ntc; 533 + rxq->xsk = xdp; 534 + 535 + fail |= !idpf_xskfq_refill_set(&set); 536 + 537 + u64_stats_update_begin(&rxq->stats_sync); 538 + u64_stats_add(&rxq->q_stats.packets, rs.packets); 539 + u64_stats_add(&rxq->q_stats.bytes, rs.bytes); 540 + u64_stats_update_end(&rxq->stats_sync); 541 + 542 + if (!wake) 543 + return unlikely(fail) ? budget : rs.packets; 544 + 545 + if (unlikely(fail)) 546 + xsk_set_rx_need_wakeup(rxq->pool); 547 + 548 + return rs.packets; 549 + } 550 + 551 + int idpf_xsk_pool_setup(struct idpf_vport *vport, struct netdev_bpf *bpf) 552 + { 553 + struct xsk_buff_pool *pool = bpf->xsk.pool; 554 + u32 qid = bpf->xsk.queue_id; 555 + bool restart; 556 + int ret; 557 + 558 + if (pool && !IS_ALIGNED(xsk_pool_get_rx_frame_size(pool), 559 + LIBETH_RX_BUF_STRIDE)) { 560 + NL_SET_ERR_MSG_FMT_MOD(bpf->extack, 561 + "%s: HW doesn't support frames sizes not aligned to %u (qid %u: %u)", 562 + netdev_name(vport->netdev), 563 + LIBETH_RX_BUF_STRIDE, qid, 564 + xsk_pool_get_rx_frame_size(pool)); 565 + return -EINVAL; 566 + } 567 + 568 + restart = idpf_xdp_enabled(vport) && netif_running(vport->netdev); 569 + if (!restart) 570 + goto pool; 571 + 572 + ret = idpf_qp_switch(vport, qid, false); 573 + if (ret) { 574 + NL_SET_ERR_MSG_FMT_MOD(bpf->extack, 575 + "%s: failed to disable queue pair %u: %pe", 576 + netdev_name(vport->netdev), qid, 577 + ERR_PTR(ret)); 578 + return ret; 579 + } 580 + 581 + pool: 582 + ret = libeth_xsk_setup_pool(vport->netdev, qid, pool); 583 + if (ret) { 584 + NL_SET_ERR_MSG_FMT_MOD(bpf->extack, 585 + "%s: failed to configure XSk pool for pair %u: %pe", 586 + netdev_name(vport->netdev), qid, 587 + ERR_PTR(ret)); 588 + return ret; 589 + } 590 + 591 + if (!restart) 592 + return 0; 593 + 594 + ret = idpf_qp_switch(vport, qid, true); 595 + if (ret) { 596 + NL_SET_ERR_MSG_FMT_MOD(bpf->extack, 597 + "%s: failed to enable queue pair %u: %pe", 598 + netdev_name(vport->netdev), qid, 599 + ERR_PTR(ret)); 600 + goto err_dis; 601 + } 602 + 603 + return 0; 604 + 605 + err_dis: 606 + libeth_xsk_setup_pool(vport->netdev, qid, false); 607 + 608 + return ret; 609 + } 610 + 611 + int idpf_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags) 612 + { 613 + const struct idpf_netdev_priv *np = netdev_priv(dev); 614 + const struct idpf_vport *vport = np->vport; 615 + struct idpf_q_vector *q_vector; 616 + 617 + if (unlikely(idpf_vport_ctrl_is_locked(dev))) 618 + return -EBUSY; 619 + 620 + if (unlikely(!vport->link_up)) 621 + return -ENETDOWN; 622 + 623 + if (unlikely(!vport->num_xdp_txq)) 624 + return -ENXIO; 625 + 626 + q_vector = idpf_find_rxq_vec(vport, qid); 627 + if (unlikely(!q_vector->xsksq)) 628 + return -ENXIO; 629 + 630 + libeth_xsk_wakeup(&q_vector->csd, qid); 631 + 632 + return 0; 633 + }

+33

drivers/net/ethernet/intel/idpf/xsk.h

··· 1 + /* SPDX-License-Identifier: GPL-2.0-only */ 2 + /* Copyright (C) 2025 Intel Corporation */ 3 + 4 + #ifndef _IDPF_XSK_H_ 5 + #define _IDPF_XSK_H_ 6 + 7 + #include <linux/types.h> 8 + 9 + enum virtchnl2_queue_type; 10 + struct idpf_buf_queue; 11 + struct idpf_q_vector; 12 + struct idpf_rx_queue; 13 + struct idpf_tx_queue; 14 + struct idpf_vport; 15 + struct net_device; 16 + struct netdev_bpf; 17 + 18 + void idpf_xsk_setup_queue(const struct idpf_vport *vport, void *q, 19 + enum virtchnl2_queue_type type); 20 + void idpf_xsk_clear_queue(void *q, enum virtchnl2_queue_type type); 21 + void idpf_xsk_init_wakeup(struct idpf_q_vector *qv); 22 + 23 + int idpf_xskfq_init(struct idpf_buf_queue *bufq); 24 + void idpf_xskfq_rel(struct idpf_buf_queue *bufq); 25 + void idpf_xsksq_clean(struct idpf_tx_queue *xdpq); 26 + 27 + int idpf_xskrq_poll(struct idpf_rx_queue *rxq, u32 budget); 28 + bool idpf_xsk_xmit(struct idpf_tx_queue *xsksq); 29 + 30 + int idpf_xsk_pool_setup(struct idpf_vport *vport, struct netdev_bpf *xdp); 31 + int idpf_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags); 32 + 33 + #endif /* !_IDPF_XSK_H_ */

Configure Feed

Configure Feed