Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge branch 'bng_en-enhancements-for-rx-and-tx-datapath'
Bhargava Marreddy says:
====================
bng_en: enhancements for RX and TX datapath
This series enhances the bng_en driver by adding:
1. Tx support (standard + TSO)
2. Rx support (standard + LRO/TPA)
====================
Link: https://patch.msgid.link/20260128185623.26559-1-bhargava.marreddy@broadcom.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
+2776
-41
+2
-1
drivers/net/ethernet/broadcom/bnge/Makefile
+2
-1
drivers/net/ethernet/broadcom/bnge/Makefile
+446
drivers/net/ethernet/broadcom/bnge/bnge_hw_def.h
+446
drivers/net/ethernet/broadcom/bnge/bnge_hw_def.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0 */
2
+
/* Copyright (c) 2025 Broadcom */
3
+
4
+
#ifndef _BNGE_HW_DEF_H_
5
+
#define _BNGE_HW_DEF_H_
6
+
7
+
#define TX_BD_FLAGS_TCP_UDP_CHKSUM BIT(0)
8
+
#define TX_BD_FLAGS_IP_CKSUM BIT(1)
9
+
#define TX_BD_FLAGS_NO_CRC BIT(2)
10
+
#define TX_BD_FLAGS_STAMP BIT(3)
11
+
#define TX_BD_FLAGS_T_IP_CHKSUM BIT(4)
12
+
#define TX_BD_FLAGS_LSO BIT(5)
13
+
#define TX_BD_FLAGS_IPID_FMT BIT(6)
14
+
#define TX_BD_FLAGS_T_IPID BIT(7)
15
+
#define TX_BD_HSIZE GENMASK(23, 16)
16
+
#define TX_BD_HSIZE_SHIFT 16
17
+
18
+
#define TX_BD_CFA_ACTION GENMASK(31, 16)
19
+
#define TX_BD_CFA_ACTION_SHIFT 16
20
+
21
+
#define TX_BD_CFA_META_MASK 0xfffffff
22
+
#define TX_BD_CFA_META_VID_MASK 0xfff
23
+
#define TX_BD_CFA_META_PRI_MASK GENMASK(15, 12)
24
+
#define TX_BD_CFA_META_PRI_SHIFT 12
25
+
#define TX_BD_CFA_META_TPID_MASK GENMASK(17, 16)
26
+
#define TX_BD_CFA_META_TPID_SHIFT 16
27
+
#define TX_BD_CFA_META_KEY GENMASK(31, 28)
28
+
#define TX_BD_CFA_META_KEY_SHIFT 28
29
+
#define TX_BD_CFA_META_KEY_VLAN BIT(28)
30
+
31
+
struct tx_bd_ext {
32
+
__le32 tx_bd_hsize_lflags;
33
+
__le32 tx_bd_mss;
34
+
__le32 tx_bd_cfa_action;
35
+
__le32 tx_bd_cfa_meta;
36
+
};
37
+
38
+
#define TX_CMP_SQ_CONS_IDX(txcmp) \
39
+
(le32_to_cpu((txcmp)->sq_cons_idx) & TX_CMP_SQ_CONS_IDX_MASK)
40
+
41
+
#define RX_CMP_CMP_TYPE GENMASK(5, 0)
42
+
#define RX_CMP_FLAGS_ERROR BIT(6)
43
+
#define RX_CMP_FLAGS_PLACEMENT GENMASK(9, 7)
44
+
#define RX_CMP_FLAGS_RSS_VALID BIT(10)
45
+
#define RX_CMP_FLAGS_PKT_METADATA_PRESENT BIT(11)
46
+
#define RX_CMP_FLAGS_ITYPES_SHIFT 12
47
+
#define RX_CMP_FLAGS_ITYPES_MASK 0xf000
48
+
#define RX_CMP_FLAGS_ITYPE_UNKNOWN (0 << 12)
49
+
#define RX_CMP_FLAGS_ITYPE_IP (1 << 12)
50
+
#define RX_CMP_FLAGS_ITYPE_TCP (2 << 12)
51
+
#define RX_CMP_FLAGS_ITYPE_UDP (3 << 12)
52
+
#define RX_CMP_FLAGS_ITYPE_FCOE (4 << 12)
53
+
#define RX_CMP_FLAGS_ITYPE_ROCE (5 << 12)
54
+
#define RX_CMP_FLAGS_ITYPE_PTP_WO_TS (8 << 12)
55
+
#define RX_CMP_FLAGS_ITYPE_PTP_W_TS (9 << 12)
56
+
#define RX_CMP_LEN GENMASK(31, 16)
57
+
#define RX_CMP_LEN_SHIFT 16
58
+
59
+
#define RX_CMP_V1 BIT(0)
60
+
#define RX_CMP_AGG_BUFS GENMASK(5, 1)
61
+
#define RX_CMP_AGG_BUFS_SHIFT 1
62
+
#define RX_CMP_RSS_HASH_TYPE GENMASK(15, 9)
63
+
#define RX_CMP_RSS_HASH_TYPE_SHIFT 9
64
+
#define RX_CMP_V3_RSS_EXT_OP_LEGACY GENMASK(15, 12)
65
+
#define RX_CMP_V3_RSS_EXT_OP_LEGACY_SHIFT 12
66
+
#define RX_CMP_V3_RSS_EXT_OP_NEW GENMASK(11, 8)
67
+
#define RX_CMP_V3_RSS_EXT_OP_NEW_SHIFT 8
68
+
#define RX_CMP_PAYLOAD_OFFSET GENMASK(23, 16)
69
+
#define RX_CMP_PAYLOAD_OFFSET_SHIFT 16
70
+
#define RX_CMP_SUB_NS_TS GENMASK(19, 16)
71
+
#define RX_CMP_SUB_NS_TS_SHIFT 16
72
+
#define RX_CMP_METADATA1 GENMASK(31, 28)
73
+
#define RX_CMP_METADATA1_SHIFT 28
74
+
#define RX_CMP_METADATA1_TPID_SEL GENMASK(30, 28)
75
+
#define RX_CMP_METADATA1_TPID_8021Q BIT(28)
76
+
#define RX_CMP_METADATA1_TPID_8021AD (0x0 << 28)
77
+
#define RX_CMP_METADATA1_VALID BIT(31)
78
+
79
+
struct rx_cmp {
80
+
__le32 rx_cmp_len_flags_type;
81
+
u32 rx_cmp_opaque;
82
+
__le32 rx_cmp_misc_v1;
83
+
__le32 rx_cmp_rss_hash;
84
+
};
85
+
86
+
#define RX_CMP_FLAGS2_IP_CS_CALC BIT(0)
87
+
#define RX_CMP_FLAGS2_L4_CS_CALC BIT(1)
88
+
#define RX_CMP_FLAGS2_T_IP_CS_CALC BIT(2)
89
+
#define RX_CMP_FLAGS2_T_L4_CS_CALC BIT(3)
90
+
#define RX_CMP_FLAGS2_META_FORMAT_VLAN BIT(4)
91
+
92
+
#define RX_CMP_FLAGS2_METADATA_TCI_MASK GENMASK(15, 0)
93
+
#define RX_CMP_FLAGS2_METADATA_VID_MASK GENMASK(11, 0)
94
+
#define RX_CMP_FLAGS2_METADATA_TPID_MASK GENMASK(31, 16)
95
+
#define RX_CMP_FLAGS2_METADATA_TPID_SFT 16
96
+
97
+
#define RX_CMP_V BIT(0)
98
+
#define RX_CMPL_ERRORS_MASK GENMASK(15, 1)
99
+
#define RX_CMPL_ERRORS_SFT 1
100
+
#define RX_CMPL_ERRORS_BUFFER_ERROR_MASK GENMASK(3, 1)
101
+
#define RX_CMPL_ERRORS_BUFFER_ERROR_NO_BUFFER (0x0 << 1)
102
+
#define RX_CMPL_ERRORS_BUFFER_ERROR_DID_NOT_FIT (0x1 << 1)
103
+
#define RX_CMPL_ERRORS_BUFFER_ERROR_NOT_ON_CHIP (0x2 << 1)
104
+
#define RX_CMPL_ERRORS_BUFFER_ERROR_BAD_FORMAT (0x3 << 1)
105
+
#define RX_CMPL_ERRORS_IP_CS_ERROR BIT(4)
106
+
#define RX_CMPL_ERRORS_L4_CS_ERROR BIT(5)
107
+
#define RX_CMPL_ERRORS_T_IP_CS_ERROR BIT(6)
108
+
#define RX_CMPL_ERRORS_T_L4_CS_ERROR BIT(7)
109
+
#define RX_CMPL_ERRORS_CRC_ERROR BIT(8)
110
+
#define RX_CMPL_ERRORS_T_PKT_ERROR_MASK GENMASK(11, 9)
111
+
#define RX_CMPL_ERRORS_T_PKT_ERROR_NO_ERROR (0x0 << 9)
112
+
#define RX_CMPL_ERRORS_T_PKT_ERROR_T_L3_BAD_VERSION (0x1 << 9)
113
+
#define RX_CMPL_ERRORS_T_PKT_ERROR_T_L3_BAD_HDR_LEN (0x2 << 9)
114
+
#define RX_CMPL_ERRORS_T_PKT_ERROR_TUNNEL_TOTAL_ERROR (0x3 << 9)
115
+
#define RX_CMPL_ERRORS_T_PKT_ERROR_T_IP_TOTAL_ERROR (0x4 << 9)
116
+
#define RX_CMPL_ERRORS_T_PKT_ERROR_T_UDP_TOTAL_ERROR (0x5 << 9)
117
+
#define RX_CMPL_ERRORS_T_PKT_ERROR_T_L3_BAD_TTL (0x6 << 9)
118
+
#define RX_CMPL_ERRORS_PKT_ERROR_MASK GENMASK(15, 12)
119
+
#define RX_CMPL_ERRORS_PKT_ERROR_NO_ERROR (0x0 << 12)
120
+
#define RX_CMPL_ERRORS_PKT_ERROR_L3_BAD_VERSION (0x1 << 12)
121
+
#define RX_CMPL_ERRORS_PKT_ERROR_L3_BAD_HDR_LEN (0x2 << 12)
122
+
#define RX_CMPL_ERRORS_PKT_ERROR_L3_BAD_TTL (0x3 << 12)
123
+
#define RX_CMPL_ERRORS_PKT_ERROR_IP_TOTAL_ERROR (0x4 << 12)
124
+
#define RX_CMPL_ERRORS_PKT_ERROR_UDP_TOTAL_ERROR (0x5 << 12)
125
+
#define RX_CMPL_ERRORS_PKT_ERROR_L4_BAD_HDR_LEN (0x6 << 12)
126
+
#define RX_CMPL_ERRORS_PKT_ERROR_L4_BAD_HDR_LEN_TOO_SMALL (0x7 << 12)
127
+
#define RX_CMPL_ERRORS_PKT_ERROR_L4_BAD_OPT_LEN (0x8 << 12)
128
+
129
+
#define RX_CMPL_CFA_CODE_MASK GENMASK(31, 16)
130
+
#define RX_CMPL_CFA_CODE_SFT 16
131
+
#define RX_CMPL_METADATA0_TCI_MASK GENMASK(31, 16)
132
+
#define RX_CMPL_METADATA0_VID_MASK GENMASK(27, 16)
133
+
#define RX_CMPL_METADATA0_SFT 16
134
+
135
+
struct rx_cmp_ext {
136
+
__le32 rx_cmp_flags2;
137
+
__le32 rx_cmp_meta_data;
138
+
__le32 rx_cmp_cfa_code_errors_v2;
139
+
__le32 rx_cmp_timestamp;
140
+
};
141
+
142
+
#define RX_AGG_CMP_TYPE GENMASK(5, 0)
143
+
#define RX_AGG_CMP_LEN GENMASK(31, 16)
144
+
#define RX_AGG_CMP_LEN_SHIFT 16
145
+
#define RX_AGG_CMP_V BIT(0)
146
+
#define RX_AGG_CMP_AGG_ID GENMASK(25, 16)
147
+
#define RX_AGG_CMP_AGG_ID_SHIFT 16
148
+
149
+
struct rx_agg_cmp {
150
+
__le32 rx_agg_cmp_len_flags_type;
151
+
u32 rx_agg_cmp_opaque;
152
+
__le32 rx_agg_cmp_v;
153
+
__le32 rx_agg_cmp_unused;
154
+
};
155
+
156
+
#define RX_CMP_L2_ERRORS \
157
+
cpu_to_le32(RX_CMPL_ERRORS_BUFFER_ERROR_MASK | RX_CMPL_ERRORS_CRC_ERROR)
158
+
159
+
#define RX_CMP_L4_CS_BITS \
160
+
(cpu_to_le32(RX_CMP_FLAGS2_L4_CS_CALC | RX_CMP_FLAGS2_T_L4_CS_CALC))
161
+
162
+
#define RX_CMP_L4_CS_ERR_BITS \
163
+
(cpu_to_le32(RX_CMPL_ERRORS_L4_CS_ERROR | RX_CMPL_ERRORS_T_L4_CS_ERROR))
164
+
165
+
#define RX_CMP_L4_CS_OK(rxcmp1) \
166
+
(((rxcmp1)->rx_cmp_flags2 & RX_CMP_L4_CS_BITS) && \
167
+
!((rxcmp1)->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS))
168
+
169
+
#define RX_CMP_METADATA0_TCI(rxcmp1) \
170
+
((le32_to_cpu((rxcmp1)->rx_cmp_cfa_code_errors_v2) & \
171
+
RX_CMPL_METADATA0_TCI_MASK) >> RX_CMPL_METADATA0_SFT)
172
+
173
+
#define RX_CMP_ENCAP(rxcmp1) \
174
+
((le32_to_cpu((rxcmp1)->rx_cmp_flags2) & \
175
+
RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3)
176
+
177
+
#define RX_CMP_V3_HASH_TYPE_LEGACY(rxcmp) \
178
+
((le32_to_cpu((rxcmp)->rx_cmp_misc_v1) & \
179
+
RX_CMP_V3_RSS_EXT_OP_LEGACY) >> RX_CMP_V3_RSS_EXT_OP_LEGACY_SHIFT)
180
+
181
+
#define RX_CMP_V3_HASH_TYPE_NEW(rxcmp) \
182
+
((le32_to_cpu((rxcmp)->rx_cmp_misc_v1) & RX_CMP_V3_RSS_EXT_OP_NEW) >>\
183
+
RX_CMP_V3_RSS_EXT_OP_NEW_SHIFT)
184
+
185
+
#define RX_CMP_V3_HASH_TYPE(bd, rxcmp) \
186
+
(((bd)->rss_cap & BNGE_RSS_CAP_RSS_TCAM) ? \
187
+
RX_CMP_V3_HASH_TYPE_NEW(rxcmp) : \
188
+
RX_CMP_V3_HASH_TYPE_LEGACY(rxcmp))
189
+
190
+
#define EXT_OP_INNER_4 0x0
191
+
#define EXT_OP_OUTER_4 0x2
192
+
#define EXT_OP_INNFL_3 0x8
193
+
#define EXT_OP_OUTFL_3 0xa
194
+
195
+
#define RX_CMP_VLAN_VALID(rxcmp) \
196
+
((rxcmp)->rx_cmp_misc_v1 & cpu_to_le32(RX_CMP_METADATA1_VALID))
197
+
198
+
#define RX_CMP_VLAN_TPID_SEL(rxcmp) \
199
+
(le32_to_cpu((rxcmp)->rx_cmp_misc_v1) & RX_CMP_METADATA1_TPID_SEL)
200
+
201
+
#define RSS_PROFILE_ID_MASK GENMASK(4, 0)
202
+
203
+
#define RX_CMP_HASH_TYPE(rxcmp) \
204
+
(((le32_to_cpu((rxcmp)->rx_cmp_misc_v1) & RX_CMP_RSS_HASH_TYPE) >>\
205
+
RX_CMP_RSS_HASH_TYPE_SHIFT) & RSS_PROFILE_ID_MASK)
206
+
207
+
#define RX_CMP_HASH_VALID(rxcmp) \
208
+
((rxcmp)->rx_cmp_len_flags_type & cpu_to_le32(RX_CMP_FLAGS_RSS_VALID))
209
+
210
+
#define TPA_AGG_AGG_ID(rx_agg) \
211
+
((le32_to_cpu((rx_agg)->rx_agg_cmp_v) & \
212
+
RX_AGG_CMP_AGG_ID) >> RX_AGG_CMP_AGG_ID_SHIFT)
213
+
214
+
#define RX_TPA_START_CMP_TYPE GENMASK(5, 0)
215
+
#define RX_TPA_START_CMP_FLAGS GENMASK(15, 6)
216
+
#define RX_TPA_START_CMP_FLAGS_SHIFT 6
217
+
#define RX_TPA_START_CMP_FLAGS_ERROR BIT(6)
218
+
#define RX_TPA_START_CMP_FLAGS_PLACEMENT GENMASK(9, 7)
219
+
#define RX_TPA_START_CMP_FLAGS_PLACEMENT_SHIFT 7
220
+
#define RX_TPA_START_CMP_FLAGS_PLACEMENT_JUMBO BIT(7)
221
+
#define RX_TPA_START_CMP_FLAGS_PLACEMENT_HDS (0x2 << 7)
222
+
#define RX_TPA_START_CMP_FLAGS_PLACEMENT_GRO_JUMBO (0x5 << 7)
223
+
#define RX_TPA_START_CMP_FLAGS_PLACEMENT_GRO_HDS (0x6 << 7)
224
+
#define RX_TPA_START_CMP_FLAGS_RSS_VALID BIT(10)
225
+
#define RX_TPA_START_CMP_FLAGS_TIMESTAMP BIT(11)
226
+
#define RX_TPA_START_CMP_FLAGS_ITYPES GENMASK(15, 12)
227
+
#define RX_TPA_START_CMP_FLAGS_ITYPES_SHIFT 12
228
+
#define RX_TPA_START_CMP_FLAGS_ITYPE_TCP (0x2 << 12)
229
+
#define RX_TPA_START_CMP_LEN GENMASK(31, 16)
230
+
#define RX_TPA_START_CMP_LEN_SHIFT 16
231
+
#define RX_TPA_START_CMP_V1 BIT(0)
232
+
#define RX_TPA_START_CMP_RSS_HASH_TYPE GENMASK(15, 9)
233
+
#define RX_TPA_START_CMP_RSS_HASH_TYPE_SHIFT 9
234
+
#define RX_TPA_START_CMP_V3_RSS_HASH_TYPE GENMASK(15, 7)
235
+
#define RX_TPA_START_CMP_V3_RSS_HASH_TYPE_SHIFT 7
236
+
#define RX_TPA_START_CMP_AGG_ID GENMASK(25, 16)
237
+
#define RX_TPA_START_CMP_AGG_ID_SHIFT 16
238
+
#define RX_TPA_START_CMP_METADATA1 GENMASK(31, 28)
239
+
#define RX_TPA_START_CMP_METADATA1_SHIFT 28
240
+
#define RX_TPA_START_METADATA1_TPID_SEL GENMASK(30, 28)
241
+
#define RX_TPA_START_METADATA1_TPID_8021Q BIT(28)
242
+
#define RX_TPA_START_METADATA1_TPID_8021AD (0x0 << 28)
243
+
#define RX_TPA_START_METADATA1_VALID BIT(31)
244
+
245
+
struct rx_tpa_start_cmp {
246
+
__le32 rx_tpa_start_cmp_len_flags_type;
247
+
u32 rx_tpa_start_cmp_opaque;
248
+
__le32 rx_tpa_start_cmp_misc_v1;
249
+
__le32 rx_tpa_start_cmp_rss_hash;
250
+
};
251
+
252
+
#define TPA_START_HASH_VALID(rx_tpa_start) \
253
+
((rx_tpa_start)->rx_tpa_start_cmp_len_flags_type & \
254
+
cpu_to_le32(RX_TPA_START_CMP_FLAGS_RSS_VALID))
255
+
256
+
#define TPA_START_HASH_TYPE(rx_tpa_start) \
257
+
(((le32_to_cpu((rx_tpa_start)->rx_tpa_start_cmp_misc_v1) & \
258
+
RX_TPA_START_CMP_RSS_HASH_TYPE) >> \
259
+
RX_TPA_START_CMP_RSS_HASH_TYPE_SHIFT) & RSS_PROFILE_ID_MASK)
260
+
261
+
#define TPA_START_V3_HASH_TYPE(rx_tpa_start) \
262
+
(((le32_to_cpu((rx_tpa_start)->rx_tpa_start_cmp_misc_v1) & \
263
+
RX_TPA_START_CMP_V3_RSS_HASH_TYPE) >> \
264
+
RX_TPA_START_CMP_V3_RSS_HASH_TYPE_SHIFT) & RSS_PROFILE_ID_MASK)
265
+
266
+
#define TPA_START_AGG_ID(rx_tpa_start) \
267
+
((le32_to_cpu((rx_tpa_start)->rx_tpa_start_cmp_misc_v1) & \
268
+
RX_TPA_START_CMP_AGG_ID) >> RX_TPA_START_CMP_AGG_ID_SHIFT)
269
+
270
+
#define TPA_START_ERROR(rx_tpa_start) \
271
+
((rx_tpa_start)->rx_tpa_start_cmp_len_flags_type & \
272
+
cpu_to_le32(RX_TPA_START_CMP_FLAGS_ERROR))
273
+
274
+
#define TPA_START_VLAN_VALID(rx_tpa_start) \
275
+
((rx_tpa_start)->rx_tpa_start_cmp_misc_v1 & \
276
+
cpu_to_le32(RX_TPA_START_METADATA1_VALID))
277
+
278
+
#define TPA_START_VLAN_TPID_SEL(rx_tpa_start) \
279
+
(le32_to_cpu((rx_tpa_start)->rx_tpa_start_cmp_misc_v1) & \
280
+
RX_TPA_START_METADATA1_TPID_SEL)
281
+
282
+
#define RX_TPA_START_CMP_FLAGS2_IP_CS_CALC BIT(0)
283
+
#define RX_TPA_START_CMP_FLAGS2_L4_CS_CALC BIT(1)
284
+
#define RX_TPA_START_CMP_FLAGS2_T_IP_CS_CALC BIT(2)
285
+
#define RX_TPA_START_CMP_FLAGS2_T_L4_CS_CALC BIT(3)
286
+
#define RX_TPA_START_CMP_FLAGS2_IP_TYPE BIT(8)
287
+
#define RX_TPA_START_CMP_FLAGS2_CSUM_CMPL_VALID BIT(9)
288
+
#define RX_TPA_START_CMP_FLAGS2_EXT_META_FORMAT GENMASK(11, 10)
289
+
#define RX_TPA_START_CMP_FLAGS2_EXT_META_FORMAT_SHIFT 10
290
+
#define RX_TPA_START_CMP_V3_FLAGS2_T_IP_TYPE BIT(10)
291
+
#define RX_TPA_START_CMP_V3_FLAGS2_AGG_GRO BIT(11)
292
+
#define RX_TPA_START_CMP_FLAGS2_CSUM_CMPL GENMASK(31, 16)
293
+
#define RX_TPA_START_CMP_FLAGS2_CSUM_CMPL_SHIFT 16
294
+
#define RX_TPA_START_CMP_V2 BIT(0)
295
+
#define RX_TPA_START_CMP_ERRORS_BUFFER_ERROR_MASK GENMASK(3, 1)
296
+
#define RX_TPA_START_CMP_ERRORS_BUFFER_ERROR_SHIFT 1
297
+
#define RX_TPA_START_CMP_ERRORS_BUFFER_ERROR_NO_BUFFER (0x0 << 1)
298
+
#define RX_TPA_START_CMP_ERRORS_BUFFER_ERROR_BAD_FORMAT (0x3 << 1)
299
+
#define RX_TPA_START_CMP_ERRORS_BUFFER_ERROR_FLUSH (0x5 << 1)
300
+
#define RX_TPA_START_CMP_CFA_CODE GENMASK(31, 16)
301
+
#define RX_TPA_START_CMPL_CFA_CODE_SHIFT 16
302
+
#define RX_TPA_START_CMP_METADATA0_TCI_MASK GENMASK(31, 16)
303
+
#define RX_TPA_START_CMP_METADATA0_VID_MASK GENMASK(27, 16)
304
+
#define RX_TPA_START_CMP_METADATA0_SFT 16
305
+
306
+
struct rx_tpa_start_cmp_ext {
307
+
__le32 rx_tpa_start_cmp_flags2;
308
+
__le32 rx_tpa_start_cmp_metadata;
309
+
__le32 rx_tpa_start_cmp_cfa_code_v2;
310
+
__le32 rx_tpa_start_cmp_hdr_info;
311
+
};
312
+
313
+
#define TPA_START_CFA_CODE(rx_tpa_start) \
314
+
((le32_to_cpu((rx_tpa_start)->rx_tpa_start_cmp_cfa_code_v2) & \
315
+
RX_TPA_START_CMP_CFA_CODE) >> RX_TPA_START_CMPL_CFA_CODE_SHIFT)
316
+
317
+
#define TPA_START_IS_IPV6(rx_tpa_start) \
318
+
(!!((rx_tpa_start)->rx_tpa_start_cmp_flags2 & \
319
+
cpu_to_le32(RX_TPA_START_CMP_FLAGS2_IP_TYPE)))
320
+
321
+
#define TPA_START_ERROR_CODE(rx_tpa_start) \
322
+
((le32_to_cpu((rx_tpa_start)->rx_tpa_start_cmp_cfa_code_v2) & \
323
+
RX_TPA_START_CMP_ERRORS_BUFFER_ERROR_MASK) >> \
324
+
RX_TPA_START_CMP_ERRORS_BUFFER_ERROR_SHIFT)
325
+
326
+
#define TPA_START_METADATA0_TCI(rx_tpa_start) \
327
+
((le32_to_cpu((rx_tpa_start)->rx_tpa_start_cmp_cfa_code_v2) & \
328
+
RX_TPA_START_CMP_METADATA0_TCI_MASK) >> \
329
+
RX_TPA_START_CMP_METADATA0_SFT)
330
+
331
+
#define RX_TPA_END_CMP_TYPE GENMASK(5, 0)
332
+
#define RX_TPA_END_CMP_FLAGS GENMASK(15, 6)
333
+
#define RX_TPA_END_CMP_FLAGS_SHIFT 6
334
+
#define RX_TPA_END_CMP_FLAGS_PLACEMENT GENMASK(9, 7)
335
+
#define RX_TPA_END_CMP_FLAGS_PLACEMENT_SHIFT 7
336
+
#define RX_TPA_END_CMP_FLAGS_PLACEMENT_JUMBO BIT(7)
337
+
#define RX_TPA_END_CMP_FLAGS_PLACEMENT_HDS (0x2 << 7)
338
+
#define RX_TPA_END_CMP_FLAGS_PLACEMENT_GRO_JUMBO (0x5 << 7)
339
+
#define RX_TPA_END_CMP_FLAGS_PLACEMENT_GRO_HDS (0x6 << 7)
340
+
#define RX_TPA_END_CMP_FLAGS_RSS_VALID BIT(10)
341
+
#define RX_TPA_END_CMP_FLAGS_ITYPES GENMASK(15, 12)
342
+
#define RX_TPA_END_CMP_FLAGS_ITYPES_SHIFT 12
343
+
#define RX_TPA_END_CMP_FLAGS_ITYPE_TCP (0x2 << 12)
344
+
#define RX_TPA_END_CMP_LEN GENMASK(31, 16)
345
+
#define RX_TPA_END_CMP_LEN_SHIFT 16
346
+
#define RX_TPA_END_CMP_V1 BIT(0)
347
+
#define RX_TPA_END_CMP_TPA_SEGS GENMASK(15, 8)
348
+
#define RX_TPA_END_CMP_TPA_SEGS_SHIFT 8
349
+
#define RX_TPA_END_CMP_AGG_ID GENMASK(25, 16)
350
+
#define RX_TPA_END_CMP_AGG_ID_SHIFT 16
351
+
#define RX_TPA_END_GRO_TS BIT(31)
352
+
353
+
struct rx_tpa_end_cmp {
354
+
__le32 rx_tpa_end_cmp_len_flags_type;
355
+
u32 rx_tpa_end_cmp_opaque;
356
+
__le32 rx_tpa_end_cmp_misc_v1;
357
+
__le32 rx_tpa_end_cmp_tsdelta;
358
+
};
359
+
360
+
#define TPA_END_AGG_ID(rx_tpa_end) \
361
+
((le32_to_cpu((rx_tpa_end)->rx_tpa_end_cmp_misc_v1) & \
362
+
RX_TPA_END_CMP_AGG_ID) >> RX_TPA_END_CMP_AGG_ID_SHIFT)
363
+
364
+
#define TPA_END_TPA_SEGS(rx_tpa_end) \
365
+
((le32_to_cpu((rx_tpa_end)->rx_tpa_end_cmp_misc_v1) & \
366
+
RX_TPA_END_CMP_TPA_SEGS) >> RX_TPA_END_CMP_TPA_SEGS_SHIFT)
367
+
368
+
#define RX_TPA_END_CMP_FLAGS_PLACEMENT_ANY_GRO \
369
+
cpu_to_le32(RX_TPA_END_CMP_FLAGS_PLACEMENT_GRO_JUMBO & \
370
+
RX_TPA_END_CMP_FLAGS_PLACEMENT_GRO_HDS)
371
+
372
+
#define TPA_END_GRO(rx_tpa_end) \
373
+
((rx_tpa_end)->rx_tpa_end_cmp_len_flags_type & \
374
+
RX_TPA_END_CMP_FLAGS_PLACEMENT_ANY_GRO)
375
+
376
+
#define TPA_END_GRO_TS(rx_tpa_end) \
377
+
(!!((rx_tpa_end)->rx_tpa_end_cmp_tsdelta & \
378
+
cpu_to_le32(RX_TPA_END_GRO_TS)))
379
+
380
+
#define RX_TPA_END_CMP_TPA_DUP_ACKS GENMASK(3, 0)
381
+
#define RX_TPA_END_CMP_PAYLOAD_OFFSET GENMASK(23, 16)
382
+
#define RX_TPA_END_CMP_PAYLOAD_OFFSET_SHIFT 16
383
+
#define RX_TPA_END_CMP_AGG_BUFS GENMASK(31, 24)
384
+
#define RX_TPA_END_CMP_AGG_BUFS_SHIFT 24
385
+
#define RX_TPA_END_CMP_TPA_SEG_LEN GENMASK(15, 0)
386
+
#define RX_TPA_END_CMP_V2 BIT(0)
387
+
#define RX_TPA_END_CMP_ERRORS GENMASK(2, 1)
388
+
#define RX_TPA_END_CMPL_ERRORS_SHIFT 1
389
+
#define RX_TPA_END_CMP_ERRORS_BUFFER_ERROR_NO_BUFFER (0x0 << 1)
390
+
#define RX_TPA_END_CMP_ERRORS_BUFFER_ERROR_NOT_ON_CHIP (0x2 << 1)
391
+
#define RX_TPA_END_CMP_ERRORS_BUFFER_ERROR_BAD_FORMAT (0x3 << 1)
392
+
#define RX_TPA_END_CMP_ERRORS_BUFFER_ERROR_RSV_ERROR (0x4 << 1)
393
+
#define RX_TPA_END_CMP_ERRORS_BUFFER_ERROR_FLUSH (0x5 << 1)
394
+
395
+
struct rx_tpa_end_cmp_ext {
396
+
__le32 rx_tpa_end_cmp_dup_acks;
397
+
__le32 rx_tpa_end_cmp_seg_len;
398
+
__le32 rx_tpa_end_cmp_errors_v2;
399
+
u32 rx_tpa_end_cmp_start_opaque;
400
+
};
401
+
402
+
#define TPA_END_ERRORS(rx_tpa_end_ext) \
403
+
((rx_tpa_end_ext)->rx_tpa_end_cmp_errors_v2 & \
404
+
cpu_to_le32(RX_TPA_END_CMP_ERRORS))
405
+
406
+
#define TPA_END_PAYLOAD_OFF(rx_tpa_end_ext) \
407
+
((le32_to_cpu((rx_tpa_end_ext)->rx_tpa_end_cmp_dup_acks) & \
408
+
RX_TPA_END_CMP_PAYLOAD_OFFSET) >> \
409
+
RX_TPA_END_CMP_PAYLOAD_OFFSET_SHIFT)
410
+
411
+
#define TPA_END_AGG_BUFS(rx_tpa_end_ext) \
412
+
((le32_to_cpu((rx_tpa_end_ext)->rx_tpa_end_cmp_dup_acks) & \
413
+
RX_TPA_END_CMP_AGG_BUFS) >> RX_TPA_END_CMP_AGG_BUFS_SHIFT)
414
+
415
+
#define EVENT_DATA1_RESET_NOTIFY_FATAL(data1) \
416
+
(((data1) & \
417
+
ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA1_REASON_CODE_MASK) ==\
418
+
ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA1_REASON_CODE_FW_EXCEPTION_FATAL)
419
+
420
+
#define EVENT_DATA1_RESET_NOTIFY_FW_ACTIVATION(data1) \
421
+
(((data1) & \
422
+
ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA1_REASON_CODE_MASK) ==\
423
+
ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA1_REASON_CODE_FW_ACTIVATION)
424
+
425
+
#define EVENT_DATA2_RESET_NOTIFY_FW_STATUS_CODE(data2) \
426
+
((data2) & \
427
+
ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA2_FW_STATUS_CODE_MASK)
428
+
429
+
#define EVENT_DATA1_RECOVERY_MASTER_FUNC(data1) \
430
+
(!!((data1) & \
431
+
ASYNC_EVENT_CMPL_ERROR_RECOVERY_EVENT_DATA1_FLAGS_MASTER_FUNC))
432
+
433
+
#define EVENT_DATA1_RECOVERY_ENABLED(data1) \
434
+
(!!((data1) & \
435
+
ASYNC_EVENT_CMPL_ERROR_RECOVERY_EVENT_DATA1_FLAGS_RECOVERY_ENABLED))
436
+
437
+
#define BNGE_EVENT_ERROR_REPORT_TYPE(data1) \
438
+
(((data1) & \
439
+
ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_MASK) >>\
440
+
ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_SFT)
441
+
442
+
#define BNGE_EVENT_INVALID_SIGNAL_DATA(data2) \
443
+
(((data2) & \
444
+
ASYNC_EVENT_CMPL_ERROR_REPORT_INVALID_SIGNAL_EVENT_DATA2_PIN_ID_MASK) >>\
445
+
ASYNC_EVENT_CMPL_ERROR_REPORT_INVALID_SIGNAL_EVENT_DATA2_PIN_ID_SFT)
446
+
#endif /* _BNGE_HW_DEF_H_ */
+65
drivers/net/ethernet/broadcom/bnge/bnge_hwrm_lib.c
+65
drivers/net/ethernet/broadcom/bnge/bnge_hwrm_lib.c
···
1183
1183
req->async_event_cr = cpu_to_le16(idx);
1184
1184
return bnge_hwrm_req_send(bd, req);
1185
1185
}
1186
+
1187
+
#define BNGE_DFLT_TUNL_TPA_BMAP \
1188
+
(VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_GRE | \
1189
+
VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_IPV4 | \
1190
+
VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_IPV6)
1191
+
1192
+
static void bnge_hwrm_vnic_update_tunl_tpa(struct bnge_dev *bd,
1193
+
struct hwrm_vnic_tpa_cfg_input *req)
1194
+
{
1195
+
struct bnge_net *bn = netdev_priv(bd->netdev);
1196
+
u32 tunl_tpa_bmap = BNGE_DFLT_TUNL_TPA_BMAP;
1197
+
1198
+
if (!(bd->fw_cap & BNGE_FW_CAP_VNIC_TUNNEL_TPA))
1199
+
return;
1200
+
1201
+
if (bn->vxlan_port)
1202
+
tunl_tpa_bmap |= VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_VXLAN;
1203
+
if (bn->vxlan_gpe_port)
1204
+
tunl_tpa_bmap |= VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_VXLAN_GPE;
1205
+
if (bn->nge_port)
1206
+
tunl_tpa_bmap |= VNIC_TPA_CFG_REQ_TNL_TPA_EN_BITMAP_GENEVE;
1207
+
1208
+
req->enables |= cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_TNL_TPA_EN);
1209
+
req->tnl_tpa_en_bitmap = cpu_to_le32(tunl_tpa_bmap);
1210
+
}
1211
+
1212
+
int bnge_hwrm_vnic_set_tpa(struct bnge_dev *bd, struct bnge_vnic_info *vnic,
1213
+
u32 tpa_flags)
1214
+
{
1215
+
struct bnge_net *bn = netdev_priv(bd->netdev);
1216
+
struct hwrm_vnic_tpa_cfg_input *req;
1217
+
int rc;
1218
+
1219
+
if (vnic->fw_vnic_id == INVALID_HW_RING_ID)
1220
+
return 0;
1221
+
1222
+
rc = bnge_hwrm_req_init(bd, req, HWRM_VNIC_TPA_CFG);
1223
+
if (rc)
1224
+
return rc;
1225
+
1226
+
if (tpa_flags) {
1227
+
u32 flags;
1228
+
1229
+
flags = VNIC_TPA_CFG_REQ_FLAGS_TPA |
1230
+
VNIC_TPA_CFG_REQ_FLAGS_ENCAP_TPA |
1231
+
VNIC_TPA_CFG_REQ_FLAGS_RSC_WND_UPDATE |
1232
+
VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_ECN |
1233
+
VNIC_TPA_CFG_REQ_FLAGS_AGG_WITH_SAME_GRE_SEQ;
1234
+
if (tpa_flags & BNGE_NET_EN_GRO)
1235
+
flags |= VNIC_TPA_CFG_REQ_FLAGS_GRO;
1236
+
1237
+
req->flags = cpu_to_le32(flags);
1238
+
req->enables =
1239
+
cpu_to_le32(VNIC_TPA_CFG_REQ_ENABLES_MAX_AGG_SEGS |
1240
+
VNIC_TPA_CFG_REQ_ENABLES_MAX_AGGS |
1241
+
VNIC_TPA_CFG_REQ_ENABLES_MIN_AGG_LEN);
1242
+
req->max_agg_segs = cpu_to_le16(MAX_TPA_SEGS);
1243
+
req->max_aggs = cpu_to_le16(bn->max_tpa);
1244
+
req->min_agg_len = cpu_to_le32(512);
1245
+
bnge_hwrm_vnic_update_tunl_tpa(bd, req);
1246
+
}
1247
+
req->vnic_id = cpu_to_le16(vnic->fw_vnic_id);
1248
+
1249
+
return bnge_hwrm_req_send(bd, req);
1250
+
}
+2
drivers/net/ethernet/broadcom/bnge/bnge_hwrm_lib.h
+2
drivers/net/ethernet/broadcom/bnge/bnge_hwrm_lib.h
+372
-37
drivers/net/ethernet/broadcom/bnge/bnge_netdev.c
+372
-37
drivers/net/ethernet/broadcom/bnge/bnge_netdev.c
···
10
10
#include <linux/list.h>
11
11
#include <linux/pci.h>
12
12
#include <linux/netdevice.h>
13
+
#include <net/netdev_lock.h>
14
+
#include <net/netdev_queues.h>
15
+
#include <net/netdev_rx_queue.h>
13
16
#include <linux/etherdevice.h>
14
17
#include <linux/if.h>
15
18
#include <net/ip.h>
19
+
#include <net/netdev_queues.h>
16
20
#include <linux/skbuff.h>
17
21
#include <net/page_pool/helpers.h>
18
22
···
24
20
#include "bnge_hwrm_lib.h"
25
21
#include "bnge_ethtool.h"
26
22
#include "bnge_rmem.h"
23
+
#include "bnge_txrx.h"
27
24
28
25
#define BNGE_RING_TO_TC_OFF(bd, tx) \
29
26
((tx) % (bd)->tx_nr_rings_per_tc)
···
377
372
}
378
373
}
379
374
375
+
static void bnge_free_one_tpa_info_data(struct bnge_net *bn,
376
+
struct bnge_rx_ring_info *rxr)
377
+
{
378
+
int i;
379
+
380
+
for (i = 0; i < bn->max_tpa; i++) {
381
+
struct bnge_tpa_info *tpa_info = &rxr->rx_tpa[i];
382
+
u8 *data = tpa_info->data;
383
+
384
+
if (!data)
385
+
continue;
386
+
387
+
tpa_info->data = NULL;
388
+
page_pool_free_va(rxr->head_pool, data, false);
389
+
}
390
+
}
391
+
380
392
static void bnge_free_one_rx_ring_pair_bufs(struct bnge_net *bn,
381
393
struct bnge_rx_ring_info *rxr)
382
394
{
395
+
struct bnge_tpa_idx_map *map;
396
+
397
+
if (rxr->rx_tpa)
398
+
bnge_free_one_tpa_info_data(bn, rxr);
399
+
383
400
bnge_free_one_rx_ring_bufs(bn, rxr);
384
401
bnge_free_one_agg_ring_bufs(bn, rxr);
402
+
403
+
map = rxr->rx_tpa_idx_map;
404
+
if (map)
405
+
memset(map->agg_idx_bmap, 0, sizeof(map->agg_idx_bmap));
385
406
}
386
407
387
408
static void bnge_free_rx_ring_pair_bufs(struct bnge_net *bn)
···
422
391
bnge_free_one_rx_ring_pair_bufs(bn, &bn->rx_ring[i]);
423
392
}
424
393
394
+
static void bnge_free_tx_skbs(struct bnge_net *bn)
395
+
{
396
+
struct bnge_dev *bd = bn->bd;
397
+
u16 max_idx;
398
+
int i;
399
+
400
+
max_idx = bn->tx_nr_pages * TX_DESC_CNT;
401
+
for (i = 0; i < bd->tx_nr_rings; i++) {
402
+
struct bnge_tx_ring_info *txr = &bn->tx_ring[i];
403
+
int j;
404
+
405
+
if (!txr->tx_buf_ring)
406
+
continue;
407
+
408
+
for (j = 0; j < max_idx;) {
409
+
struct bnge_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j];
410
+
struct sk_buff *skb;
411
+
int k, last;
412
+
413
+
skb = tx_buf->skb;
414
+
if (!skb) {
415
+
j++;
416
+
continue;
417
+
}
418
+
419
+
tx_buf->skb = NULL;
420
+
421
+
dma_unmap_single(bd->dev,
422
+
dma_unmap_addr(tx_buf, mapping),
423
+
skb_headlen(skb),
424
+
DMA_TO_DEVICE);
425
+
426
+
last = tx_buf->nr_frags;
427
+
j += 2;
428
+
for (k = 0; k < last; k++, j++) {
429
+
int ring_idx = j & bn->tx_ring_mask;
430
+
skb_frag_t *frag = &skb_shinfo(skb)->frags[k];
431
+
432
+
tx_buf = &txr->tx_buf_ring[ring_idx];
433
+
dma_unmap_page(bd->dev,
434
+
dma_unmap_addr(tx_buf, mapping),
435
+
skb_frag_size(frag),
436
+
DMA_TO_DEVICE);
437
+
}
438
+
dev_kfree_skb(skb);
439
+
}
440
+
netdev_tx_reset_queue(netdev_get_tx_queue(bd->netdev, i));
441
+
}
442
+
}
443
+
425
444
static void bnge_free_all_rings_bufs(struct bnge_net *bn)
426
445
{
427
446
bnge_free_rx_ring_pair_bufs(bn);
447
+
bnge_free_tx_skbs(bn);
448
+
}
449
+
450
+
static void bnge_free_tpa_info(struct bnge_net *bn)
451
+
{
452
+
struct bnge_dev *bd = bn->bd;
453
+
int i, j;
454
+
455
+
for (i = 0; i < bd->rx_nr_rings; i++) {
456
+
struct bnge_rx_ring_info *rxr = &bn->rx_ring[i];
457
+
458
+
kfree(rxr->rx_tpa_idx_map);
459
+
rxr->rx_tpa_idx_map = NULL;
460
+
if (rxr->rx_tpa) {
461
+
for (j = 0; j < bn->max_tpa; j++) {
462
+
kfree(rxr->rx_tpa[j].agg_arr);
463
+
rxr->rx_tpa[j].agg_arr = NULL;
464
+
}
465
+
}
466
+
kfree(rxr->rx_tpa);
467
+
rxr->rx_tpa = NULL;
468
+
}
469
+
}
470
+
471
+
static int bnge_alloc_tpa_info(struct bnge_net *bn)
472
+
{
473
+
struct bnge_dev *bd = bn->bd;
474
+
int i, j;
475
+
476
+
if (!bd->max_tpa_v2)
477
+
return 0;
478
+
479
+
bn->max_tpa = max_t(u16, bd->max_tpa_v2, MAX_TPA);
480
+
for (i = 0; i < bd->rx_nr_rings; i++) {
481
+
struct bnge_rx_ring_info *rxr = &bn->rx_ring[i];
482
+
483
+
rxr->rx_tpa = kcalloc(bn->max_tpa, sizeof(struct bnge_tpa_info),
484
+
GFP_KERNEL);
485
+
if (!rxr->rx_tpa)
486
+
goto err_free_tpa_info;
487
+
488
+
for (j = 0; j < bn->max_tpa; j++) {
489
+
struct rx_agg_cmp *agg;
490
+
491
+
agg = kcalloc(MAX_SKB_FRAGS, sizeof(*agg), GFP_KERNEL);
492
+
if (!agg)
493
+
goto err_free_tpa_info;
494
+
rxr->rx_tpa[j].agg_arr = agg;
495
+
}
496
+
rxr->rx_tpa_idx_map = kzalloc(sizeof(*rxr->rx_tpa_idx_map),
497
+
GFP_KERNEL);
498
+
if (!rxr->rx_tpa_idx_map)
499
+
goto err_free_tpa_info;
500
+
}
501
+
return 0;
502
+
503
+
err_free_tpa_info:
504
+
bnge_free_tpa_info(bn);
505
+
return -ENOMEM;
428
506
}
429
507
430
508
static void bnge_free_rx_rings(struct bnge_net *bn)
···
541
401
struct bnge_dev *bd = bn->bd;
542
402
int i;
543
403
404
+
bnge_free_tpa_info(bn);
544
405
for (i = 0; i < bd->rx_nr_rings; i++) {
545
406
struct bnge_rx_ring_info *rxr = &bn->rx_ring[i];
546
407
struct bnge_ring_struct *ring;
···
665
524
if (rc)
666
525
goto err_free_rx_rings;
667
526
}
527
+
}
528
+
529
+
if (bn->priv_flags & BNGE_NET_EN_TPA) {
530
+
rc = bnge_alloc_tpa_info(bn);
531
+
if (rc)
532
+
goto err_free_rx_rings;
668
533
}
669
534
return rc;
670
535
···
1003
856
return txr->tx_cpr->ring_struct.fw_ring_id;
1004
857
}
1005
858
859
+
static void bnge_db_nq_arm(struct bnge_net *bn,
860
+
struct bnge_db_info *db, u32 idx)
861
+
{
862
+
bnge_writeq(bn->bd, db->db_key64 | DBR_TYPE_NQ_ARM |
863
+
DB_RING_IDX(db, idx), db->doorbell);
864
+
}
865
+
1006
866
static void bnge_db_nq(struct bnge_net *bn, struct bnge_db_info *db, u32 idx)
1007
867
{
1008
868
bnge_writeq(bn->bd, db->db_key64 | DBR_TYPE_NQ_MASK |
···
1030
876
nqr = &bnapi->nq_ring;
1031
877
1032
878
return nqr->ring_struct.map_idx;
1033
-
}
1034
-
1035
-
static irqreturn_t bnge_msix(int irq, void *dev_instance)
1036
-
{
1037
-
/* NAPI scheduling to be added in a future patch */
1038
-
return IRQ_HANDLED;
1039
879
}
1040
880
1041
881
static void bnge_init_nq_tree(struct bnge_net *bn)
···
1073
925
return netmem;
1074
926
}
1075
927
1076
-
static u8 *__bnge_alloc_rx_frag(struct bnge_net *bn, dma_addr_t *mapping,
1077
-
struct bnge_rx_ring_info *rxr,
1078
-
gfp_t gfp)
928
+
u8 *__bnge_alloc_rx_frag(struct bnge_net *bn, dma_addr_t *mapping,
929
+
struct bnge_rx_ring_info *rxr,
930
+
gfp_t gfp)
1079
931
{
1080
932
unsigned int offset;
1081
933
struct page *page;
···
1089
941
return page_address(page) + offset;
1090
942
}
1091
943
1092
-
static int bnge_alloc_rx_data(struct bnge_net *bn,
1093
-
struct bnge_rx_ring_info *rxr,
1094
-
u16 prod, gfp_t gfp)
944
+
int bnge_alloc_rx_data(struct bnge_net *bn, struct bnge_rx_ring_info *rxr,
945
+
u16 prod, gfp_t gfp)
1095
946
{
1096
947
struct bnge_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[RING_RX(bn, prod)];
1097
948
struct rx_bd *rxbd;
···
1142
995
return 0;
1143
996
}
1144
997
1145
-
static u16 bnge_find_next_agg_idx(struct bnge_rx_ring_info *rxr, u16 idx)
998
+
u16 bnge_find_next_agg_idx(struct bnge_rx_ring_info *rxr, u16 idx)
1146
999
{
1147
1000
u16 next, max = rxr->rx_agg_bmap_size;
1148
1001
···
1152
1005
return next;
1153
1006
}
1154
1007
1155
-
static int bnge_alloc_rx_netmem(struct bnge_net *bn,
1156
-
struct bnge_rx_ring_info *rxr,
1157
-
u16 prod, gfp_t gfp)
1008
+
int bnge_alloc_rx_netmem(struct bnge_net *bn,
1009
+
struct bnge_rx_ring_info *rxr,
1010
+
u16 prod, gfp_t gfp)
1158
1011
{
1159
1012
struct bnge_sw_rx_agg_bd *rx_agg_buf;
1160
1013
u16 sw_prod = rxr->rx_sw_agg_prod;
···
1217
1070
return -ENOMEM;
1218
1071
}
1219
1072
1073
+
static int bnge_alloc_one_tpa_info_data(struct bnge_net *bn,
1074
+
struct bnge_rx_ring_info *rxr)
1075
+
{
1076
+
dma_addr_t mapping;
1077
+
u8 *data;
1078
+
int i;
1079
+
1080
+
for (i = 0; i < bn->max_tpa; i++) {
1081
+
data = __bnge_alloc_rx_frag(bn, &mapping, rxr,
1082
+
GFP_KERNEL);
1083
+
if (!data)
1084
+
goto err_free_tpa_info_data;
1085
+
1086
+
rxr->rx_tpa[i].data = data;
1087
+
rxr->rx_tpa[i].data_ptr = data + bn->rx_offset;
1088
+
rxr->rx_tpa[i].mapping = mapping;
1089
+
}
1090
+
return 0;
1091
+
1092
+
err_free_tpa_info_data:
1093
+
bnge_free_one_tpa_info_data(bn, rxr);
1094
+
return -ENOMEM;
1095
+
}
1096
+
1220
1097
static int bnge_alloc_one_rx_ring_pair_bufs(struct bnge_net *bn, int ring_nr)
1221
1098
{
1222
1099
struct bnge_rx_ring_info *rxr = &bn->rx_ring[ring_nr];
···
1255
1084
if (rc)
1256
1085
goto err_free_one_rx_ring_bufs;
1257
1086
}
1087
+
1088
+
if (rxr->rx_tpa) {
1089
+
rc = bnge_alloc_one_tpa_info_data(bn, rxr);
1090
+
if (rc)
1091
+
goto err_free_one_agg_ring_bufs;
1092
+
}
1093
+
1258
1094
return 0;
1259
1095
1096
+
err_free_one_agg_ring_bufs:
1097
+
bnge_free_one_agg_ring_bufs(bn, rxr);
1260
1098
err_free_one_rx_ring_bufs:
1261
1099
bnge_free_one_rx_ring_bufs(bn, rxr);
1262
1100
return rc;
···
1935
1755
return rc;
1936
1756
}
1937
1757
1758
+
static void bnge_disable_int(struct bnge_net *bn)
1759
+
{
1760
+
struct bnge_dev *bd = bn->bd;
1761
+
int i;
1762
+
1763
+
if (!bn->bnapi)
1764
+
return;
1765
+
1766
+
for (i = 0; i < bd->nq_nr_rings; i++) {
1767
+
struct bnge_napi *bnapi = bn->bnapi[i];
1768
+
struct bnge_nq_ring_info *nqr;
1769
+
struct bnge_ring_struct *ring;
1770
+
1771
+
nqr = &bnapi->nq_ring;
1772
+
ring = &nqr->ring_struct;
1773
+
1774
+
if (ring->fw_ring_id != INVALID_HW_RING_ID)
1775
+
bnge_db_nq(bn, &nqr->nq_db, nqr->nq_raw_cons);
1776
+
}
1777
+
}
1778
+
1779
+
static void bnge_disable_int_sync(struct bnge_net *bn)
1780
+
{
1781
+
struct bnge_dev *bd = bn->bd;
1782
+
int i;
1783
+
1784
+
bnge_disable_int(bn);
1785
+
for (i = 0; i < bd->nq_nr_rings; i++) {
1786
+
int map_idx = bnge_cp_num_to_irq_num(bn, i);
1787
+
1788
+
synchronize_irq(bd->irq_tbl[map_idx].vector);
1789
+
}
1790
+
}
1791
+
1792
+
static void bnge_enable_int(struct bnge_net *bn)
1793
+
{
1794
+
struct bnge_dev *bd = bn->bd;
1795
+
int i;
1796
+
1797
+
for (i = 0; i < bd->nq_nr_rings; i++) {
1798
+
struct bnge_napi *bnapi = bn->bnapi[i];
1799
+
struct bnge_nq_ring_info *nqr;
1800
+
1801
+
nqr = &bnapi->nq_ring;
1802
+
bnge_db_nq_arm(bn, &nqr->nq_db, nqr->nq_raw_cons);
1803
+
}
1804
+
}
1805
+
1806
+
static void bnge_disable_napi(struct bnge_net *bn)
1807
+
{
1808
+
struct bnge_dev *bd = bn->bd;
1809
+
int i;
1810
+
1811
+
if (test_and_set_bit(BNGE_STATE_NAPI_DISABLED, &bn->state))
1812
+
return;
1813
+
1814
+
for (i = 0; i < bd->nq_nr_rings; i++) {
1815
+
struct bnge_napi *bnapi = bn->bnapi[i];
1816
+
1817
+
napi_disable_locked(&bnapi->napi);
1818
+
}
1819
+
}
1820
+
1821
+
static void bnge_enable_napi(struct bnge_net *bn)
1822
+
{
1823
+
struct bnge_dev *bd = bn->bd;
1824
+
int i;
1825
+
1826
+
clear_bit(BNGE_STATE_NAPI_DISABLED, &bn->state);
1827
+
for (i = 0; i < bd->nq_nr_rings; i++) {
1828
+
struct bnge_napi *bnapi = bn->bnapi[i];
1829
+
1830
+
bnapi->in_reset = false;
1831
+
bnapi->tx_fault = 0;
1832
+
1833
+
napi_enable_locked(&bnapi->napi);
1834
+
}
1835
+
}
1836
+
1938
1837
static void bnge_hwrm_vnic_free(struct bnge_net *bn)
1939
1838
{
1940
1839
int i;
···
2145
1886
bnge_hwrm_rx_agg_ring_free(bn, &bn->rx_ring[i], close_path);
2146
1887
}
2147
1888
1889
+
/* The completion rings are about to be freed. After that the
1890
+
* IRQ doorbell will not work anymore. So we need to disable
1891
+
* IRQ here.
1892
+
*/
1893
+
bnge_disable_int_sync(bn);
1894
+
2148
1895
for (i = 0; i < bd->nq_nr_rings; i++) {
2149
1896
struct bnge_napi *bnapi = bn->bnapi[i];
2150
1897
struct bnge_nq_ring_info *nqr;
···
2280
2015
return rc;
2281
2016
}
2282
2017
2018
+
static int bnge_set_tpa(struct bnge_net *bn, bool set_tpa)
2019
+
{
2020
+
u32 tpa_flags = 0;
2021
+
int rc, i;
2022
+
2023
+
if (set_tpa)
2024
+
tpa_flags = bn->priv_flags & BNGE_NET_EN_TPA;
2025
+
else if (BNGE_NO_FW_ACCESS(bn->bd))
2026
+
return 0;
2027
+
for (i = 0; i < bn->nr_vnics; i++) {
2028
+
rc = bnge_hwrm_vnic_set_tpa(bn->bd, &bn->vnic_info[i],
2029
+
tpa_flags);
2030
+
if (rc) {
2031
+
netdev_err(bn->netdev, "hwrm vnic set tpa failure rc for vnic %d: %x\n",
2032
+
i, rc);
2033
+
return rc;
2034
+
}
2035
+
}
2036
+
return 0;
2037
+
}
2038
+
2283
2039
static int bnge_init_chip(struct bnge_net *bn)
2284
2040
{
2285
2041
struct bnge_vnic_info *vnic = &bn->vnic_info[BNGE_VNIC_DEFAULT];
···
2334
2048
2335
2049
if (bd->rss_cap & BNGE_RSS_CAP_RSS_HASH_TYPE_DELTA)
2336
2050
bnge_hwrm_update_rss_hash_cfg(bn);
2051
+
2052
+
if (bn->priv_flags & BNGE_NET_EN_TPA) {
2053
+
rc = bnge_set_tpa(bn, true);
2054
+
if (rc)
2055
+
goto err_out;
2056
+
}
2337
2057
2338
2058
/* Filter for default vnic 0 */
2339
2059
rc = bnge_hwrm_set_vnic_filter(bn, 0, 0, bn->netdev->dev_addr);
···
2375
2083
err_out:
2376
2084
bnge_hwrm_resource_free(bn, 0);
2377
2085
return rc;
2378
-
}
2379
-
2380
-
static int bnge_napi_poll(struct napi_struct *napi, int budget)
2381
-
{
2382
-
int work_done = 0;
2383
-
2384
-
/* defer NAPI implementation to next patch series */
2385
-
napi_complete_done(napi, work_done);
2386
-
2387
-
return work_done;
2388
2086
}
2389
2087
2390
2088
static void bnge_init_napi(struct bnge_net *bn)
···
2443
2161
return rc;
2444
2162
}
2445
2163
2164
+
static void bnge_tx_disable(struct bnge_net *bn)
2165
+
{
2166
+
struct bnge_tx_ring_info *txr;
2167
+
int i;
2168
+
2169
+
if (bn->tx_ring) {
2170
+
for (i = 0; i < bn->bd->tx_nr_rings; i++) {
2171
+
txr = &bn->tx_ring[i];
2172
+
WRITE_ONCE(txr->dev_state, BNGE_DEV_STATE_CLOSING);
2173
+
}
2174
+
}
2175
+
/* Make sure napi polls see @dev_state change */
2176
+
synchronize_net();
2177
+
2178
+
if (!bn->netdev)
2179
+
return;
2180
+
/* Drop carrier first to prevent TX timeout */
2181
+
netif_carrier_off(bn->netdev);
2182
+
/* Stop all TX queues */
2183
+
netif_tx_disable(bn->netdev);
2184
+
}
2185
+
2186
+
static void bnge_tx_enable(struct bnge_net *bn)
2187
+
{
2188
+
struct bnge_tx_ring_info *txr;
2189
+
int i;
2190
+
2191
+
for (i = 0; i < bn->bd->tx_nr_rings; i++) {
2192
+
txr = &bn->tx_ring[i];
2193
+
WRITE_ONCE(txr->dev_state, 0);
2194
+
}
2195
+
/* Make sure napi polls see @dev_state change */
2196
+
synchronize_net();
2197
+
netif_tx_wake_all_queues(bn->netdev);
2198
+
}
2199
+
2446
2200
static int bnge_open_core(struct bnge_net *bn)
2447
2201
{
2448
2202
struct bnge_dev *bd = bn->bd;
···
2510
2192
netdev_err(bn->netdev, "bnge_init_nic err: %d\n", rc);
2511
2193
goto err_free_irq;
2512
2194
}
2195
+
2196
+
bnge_enable_napi(bn);
2197
+
2513
2198
set_bit(BNGE_STATE_OPEN, &bd->state);
2199
+
2200
+
bnge_enable_int(bn);
2201
+
2202
+
bnge_tx_enable(bn);
2514
2203
return 0;
2515
2204
2516
2205
err_free_irq:
···
2526
2201
bnge_del_napi(bn);
2527
2202
bnge_free_core(bn);
2528
2203
return rc;
2529
-
}
2530
-
2531
-
static netdev_tx_t bnge_start_xmit(struct sk_buff *skb, struct net_device *dev)
2532
-
{
2533
-
dev_kfree_skb_any(skb);
2534
-
2535
-
return NETDEV_TX_OK;
2536
2204
}
2537
2205
2538
2206
static int bnge_open(struct net_device *dev)
···
2542
2224
2543
2225
static int bnge_shutdown_nic(struct bnge_net *bn)
2544
2226
{
2545
-
/* TODO: close_path = 0 until we make NAPI functional */
2546
-
bnge_hwrm_resource_free(bn, 0);
2227
+
bnge_hwrm_resource_free(bn, 1);
2547
2228
return 0;
2548
2229
}
2549
2230
···
2550
2233
{
2551
2234
struct bnge_dev *bd = bn->bd;
2552
2235
2236
+
bnge_tx_disable(bn);
2237
+
2553
2238
clear_bit(BNGE_STATE_OPEN, &bd->state);
2554
2239
bnge_shutdown_nic(bn);
2240
+
bnge_disable_napi(bn);
2555
2241
bnge_free_all_rings_bufs(bn);
2556
2242
bnge_free_irq(bn);
2557
2243
bnge_del_napi(bn);
···
2575
2255
.ndo_open = bnge_open,
2576
2256
.ndo_stop = bnge_close,
2577
2257
.ndo_start_xmit = bnge_start_xmit,
2258
+
.ndo_features_check = bnge_features_check,
2578
2259
};
2579
2260
2580
2261
static void bnge_init_mac_addr(struct bnge_dev *bd)
···
2616
2295
rx_space = rx_size + ALIGN(NET_SKB_PAD, 8) +
2617
2296
SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
2618
2297
2619
-
bn->rx_copy_thresh = BNGE_RX_COPY_THRESH;
2620
2298
ring_size = bn->rx_ring_size;
2621
2299
bn->rx_agg_ring_size = 0;
2622
2300
bn->rx_agg_nr_pages = 0;
···
2654
2334
bn->rx_agg_ring_size = agg_ring_size;
2655
2335
bn->rx_agg_ring_mask = (bn->rx_agg_nr_pages * RX_DESC_CNT) - 1;
2656
2336
2657
-
rx_size = SKB_DATA_ALIGN(BNGE_RX_COPY_THRESH + NET_IP_ALIGN);
2337
+
rx_size = max3(BNGE_DEFAULT_RX_COPYBREAK,
2338
+
bn->rx_copybreak,
2339
+
bn->netdev->cfg_pending->hds_thresh);
2340
+
rx_size = SKB_DATA_ALIGN(rx_size + NET_IP_ALIGN);
2658
2341
rx_space = rx_size + NET_SKB_PAD +
2659
2342
SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
2660
2343
}
···
2688
2365
}
2689
2366
bn->cp_bit = bn->cp_nr_pages * CP_DESC_CNT;
2690
2367
bn->cp_ring_mask = bn->cp_bit - 1;
2368
+
}
2369
+
2370
+
static void bnge_init_ring_params(struct bnge_net *bn)
2371
+
{
2372
+
u32 rx_size;
2373
+
2374
+
bn->rx_copybreak = BNGE_DEFAULT_RX_COPYBREAK;
2375
+
/* Try to fit 4 chunks into a 4k page */
2376
+
rx_size = SZ_1K -
2377
+
NET_SKB_PAD - SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
2378
+
bn->netdev->cfg->hds_thresh = max(BNGE_DEFAULT_RX_COPYBREAK, rx_size);
2691
2379
}
2692
2380
2693
2381
int bnge_netdev_alloc(struct bnge_dev *bd, int max_irqs)
···
2790
2456
bn->rx_dir = DMA_FROM_DEVICE;
2791
2457
2792
2458
bnge_set_tpa_flags(bd);
2459
+
bnge_init_ring_params(bn);
2793
2460
bnge_set_ring_params(bd);
2794
2461
2795
2462
bnge_init_l2_fltr_tbl(bn);
+118
-3
drivers/net/ethernet/broadcom/bnge/bnge_netdev.h
+118
-3
drivers/net/ethernet/broadcom/bnge/bnge_netdev.h
···
8
8
#include <linux/io-64-nonatomic-lo-hi.h>
9
9
#include <linux/refcount.h>
10
10
#include "bnge_db.h"
11
+
#include "bnge_hw_def.h"
11
12
12
13
struct tx_bd {
13
14
__le32 tx_bd_len_flags_type;
···
77
76
#define CMPL_BASE_TYPE_HWRM_FWD_REQ 0x22UL
78
77
#define CMPL_BASE_TYPE_HWRM_FWD_RESP 0x24UL
79
78
#define CMPL_BASE_TYPE_HWRM_ASYNC_EVENT 0x2eUL
79
+
#define CMPL_BA_TY_HWRM_ASY_EVT CMPL_BASE_TYPE_HWRM_ASYNC_EVENT
80
80
#define TX_CMP_FLAGS_ERROR (1 << 6)
81
81
#define TX_CMP_FLAGS_PUSH (1 << 7)
82
82
u32 tx_cmp_opaque;
···
137
135
u16 nq_fw_ring_id;
138
136
};
139
137
140
-
#define BNGE_RX_COPY_THRESH 256
138
+
#define BNGE_DEFAULT_RX_COPYBREAK 256
139
+
#define BNGE_MAX_RX_COPYBREAK 1024
141
140
142
141
#define BNGE_HW_FEATURE_VLAN_ALL_RX \
143
142
(NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX)
···
152
149
};
153
150
154
151
#define BNGE_NET_EN_TPA (BNGE_NET_EN_GRO | BNGE_NET_EN_LRO)
152
+
153
+
#define BNGE_NO_FW_ACCESS(bd) (pci_channel_offline((bd)->pdev))
154
+
155
+
#define MAX_TPA 256
156
+
#define MAX_TPA_MASK (MAX_TPA - 1)
157
+
#define MAX_TPA_SEGS 0x3f
158
+
159
+
#define BNGE_TPA_INNER_L3_OFF(hdr_info) \
160
+
(((hdr_info) >> 18) & 0x1ff)
161
+
162
+
#define BNGE_TPA_INNER_L2_OFF(hdr_info) \
163
+
(((hdr_info) >> 9) & 0x1ff)
164
+
165
+
#define BNGE_TPA_OUTER_L3_OFF(hdr_info) \
166
+
((hdr_info) & 0x1ff)
167
+
168
+
struct bnge_tpa_idx_map {
169
+
u16 agg_id_tbl[1024];
170
+
DECLARE_BITMAP(agg_idx_bmap, MAX_TPA);
171
+
};
172
+
173
+
struct bnge_tpa_info {
174
+
void *data;
175
+
u8 *data_ptr;
176
+
dma_addr_t mapping;
177
+
u16 len;
178
+
unsigned short gso_type;
179
+
u32 flags2;
180
+
u32 metadata;
181
+
enum pkt_hash_types hash_type;
182
+
u32 rss_hash;
183
+
u32 hdr_info;
184
+
185
+
u16 cfa_code; /* cfa_code in TPA start compl */
186
+
u8 agg_count;
187
+
bool vlan_valid;
188
+
bool cfa_code_valid;
189
+
struct rx_agg_cmp *agg_arr;
190
+
};
155
191
156
192
/* Minimum TX BDs for a TX packet with MAX_SKB_FRAGS + 1. We need one extra
157
193
* BD because the first TX BD is always a long BD.
···
214
172
#define RING_RX_AGG(bn, idx) ((idx) & (bn)->rx_agg_ring_mask)
215
173
#define NEXT_RX_AGG(idx) ((idx) + 1)
216
174
175
+
#define BNGE_NQ_HDL_IDX_MASK 0x00ffffff
176
+
#define BNGE_NQ_HDL_TYPE_MASK 0xff000000
217
177
#define BNGE_NQ_HDL_TYPE_SHIFT 24
218
178
#define BNGE_NQ_HDL_TYPE_RX 0x00
219
179
#define BNGE_NQ_HDL_TYPE_TX 0x01
180
+
181
+
#define BNGE_NQ_HDL_IDX(hdl) ((hdl) & BNGE_NQ_HDL_IDX_MASK)
182
+
#define BNGE_NQ_HDL_TYPE(hdl) (((hdl) & BNGE_NQ_HDL_TYPE_MASK) >> \
183
+
BNGE_NQ_HDL_TYPE_SHIFT)
220
184
221
185
struct bnge_net {
222
186
struct bnge_dev *bd;
···
234
186
u32 rx_buf_size;
235
187
u32 rx_buf_use_size; /* usable size */
236
188
u32 rx_agg_ring_size;
237
-
u32 rx_copy_thresh;
189
+
u32 rx_copybreak;
238
190
u32 rx_ring_mask;
239
191
u32 rx_agg_ring_mask;
240
192
u16 rx_nr_pages;
···
279
231
u8 rss_hash_key_updated:1;
280
232
int rsscos_nr_ctxs;
281
233
u32 stats_coal_ticks;
234
+
235
+
unsigned long state;
236
+
#define BNGE_STATE_NAPI_DISABLED 0
237
+
238
+
u32 msg_enable;
239
+
u16 max_tpa;
240
+
__be16 vxlan_port;
241
+
__be16 nge_port;
242
+
__be16 vxlan_gpe_port;
282
243
};
283
244
284
245
#define BNGE_DEFAULT_RX_RING_SIZE 511
···
334
277
txr = (iter < BNGE_MAX_TXR_PER_NAPI - 1) ? \
335
278
(bnapi)->tx_ring[++iter] : NULL)
336
279
280
+
#define DB_EPOCH(db, idx) (((idx) & (db)->db_epoch_mask) << \
281
+
((db)->db_epoch_shift))
282
+
283
+
#define DB_TOGGLE(tgl) ((tgl) << DBR_TOGGLE_SFT)
284
+
285
+
#define DB_RING_IDX(db, idx) (((idx) & (db)->db_ring_mask) | \
286
+
DB_EPOCH(db, idx))
287
+
337
288
#define BNGE_SET_NQ_HDL(cpr) \
338
289
(((cpr)->cp_ring_type << BNGE_NQ_HDL_TYPE_SHIFT) | (cpr)->cp_idx)
290
+
291
+
#define BNGE_DB_NQ(bd, db, idx) \
292
+
bnge_writeq(bd, (db)->db_key64 | DBR_TYPE_NQ | DB_RING_IDX(db, idx),\
293
+
(db)->doorbell)
294
+
295
+
#define BNGE_DB_NQ_ARM(bd, db, idx) \
296
+
bnge_writeq(bd, (db)->db_key64 | DBR_TYPE_NQ_ARM | \
297
+
DB_RING_IDX(db, idx), (db)->doorbell)
339
298
340
299
struct bnge_stats_mem {
341
300
u64 *sw_stats;
···
360
287
dma_addr_t hw_stats_map;
361
288
int len;
362
289
};
290
+
291
+
struct nqe_cn {
292
+
__le16 type;
293
+
#define NQ_CN_TYPE_MASK 0x3fUL
294
+
#define NQ_CN_TYPE_SFT 0
295
+
#define NQ_CN_TYPE_CQ_NOTIFICATION 0x30UL
296
+
#define NQ_CN_TYPE_LAST NQ_CN_TYPE_CQ_NOTIFICATION
297
+
#define NQ_CN_TOGGLE_MASK 0xc0UL
298
+
#define NQ_CN_TOGGLE_SFT 6
299
+
__le16 reserved16;
300
+
__le32 cq_handle_low;
301
+
__le32 v;
302
+
#define NQ_CN_V 0x1UL
303
+
__le32 cq_handle_high;
304
+
};
305
+
306
+
#define NQE_CN_TYPE(type) ((type) & NQ_CN_TYPE_MASK)
307
+
#define NQE_CN_TOGGLE(type) (((type) & NQ_CN_TOGGLE_MASK) >> \
308
+
NQ_CN_TOGGLE_SFT)
363
309
364
310
struct bnge_cp_ring_info {
365
311
struct bnge_napi *bnapi;
···
389
297
u8 cp_idx;
390
298
u32 cp_raw_cons;
391
299
struct bnge_db_info cp_db;
300
+
bool had_work_done;
301
+
bool has_more_work;
302
+
bool had_nqe_notify;
303
+
u8 toggle;
392
304
};
393
305
394
306
struct bnge_nq_ring_info {
···
405
309
406
310
struct bnge_stats_mem stats;
407
311
u32 hw_stats_ctx_id;
312
+
bool has_more_work;
408
313
409
-
int cp_ring_count;
314
+
u16 cp_ring_count;
410
315
struct bnge_cp_ring_info *cp_ring_arr;
411
316
};
412
317
···
432
335
433
336
dma_addr_t rx_desc_mapping[MAX_RX_PAGES];
434
337
dma_addr_t rx_agg_desc_mapping[MAX_RX_AGG_PAGES];
338
+
339
+
struct bnge_tpa_info *rx_tpa;
340
+
struct bnge_tpa_idx_map *rx_tpa_idx_map;
435
341
436
342
struct bnge_ring_struct rx_ring_struct;
437
343
struct bnge_ring_struct rx_agg_ring_struct;
···
473
373
struct bnge_nq_ring_info nq_ring;
474
374
struct bnge_rx_ring_info *rx_ring;
475
375
struct bnge_tx_ring_info *tx_ring[BNGE_MAX_TXR_PER_NAPI];
376
+
u8 events;
377
+
#define BNGE_RX_EVENT 1
378
+
#define BNGE_AGG_EVENT 2
379
+
#define BNGE_TX_EVENT 4
380
+
#define BNGE_REDIRECT_EVENT 8
381
+
#define BNGE_TX_CMP_EVENT 0x10
382
+
bool in_reset;
383
+
bool tx_fault;
476
384
};
477
385
478
386
#define INVALID_STATS_CTX_ID -1
···
559
451
u16 bnge_cp_ring_for_rx(struct bnge_rx_ring_info *rxr);
560
452
u16 bnge_cp_ring_for_tx(struct bnge_tx_ring_info *txr);
561
453
void bnge_fill_hw_rss_tbl(struct bnge_net *bn, struct bnge_vnic_info *vnic);
454
+
int bnge_alloc_rx_data(struct bnge_net *bn, struct bnge_rx_ring_info *rxr,
455
+
u16 prod, gfp_t gfp);
456
+
u16 bnge_find_next_agg_idx(struct bnge_rx_ring_info *rxr, u16 idx);
457
+
u8 *__bnge_alloc_rx_frag(struct bnge_net *bn, dma_addr_t *mapping,
458
+
struct bnge_rx_ring_info *rxr, gfp_t gfp);
459
+
int bnge_alloc_rx_netmem(struct bnge_net *bn, struct bnge_rx_ring_info *rxr,
460
+
u16 prod, gfp_t gfp);
562
461
#endif /* _BNGE_NETDEV_H_ */
+1645
drivers/net/ethernet/broadcom/bnge/bnge_txrx.c
+1645
drivers/net/ethernet/broadcom/bnge/bnge_txrx.c
···
1
+
// SPDX-License-Identifier: GPL-2.0
2
+
// Copyright (c) 2025 Broadcom.
3
+
4
+
#include <asm/byteorder.h>
5
+
#include <linux/dma-mapping.h>
6
+
#include <linux/dmapool.h>
7
+
#include <linux/delay.h>
8
+
#include <linux/errno.h>
9
+
#include <linux/kernel.h>
10
+
#include <linux/list.h>
11
+
#include <linux/pci.h>
12
+
#include <linux/netdevice.h>
13
+
#include <linux/etherdevice.h>
14
+
#include <linux/if.h>
15
+
#include <net/ip.h>
16
+
#include <net/tcp.h>
17
+
#include <net/gro.h>
18
+
#include <linux/skbuff.h>
19
+
#include <net/page_pool/helpers.h>
20
+
#include <linux/if_vlan.h>
21
+
#include <net/udp_tunnel.h>
22
+
#include <net/dst_metadata.h>
23
+
#include <net/netdev_queues.h>
24
+
25
+
#include "bnge.h"
26
+
#include "bnge_hwrm.h"
27
+
#include "bnge_hwrm_lib.h"
28
+
#include "bnge_netdev.h"
29
+
#include "bnge_rmem.h"
30
+
#include "bnge_txrx.h"
31
+
32
+
irqreturn_t bnge_msix(int irq, void *dev_instance)
33
+
{
34
+
struct bnge_napi *bnapi = dev_instance;
35
+
struct bnge_nq_ring_info *nqr;
36
+
struct bnge_net *bn;
37
+
u32 cons;
38
+
39
+
bn = bnapi->bn;
40
+
nqr = &bnapi->nq_ring;
41
+
cons = RING_CMP(bn, nqr->nq_raw_cons);
42
+
43
+
prefetch(&nqr->desc_ring[CP_RING(cons)][CP_IDX(cons)]);
44
+
napi_schedule(&bnapi->napi);
45
+
return IRQ_HANDLED;
46
+
}
47
+
48
+
static struct rx_agg_cmp *bnge_get_tpa_agg(struct bnge_net *bn,
49
+
struct bnge_rx_ring_info *rxr,
50
+
u16 agg_id, u16 curr)
51
+
{
52
+
struct bnge_tpa_info *tpa_info = &rxr->rx_tpa[agg_id];
53
+
54
+
return &tpa_info->agg_arr[curr];
55
+
}
56
+
57
+
static struct rx_agg_cmp *bnge_get_agg(struct bnge_net *bn,
58
+
struct bnge_cp_ring_info *cpr,
59
+
u16 cp_cons, u16 curr)
60
+
{
61
+
struct rx_agg_cmp *agg;
62
+
63
+
cp_cons = RING_CMP(bn, ADV_RAW_CMP(cp_cons, curr));
64
+
agg = (struct rx_agg_cmp *)
65
+
&cpr->desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
66
+
return agg;
67
+
}
68
+
69
+
static void bnge_reuse_rx_agg_bufs(struct bnge_cp_ring_info *cpr, u16 idx,
70
+
u16 start, u32 agg_bufs, bool tpa)
71
+
{
72
+
struct bnge_napi *bnapi = cpr->bnapi;
73
+
struct bnge_net *bn = bnapi->bn;
74
+
struct bnge_rx_ring_info *rxr;
75
+
u16 prod, sw_prod;
76
+
u32 i;
77
+
78
+
rxr = bnapi->rx_ring;
79
+
sw_prod = rxr->rx_sw_agg_prod;
80
+
prod = rxr->rx_agg_prod;
81
+
82
+
for (i = 0; i < agg_bufs; i++) {
83
+
struct bnge_sw_rx_agg_bd *cons_rx_buf, *prod_rx_buf;
84
+
struct rx_agg_cmp *agg;
85
+
struct rx_bd *prod_bd;
86
+
netmem_ref netmem;
87
+
u16 cons;
88
+
89
+
if (tpa)
90
+
agg = bnge_get_tpa_agg(bn, rxr, idx, start + i);
91
+
else
92
+
agg = bnge_get_agg(bn, cpr, idx, start + i);
93
+
cons = agg->rx_agg_cmp_opaque;
94
+
__clear_bit(cons, rxr->rx_agg_bmap);
95
+
96
+
if (unlikely(test_bit(sw_prod, rxr->rx_agg_bmap)))
97
+
sw_prod = bnge_find_next_agg_idx(rxr, sw_prod);
98
+
99
+
__set_bit(sw_prod, rxr->rx_agg_bmap);
100
+
prod_rx_buf = &rxr->rx_agg_buf_ring[sw_prod];
101
+
cons_rx_buf = &rxr->rx_agg_buf_ring[cons];
102
+
103
+
/* It is possible for sw_prod to be equal to cons, so
104
+
* set cons_rx_buf->netmem to 0 first.
105
+
*/
106
+
netmem = cons_rx_buf->netmem;
107
+
cons_rx_buf->netmem = 0;
108
+
prod_rx_buf->netmem = netmem;
109
+
prod_rx_buf->offset = cons_rx_buf->offset;
110
+
111
+
prod_rx_buf->mapping = cons_rx_buf->mapping;
112
+
113
+
prod_bd = &rxr->rx_agg_desc_ring[RX_AGG_RING(bn, prod)]
114
+
[RX_IDX(prod)];
115
+
116
+
prod_bd->rx_bd_haddr = cpu_to_le64(cons_rx_buf->mapping);
117
+
prod_bd->rx_bd_opaque = sw_prod;
118
+
119
+
prod = NEXT_RX_AGG(prod);
120
+
sw_prod = RING_RX_AGG(bn, NEXT_RX_AGG(sw_prod));
121
+
}
122
+
rxr->rx_agg_prod = prod;
123
+
rxr->rx_sw_agg_prod = sw_prod;
124
+
}
125
+
126
+
static int bnge_agg_bufs_valid(struct bnge_net *bn,
127
+
struct bnge_cp_ring_info *cpr,
128
+
u8 agg_bufs, u32 *raw_cons)
129
+
{
130
+
struct rx_agg_cmp *agg;
131
+
u16 last;
132
+
133
+
*raw_cons = ADV_RAW_CMP(*raw_cons, agg_bufs);
134
+
last = RING_CMP(bn, *raw_cons);
135
+
agg = (struct rx_agg_cmp *)
136
+
&cpr->desc_ring[CP_RING(last)][CP_IDX(last)];
137
+
return RX_AGG_CMP_VALID(bn, agg, *raw_cons);
138
+
}
139
+
140
+
static int bnge_discard_rx(struct bnge_net *bn, struct bnge_cp_ring_info *cpr,
141
+
u32 *raw_cons, void *cmp)
142
+
{
143
+
u32 tmp_raw_cons = *raw_cons;
144
+
struct rx_cmp *rxcmp = cmp;
145
+
u8 cmp_type, agg_bufs = 0;
146
+
147
+
cmp_type = RX_CMP_TYPE(rxcmp);
148
+
149
+
if (cmp_type == CMP_TYPE_RX_L2_CMP) {
150
+
agg_bufs = (le32_to_cpu(rxcmp->rx_cmp_misc_v1) &
151
+
RX_CMP_AGG_BUFS) >>
152
+
RX_CMP_AGG_BUFS_SHIFT;
153
+
} else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
154
+
return 0;
155
+
}
156
+
157
+
if (agg_bufs) {
158
+
if (!bnge_agg_bufs_valid(bn, cpr, agg_bufs, &tmp_raw_cons))
159
+
return -EBUSY;
160
+
}
161
+
*raw_cons = tmp_raw_cons;
162
+
return 0;
163
+
}
164
+
165
+
static u32 __bnge_rx_agg_netmems(struct bnge_net *bn,
166
+
struct bnge_cp_ring_info *cpr,
167
+
u16 idx, u32 agg_bufs, bool tpa,
168
+
struct sk_buff *skb)
169
+
{
170
+
struct bnge_napi *bnapi = cpr->bnapi;
171
+
struct skb_shared_info *shinfo;
172
+
struct bnge_rx_ring_info *rxr;
173
+
u32 i, total_frag_len = 0;
174
+
u16 prod;
175
+
176
+
rxr = bnapi->rx_ring;
177
+
prod = rxr->rx_agg_prod;
178
+
shinfo = skb_shinfo(skb);
179
+
180
+
for (i = 0; i < agg_bufs; i++) {
181
+
struct bnge_sw_rx_agg_bd *cons_rx_buf;
182
+
struct rx_agg_cmp *agg;
183
+
u16 cons, frag_len;
184
+
netmem_ref netmem;
185
+
186
+
if (tpa)
187
+
agg = bnge_get_tpa_agg(bn, rxr, idx, i);
188
+
else
189
+
agg = bnge_get_agg(bn, cpr, idx, i);
190
+
cons = agg->rx_agg_cmp_opaque;
191
+
frag_len = (le32_to_cpu(agg->rx_agg_cmp_len_flags_type) &
192
+
RX_AGG_CMP_LEN) >> RX_AGG_CMP_LEN_SHIFT;
193
+
194
+
cons_rx_buf = &rxr->rx_agg_buf_ring[cons];
195
+
skb_add_rx_frag_netmem(skb, i, cons_rx_buf->netmem,
196
+
cons_rx_buf->offset,
197
+
frag_len, BNGE_RX_PAGE_SIZE);
198
+
__clear_bit(cons, rxr->rx_agg_bmap);
199
+
200
+
/* It is possible for bnge_alloc_rx_netmem() to allocate
201
+
* a sw_prod index that equals the cons index, so we
202
+
* need to clear the cons entry now.
203
+
*/
204
+
netmem = cons_rx_buf->netmem;
205
+
cons_rx_buf->netmem = 0;
206
+
207
+
if (bnge_alloc_rx_netmem(bn, rxr, prod, GFP_ATOMIC) != 0) {
208
+
skb->len -= frag_len;
209
+
skb->data_len -= frag_len;
210
+
skb->truesize -= BNGE_RX_PAGE_SIZE;
211
+
212
+
--shinfo->nr_frags;
213
+
cons_rx_buf->netmem = netmem;
214
+
215
+
/* Update prod since possibly some netmems have been
216
+
* allocated already.
217
+
*/
218
+
rxr->rx_agg_prod = prod;
219
+
bnge_reuse_rx_agg_bufs(cpr, idx, i, agg_bufs - i, tpa);
220
+
return 0;
221
+
}
222
+
223
+
page_pool_dma_sync_netmem_for_cpu(rxr->page_pool, netmem, 0,
224
+
BNGE_RX_PAGE_SIZE);
225
+
226
+
total_frag_len += frag_len;
227
+
prod = NEXT_RX_AGG(prod);
228
+
}
229
+
rxr->rx_agg_prod = prod;
230
+
return total_frag_len;
231
+
}
232
+
233
+
static struct sk_buff *bnge_rx_agg_netmems_skb(struct bnge_net *bn,
234
+
struct bnge_cp_ring_info *cpr,
235
+
struct sk_buff *skb, u16 idx,
236
+
u32 agg_bufs, bool tpa)
237
+
{
238
+
u32 total_frag_len;
239
+
240
+
total_frag_len = __bnge_rx_agg_netmems(bn, cpr, idx, agg_bufs,
241
+
tpa, skb);
242
+
if (!total_frag_len) {
243
+
skb_mark_for_recycle(skb);
244
+
dev_kfree_skb(skb);
245
+
return NULL;
246
+
}
247
+
248
+
return skb;
249
+
}
250
+
251
+
static void bnge_sched_reset_rxr(struct bnge_net *bn,
252
+
struct bnge_rx_ring_info *rxr)
253
+
{
254
+
if (!rxr->bnapi->in_reset) {
255
+
rxr->bnapi->in_reset = true;
256
+
257
+
/* TODO: Initiate reset task */
258
+
}
259
+
rxr->rx_next_cons = 0xffff;
260
+
}
261
+
262
+
static void bnge_sched_reset_txr(struct bnge_net *bn,
263
+
struct bnge_tx_ring_info *txr,
264
+
u16 curr)
265
+
{
266
+
struct bnge_napi *bnapi = txr->bnapi;
267
+
268
+
if (bnapi->tx_fault)
269
+
return;
270
+
271
+
netdev_err(bn->netdev, "Invalid Tx completion (ring:%d tx_hw_cons:%u cons:%u prod:%u curr:%u)",
272
+
txr->txq_index, txr->tx_hw_cons,
273
+
txr->tx_cons, txr->tx_prod, curr);
274
+
WARN_ON_ONCE(1);
275
+
bnapi->tx_fault = 1;
276
+
/* TODO: Initiate reset task */
277
+
}
278
+
279
+
static u16 bnge_tpa_alloc_agg_idx(struct bnge_rx_ring_info *rxr, u16 agg_id)
280
+
{
281
+
struct bnge_tpa_idx_map *map = rxr->rx_tpa_idx_map;
282
+
u16 idx = agg_id & MAX_TPA_MASK;
283
+
284
+
if (test_bit(idx, map->agg_idx_bmap)) {
285
+
idx = find_first_zero_bit(map->agg_idx_bmap, MAX_TPA);
286
+
if (idx >= MAX_TPA)
287
+
return INVALID_HW_RING_ID;
288
+
}
289
+
__set_bit(idx, map->agg_idx_bmap);
290
+
map->agg_id_tbl[agg_id] = idx;
291
+
return idx;
292
+
}
293
+
294
+
static void bnge_free_agg_idx(struct bnge_rx_ring_info *rxr, u16 idx)
295
+
{
296
+
struct bnge_tpa_idx_map *map = rxr->rx_tpa_idx_map;
297
+
298
+
__clear_bit(idx, map->agg_idx_bmap);
299
+
}
300
+
301
+
static u16 bnge_lookup_agg_idx(struct bnge_rx_ring_info *rxr, u16 agg_id)
302
+
{
303
+
struct bnge_tpa_idx_map *map = rxr->rx_tpa_idx_map;
304
+
305
+
return map->agg_id_tbl[agg_id];
306
+
}
307
+
308
+
static void bnge_tpa_metadata(struct bnge_tpa_info *tpa_info,
309
+
struct rx_tpa_start_cmp *tpa_start,
310
+
struct rx_tpa_start_cmp_ext *tpa_start1)
311
+
{
312
+
tpa_info->cfa_code_valid = 1;
313
+
tpa_info->cfa_code = TPA_START_CFA_CODE(tpa_start1);
314
+
tpa_info->vlan_valid = 0;
315
+
if (tpa_info->flags2 & RX_CMP_FLAGS2_META_FORMAT_VLAN) {
316
+
tpa_info->vlan_valid = 1;
317
+
tpa_info->metadata =
318
+
le32_to_cpu(tpa_start1->rx_tpa_start_cmp_metadata);
319
+
}
320
+
}
321
+
322
+
static void bnge_tpa_metadata_v2(struct bnge_tpa_info *tpa_info,
323
+
struct rx_tpa_start_cmp *tpa_start,
324
+
struct rx_tpa_start_cmp_ext *tpa_start1)
325
+
{
326
+
tpa_info->vlan_valid = 0;
327
+
if (TPA_START_VLAN_VALID(tpa_start)) {
328
+
u32 tpid_sel = TPA_START_VLAN_TPID_SEL(tpa_start);
329
+
u32 vlan_proto = ETH_P_8021Q;
330
+
331
+
tpa_info->vlan_valid = 1;
332
+
if (tpid_sel == RX_TPA_START_METADATA1_TPID_8021AD)
333
+
vlan_proto = ETH_P_8021AD;
334
+
tpa_info->metadata = vlan_proto << 16 |
335
+
TPA_START_METADATA0_TCI(tpa_start1);
336
+
}
337
+
}
338
+
339
+
static void bnge_tpa_start(struct bnge_net *bn, struct bnge_rx_ring_info *rxr,
340
+
u8 cmp_type, struct rx_tpa_start_cmp *tpa_start,
341
+
struct rx_tpa_start_cmp_ext *tpa_start1)
342
+
{
343
+
struct bnge_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
344
+
struct bnge_tpa_info *tpa_info;
345
+
u16 cons, prod, agg_id;
346
+
struct rx_bd *prod_bd;
347
+
dma_addr_t mapping;
348
+
349
+
agg_id = TPA_START_AGG_ID(tpa_start);
350
+
agg_id = bnge_tpa_alloc_agg_idx(rxr, agg_id);
351
+
if (unlikely(agg_id == INVALID_HW_RING_ID)) {
352
+
netdev_warn(bn->netdev, "Unable to allocate agg ID for ring %d, agg 0x%lx\n",
353
+
rxr->bnapi->index, TPA_START_AGG_ID(tpa_start));
354
+
bnge_sched_reset_rxr(bn, rxr);
355
+
return;
356
+
}
357
+
cons = tpa_start->rx_tpa_start_cmp_opaque;
358
+
prod = rxr->rx_prod;
359
+
cons_rx_buf = &rxr->rx_buf_ring[cons];
360
+
prod_rx_buf = &rxr->rx_buf_ring[RING_RX(bn, prod)];
361
+
tpa_info = &rxr->rx_tpa[agg_id];
362
+
363
+
if (unlikely(cons != rxr->rx_next_cons ||
364
+
TPA_START_ERROR(tpa_start))) {
365
+
netdev_warn(bn->netdev, "TPA cons %x, expected cons %x, error code %lx\n",
366
+
cons, rxr->rx_next_cons,
367
+
TPA_START_ERROR_CODE(tpa_start1));
368
+
bnge_sched_reset_rxr(bn, rxr);
369
+
return;
370
+
}
371
+
prod_rx_buf->data = tpa_info->data;
372
+
prod_rx_buf->data_ptr = tpa_info->data_ptr;
373
+
374
+
mapping = tpa_info->mapping;
375
+
prod_rx_buf->mapping = mapping;
376
+
377
+
prod_bd = &rxr->rx_desc_ring[RX_RING(bn, prod)][RX_IDX(prod)];
378
+
379
+
prod_bd->rx_bd_haddr = cpu_to_le64(mapping);
380
+
381
+
tpa_info->data = cons_rx_buf->data;
382
+
tpa_info->data_ptr = cons_rx_buf->data_ptr;
383
+
cons_rx_buf->data = NULL;
384
+
tpa_info->mapping = cons_rx_buf->mapping;
385
+
386
+
tpa_info->len =
387
+
le32_to_cpu(tpa_start->rx_tpa_start_cmp_len_flags_type) >>
388
+
RX_TPA_START_CMP_LEN_SHIFT;
389
+
if (likely(TPA_START_HASH_VALID(tpa_start))) {
390
+
tpa_info->hash_type = PKT_HASH_TYPE_L4;
391
+
if (TPA_START_IS_IPV6(tpa_start1))
392
+
tpa_info->gso_type = SKB_GSO_TCPV6;
393
+
else
394
+
tpa_info->gso_type = SKB_GSO_TCPV4;
395
+
tpa_info->rss_hash =
396
+
le32_to_cpu(tpa_start->rx_tpa_start_cmp_rss_hash);
397
+
} else {
398
+
tpa_info->hash_type = PKT_HASH_TYPE_NONE;
399
+
tpa_info->gso_type = 0;
400
+
netif_warn(bn, rx_err, bn->netdev, "TPA packet without valid hash\n");
401
+
}
402
+
tpa_info->flags2 = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_flags2);
403
+
tpa_info->hdr_info = le32_to_cpu(tpa_start1->rx_tpa_start_cmp_hdr_info);
404
+
if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP)
405
+
bnge_tpa_metadata(tpa_info, tpa_start, tpa_start1);
406
+
else
407
+
bnge_tpa_metadata_v2(tpa_info, tpa_start, tpa_start1);
408
+
tpa_info->agg_count = 0;
409
+
410
+
rxr->rx_prod = NEXT_RX(prod);
411
+
cons = RING_RX(bn, NEXT_RX(cons));
412
+
rxr->rx_next_cons = RING_RX(bn, NEXT_RX(cons));
413
+
cons_rx_buf = &rxr->rx_buf_ring[cons];
414
+
415
+
bnge_reuse_rx_data(rxr, cons, cons_rx_buf->data);
416
+
rxr->rx_prod = NEXT_RX(rxr->rx_prod);
417
+
cons_rx_buf->data = NULL;
418
+
}
419
+
420
+
static void bnge_abort_tpa(struct bnge_cp_ring_info *cpr, u16 idx, u32 agg_bufs)
421
+
{
422
+
if (agg_bufs)
423
+
bnge_reuse_rx_agg_bufs(cpr, idx, 0, agg_bufs, true);
424
+
}
425
+
426
+
static void bnge_tpa_agg(struct bnge_net *bn, struct bnge_rx_ring_info *rxr,
427
+
struct rx_agg_cmp *rx_agg)
428
+
{
429
+
u16 agg_id = TPA_AGG_AGG_ID(rx_agg);
430
+
struct bnge_tpa_info *tpa_info;
431
+
432
+
agg_id = bnge_lookup_agg_idx(rxr, agg_id);
433
+
tpa_info = &rxr->rx_tpa[agg_id];
434
+
435
+
if (unlikely(tpa_info->agg_count >= MAX_SKB_FRAGS)) {
436
+
netdev_warn(bn->netdev,
437
+
"TPA completion count %d exceeds limit for ring %d\n",
438
+
tpa_info->agg_count, rxr->bnapi->index);
439
+
440
+
bnge_sched_reset_rxr(bn, rxr);
441
+
return;
442
+
}
443
+
444
+
tpa_info->agg_arr[tpa_info->agg_count++] = *rx_agg;
445
+
}
446
+
447
+
void bnge_reuse_rx_data(struct bnge_rx_ring_info *rxr, u16 cons, void *data)
448
+
{
449
+
struct bnge_sw_rx_bd *cons_rx_buf, *prod_rx_buf;
450
+
struct bnge_net *bn = rxr->bnapi->bn;
451
+
struct rx_bd *cons_bd, *prod_bd;
452
+
u16 prod = rxr->rx_prod;
453
+
454
+
prod_rx_buf = &rxr->rx_buf_ring[RING_RX(bn, prod)];
455
+
cons_rx_buf = &rxr->rx_buf_ring[cons];
456
+
457
+
prod_rx_buf->data = data;
458
+
prod_rx_buf->data_ptr = cons_rx_buf->data_ptr;
459
+
460
+
prod_rx_buf->mapping = cons_rx_buf->mapping;
461
+
462
+
prod_bd = &rxr->rx_desc_ring[RX_RING(bn, prod)][RX_IDX(prod)];
463
+
cons_bd = &rxr->rx_desc_ring[RX_RING(bn, cons)][RX_IDX(cons)];
464
+
465
+
prod_bd->rx_bd_haddr = cons_bd->rx_bd_haddr;
466
+
}
467
+
468
+
static void bnge_deliver_skb(struct bnge_net *bn, struct bnge_napi *bnapi,
469
+
struct sk_buff *skb)
470
+
{
471
+
skb_mark_for_recycle(skb);
472
+
skb_record_rx_queue(skb, bnapi->index);
473
+
napi_gro_receive(&bnapi->napi, skb);
474
+
}
475
+
476
+
static struct sk_buff *bnge_copy_skb(struct bnge_napi *bnapi, u8 *data,
477
+
unsigned int len, dma_addr_t mapping)
478
+
{
479
+
struct bnge_net *bn = bnapi->bn;
480
+
struct bnge_dev *bd = bn->bd;
481
+
struct sk_buff *skb;
482
+
483
+
skb = napi_alloc_skb(&bnapi->napi, len);
484
+
if (!skb)
485
+
return NULL;
486
+
487
+
dma_sync_single_for_cpu(bd->dev, mapping, len, bn->rx_dir);
488
+
489
+
memcpy(skb->data - NET_IP_ALIGN, data - NET_IP_ALIGN,
490
+
len + NET_IP_ALIGN);
491
+
492
+
dma_sync_single_for_device(bd->dev, mapping, len, bn->rx_dir);
493
+
494
+
skb_put(skb, len);
495
+
496
+
return skb;
497
+
}
498
+
499
+
#ifdef CONFIG_INET
500
+
static void bnge_gro_tunnel(struct sk_buff *skb, __be16 ip_proto)
501
+
{
502
+
struct udphdr *uh = NULL;
503
+
504
+
if (ip_proto == htons(ETH_P_IP)) {
505
+
struct iphdr *iph = (struct iphdr *)skb->data;
506
+
507
+
if (iph->protocol == IPPROTO_UDP)
508
+
uh = (struct udphdr *)(iph + 1);
509
+
} else {
510
+
struct ipv6hdr *iph = (struct ipv6hdr *)skb->data;
511
+
512
+
if (iph->nexthdr == IPPROTO_UDP)
513
+
uh = (struct udphdr *)(iph + 1);
514
+
}
515
+
if (uh) {
516
+
if (uh->check)
517
+
skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM;
518
+
else
519
+
skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
520
+
}
521
+
}
522
+
523
+
static struct sk_buff *bnge_gro_func(struct bnge_tpa_info *tpa_info,
524
+
int payload_off, int tcp_ts,
525
+
struct sk_buff *skb)
526
+
{
527
+
u16 outer_ip_off, inner_ip_off, inner_mac_off;
528
+
u32 hdr_info = tpa_info->hdr_info;
529
+
int iphdr_len, nw_off;
530
+
531
+
inner_ip_off = BNGE_TPA_INNER_L3_OFF(hdr_info);
532
+
inner_mac_off = BNGE_TPA_INNER_L2_OFF(hdr_info);
533
+
outer_ip_off = BNGE_TPA_OUTER_L3_OFF(hdr_info);
534
+
535
+
nw_off = inner_ip_off - ETH_HLEN;
536
+
skb_set_network_header(skb, nw_off);
537
+
iphdr_len = (tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_IP_TYPE) ?
538
+
sizeof(struct ipv6hdr) : sizeof(struct iphdr);
539
+
skb_set_transport_header(skb, nw_off + iphdr_len);
540
+
541
+
if (inner_mac_off) { /* tunnel */
542
+
__be16 proto = *((__be16 *)(skb->data + outer_ip_off -
543
+
ETH_HLEN - 2));
544
+
545
+
bnge_gro_tunnel(skb, proto);
546
+
}
547
+
548
+
return skb;
549
+
}
550
+
551
+
static struct sk_buff *bnge_gro_skb(struct bnge_net *bn,
552
+
struct bnge_tpa_info *tpa_info,
553
+
struct rx_tpa_end_cmp *tpa_end,
554
+
struct rx_tpa_end_cmp_ext *tpa_end1,
555
+
struct sk_buff *skb)
556
+
{
557
+
int payload_off;
558
+
u16 segs;
559
+
560
+
segs = TPA_END_TPA_SEGS(tpa_end);
561
+
if (segs == 1)
562
+
return skb;
563
+
564
+
NAPI_GRO_CB(skb)->count = segs;
565
+
skb_shinfo(skb)->gso_size =
566
+
le32_to_cpu(tpa_end1->rx_tpa_end_cmp_seg_len);
567
+
skb_shinfo(skb)->gso_type = tpa_info->gso_type;
568
+
payload_off = TPA_END_PAYLOAD_OFF(tpa_end1);
569
+
skb = bnge_gro_func(tpa_info, payload_off,
570
+
TPA_END_GRO_TS(tpa_end), skb);
571
+
if (likely(skb))
572
+
tcp_gro_complete(skb);
573
+
574
+
return skb;
575
+
}
576
+
#endif
577
+
578
+
static struct sk_buff *bnge_tpa_end(struct bnge_net *bn,
579
+
struct bnge_cp_ring_info *cpr,
580
+
u32 *raw_cons,
581
+
struct rx_tpa_end_cmp *tpa_end,
582
+
struct rx_tpa_end_cmp_ext *tpa_end1,
583
+
u8 *event)
584
+
{
585
+
struct bnge_napi *bnapi = cpr->bnapi;
586
+
struct net_device *dev = bn->netdev;
587
+
struct bnge_tpa_info *tpa_info;
588
+
struct bnge_rx_ring_info *rxr;
589
+
u8 *data_ptr, agg_bufs;
590
+
struct sk_buff *skb;
591
+
u16 idx = 0, agg_id;
592
+
dma_addr_t mapping;
593
+
unsigned int len;
594
+
void *data;
595
+
596
+
if (unlikely(bnapi->in_reset)) {
597
+
int rc = bnge_discard_rx(bn, cpr, raw_cons, tpa_end);
598
+
599
+
if (rc < 0)
600
+
return ERR_PTR(-EBUSY);
601
+
return NULL;
602
+
}
603
+
604
+
rxr = bnapi->rx_ring;
605
+
agg_id = TPA_END_AGG_ID(tpa_end);
606
+
agg_id = bnge_lookup_agg_idx(rxr, agg_id);
607
+
agg_bufs = TPA_END_AGG_BUFS(tpa_end1);
608
+
tpa_info = &rxr->rx_tpa[agg_id];
609
+
if (unlikely(agg_bufs != tpa_info->agg_count)) {
610
+
netdev_warn(bn->netdev, "TPA end agg_buf %d != expected agg_bufs %d\n",
611
+
agg_bufs, tpa_info->agg_count);
612
+
agg_bufs = tpa_info->agg_count;
613
+
}
614
+
tpa_info->agg_count = 0;
615
+
*event |= BNGE_AGG_EVENT;
616
+
bnge_free_agg_idx(rxr, agg_id);
617
+
idx = agg_id;
618
+
data = tpa_info->data;
619
+
data_ptr = tpa_info->data_ptr;
620
+
prefetch(data_ptr);
621
+
len = tpa_info->len;
622
+
mapping = tpa_info->mapping;
623
+
624
+
if (unlikely(agg_bufs > MAX_SKB_FRAGS || TPA_END_ERRORS(tpa_end1))) {
625
+
bnge_abort_tpa(cpr, idx, agg_bufs);
626
+
if (agg_bufs > MAX_SKB_FRAGS)
627
+
netdev_warn(bn->netdev, "TPA frags %d exceeded MAX_SKB_FRAGS %d\n",
628
+
agg_bufs, (int)MAX_SKB_FRAGS);
629
+
return NULL;
630
+
}
631
+
632
+
if (len <= bn->rx_copybreak) {
633
+
skb = bnge_copy_skb(bnapi, data_ptr, len, mapping);
634
+
if (!skb) {
635
+
bnge_abort_tpa(cpr, idx, agg_bufs);
636
+
return NULL;
637
+
}
638
+
} else {
639
+
dma_addr_t new_mapping;
640
+
u8 *new_data;
641
+
642
+
new_data = __bnge_alloc_rx_frag(bn, &new_mapping, rxr,
643
+
GFP_ATOMIC);
644
+
if (!new_data) {
645
+
bnge_abort_tpa(cpr, idx, agg_bufs);
646
+
return NULL;
647
+
}
648
+
649
+
tpa_info->data = new_data;
650
+
tpa_info->data_ptr = new_data + bn->rx_offset;
651
+
tpa_info->mapping = new_mapping;
652
+
653
+
skb = napi_build_skb(data, bn->rx_buf_size);
654
+
dma_sync_single_for_cpu(bn->bd->dev, mapping,
655
+
bn->rx_buf_use_size, bn->rx_dir);
656
+
657
+
if (!skb) {
658
+
page_pool_free_va(rxr->head_pool, data, true);
659
+
bnge_abort_tpa(cpr, idx, agg_bufs);
660
+
return NULL;
661
+
}
662
+
skb_mark_for_recycle(skb);
663
+
skb_reserve(skb, bn->rx_offset);
664
+
skb_put(skb, len);
665
+
}
666
+
667
+
if (agg_bufs) {
668
+
skb = bnge_rx_agg_netmems_skb(bn, cpr, skb, idx, agg_bufs,
669
+
true);
670
+
/* Page reuse already handled by bnge_rx_agg_netmems_skb(). */
671
+
if (!skb)
672
+
return NULL;
673
+
}
674
+
675
+
skb->protocol = eth_type_trans(skb, dev);
676
+
677
+
if (tpa_info->hash_type != PKT_HASH_TYPE_NONE)
678
+
skb_set_hash(skb, tpa_info->rss_hash, tpa_info->hash_type);
679
+
680
+
if (tpa_info->vlan_valid &&
681
+
(dev->features & BNGE_HW_FEATURE_VLAN_ALL_RX)) {
682
+
__be16 vlan_proto = htons(tpa_info->metadata >>
683
+
RX_CMP_FLAGS2_METADATA_TPID_SFT);
684
+
u16 vtag = tpa_info->metadata & RX_CMP_FLAGS2_METADATA_TCI_MASK;
685
+
686
+
if (eth_type_vlan(vlan_proto)) {
687
+
__vlan_hwaccel_put_tag(skb, vlan_proto, vtag);
688
+
} else {
689
+
dev_kfree_skb(skb);
690
+
return NULL;
691
+
}
692
+
}
693
+
694
+
skb_checksum_none_assert(skb);
695
+
if (likely(tpa_info->flags2 & RX_TPA_START_CMP_FLAGS2_L4_CS_CALC)) {
696
+
skb->ip_summed = CHECKSUM_UNNECESSARY;
697
+
skb->csum_level =
698
+
(tpa_info->flags2 & RX_CMP_FLAGS2_T_L4_CS_CALC) >> 3;
699
+
}
700
+
701
+
#ifdef CONFIG_INET
702
+
if (bn->priv_flags & BNGE_NET_EN_GRO)
703
+
skb = bnge_gro_skb(bn, tpa_info, tpa_end, tpa_end1, skb);
704
+
#endif
705
+
706
+
return skb;
707
+
}
708
+
709
+
static enum pkt_hash_types bnge_rss_ext_op(struct bnge_net *bn,
710
+
struct rx_cmp *rxcmp)
711
+
{
712
+
u8 ext_op = RX_CMP_V3_HASH_TYPE(bn->bd, rxcmp);
713
+
714
+
switch (ext_op) {
715
+
case EXT_OP_INNER_4:
716
+
case EXT_OP_OUTER_4:
717
+
case EXT_OP_INNFL_3:
718
+
case EXT_OP_OUTFL_3:
719
+
return PKT_HASH_TYPE_L4;
720
+
default:
721
+
return PKT_HASH_TYPE_L3;
722
+
}
723
+
}
724
+
725
+
static struct sk_buff *bnge_rx_vlan(struct sk_buff *skb, u8 cmp_type,
726
+
struct rx_cmp *rxcmp,
727
+
struct rx_cmp_ext *rxcmp1)
728
+
{
729
+
__be16 vlan_proto;
730
+
u16 vtag;
731
+
732
+
if (cmp_type == CMP_TYPE_RX_L2_CMP) {
733
+
__le32 flags2 = rxcmp1->rx_cmp_flags2;
734
+
u32 meta_data;
735
+
736
+
if (!(flags2 & cpu_to_le32(RX_CMP_FLAGS2_META_FORMAT_VLAN)))
737
+
return skb;
738
+
739
+
meta_data = le32_to_cpu(rxcmp1->rx_cmp_meta_data);
740
+
vtag = meta_data & RX_CMP_FLAGS2_METADATA_TCI_MASK;
741
+
vlan_proto =
742
+
htons(meta_data >> RX_CMP_FLAGS2_METADATA_TPID_SFT);
743
+
if (eth_type_vlan(vlan_proto))
744
+
__vlan_hwaccel_put_tag(skb, vlan_proto, vtag);
745
+
else
746
+
goto vlan_err;
747
+
} else if (cmp_type == CMP_TYPE_RX_L2_V3_CMP) {
748
+
if (RX_CMP_VLAN_VALID(rxcmp)) {
749
+
u32 tpid_sel = RX_CMP_VLAN_TPID_SEL(rxcmp);
750
+
751
+
if (tpid_sel == RX_CMP_METADATA1_TPID_8021Q)
752
+
vlan_proto = htons(ETH_P_8021Q);
753
+
else if (tpid_sel == RX_CMP_METADATA1_TPID_8021AD)
754
+
vlan_proto = htons(ETH_P_8021AD);
755
+
else
756
+
goto vlan_err;
757
+
vtag = RX_CMP_METADATA0_TCI(rxcmp1);
758
+
__vlan_hwaccel_put_tag(skb, vlan_proto, vtag);
759
+
}
760
+
}
761
+
return skb;
762
+
763
+
vlan_err:
764
+
skb_mark_for_recycle(skb);
765
+
dev_kfree_skb(skb);
766
+
return NULL;
767
+
}
768
+
769
+
static struct sk_buff *bnge_rx_skb(struct bnge_net *bn,
770
+
struct bnge_rx_ring_info *rxr, u16 cons,
771
+
void *data, u8 *data_ptr,
772
+
dma_addr_t dma_addr,
773
+
unsigned int len)
774
+
{
775
+
struct bnge_dev *bd = bn->bd;
776
+
u16 prod = rxr->rx_prod;
777
+
struct sk_buff *skb;
778
+
int err;
779
+
780
+
err = bnge_alloc_rx_data(bn, rxr, prod, GFP_ATOMIC);
781
+
if (unlikely(err)) {
782
+
bnge_reuse_rx_data(rxr, cons, data);
783
+
return NULL;
784
+
}
785
+
786
+
dma_sync_single_for_cpu(bd->dev, dma_addr, len, bn->rx_dir);
787
+
skb = napi_build_skb(data, bn->rx_buf_size);
788
+
if (!skb) {
789
+
page_pool_free_va(rxr->head_pool, data, true);
790
+
return NULL;
791
+
}
792
+
793
+
skb_mark_for_recycle(skb);
794
+
skb_reserve(skb, bn->rx_offset);
795
+
skb_put(skb, len);
796
+
return skb;
797
+
}
798
+
799
+
/* returns the following:
800
+
* 1 - 1 packet successfully received
801
+
* 0 - successful TPA_START, packet not completed yet
802
+
* -EBUSY - completion ring does not have all the agg buffers yet
803
+
* -ENOMEM - packet aborted due to out of memory
804
+
* -EIO - packet aborted due to hw error indicated in BD
805
+
*/
806
+
static int bnge_rx_pkt(struct bnge_net *bn, struct bnge_cp_ring_info *cpr,
807
+
u32 *raw_cons, u8 *event)
808
+
{
809
+
struct bnge_napi *bnapi = cpr->bnapi;
810
+
struct net_device *dev = bn->netdev;
811
+
struct bnge_rx_ring_info *rxr;
812
+
u32 tmp_raw_cons, flags, misc;
813
+
struct bnge_sw_rx_bd *rx_buf;
814
+
struct rx_cmp_ext *rxcmp1;
815
+
u16 cons, prod, cp_cons;
816
+
u8 *data_ptr, cmp_type;
817
+
struct rx_cmp *rxcmp;
818
+
dma_addr_t dma_addr;
819
+
struct sk_buff *skb;
820
+
unsigned int len;
821
+
u8 agg_bufs;
822
+
void *data;
823
+
int rc = 0;
824
+
825
+
rxr = bnapi->rx_ring;
826
+
827
+
tmp_raw_cons = *raw_cons;
828
+
cp_cons = RING_CMP(bn, tmp_raw_cons);
829
+
rxcmp = (struct rx_cmp *)
830
+
&cpr->desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
831
+
832
+
cmp_type = RX_CMP_TYPE(rxcmp);
833
+
834
+
if (cmp_type == CMP_TYPE_RX_TPA_AGG_CMP) {
835
+
bnge_tpa_agg(bn, rxr, (struct rx_agg_cmp *)rxcmp);
836
+
goto next_rx_no_prod_no_len;
837
+
}
838
+
839
+
tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
840
+
cp_cons = RING_CMP(bn, tmp_raw_cons);
841
+
rxcmp1 = (struct rx_cmp_ext *)
842
+
&cpr->desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
843
+
844
+
if (!RX_CMP_VALID(bn, rxcmp1, tmp_raw_cons))
845
+
return -EBUSY;
846
+
847
+
/* The valid test of the entry must be done first before
848
+
* reading any further.
849
+
*/
850
+
dma_rmb();
851
+
prod = rxr->rx_prod;
852
+
853
+
if (cmp_type == CMP_TYPE_RX_L2_TPA_START_CMP ||
854
+
cmp_type == CMP_TYPE_RX_L2_TPA_START_V3_CMP) {
855
+
bnge_tpa_start(bn, rxr, cmp_type,
856
+
(struct rx_tpa_start_cmp *)rxcmp,
857
+
(struct rx_tpa_start_cmp_ext *)rxcmp1);
858
+
859
+
*event |= BNGE_RX_EVENT;
860
+
goto next_rx_no_prod_no_len;
861
+
862
+
} else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
863
+
skb = bnge_tpa_end(bn, cpr, &tmp_raw_cons,
864
+
(struct rx_tpa_end_cmp *)rxcmp,
865
+
(struct rx_tpa_end_cmp_ext *)rxcmp1, event);
866
+
if (IS_ERR(skb))
867
+
return -EBUSY;
868
+
869
+
rc = -ENOMEM;
870
+
if (likely(skb)) {
871
+
bnge_deliver_skb(bn, bnapi, skb);
872
+
rc = 1;
873
+
}
874
+
*event |= BNGE_RX_EVENT;
875
+
goto next_rx_no_prod_no_len;
876
+
}
877
+
878
+
cons = rxcmp->rx_cmp_opaque;
879
+
if (unlikely(cons != rxr->rx_next_cons)) {
880
+
int rc1 = bnge_discard_rx(bn, cpr, &tmp_raw_cons, rxcmp);
881
+
882
+
/* 0xffff is forced error, don't print it */
883
+
if (rxr->rx_next_cons != 0xffff)
884
+
netdev_warn(bn->netdev, "RX cons %x != expected cons %x\n",
885
+
cons, rxr->rx_next_cons);
886
+
bnge_sched_reset_rxr(bn, rxr);
887
+
if (rc1)
888
+
return rc1;
889
+
goto next_rx_no_prod_no_len;
890
+
}
891
+
rx_buf = &rxr->rx_buf_ring[cons];
892
+
data = rx_buf->data;
893
+
data_ptr = rx_buf->data_ptr;
894
+
prefetch(data_ptr);
895
+
896
+
misc = le32_to_cpu(rxcmp->rx_cmp_misc_v1);
897
+
agg_bufs = (misc & RX_CMP_AGG_BUFS) >> RX_CMP_AGG_BUFS_SHIFT;
898
+
899
+
if (agg_bufs) {
900
+
if (!bnge_agg_bufs_valid(bn, cpr, agg_bufs, &tmp_raw_cons))
901
+
return -EBUSY;
902
+
903
+
cp_cons = NEXT_CMP(bn, cp_cons);
904
+
*event |= BNGE_AGG_EVENT;
905
+
}
906
+
*event |= BNGE_RX_EVENT;
907
+
908
+
rx_buf->data = NULL;
909
+
if (rxcmp1->rx_cmp_cfa_code_errors_v2 & RX_CMP_L2_ERRORS) {
910
+
bnge_reuse_rx_data(rxr, cons, data);
911
+
if (agg_bufs)
912
+
bnge_reuse_rx_agg_bufs(cpr, cp_cons, 0, agg_bufs,
913
+
false);
914
+
rc = -EIO;
915
+
goto next_rx_no_len;
916
+
}
917
+
918
+
flags = le32_to_cpu(rxcmp->rx_cmp_len_flags_type);
919
+
len = flags >> RX_CMP_LEN_SHIFT;
920
+
dma_addr = rx_buf->mapping;
921
+
922
+
if (len <= bn->rx_copybreak) {
923
+
skb = bnge_copy_skb(bnapi, data_ptr, len, dma_addr);
924
+
bnge_reuse_rx_data(rxr, cons, data);
925
+
} else {
926
+
skb = bnge_rx_skb(bn, rxr, cons, data, data_ptr, dma_addr, len);
927
+
}
928
+
929
+
if (!skb) {
930
+
if (agg_bufs)
931
+
bnge_reuse_rx_agg_bufs(cpr, cp_cons, 0,
932
+
agg_bufs, false);
933
+
goto oom_next_rx;
934
+
}
935
+
936
+
if (agg_bufs) {
937
+
skb = bnge_rx_agg_netmems_skb(bn, cpr, skb, cp_cons,
938
+
agg_bufs, false);
939
+
if (!skb)
940
+
goto oom_next_rx;
941
+
}
942
+
943
+
if (RX_CMP_HASH_VALID(rxcmp)) {
944
+
enum pkt_hash_types type;
945
+
946
+
if (cmp_type == CMP_TYPE_RX_L2_V3_CMP) {
947
+
type = bnge_rss_ext_op(bn, rxcmp);
948
+
} else {
949
+
u32 itypes = RX_CMP_ITYPES(rxcmp);
950
+
951
+
if (itypes == RX_CMP_FLAGS_ITYPE_TCP ||
952
+
itypes == RX_CMP_FLAGS_ITYPE_UDP)
953
+
type = PKT_HASH_TYPE_L4;
954
+
else
955
+
type = PKT_HASH_TYPE_L3;
956
+
}
957
+
skb_set_hash(skb, le32_to_cpu(rxcmp->rx_cmp_rss_hash), type);
958
+
}
959
+
960
+
skb->protocol = eth_type_trans(skb, dev);
961
+
962
+
if (skb->dev->features & BNGE_HW_FEATURE_VLAN_ALL_RX) {
963
+
skb = bnge_rx_vlan(skb, cmp_type, rxcmp, rxcmp1);
964
+
if (!skb)
965
+
goto next_rx;
966
+
}
967
+
968
+
skb_checksum_none_assert(skb);
969
+
if (RX_CMP_L4_CS_OK(rxcmp1)) {
970
+
if (dev->features & NETIF_F_RXCSUM) {
971
+
skb->ip_summed = CHECKSUM_UNNECESSARY;
972
+
skb->csum_level = RX_CMP_ENCAP(rxcmp1);
973
+
}
974
+
}
975
+
976
+
bnge_deliver_skb(bn, bnapi, skb);
977
+
rc = 1;
978
+
979
+
next_rx:
980
+
/* Update Stats */
981
+
next_rx_no_len:
982
+
rxr->rx_prod = NEXT_RX(prod);
983
+
rxr->rx_next_cons = RING_RX(bn, NEXT_RX(cons));
984
+
985
+
next_rx_no_prod_no_len:
986
+
*raw_cons = tmp_raw_cons;
987
+
return rc;
988
+
989
+
oom_next_rx:
990
+
rc = -ENOMEM;
991
+
goto next_rx;
992
+
}
993
+
994
+
/* In netpoll mode, if we are using a combined completion ring, we need to
995
+
* discard the rx packets and recycle the buffers.
996
+
*/
997
+
static int bnge_force_rx_discard(struct bnge_net *bn,
998
+
struct bnge_cp_ring_info *cpr,
999
+
u32 *raw_cons, u8 *event)
1000
+
{
1001
+
u32 tmp_raw_cons = *raw_cons;
1002
+
struct rx_cmp_ext *rxcmp1;
1003
+
struct rx_cmp *rxcmp;
1004
+
u16 cp_cons;
1005
+
u8 cmp_type;
1006
+
int rc;
1007
+
1008
+
cp_cons = RING_CMP(bn, tmp_raw_cons);
1009
+
rxcmp = (struct rx_cmp *)
1010
+
&cpr->desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1011
+
1012
+
tmp_raw_cons = NEXT_RAW_CMP(tmp_raw_cons);
1013
+
cp_cons = RING_CMP(bn, tmp_raw_cons);
1014
+
rxcmp1 = (struct rx_cmp_ext *)
1015
+
&cpr->desc_ring[CP_RING(cp_cons)][CP_IDX(cp_cons)];
1016
+
1017
+
if (!RX_CMP_VALID(bn, rxcmp1, tmp_raw_cons))
1018
+
return -EBUSY;
1019
+
1020
+
/* The valid test of the entry must be done first before
1021
+
* reading any further.
1022
+
*/
1023
+
dma_rmb();
1024
+
cmp_type = RX_CMP_TYPE(rxcmp);
1025
+
if (cmp_type == CMP_TYPE_RX_L2_CMP ||
1026
+
cmp_type == CMP_TYPE_RX_L2_V3_CMP) {
1027
+
rxcmp1->rx_cmp_cfa_code_errors_v2 |=
1028
+
cpu_to_le32(RX_CMPL_ERRORS_CRC_ERROR);
1029
+
} else if (cmp_type == CMP_TYPE_RX_L2_TPA_END_CMP) {
1030
+
struct rx_tpa_end_cmp_ext *tpa_end1;
1031
+
1032
+
tpa_end1 = (struct rx_tpa_end_cmp_ext *)rxcmp1;
1033
+
tpa_end1->rx_tpa_end_cmp_errors_v2 |=
1034
+
cpu_to_le32(RX_TPA_END_CMP_ERRORS);
1035
+
}
1036
+
rc = bnge_rx_pkt(bn, cpr, raw_cons, event);
1037
+
return rc;
1038
+
}
1039
+
1040
+
static void __bnge_tx_int(struct bnge_net *bn, struct bnge_tx_ring_info *txr,
1041
+
int budget)
1042
+
{
1043
+
u16 hw_cons = txr->tx_hw_cons;
1044
+
struct bnge_dev *bd = bn->bd;
1045
+
unsigned int tx_bytes = 0;
1046
+
unsigned int tx_pkts = 0;
1047
+
struct netdev_queue *txq;
1048
+
u16 cons = txr->tx_cons;
1049
+
skb_frag_t *frag;
1050
+
1051
+
txq = netdev_get_tx_queue(bn->netdev, txr->txq_index);
1052
+
1053
+
while (SW_TX_RING(bn, cons) != hw_cons) {
1054
+
struct bnge_sw_tx_bd *tx_buf;
1055
+
struct sk_buff *skb;
1056
+
int j, last;
1057
+
1058
+
tx_buf = &txr->tx_buf_ring[SW_TX_RING(bn, cons)];
1059
+
skb = tx_buf->skb;
1060
+
if (unlikely(!skb)) {
1061
+
bnge_sched_reset_txr(bn, txr, cons);
1062
+
return;
1063
+
}
1064
+
1065
+
cons = NEXT_TX(cons);
1066
+
tx_pkts++;
1067
+
tx_bytes += skb->len;
1068
+
tx_buf->skb = NULL;
1069
+
1070
+
dma_unmap_single(bd->dev, dma_unmap_addr(tx_buf, mapping),
1071
+
skb_headlen(skb), DMA_TO_DEVICE);
1072
+
last = tx_buf->nr_frags;
1073
+
1074
+
for (j = 0; j < last; j++) {
1075
+
frag = &skb_shinfo(skb)->frags[j];
1076
+
cons = NEXT_TX(cons);
1077
+
tx_buf = &txr->tx_buf_ring[SW_TX_RING(bn, cons)];
1078
+
netmem_dma_unmap_page_attrs(bd->dev,
1079
+
dma_unmap_addr(tx_buf,
1080
+
mapping),
1081
+
skb_frag_size(frag),
1082
+
DMA_TO_DEVICE, 0);
1083
+
}
1084
+
1085
+
cons = NEXT_TX(cons);
1086
+
1087
+
napi_consume_skb(skb, budget);
1088
+
}
1089
+
1090
+
WRITE_ONCE(txr->tx_cons, cons);
1091
+
1092
+
__netif_txq_completed_wake(txq, tx_pkts, tx_bytes,
1093
+
bnge_tx_avail(bn, txr), bn->tx_wake_thresh,
1094
+
(READ_ONCE(txr->dev_state) ==
1095
+
BNGE_DEV_STATE_CLOSING));
1096
+
}
1097
+
1098
+
static void bnge_tx_int(struct bnge_net *bn, struct bnge_napi *bnapi,
1099
+
int budget)
1100
+
{
1101
+
struct bnge_tx_ring_info *txr;
1102
+
int i;
1103
+
1104
+
bnge_for_each_napi_tx(i, bnapi, txr) {
1105
+
if (txr->tx_hw_cons != SW_TX_RING(bn, txr->tx_cons))
1106
+
__bnge_tx_int(bn, txr, budget);
1107
+
}
1108
+
1109
+
bnapi->events &= ~BNGE_TX_CMP_EVENT;
1110
+
}
1111
+
1112
+
static void __bnge_poll_work_done(struct bnge_net *bn, struct bnge_napi *bnapi,
1113
+
int budget)
1114
+
{
1115
+
struct bnge_rx_ring_info *rxr = bnapi->rx_ring;
1116
+
1117
+
if ((bnapi->events & BNGE_TX_CMP_EVENT) && !bnapi->tx_fault)
1118
+
bnge_tx_int(bn, bnapi, budget);
1119
+
1120
+
if ((bnapi->events & BNGE_RX_EVENT)) {
1121
+
bnge_db_write(bn->bd, &rxr->rx_db, rxr->rx_prod);
1122
+
bnapi->events &= ~BNGE_RX_EVENT;
1123
+
}
1124
+
1125
+
if (bnapi->events & BNGE_AGG_EVENT) {
1126
+
bnge_db_write(bn->bd, &rxr->rx_agg_db, rxr->rx_agg_prod);
1127
+
bnapi->events &= ~BNGE_AGG_EVENT;
1128
+
}
1129
+
}
1130
+
1131
+
static void
1132
+
bnge_hwrm_update_token(struct bnge_dev *bd, u16 seq_id,
1133
+
enum bnge_hwrm_wait_state state)
1134
+
{
1135
+
struct bnge_hwrm_wait_token *token;
1136
+
1137
+
rcu_read_lock();
1138
+
hlist_for_each_entry_rcu(token, &bd->hwrm_pending_list, node) {
1139
+
if (token->seq_id == seq_id) {
1140
+
WRITE_ONCE(token->state, state);
1141
+
rcu_read_unlock();
1142
+
return;
1143
+
}
1144
+
}
1145
+
rcu_read_unlock();
1146
+
dev_err(bd->dev, "Invalid hwrm seq id %d\n", seq_id);
1147
+
}
1148
+
1149
+
static int bnge_hwrm_handler(struct bnge_dev *bd, struct tx_cmp *txcmp)
1150
+
{
1151
+
struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp;
1152
+
u16 cmpl_type = TX_CMP_TYPE(txcmp), seq_id;
1153
+
1154
+
switch (cmpl_type) {
1155
+
case CMPL_BASE_TYPE_HWRM_DONE:
1156
+
seq_id = le16_to_cpu(h_cmpl->sequence_id);
1157
+
bnge_hwrm_update_token(bd, seq_id, BNGE_HWRM_COMPLETE);
1158
+
break;
1159
+
1160
+
case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT:
1161
+
default:
1162
+
break;
1163
+
}
1164
+
1165
+
return 0;
1166
+
}
1167
+
1168
+
static int __bnge_poll_work(struct bnge_net *bn, struct bnge_cp_ring_info *cpr,
1169
+
int budget)
1170
+
{
1171
+
struct bnge_napi *bnapi = cpr->bnapi;
1172
+
u32 raw_cons = cpr->cp_raw_cons;
1173
+
struct tx_cmp *txcmp;
1174
+
int rx_pkts = 0;
1175
+
u8 event = 0;
1176
+
u32 cons;
1177
+
1178
+
cpr->has_more_work = 0;
1179
+
cpr->had_work_done = 1;
1180
+
while (1) {
1181
+
u8 cmp_type;
1182
+
int rc;
1183
+
1184
+
cons = RING_CMP(bn, raw_cons);
1185
+
txcmp = &cpr->desc_ring[CP_RING(cons)][CP_IDX(cons)];
1186
+
1187
+
if (!TX_CMP_VALID(bn, txcmp, raw_cons))
1188
+
break;
1189
+
1190
+
/* The valid test of the entry must be done first before
1191
+
* reading any further.
1192
+
*/
1193
+
dma_rmb();
1194
+
cmp_type = TX_CMP_TYPE(txcmp);
1195
+
if (cmp_type == CMP_TYPE_TX_L2_CMP ||
1196
+
cmp_type == CMP_TYPE_TX_L2_COAL_CMP) {
1197
+
u32 opaque = txcmp->tx_cmp_opaque;
1198
+
struct bnge_tx_ring_info *txr;
1199
+
u16 tx_freed;
1200
+
1201
+
txr = bnapi->tx_ring[TX_OPAQUE_RING(opaque)];
1202
+
event |= BNGE_TX_CMP_EVENT;
1203
+
if (cmp_type == CMP_TYPE_TX_L2_COAL_CMP)
1204
+
txr->tx_hw_cons = TX_CMP_SQ_CONS_IDX(txcmp);
1205
+
else
1206
+
txr->tx_hw_cons = TX_OPAQUE_PROD(bn, opaque);
1207
+
tx_freed = ((txr->tx_hw_cons - txr->tx_cons) &
1208
+
bn->tx_ring_mask);
1209
+
/* return full budget so NAPI will complete. */
1210
+
if (unlikely(tx_freed >= bn->tx_wake_thresh)) {
1211
+
rx_pkts = budget;
1212
+
raw_cons = NEXT_RAW_CMP(raw_cons);
1213
+
if (budget)
1214
+
cpr->has_more_work = 1;
1215
+
break;
1216
+
}
1217
+
} else if (cmp_type >= CMP_TYPE_RX_L2_CMP &&
1218
+
cmp_type <= CMP_TYPE_RX_L2_TPA_START_V3_CMP) {
1219
+
if (likely(budget))
1220
+
rc = bnge_rx_pkt(bn, cpr, &raw_cons, &event);
1221
+
else
1222
+
rc = bnge_force_rx_discard(bn, cpr, &raw_cons,
1223
+
&event);
1224
+
if (likely(rc >= 0))
1225
+
rx_pkts += rc;
1226
+
/* Increment rx_pkts when rc is -ENOMEM to count towards
1227
+
* the NAPI budget. Otherwise, we may potentially loop
1228
+
* here forever if we consistently cannot allocate
1229
+
* buffers.
1230
+
*/
1231
+
else if (rc == -ENOMEM && budget)
1232
+
rx_pkts++;
1233
+
else if (rc == -EBUSY) /* partial completion */
1234
+
break;
1235
+
} else if (unlikely(cmp_type == CMPL_BASE_TYPE_HWRM_DONE ||
1236
+
cmp_type == CMPL_BASE_TYPE_HWRM_FWD_REQ ||
1237
+
cmp_type == CMPL_BA_TY_HWRM_ASY_EVT)) {
1238
+
bnge_hwrm_handler(bn->bd, txcmp);
1239
+
}
1240
+
raw_cons = NEXT_RAW_CMP(raw_cons);
1241
+
1242
+
if (rx_pkts && rx_pkts == budget) {
1243
+
cpr->has_more_work = 1;
1244
+
break;
1245
+
}
1246
+
}
1247
+
1248
+
cpr->cp_raw_cons = raw_cons;
1249
+
bnapi->events |= event;
1250
+
return rx_pkts;
1251
+
}
1252
+
1253
+
static void __bnge_poll_cqs_done(struct bnge_net *bn, struct bnge_napi *bnapi,
1254
+
u64 dbr_type, int budget)
1255
+
{
1256
+
struct bnge_nq_ring_info *nqr = &bnapi->nq_ring;
1257
+
int i;
1258
+
1259
+
for (i = 0; i < nqr->cp_ring_count; i++) {
1260
+
struct bnge_cp_ring_info *cpr = &nqr->cp_ring_arr[i];
1261
+
struct bnge_db_info *db;
1262
+
1263
+
if (cpr->had_work_done) {
1264
+
u32 tgl = 0;
1265
+
1266
+
if (dbr_type == DBR_TYPE_CQ_ARMALL) {
1267
+
cpr->had_nqe_notify = 0;
1268
+
tgl = cpr->toggle;
1269
+
}
1270
+
db = &cpr->cp_db;
1271
+
bnge_writeq(bn->bd,
1272
+
db->db_key64 | dbr_type | DB_TOGGLE(tgl) |
1273
+
DB_RING_IDX(db, cpr->cp_raw_cons),
1274
+
db->doorbell);
1275
+
cpr->had_work_done = 0;
1276
+
}
1277
+
}
1278
+
__bnge_poll_work_done(bn, bnapi, budget);
1279
+
}
1280
+
1281
+
static int __bnge_poll_cqs(struct bnge_net *bn, struct bnge_napi *bnapi,
1282
+
int budget)
1283
+
{
1284
+
struct bnge_nq_ring_info *nqr = &bnapi->nq_ring;
1285
+
int i, work_done = 0;
1286
+
1287
+
for (i = 0; i < nqr->cp_ring_count; i++) {
1288
+
struct bnge_cp_ring_info *cpr = &nqr->cp_ring_arr[i];
1289
+
1290
+
if (cpr->had_nqe_notify) {
1291
+
work_done += __bnge_poll_work(bn, cpr,
1292
+
budget - work_done);
1293
+
nqr->has_more_work |= cpr->has_more_work;
1294
+
}
1295
+
}
1296
+
return work_done;
1297
+
}
1298
+
1299
+
int bnge_napi_poll(struct napi_struct *napi, int budget)
1300
+
{
1301
+
struct bnge_napi *bnapi = container_of(napi, struct bnge_napi, napi);
1302
+
struct bnge_nq_ring_info *nqr = &bnapi->nq_ring;
1303
+
u32 raw_cons = nqr->nq_raw_cons;
1304
+
struct bnge_net *bn = bnapi->bn;
1305
+
struct bnge_dev *bd = bn->bd;
1306
+
struct nqe_cn *nqcmp;
1307
+
int work_done = 0;
1308
+
u32 cons;
1309
+
1310
+
if (nqr->has_more_work) {
1311
+
nqr->has_more_work = 0;
1312
+
work_done = __bnge_poll_cqs(bn, bnapi, budget);
1313
+
}
1314
+
1315
+
while (1) {
1316
+
u16 type;
1317
+
1318
+
cons = RING_CMP(bn, raw_cons);
1319
+
nqcmp = &nqr->desc_ring[CP_RING(cons)][CP_IDX(cons)];
1320
+
1321
+
if (!NQ_CMP_VALID(bn, nqcmp, raw_cons)) {
1322
+
if (nqr->has_more_work)
1323
+
break;
1324
+
1325
+
__bnge_poll_cqs_done(bn, bnapi, DBR_TYPE_CQ_ARMALL,
1326
+
budget);
1327
+
nqr->nq_raw_cons = raw_cons;
1328
+
if (napi_complete_done(napi, work_done))
1329
+
BNGE_DB_NQ_ARM(bd, &nqr->nq_db,
1330
+
nqr->nq_raw_cons);
1331
+
goto poll_done;
1332
+
}
1333
+
1334
+
/* The valid test of the entry must be done first before
1335
+
* reading any further.
1336
+
*/
1337
+
dma_rmb();
1338
+
1339
+
type = le16_to_cpu(nqcmp->type);
1340
+
if (NQE_CN_TYPE(type) == NQ_CN_TYPE_CQ_NOTIFICATION) {
1341
+
u32 idx = le32_to_cpu(nqcmp->cq_handle_low);
1342
+
u32 cq_type = BNGE_NQ_HDL_TYPE(idx);
1343
+
struct bnge_cp_ring_info *cpr;
1344
+
1345
+
/* No more budget for RX work */
1346
+
if (budget && work_done >= budget &&
1347
+
cq_type == BNGE_NQ_HDL_TYPE_RX)
1348
+
break;
1349
+
1350
+
idx = BNGE_NQ_HDL_IDX(idx);
1351
+
cpr = &nqr->cp_ring_arr[idx];
1352
+
cpr->had_nqe_notify = 1;
1353
+
cpr->toggle = NQE_CN_TOGGLE(type);
1354
+
work_done += __bnge_poll_work(bn, cpr,
1355
+
budget - work_done);
1356
+
nqr->has_more_work |= cpr->has_more_work;
1357
+
} else {
1358
+
bnge_hwrm_handler(bn->bd, (struct tx_cmp *)nqcmp);
1359
+
}
1360
+
raw_cons = NEXT_RAW_CMP(raw_cons);
1361
+
}
1362
+
1363
+
__bnge_poll_cqs_done(bn, bnapi, DBR_TYPE_CQ, budget);
1364
+
if (raw_cons != nqr->nq_raw_cons) {
1365
+
nqr->nq_raw_cons = raw_cons;
1366
+
BNGE_DB_NQ(bd, &nqr->nq_db, raw_cons);
1367
+
}
1368
+
poll_done:
1369
+
return work_done;
1370
+
}
1371
+
1372
+
static u16 bnge_xmit_get_cfa_action(struct sk_buff *skb)
1373
+
{
1374
+
struct metadata_dst *md_dst = skb_metadata_dst(skb);
1375
+
1376
+
if (!md_dst || md_dst->type != METADATA_HW_PORT_MUX)
1377
+
return 0;
1378
+
1379
+
return md_dst->u.port_info.port_id;
1380
+
}
1381
+
1382
+
static const u16 bnge_lhint_arr[] = {
1383
+
TX_BD_FLAGS_LHINT_512_AND_SMALLER,
1384
+
TX_BD_FLAGS_LHINT_512_TO_1023,
1385
+
TX_BD_FLAGS_LHINT_1024_TO_2047,
1386
+
TX_BD_FLAGS_LHINT_1024_TO_2047,
1387
+
TX_BD_FLAGS_LHINT_2048_AND_LARGER,
1388
+
TX_BD_FLAGS_LHINT_2048_AND_LARGER,
1389
+
TX_BD_FLAGS_LHINT_2048_AND_LARGER,
1390
+
TX_BD_FLAGS_LHINT_2048_AND_LARGER,
1391
+
TX_BD_FLAGS_LHINT_2048_AND_LARGER,
1392
+
TX_BD_FLAGS_LHINT_2048_AND_LARGER,
1393
+
TX_BD_FLAGS_LHINT_2048_AND_LARGER,
1394
+
TX_BD_FLAGS_LHINT_2048_AND_LARGER,
1395
+
TX_BD_FLAGS_LHINT_2048_AND_LARGER,
1396
+
TX_BD_FLAGS_LHINT_2048_AND_LARGER,
1397
+
TX_BD_FLAGS_LHINT_2048_AND_LARGER,
1398
+
TX_BD_FLAGS_LHINT_2048_AND_LARGER,
1399
+
TX_BD_FLAGS_LHINT_2048_AND_LARGER,
1400
+
TX_BD_FLAGS_LHINT_2048_AND_LARGER,
1401
+
TX_BD_FLAGS_LHINT_2048_AND_LARGER,
1402
+
};
1403
+
1404
+
static void bnge_txr_db_kick(struct bnge_net *bn, struct bnge_tx_ring_info *txr,
1405
+
u16 prod)
1406
+
{
1407
+
/* Sync BD data before updating doorbell */
1408
+
wmb();
1409
+
bnge_db_write(bn->bd, &txr->tx_db, prod);
1410
+
txr->kick_pending = 0;
1411
+
}
1412
+
1413
+
static u32 bnge_get_gso_hdr_len(struct sk_buff *skb)
1414
+
{
1415
+
bool udp_gso = !!(skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4);
1416
+
u32 hdr_len;
1417
+
1418
+
if (skb->encapsulation) {
1419
+
if (udp_gso)
1420
+
hdr_len = skb_inner_transport_offset(skb) +
1421
+
sizeof(struct udphdr);
1422
+
else
1423
+
hdr_len = skb_inner_tcp_all_headers(skb);
1424
+
} else if (udp_gso) {
1425
+
hdr_len = skb_transport_offset(skb) + sizeof(struct udphdr);
1426
+
} else {
1427
+
hdr_len = skb_tcp_all_headers(skb);
1428
+
}
1429
+
1430
+
return hdr_len;
1431
+
}
1432
+
1433
+
netdev_tx_t bnge_start_xmit(struct sk_buff *skb, struct net_device *dev)
1434
+
{
1435
+
u32 len, free_size, vlan_tag_flags, cfa_action, flags;
1436
+
struct bnge_net *bn = netdev_priv(dev);
1437
+
struct bnge_tx_ring_info *txr;
1438
+
struct bnge_dev *bd = bn->bd;
1439
+
struct bnge_sw_tx_bd *tx_buf;
1440
+
struct tx_bd *txbd, *txbd0;
1441
+
struct netdev_queue *txq;
1442
+
struct tx_bd_ext *txbd1;
1443
+
u16 prod, last_frag;
1444
+
unsigned int length;
1445
+
dma_addr_t mapping;
1446
+
__le32 lflags = 0;
1447
+
skb_frag_t *frag;
1448
+
int i;
1449
+
1450
+
i = skb_get_queue_mapping(skb);
1451
+
txq = netdev_get_tx_queue(dev, i);
1452
+
txr = &bn->tx_ring[bn->tx_ring_map[i]];
1453
+
prod = txr->tx_prod;
1454
+
1455
+
free_size = bnge_tx_avail(bn, txr);
1456
+
if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) {
1457
+
/* We must have raced with NAPI cleanup */
1458
+
if (net_ratelimit() && txr->kick_pending)
1459
+
netif_warn(bn, tx_err, dev,
1460
+
"bnge: ring busy w/ flush pending!\n");
1461
+
if (!netif_txq_try_stop(txq, bnge_tx_avail(bn, txr),
1462
+
bn->tx_wake_thresh))
1463
+
return NETDEV_TX_BUSY;
1464
+
}
1465
+
1466
+
if (unlikely(ipv6_hopopt_jumbo_remove(skb)))
1467
+
goto tx_free;
1468
+
1469
+
last_frag = skb_shinfo(skb)->nr_frags;
1470
+
1471
+
txbd = &txr->tx_desc_ring[TX_RING(bn, prod)][TX_IDX(prod)];
1472
+
1473
+
tx_buf = &txr->tx_buf_ring[SW_TX_RING(bn, prod)];
1474
+
tx_buf->skb = skb;
1475
+
tx_buf->nr_frags = last_frag;
1476
+
1477
+
vlan_tag_flags = 0;
1478
+
cfa_action = bnge_xmit_get_cfa_action(skb);
1479
+
if (skb_vlan_tag_present(skb)) {
1480
+
vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN |
1481
+
skb_vlan_tag_get(skb);
1482
+
/* Currently supports 8021Q, 8021AD vlan offloads
1483
+
* QINQ1, QINQ2, QINQ3 vlan headers are deprecated
1484
+
*/
1485
+
if (skb->vlan_proto == htons(ETH_P_8021Q))
1486
+
vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT;
1487
+
}
1488
+
1489
+
if (unlikely(skb->no_fcs))
1490
+
lflags |= cpu_to_le32(TX_BD_FLAGS_NO_CRC);
1491
+
1492
+
if (eth_skb_pad(skb))
1493
+
goto tx_kick_pending;
1494
+
1495
+
len = skb_headlen(skb);
1496
+
1497
+
mapping = dma_map_single(bd->dev, skb->data, len, DMA_TO_DEVICE);
1498
+
1499
+
if (unlikely(dma_mapping_error(bd->dev, mapping)))
1500
+
goto tx_free;
1501
+
1502
+
dma_unmap_addr_set(tx_buf, mapping, mapping);
1503
+
flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD |
1504
+
TX_BD_CNT(last_frag + 2);
1505
+
1506
+
txbd->tx_bd_haddr = cpu_to_le64(mapping);
1507
+
txbd->tx_bd_opaque = SET_TX_OPAQUE(bn, txr, prod, 2 + last_frag);
1508
+
1509
+
prod = NEXT_TX(prod);
1510
+
txbd1 = (struct tx_bd_ext *)
1511
+
&txr->tx_desc_ring[TX_RING(bn, prod)][TX_IDX(prod)];
1512
+
1513
+
if (skb_is_gso(skb)) {
1514
+
u32 hdr_len = bnge_get_gso_hdr_len(skb);
1515
+
1516
+
lflags |= cpu_to_le32(TX_BD_FLAGS_LSO | TX_BD_FLAGS_T_IPID |
1517
+
(hdr_len << (TX_BD_HSIZE_SHIFT - 1)));
1518
+
length = skb_shinfo(skb)->gso_size;
1519
+
txbd1->tx_bd_mss = cpu_to_le32(length);
1520
+
length += hdr_len;
1521
+
} else {
1522
+
length = skb->len;
1523
+
if (skb->ip_summed == CHECKSUM_PARTIAL) {
1524
+
lflags |= cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM);
1525
+
txbd1->tx_bd_mss = 0;
1526
+
}
1527
+
}
1528
+
1529
+
flags |= bnge_lhint_arr[length >> 9];
1530
+
1531
+
txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
1532
+
txbd1->tx_bd_hsize_lflags = lflags;
1533
+
txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags);
1534
+
txbd1->tx_bd_cfa_action =
1535
+
cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT);
1536
+
txbd0 = txbd;
1537
+
for (i = 0; i < last_frag; i++) {
1538
+
frag = &skb_shinfo(skb)->frags[i];
1539
+
1540
+
prod = NEXT_TX(prod);
1541
+
txbd = &txr->tx_desc_ring[TX_RING(bn, prod)][TX_IDX(prod)];
1542
+
1543
+
len = skb_frag_size(frag);
1544
+
mapping = skb_frag_dma_map(bd->dev, frag, 0, len,
1545
+
DMA_TO_DEVICE);
1546
+
1547
+
if (unlikely(dma_mapping_error(bd->dev, mapping)))
1548
+
goto tx_dma_error;
1549
+
1550
+
tx_buf = &txr->tx_buf_ring[SW_TX_RING(bn, prod)];
1551
+
netmem_dma_unmap_addr_set(skb_frag_netmem(frag), tx_buf,
1552
+
mapping, mapping);
1553
+
1554
+
txbd->tx_bd_haddr = cpu_to_le64(mapping);
1555
+
1556
+
flags = len << TX_BD_LEN_SHIFT;
1557
+
txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
1558
+
}
1559
+
1560
+
flags &= ~TX_BD_LEN;
1561
+
txbd->tx_bd_len_flags_type =
1562
+
cpu_to_le32(((len) << TX_BD_LEN_SHIFT) | flags |
1563
+
TX_BD_FLAGS_PACKET_END);
1564
+
1565
+
netdev_tx_sent_queue(txq, skb->len);
1566
+
1567
+
prod = NEXT_TX(prod);
1568
+
WRITE_ONCE(txr->tx_prod, prod);
1569
+
1570
+
if (!netdev_xmit_more() || netif_xmit_stopped(txq)) {
1571
+
bnge_txr_db_kick(bn, txr, prod);
1572
+
} else {
1573
+
if (free_size >= bn->tx_wake_thresh)
1574
+
txbd0->tx_bd_len_flags_type |=
1575
+
cpu_to_le32(TX_BD_FLAGS_NO_CMPL);
1576
+
txr->kick_pending = 1;
1577
+
}
1578
+
1579
+
if (unlikely(bnge_tx_avail(bn, txr) <= MAX_SKB_FRAGS + 1)) {
1580
+
if (netdev_xmit_more()) {
1581
+
txbd0->tx_bd_len_flags_type &=
1582
+
cpu_to_le32(~TX_BD_FLAGS_NO_CMPL);
1583
+
bnge_txr_db_kick(bn, txr, prod);
1584
+
}
1585
+
1586
+
netif_txq_try_stop(txq, bnge_tx_avail(bn, txr),
1587
+
bn->tx_wake_thresh);
1588
+
}
1589
+
return NETDEV_TX_OK;
1590
+
1591
+
tx_dma_error:
1592
+
last_frag = i;
1593
+
1594
+
/* start back at beginning and unmap skb */
1595
+
prod = txr->tx_prod;
1596
+
tx_buf = &txr->tx_buf_ring[SW_TX_RING(bn, prod)];
1597
+
dma_unmap_single(bd->dev, dma_unmap_addr(tx_buf, mapping),
1598
+
skb_headlen(skb), DMA_TO_DEVICE);
1599
+
prod = NEXT_TX(prod);
1600
+
1601
+
/* unmap remaining mapped pages */
1602
+
for (i = 0; i < last_frag; i++) {
1603
+
prod = NEXT_TX(prod);
1604
+
tx_buf = &txr->tx_buf_ring[SW_TX_RING(bn, prod)];
1605
+
frag = &skb_shinfo(skb)->frags[i];
1606
+
netmem_dma_unmap_page_attrs(bd->dev,
1607
+
dma_unmap_addr(tx_buf, mapping),
1608
+
skb_frag_size(frag),
1609
+
DMA_TO_DEVICE, 0);
1610
+
}
1611
+
1612
+
tx_free:
1613
+
dev_kfree_skb_any(skb);
1614
+
1615
+
tx_kick_pending:
1616
+
if (txr->kick_pending)
1617
+
bnge_txr_db_kick(bn, txr, txr->tx_prod);
1618
+
txr->tx_buf_ring[SW_TX_RING(bn, txr->tx_prod)].skb = NULL;
1619
+
dev_core_stats_tx_dropped_inc(dev);
1620
+
return NETDEV_TX_OK;
1621
+
}
1622
+
1623
+
netdev_features_t bnge_features_check(struct sk_buff *skb,
1624
+
struct net_device *dev,
1625
+
netdev_features_t features)
1626
+
{
1627
+
u32 len;
1628
+
1629
+
features = vlan_features_check(skb, features);
1630
+
#if (MAX_SKB_FRAGS > TX_MAX_FRAGS)
1631
+
if (skb_shinfo(skb)->nr_frags > TX_MAX_FRAGS)
1632
+
features &= ~NETIF_F_SG;
1633
+
#endif
1634
+
1635
+
if (skb_is_gso(skb))
1636
+
len = bnge_get_gso_hdr_len(skb) + skb_shinfo(skb)->gso_size;
1637
+
else
1638
+
len = skb->len;
1639
+
1640
+
len >>= 9;
1641
+
if (unlikely(len >= ARRAY_SIZE(bnge_lhint_arr)))
1642
+
features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
1643
+
1644
+
return features;
1645
+
}
+126
drivers/net/ethernet/broadcom/bnge/bnge_txrx.h
+126
drivers/net/ethernet/broadcom/bnge/bnge_txrx.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0 */
2
+
/* Copyright (c) 2025 Broadcom */
3
+
4
+
#ifndef _BNGE_TXRX_H_
5
+
#define _BNGE_TXRX_H_
6
+
7
+
#include <linux/bnxt/hsi.h>
8
+
#include "bnge_netdev.h"
9
+
10
+
static inline u32 bnge_tx_avail(struct bnge_net *bn,
11
+
const struct bnge_tx_ring_info *txr)
12
+
{
13
+
u32 used = READ_ONCE(txr->tx_prod) - READ_ONCE(txr->tx_cons);
14
+
15
+
return bn->tx_ring_size - (used & bn->tx_ring_mask);
16
+
}
17
+
18
+
static inline void bnge_writeq_relaxed(struct bnge_dev *bd, u64 val,
19
+
void __iomem *addr)
20
+
{
21
+
#if BITS_PER_LONG == 32
22
+
spin_lock(&bd->db_lock);
23
+
lo_hi_writeq_relaxed(val, addr);
24
+
spin_unlock(&bd->db_lock);
25
+
#else
26
+
writeq_relaxed(val, addr);
27
+
#endif
28
+
}
29
+
30
+
/* For TX and RX ring doorbells with no ordering guarantee*/
31
+
static inline void bnge_db_write_relaxed(struct bnge_net *bn,
32
+
struct bnge_db_info *db, u32 idx)
33
+
{
34
+
bnge_writeq_relaxed(bn->bd, db->db_key64 | DB_RING_IDX(db, idx),
35
+
db->doorbell);
36
+
}
37
+
38
+
#define TX_OPAQUE_IDX_MASK 0x0000ffff
39
+
#define TX_OPAQUE_BDS_MASK 0x00ff0000
40
+
#define TX_OPAQUE_BDS_SHIFT 16
41
+
#define TX_OPAQUE_RING_MASK 0xff000000
42
+
#define TX_OPAQUE_RING_SHIFT 24
43
+
44
+
#define SET_TX_OPAQUE(bn, txr, idx, bds) \
45
+
(((txr)->tx_napi_idx << TX_OPAQUE_RING_SHIFT) | \
46
+
((bds) << TX_OPAQUE_BDS_SHIFT) | ((idx) & (bn)->tx_ring_mask))
47
+
48
+
#define TX_OPAQUE_IDX(opq) ((opq) & TX_OPAQUE_IDX_MASK)
49
+
#define TX_OPAQUE_RING(opq) (((opq) & TX_OPAQUE_RING_MASK) >> \
50
+
TX_OPAQUE_RING_SHIFT)
51
+
#define TX_OPAQUE_BDS(opq) (((opq) & TX_OPAQUE_BDS_MASK) >> \
52
+
TX_OPAQUE_BDS_SHIFT)
53
+
#define TX_OPAQUE_PROD(bn, opq) ((TX_OPAQUE_IDX(opq) + TX_OPAQUE_BDS(opq)) &\
54
+
(bn)->tx_ring_mask)
55
+
#define TX_BD_CNT(n) (((n) << TX_BD_FLAGS_BD_CNT_SHIFT) & TX_BD_FLAGS_BD_CNT)
56
+
57
+
#define TX_MAX_BD_CNT 32
58
+
59
+
#define TX_MAX_FRAGS (TX_MAX_BD_CNT - 2)
60
+
61
+
/* Minimum TX BDs for a TX packet with MAX_SKB_FRAGS + 1. We need one extra
62
+
* BD because the first TX BD is always a long BD.
63
+
*/
64
+
#define BNGE_MIN_TX_DESC_CNT (MAX_SKB_FRAGS + 2)
65
+
66
+
#define RX_RING(bn, x) (((x) & (bn)->rx_ring_mask) >> (BNGE_PAGE_SHIFT - 4))
67
+
#define RX_AGG_RING(bn, x) (((x) & (bn)->rx_agg_ring_mask) >> \
68
+
(BNGE_PAGE_SHIFT - 4))
69
+
#define RX_IDX(x) ((x) & (RX_DESC_CNT - 1))
70
+
71
+
#define TX_RING(bn, x) (((x) & (bn)->tx_ring_mask) >> (BNGE_PAGE_SHIFT - 4))
72
+
#define TX_IDX(x) ((x) & (TX_DESC_CNT - 1))
73
+
74
+
#define CP_RING(x) (((x) & ~(CP_DESC_CNT - 1)) >> (BNGE_PAGE_SHIFT - 4))
75
+
#define CP_IDX(x) ((x) & (CP_DESC_CNT - 1))
76
+
77
+
#define TX_CMP_VALID(bn, txcmp, raw_cons) \
78
+
(!!((txcmp)->tx_cmp_errors_v & cpu_to_le32(TX_CMP_V)) == \
79
+
!((raw_cons) & (bn)->cp_bit))
80
+
81
+
#define RX_CMP_VALID(bn, rxcmp1, raw_cons) \
82
+
(!!((rxcmp1)->rx_cmp_cfa_code_errors_v2 & cpu_to_le32(RX_CMP_V)) ==\
83
+
!((raw_cons) & (bn)->cp_bit))
84
+
85
+
#define RX_AGG_CMP_VALID(bn, agg, raw_cons) \
86
+
(!!((agg)->rx_agg_cmp_v & cpu_to_le32(RX_AGG_CMP_V)) == \
87
+
!((raw_cons) & (bn)->cp_bit))
88
+
89
+
#define NQ_CMP_VALID(bn, nqcmp, raw_cons) \
90
+
(!!((nqcmp)->v & cpu_to_le32(NQ_CN_V)) == !((raw_cons) & (bn)->cp_bit))
91
+
92
+
#define TX_CMP_TYPE(txcmp) \
93
+
(le32_to_cpu((txcmp)->tx_cmp_flags_type) & CMP_TYPE)
94
+
95
+
#define RX_CMP_TYPE(rxcmp) \
96
+
(le32_to_cpu((rxcmp)->rx_cmp_len_flags_type) & RX_CMP_CMP_TYPE)
97
+
98
+
#define RING_RX(bn, idx) ((idx) & (bn)->rx_ring_mask)
99
+
#define NEXT_RX(idx) ((idx) + 1)
100
+
101
+
#define RING_RX_AGG(bn, idx) ((idx) & (bn)->rx_agg_ring_mask)
102
+
#define NEXT_RX_AGG(idx) ((idx) + 1)
103
+
104
+
#define SW_TX_RING(bn, idx) ((idx) & (bn)->tx_ring_mask)
105
+
#define NEXT_TX(idx) ((idx) + 1)
106
+
107
+
#define ADV_RAW_CMP(idx, n) ((idx) + (n))
108
+
#define NEXT_RAW_CMP(idx) ADV_RAW_CMP(idx, 1)
109
+
#define RING_CMP(bn, idx) ((idx) & (bn)->cp_ring_mask)
110
+
#define NEXT_CMP(bn, idx) RING_CMP(bn, ADV_RAW_CMP(idx, 1))
111
+
112
+
#define RX_CMP_ITYPES(rxcmp) \
113
+
(le32_to_cpu((rxcmp)->rx_cmp_len_flags_type) & RX_CMP_FLAGS_ITYPES_MASK)
114
+
115
+
#define RX_CMP_CFA_CODE(rxcmpl1) \
116
+
((le32_to_cpu((rxcmpl1)->rx_cmp_cfa_code_errors_v2) & \
117
+
RX_CMPL_CFA_CODE_MASK) >> RX_CMPL_CFA_CODE_SFT)
118
+
119
+
irqreturn_t bnge_msix(int irq, void *dev_instance);
120
+
netdev_tx_t bnge_start_xmit(struct sk_buff *skb, struct net_device *dev);
121
+
void bnge_reuse_rx_data(struct bnge_rx_ring_info *rxr, u16 cons, void *data);
122
+
int bnge_napi_poll(struct napi_struct *napi, int budget);
123
+
netdev_features_t bnge_features_check(struct sk_buff *skb,
124
+
struct net_device *dev,
125
+
netdev_features_t features);
126
+
#endif /* _BNGE_TXRX_H_ */