Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge branch 'net-mlx5e-support-rss-for-ipsec-offload'
Tariq Toukan says:
====================
net/mlx5e: Support RSS for IPSec offload
The series by Jianbo uses a new firmware feature to identify the inner
protocol of decrypted packets, adding new flow groups and steering rules
to redirect them for proper L4-based RSS. This ensures traffic is spread
across multiple CPU cores.
====================
Link: https://patch.msgid.link/1758179963-649455-1-git-send-email-tariqt@nvidia.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
+453
-38
+3
-2
drivers/net/ethernet/mellanox/mlx5/core/en/fs.h
+3
-2
drivers/net/ethernet/mellanox/mlx5/core/en/fs.h
···
57
57
bool promisc_enabled;
58
58
};
59
59
60
-
#define MLX5E_NUM_INDIR_TIRS (MLX5_NUM_TT - 1)
60
+
#define MLX5E_NUM_INDIR_TIRS (MLX5_NUM_INDIR_TIRS)
61
61
62
62
#define MLX5_HASH_IP (MLX5_HASH_FIELD_SEL_SRC_IP |\
63
63
MLX5_HASH_FIELD_SEL_DST_IP)
···
132
132
133
133
void mlx5e_set_ttc_params(struct mlx5e_flow_steering *fs,
134
134
struct mlx5e_rx_res *rx_res,
135
-
struct ttc_params *ttc_params, bool tunnel);
135
+
struct ttc_params *ttc_params, bool tunnel,
136
+
bool ipsec_rss);
136
137
137
138
void mlx5e_destroy_ttc_table(struct mlx5e_flow_steering *fs);
138
139
int mlx5e_create_ttc_table(struct mlx5e_flow_steering *fs,
+32
-8
drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c
+32
-8
drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c
···
61
61
struct mlx5_flow_table *pol_miss_ft;
62
62
struct mlx5_flow_handle *pol_miss_rule;
63
63
u8 allow_tunnel_mode : 1;
64
+
u8 ttc_rules_added : 1;
64
65
};
65
66
66
67
/* IPsec RX flow steering */
···
586
585
return err;
587
586
}
588
587
588
+
static struct mlx5_flow_destination
589
+
ipsec_rx_decrypted_pkt_def_dest(struct mlx5_ttc_table *ttc, u32 family)
590
+
{
591
+
struct mlx5_flow_destination dest;
592
+
593
+
if (!mlx5_ttc_has_esp_flow_group(ttc))
594
+
return mlx5_ttc_get_default_dest(ttc, family2tt(family));
595
+
596
+
dest.ft = mlx5_get_ttc_flow_table(ttc);
597
+
dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
598
+
599
+
return dest;
600
+
}
601
+
589
602
static void ipsec_rx_update_default_dest(struct mlx5e_ipsec_rx *rx,
590
603
struct mlx5_flow_destination *old_dest,
591
604
struct mlx5_flow_destination *new_dest)
···
613
598
{
614
599
struct mlx5e_ipsec_rx *rx = ipsec_rx(ipsec, family, XFRM_DEV_OFFLOAD_PACKET);
615
600
struct mlx5_flow_namespace *ns = mlx5e_fs_get_ns(ipsec->fs, false);
601
+
struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(ipsec->fs, false);
616
602
struct mlx5_flow_destination old_dest, new_dest;
617
603
618
-
old_dest = mlx5_ttc_get_default_dest(mlx5e_fs_get_ttc(ipsec->fs, false),
619
-
family2tt(family));
604
+
old_dest = ipsec_rx_decrypted_pkt_def_dest(ttc, family);
620
605
621
606
mlx5_ipsec_fs_roce_rx_create(ipsec->mdev, ipsec->roce, ns, &old_dest, family,
622
607
MLX5E_ACCEL_FS_ESP_FT_ROCE_LEVEL, MLX5E_NIC_PRIO);
···
629
614
static void handle_ipsec_rx_cleanup(struct mlx5e_ipsec *ipsec, u32 family)
630
615
{
631
616
struct mlx5e_ipsec_rx *rx = ipsec_rx(ipsec, family, XFRM_DEV_OFFLOAD_PACKET);
617
+
struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(ipsec->fs, false);
632
618
struct mlx5_flow_destination old_dest, new_dest;
633
619
634
620
old_dest.ft = mlx5_ipsec_fs_roce_ft_get(ipsec->roce, family);
635
621
old_dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
636
-
new_dest = mlx5_ttc_get_default_dest(mlx5e_fs_get_ttc(ipsec->fs, false),
637
-
family2tt(family));
622
+
new_dest = ipsec_rx_decrypted_pkt_def_dest(ttc, family);
638
623
ipsec_rx_update_default_dest(rx, &old_dest, &new_dest);
639
624
640
625
mlx5_ipsec_fs_roce_rx_destroy(ipsec->roce, family, ipsec->mdev);
···
684
669
complete(&work->master_priv->ipsec->comp);
685
670
}
686
671
687
-
static void ipsec_rx_ft_disconnect(struct mlx5e_ipsec *ipsec, u32 family)
672
+
static void ipsec_rx_ft_disconnect(struct mlx5e_ipsec *ipsec,
673
+
struct mlx5e_ipsec_rx *rx, u32 family)
688
674
{
689
675
struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(ipsec->fs, false);
690
676
677
+
if (rx->ttc_rules_added)
678
+
mlx5_ttc_destroy_ipsec_rules(ttc);
691
679
mlx5_ttc_fwd_default_dest(ttc, family2tt(family));
692
680
}
693
681
···
725
707
{
726
708
/* disconnect */
727
709
if (rx != ipsec->rx_esw)
728
-
ipsec_rx_ft_disconnect(ipsec, family);
710
+
ipsec_rx_ft_disconnect(ipsec, rx, family);
729
711
730
712
mlx5_del_flow_rules(rx->sa.rule);
731
713
mlx5_destroy_flow_group(rx->sa.group);
···
782
764
if (rx == ipsec->rx_esw)
783
765
return mlx5_esw_ipsec_rx_status_pass_dest_get(ipsec, dest);
784
766
785
-
*dest = mlx5_ttc_get_default_dest(attr->ttc, family2tt(attr->family));
767
+
*dest = ipsec_rx_decrypted_pkt_def_dest(attr->ttc, attr->family);
786
768
err = mlx5_ipsec_fs_roce_rx_create(ipsec->mdev, ipsec->roce, attr->ns, dest,
787
769
attr->family, MLX5E_ACCEL_FS_ESP_FT_ROCE_LEVEL,
788
770
attr->prio);
···
825
807
struct mlx5e_ipsec_rx_create_attr *attr)
826
808
{
827
809
struct mlx5_flow_destination dest = {};
810
+
struct mlx5_ttc_table *ttc, *inner_ttc;
828
811
829
812
dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
830
813
dest.ft = rx->ft.sa;
831
-
mlx5_ttc_fwd_dest(attr->ttc, family2tt(attr->family), &dest);
814
+
if (mlx5_ttc_fwd_dest(attr->ttc, family2tt(attr->family), &dest))
815
+
return;
816
+
817
+
ttc = mlx5e_fs_get_ttc(ipsec->fs, false);
818
+
inner_ttc = mlx5e_fs_get_ttc(ipsec->fs, true);
819
+
rx->ttc_rules_added = !mlx5_ttc_create_ipsec_rules(ttc, inner_ttc);
832
820
}
833
821
834
822
static int ipsec_rx_chains_create_miss(struct mlx5e_ipsec *ipsec,
+19
-2
drivers/net/ethernet/mellanox/mlx5/core/en_fs.c
+19
-2
drivers/net/ethernet/mellanox/mlx5/core/en_fs.c
···
905
905
ft_attr->prio = MLX5E_NIC_PRIO;
906
906
907
907
for (tt = 0; tt < MLX5_NUM_TT; tt++) {
908
+
if (mlx5_ttc_is_decrypted_esp_tt(tt))
909
+
continue;
910
+
908
911
ttc_params->dests[tt].type = MLX5_FLOW_DESTINATION_TYPE_TIR;
909
912
ttc_params->dests[tt].tir_num =
910
913
tt == MLX5_TT_ANY ?
···
917
914
}
918
915
}
919
916
917
+
static bool mlx5e_ipsec_rss_supported(struct mlx5_core_dev *mdev)
918
+
{
919
+
return MLX5_CAP_NIC_RX_FT_FIELD_SUPPORT_2(mdev, ipsec_next_header) &&
920
+
MLX5_CAP_NIC_RX_FT_FIELD_SUPPORT_2(mdev, outer_l4_type_ext) &&
921
+
MLX5_CAP_NIC_RX_FT_FIELD_SUPPORT_2(mdev, inner_l4_type_ext);
922
+
}
923
+
920
924
void mlx5e_set_ttc_params(struct mlx5e_flow_steering *fs,
921
925
struct mlx5e_rx_res *rx_res,
922
-
struct ttc_params *ttc_params, bool tunnel)
926
+
struct ttc_params *ttc_params, bool tunnel,
927
+
bool ipsec_rss)
923
928
924
929
{
925
930
struct mlx5_flow_table_attr *ft_attr = &ttc_params->ft_attr;
···
938
927
ft_attr->level = MLX5E_TTC_FT_LEVEL;
939
928
ft_attr->prio = MLX5E_NIC_PRIO;
940
929
930
+
ttc_params->ipsec_rss = ipsec_rss &&
931
+
mlx5e_ipsec_rss_supported(fs->mdev);
932
+
941
933
for (tt = 0; tt < MLX5_NUM_TT; tt++) {
934
+
if (mlx5_ttc_is_decrypted_esp_tt(tt))
935
+
continue;
936
+
942
937
ttc_params->dests[tt].type = MLX5_FLOW_DESTINATION_TYPE_TIR;
943
938
ttc_params->dests[tt].tir_num =
944
939
tt == MLX5_TT_ANY ?
···
1310
1293
{
1311
1294
struct ttc_params ttc_params = {};
1312
1295
1313
-
mlx5e_set_ttc_params(fs, rx_res, &ttc_params, true);
1296
+
mlx5e_set_ttc_params(fs, rx_res, &ttc_params, true, true);
1314
1297
fs->ttc = mlx5_create_ttc_table(fs->mdev, &ttc_params);
1315
1298
return PTR_ERR_OR_ZERO(fs->ttc);
1316
1299
}
+1
-1
drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
+1
-1
drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
···
974
974
MLX5_FLOW_NAMESPACE_KERNEL), false);
975
975
976
976
/* The inner_ttc in the ttc params is intentionally not set */
977
-
mlx5e_set_ttc_params(priv->fs, priv->rx_res, &ttc_params, false);
977
+
mlx5e_set_ttc_params(priv->fs, priv->rx_res, &ttc_params, false, false);
978
978
979
979
if (rep->vport != MLX5_VPORT_UPLINK)
980
980
/* To give uplik rep TTC a lower level for chaining from root ft */
+3
drivers/net/ethernet/mellanox/mlx5/core/en_tc.c
+3
drivers/net/ethernet/mellanox/mlx5/core/en_tc.c
···
838
838
839
839
ttc_params->ns_type = MLX5_FLOW_NAMESPACE_KERNEL;
840
840
for (tt = 0; tt < MLX5_NUM_TT; tt++) {
841
+
if (mlx5_ttc_is_decrypted_esp_tt(tt))
842
+
continue;
843
+
841
844
ttc_params->dests[tt].type = MLX5_FLOW_DESTINATION_TYPE_TIR;
842
845
ttc_params->dests[tt].tir_num =
843
846
tt == MLX5_TT_ANY ?
+372
-25
drivers/net/ethernet/mellanox/mlx5/core/lib/fs_ttc.c
+372
-25
drivers/net/ethernet/mellanox/mlx5/core/lib/fs_ttc.c
···
9
9
#include "mlx5_core.h"
10
10
#include "lib/fs_ttc.h"
11
11
12
-
#define MLX5_TTC_MAX_NUM_GROUPS 4
12
+
#define MLX5_TTC_MAX_NUM_GROUPS 7
13
13
#define MLX5_TTC_GROUP_TCPUDP_SIZE (MLX5_TT_IPV6_UDP + 1)
14
14
15
15
struct mlx5_fs_ttc_groups {
···
31
31
/* L3/L4 traffic type classifier */
32
32
struct mlx5_ttc_table {
33
33
int num_groups;
34
+
const struct mlx5_fs_ttc_groups *groups;
35
+
struct mlx5_core_dev *mdev;
34
36
struct mlx5_flow_table *t;
35
37
struct mlx5_flow_group **g;
36
38
struct mlx5_ttc_rule rules[MLX5_NUM_TT];
37
39
struct mlx5_flow_handle *tunnel_rules[MLX5_NUM_TUNNEL_TT];
40
+
u32 refcnt;
41
+
struct mutex mutex; /* Protect adding rules for ipsec crypto offload */
38
42
};
39
43
40
44
struct mlx5_flow_table *mlx5_get_ttc_flow_table(struct mlx5_ttc_table *ttc)
···
167
163
enum TTC_GROUP_TYPE {
168
164
TTC_GROUPS_DEFAULT = 0,
169
165
TTC_GROUPS_USE_L4_TYPE = 1,
166
+
TTC_GROUPS_DEFAULT_ESP = 2,
167
+
TTC_GROUPS_USE_L4_TYPE_ESP = 3,
170
168
};
171
169
172
170
static const struct mlx5_fs_ttc_groups ttc_groups[] = {
···
186
180
.group_size = {
187
181
MLX5_TTC_GROUP_TCPUDP_SIZE,
188
182
BIT(3) + MLX5_NUM_TUNNEL_TT - MLX5_TTC_GROUP_TCPUDP_SIZE,
183
+
BIT(1),
184
+
BIT(0),
185
+
},
186
+
},
187
+
[TTC_GROUPS_DEFAULT_ESP] = {
188
+
.num_groups = 6,
189
+
.group_size = {
190
+
MLX5_TTC_GROUP_TCPUDP_SIZE + BIT(1) +
191
+
MLX5_NUM_TUNNEL_TT,
192
+
BIT(2), /* decrypted outer L4 */
193
+
BIT(2), /* decrypted inner L4 */
194
+
BIT(1), /* ESP */
195
+
BIT(1),
196
+
BIT(0),
197
+
},
198
+
},
199
+
[TTC_GROUPS_USE_L4_TYPE_ESP] = {
200
+
.use_l4_type = true,
201
+
.num_groups = 7,
202
+
.group_size = {
203
+
MLX5_TTC_GROUP_TCPUDP_SIZE,
204
+
BIT(1) + MLX5_NUM_TUNNEL_TT,
205
+
BIT(2), /* decrypted outer L4 */
206
+
BIT(2), /* decrypted inner L4 */
207
+
BIT(1), /* ESP */
189
208
BIT(1),
190
209
BIT(0),
191
210
},
···
237
206
},
238
207
},
239
208
};
209
+
210
+
static const struct mlx5_fs_ttc_groups *
211
+
mlx5_ttc_get_fs_groups(bool use_l4_type, bool ipsec_rss)
212
+
{
213
+
if (!ipsec_rss)
214
+
return use_l4_type ? &ttc_groups[TTC_GROUPS_USE_L4_TYPE] :
215
+
&ttc_groups[TTC_GROUPS_DEFAULT];
216
+
217
+
return use_l4_type ? &ttc_groups[TTC_GROUPS_USE_L4_TYPE_ESP] :
218
+
&ttc_groups[TTC_GROUPS_DEFAULT_ESP];
219
+
}
220
+
221
+
bool mlx5_ttc_has_esp_flow_group(struct mlx5_ttc_table *ttc)
222
+
{
223
+
return ttc->groups == &ttc_groups[TTC_GROUPS_DEFAULT_ESP] ||
224
+
ttc->groups == &ttc_groups[TTC_GROUPS_USE_L4_TYPE_ESP];
225
+
}
240
226
241
227
u8 mlx5_get_proto_by_tunnel_type(enum mlx5_tunnel_types tt)
242
228
{
···
305
257
return 0;
306
258
}
307
259
260
+
static void mlx5_fs_ttc_set_match_ipv_outer(struct mlx5_core_dev *mdev,
261
+
struct mlx5_flow_spec *spec,
262
+
u16 etype)
263
+
{
264
+
int match_ipv_outer =
265
+
MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
266
+
ft_field_support.outer_ip_version);
267
+
u8 ipv;
268
+
269
+
ipv = mlx5_etype_to_ipv(etype);
270
+
if (match_ipv_outer && ipv) {
271
+
MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
272
+
outer_headers.ip_version);
273
+
MLX5_SET(fte_match_param, spec->match_value,
274
+
outer_headers.ip_version, ipv);
275
+
} else {
276
+
MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
277
+
outer_headers.ethertype);
278
+
MLX5_SET(fte_match_param, spec->match_value,
279
+
outer_headers.ethertype, etype);
280
+
}
281
+
282
+
spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
283
+
}
284
+
308
285
static void mlx5_fs_ttc_set_match_proto(void *headers_c, void *headers_v,
309
286
u8 proto, bool use_l4_type)
310
287
{
···
352
279
static struct mlx5_flow_handle *
353
280
mlx5_generate_ttc_rule(struct mlx5_core_dev *dev, struct mlx5_flow_table *ft,
354
281
struct mlx5_flow_destination *dest, u16 etype, u8 proto,
355
-
bool use_l4_type)
282
+
bool use_l4_type, bool ipsec_rss)
356
283
{
357
-
int match_ipv_outer =
358
-
MLX5_CAP_FLOWTABLE_NIC_RX(dev,
359
-
ft_field_support.outer_ip_version);
360
284
MLX5_DECLARE_FLOW_ACT(flow_act);
361
285
struct mlx5_flow_handle *rule;
362
286
struct mlx5_flow_spec *spec;
363
287
int err = 0;
364
-
u8 ipv;
365
288
366
289
spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
367
290
if (!spec)
···
374
305
proto, use_l4_type);
375
306
}
376
307
377
-
ipv = mlx5_etype_to_ipv(etype);
378
-
if (match_ipv_outer && ipv) {
379
-
spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
380
-
MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version);
381
-
MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, ipv);
382
-
} else if (etype) {
383
-
spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
384
-
MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ethertype);
385
-
MLX5_SET(fte_match_param, spec->match_value, outer_headers.ethertype, etype);
308
+
if (etype)
309
+
mlx5_fs_ttc_set_match_ipv_outer(dev, spec, etype);
310
+
311
+
if (ipsec_rss && proto == IPPROTO_ESP) {
312
+
MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
313
+
misc_parameters_2.ipsec_next_header);
314
+
MLX5_SET(fte_match_param, spec->match_value,
315
+
misc_parameters_2.ipsec_next_header, 0);
316
+
spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2;
386
317
}
387
318
388
319
rule = mlx5_add_flow_rules(ft, spec, &flow_act, dest, 1);
···
411
342
for (tt = 0; tt < MLX5_NUM_TT; tt++) {
412
343
struct mlx5_ttc_rule *rule = &rules[tt];
413
344
345
+
if (mlx5_ttc_is_decrypted_esp_tt(tt))
346
+
continue;
347
+
414
348
if (test_bit(tt, params->ignore_dests))
415
349
continue;
416
350
rule->rule = mlx5_generate_ttc_rule(dev, ft, ¶ms->dests[tt],
417
351
ttc_rules[tt].etype,
418
352
ttc_rules[tt].proto,
419
-
use_l4_type);
353
+
use_l4_type,
354
+
params->ipsec_rss);
420
355
if (IS_ERR(rule->rule)) {
421
356
err = PTR_ERR(rule->rule);
422
357
rule->rule = NULL;
···
443
370
¶ms->tunnel_dests[tt],
444
371
ttc_tunnel_rules[tt].etype,
445
372
ttc_tunnel_rules[tt].proto,
446
-
use_l4_type);
373
+
use_l4_type, false);
447
374
if (IS_ERR(trules[tt])) {
448
375
err = PTR_ERR(trules[tt]);
449
376
trules[tt] = NULL;
···
458
385
return err;
459
386
}
460
387
461
-
static int mlx5_create_ttc_table_groups(struct mlx5_ttc_table *ttc,
462
-
bool use_ipv,
463
-
const struct mlx5_fs_ttc_groups *groups)
388
+
static int mlx5_create_ttc_table_ipsec_groups(struct mlx5_ttc_table *ttc,
389
+
bool use_ipv,
390
+
u32 *in, int *next_ix)
464
391
{
392
+
u8 *mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
393
+
const struct mlx5_fs_ttc_groups *groups = ttc->groups;
394
+
int ix = *next_ix;
395
+
396
+
MLX5_SET(fte_match_param, mc, outer_headers.ip_protocol, 0);
397
+
398
+
/* decrypted ESP outer group */
399
+
MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
400
+
MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.l4_type_ext);
401
+
MLX5_SET_CFG(in, start_flow_index, ix);
402
+
ix += groups->group_size[ttc->num_groups];
403
+
MLX5_SET_CFG(in, end_flow_index, ix - 1);
404
+
ttc->g[ttc->num_groups] = mlx5_create_flow_group(ttc->t, in);
405
+
if (IS_ERR(ttc->g[ttc->num_groups]))
406
+
goto err;
407
+
ttc->num_groups++;
408
+
409
+
MLX5_SET(fte_match_param, mc, outer_headers.l4_type_ext, 0);
410
+
411
+
/* decrypted ESP inner group */
412
+
MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_INNER_HEADERS);
413
+
if (use_ipv)
414
+
MLX5_SET(fte_match_param, mc, outer_headers.ip_version, 0);
415
+
else
416
+
MLX5_SET(fte_match_param, mc, outer_headers.ethertype, 0);
417
+
MLX5_SET_TO_ONES(fte_match_param, mc, inner_headers.ip_version);
418
+
MLX5_SET_TO_ONES(fte_match_param, mc, inner_headers.l4_type_ext);
419
+
MLX5_SET_CFG(in, start_flow_index, ix);
420
+
ix += groups->group_size[ttc->num_groups];
421
+
MLX5_SET_CFG(in, end_flow_index, ix - 1);
422
+
ttc->g[ttc->num_groups] = mlx5_create_flow_group(ttc->t, in);
423
+
if (IS_ERR(ttc->g[ttc->num_groups]))
424
+
goto err;
425
+
ttc->num_groups++;
426
+
427
+
MLX5_SET(fte_match_param, mc, inner_headers.ip_version, 0);
428
+
MLX5_SET(fte_match_param, mc, inner_headers.l4_type_ext, 0);
429
+
430
+
/* undecrypted ESP group */
431
+
MLX5_SET_CFG(in, match_criteria_enable,
432
+
MLX5_MATCH_OUTER_HEADERS | MLX5_MATCH_MISC_PARAMETERS_2);
433
+
if (use_ipv)
434
+
MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ip_version);
435
+
else
436
+
MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype);
437
+
MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ip_protocol);
438
+
MLX5_SET_TO_ONES(fte_match_param, mc,
439
+
misc_parameters_2.ipsec_next_header);
440
+
MLX5_SET_CFG(in, start_flow_index, ix);
441
+
ix += groups->group_size[ttc->num_groups];
442
+
MLX5_SET_CFG(in, end_flow_index, ix - 1);
443
+
ttc->g[ttc->num_groups] = mlx5_create_flow_group(ttc->t, in);
444
+
if (IS_ERR(ttc->g[ttc->num_groups]))
445
+
goto err;
446
+
ttc->num_groups++;
447
+
448
+
*next_ix = ix;
449
+
450
+
return 0;
451
+
452
+
err:
453
+
return PTR_ERR(ttc->g[ttc->num_groups]);
454
+
}
455
+
456
+
static int mlx5_create_ttc_table_groups(struct mlx5_ttc_table *ttc,
457
+
bool use_ipv)
458
+
{
459
+
const struct mlx5_fs_ttc_groups *groups = ttc->groups;
465
460
int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
466
461
int ix = 0;
467
462
u32 *in;
···
577
436
goto err;
578
437
ttc->num_groups++;
579
438
439
+
if (mlx5_ttc_has_esp_flow_group(ttc)) {
440
+
err = mlx5_create_ttc_table_ipsec_groups(ttc, use_ipv, in, &ix);
441
+
if (err)
442
+
goto err;
443
+
444
+
MLX5_SET(fte_match_param, mc,
445
+
misc_parameters_2.ipsec_next_header, 0);
446
+
}
447
+
580
448
/* L3 Group */
581
449
MLX5_SET(fte_match_param, mc, outer_headers.ip_protocol, 0);
450
+
MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
582
451
MLX5_SET_CFG(in, start_flow_index, ix);
583
452
ix += groups->group_size[ttc->num_groups];
584
453
MLX5_SET_CFG(in, end_flow_index, ix - 1);
···
677
526
678
527
for (tt = 0; tt < MLX5_NUM_TT; tt++) {
679
528
struct mlx5_ttc_rule *rule = &rules[tt];
529
+
530
+
if (mlx5_ttc_is_decrypted_esp_tt(tt))
531
+
continue;
680
532
681
533
if (test_bit(tt, params->ignore_dests))
682
534
continue;
···
854
700
855
701
kfree(ttc->g);
856
702
mlx5_destroy_flow_table(ttc->t);
703
+
mutex_destroy(&ttc->mutex);
857
704
kvfree(ttc);
858
705
}
859
706
···
864
709
bool match_ipv_outer =
865
710
MLX5_CAP_FLOWTABLE_NIC_RX(dev,
866
711
ft_field_support.outer_ip_version);
867
-
const struct mlx5_fs_ttc_groups *groups;
868
712
struct mlx5_flow_namespace *ns;
869
713
struct mlx5_ttc_table *ttc;
870
714
bool use_l4_type;
···
892
738
return ERR_PTR(-EOPNOTSUPP);
893
739
}
894
740
895
-
groups = use_l4_type ? &ttc_groups[TTC_GROUPS_USE_L4_TYPE] :
896
-
&ttc_groups[TTC_GROUPS_DEFAULT];
741
+
ttc->groups = mlx5_ttc_get_fs_groups(use_l4_type, params->ipsec_rss);
897
742
898
743
WARN_ON_ONCE(params->ft_attr.max_fte);
899
-
params->ft_attr.max_fte = mlx5_fs_ttc_table_size(groups);
744
+
params->ft_attr.max_fte = mlx5_fs_ttc_table_size(ttc->groups);
900
745
ttc->t = mlx5_create_flow_table(ns, ¶ms->ft_attr);
901
746
if (IS_ERR(ttc->t)) {
902
747
err = PTR_ERR(ttc->t);
···
903
750
return ERR_PTR(err);
904
751
}
905
752
906
-
err = mlx5_create_ttc_table_groups(ttc, match_ipv_outer, groups);
753
+
err = mlx5_create_ttc_table_groups(ttc, match_ipv_outer);
907
754
if (err)
908
755
goto destroy_ft;
909
756
910
757
err = mlx5_generate_ttc_table_rules(dev, params, ttc, use_l4_type);
911
758
if (err)
912
759
goto destroy_ft;
760
+
761
+
ttc->mdev = dev;
762
+
mutex_init(&ttc->mutex);
913
763
914
764
return ttc;
915
765
···
946
790
struct mlx5_flow_destination dest = mlx5_ttc_get_default_dest(ttc, type);
947
791
948
792
return mlx5_ttc_fwd_dest(ttc, type, &dest);
793
+
}
794
+
795
+
static void _mlx5_ttc_destroy_ipsec_rules(struct mlx5_ttc_table *ttc)
796
+
{
797
+
enum mlx5_traffic_types i;
798
+
799
+
for (i = MLX5_TT_DECRYPTED_ESP_OUTER_IPV4_TCP;
800
+
i <= MLX5_TT_DECRYPTED_ESP_INNER_IPV6_UDP; i++) {
801
+
if (!ttc->rules[i].rule)
802
+
continue;
803
+
804
+
mlx5_del_flow_rules(ttc->rules[i].rule);
805
+
ttc->rules[i].rule = NULL;
806
+
}
807
+
}
808
+
809
+
void mlx5_ttc_destroy_ipsec_rules(struct mlx5_ttc_table *ttc)
810
+
{
811
+
if (!mlx5_ttc_has_esp_flow_group(ttc))
812
+
return;
813
+
814
+
mutex_lock(&ttc->mutex);
815
+
if (--ttc->refcnt)
816
+
goto unlock;
817
+
818
+
_mlx5_ttc_destroy_ipsec_rules(ttc);
819
+
unlock:
820
+
mutex_unlock(&ttc->mutex);
821
+
}
822
+
823
+
static int mlx5_ttc_get_tt_attrs(enum mlx5_traffic_types type,
824
+
u16 *etype, int *l4_type_ext,
825
+
enum mlx5_traffic_types *tir_tt)
826
+
{
827
+
switch (type) {
828
+
case MLX5_TT_DECRYPTED_ESP_OUTER_IPV4_TCP:
829
+
case MLX5_TT_DECRYPTED_ESP_INNER_IPV4_TCP:
830
+
*etype = ETH_P_IP;
831
+
*l4_type_ext = MLX5_PACKET_L4_TYPE_EXT_TCP;
832
+
*tir_tt = MLX5_TT_IPV4_TCP;
833
+
break;
834
+
case MLX5_TT_DECRYPTED_ESP_OUTER_IPV6_TCP:
835
+
case MLX5_TT_DECRYPTED_ESP_INNER_IPV6_TCP:
836
+
*etype = ETH_P_IPV6;
837
+
*l4_type_ext = MLX5_PACKET_L4_TYPE_EXT_TCP;
838
+
*tir_tt = MLX5_TT_IPV6_TCP;
839
+
break;
840
+
case MLX5_TT_DECRYPTED_ESP_OUTER_IPV4_UDP:
841
+
case MLX5_TT_DECRYPTED_ESP_INNER_IPV4_UDP:
842
+
*etype = ETH_P_IP;
843
+
*l4_type_ext = MLX5_PACKET_L4_TYPE_EXT_UDP;
844
+
*tir_tt = MLX5_TT_IPV4_UDP;
845
+
break;
846
+
case MLX5_TT_DECRYPTED_ESP_OUTER_IPV6_UDP:
847
+
case MLX5_TT_DECRYPTED_ESP_INNER_IPV6_UDP:
848
+
*etype = ETH_P_IPV6;
849
+
*l4_type_ext = MLX5_PACKET_L4_TYPE_EXT_UDP;
850
+
*tir_tt = MLX5_TT_IPV6_UDP;
851
+
break;
852
+
default:
853
+
return -EINVAL;
854
+
}
855
+
856
+
return 0;
857
+
}
858
+
859
+
static struct mlx5_flow_handle *
860
+
mlx5_ttc_create_ipsec_outer_rule(struct mlx5_ttc_table *ttc,
861
+
enum mlx5_traffic_types type)
862
+
{
863
+
struct mlx5_flow_destination dest;
864
+
MLX5_DECLARE_FLOW_ACT(flow_act);
865
+
enum mlx5_traffic_types tir_tt;
866
+
struct mlx5_flow_handle *rule;
867
+
struct mlx5_flow_spec *spec;
868
+
int l4_type_ext;
869
+
u16 etype;
870
+
int err;
871
+
872
+
err = mlx5_ttc_get_tt_attrs(type, &etype, &l4_type_ext, &tir_tt);
873
+
if (err)
874
+
return ERR_PTR(err);
875
+
876
+
spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
877
+
if (!spec)
878
+
return ERR_PTR(-ENOMEM);
879
+
880
+
mlx5_fs_ttc_set_match_ipv_outer(ttc->mdev, spec, etype);
881
+
882
+
MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
883
+
outer_headers.l4_type_ext);
884
+
MLX5_SET(fte_match_param, spec->match_value,
885
+
outer_headers.l4_type_ext, l4_type_ext);
886
+
887
+
dest = mlx5_ttc_get_default_dest(ttc, tir_tt);
888
+
889
+
rule = mlx5_add_flow_rules(ttc->t, spec, &flow_act, &dest, 1);
890
+
if (IS_ERR(rule)) {
891
+
err = PTR_ERR(rule);
892
+
mlx5_core_err(ttc->mdev, "%s: add rule failed\n", __func__);
893
+
}
894
+
895
+
kvfree(spec);
896
+
return err ? ERR_PTR(err) : rule;
897
+
}
898
+
899
+
static struct mlx5_flow_handle *
900
+
mlx5_ttc_create_ipsec_inner_rule(struct mlx5_ttc_table *ttc,
901
+
struct mlx5_ttc_table *inner_ttc,
902
+
enum mlx5_traffic_types type)
903
+
{
904
+
struct mlx5_flow_destination dest;
905
+
MLX5_DECLARE_FLOW_ACT(flow_act);
906
+
enum mlx5_traffic_types tir_tt;
907
+
struct mlx5_flow_handle *rule;
908
+
struct mlx5_flow_spec *spec;
909
+
int l4_type_ext;
910
+
u16 etype;
911
+
int err;
912
+
913
+
err = mlx5_ttc_get_tt_attrs(type, &etype, &l4_type_ext, &tir_tt);
914
+
if (err)
915
+
return ERR_PTR(err);
916
+
917
+
spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
918
+
if (!spec)
919
+
return ERR_PTR(-ENOMEM);
920
+
921
+
MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
922
+
inner_headers.ip_version);
923
+
MLX5_SET(fte_match_param, spec->match_value,
924
+
inner_headers.ip_version, mlx5_etype_to_ipv(etype));
925
+
MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
926
+
inner_headers.l4_type_ext);
927
+
MLX5_SET(fte_match_param, spec->match_value,
928
+
inner_headers.l4_type_ext, l4_type_ext);
929
+
930
+
dest = mlx5_ttc_get_default_dest(inner_ttc, tir_tt);
931
+
932
+
spec->match_criteria_enable = MLX5_MATCH_INNER_HEADERS;
933
+
934
+
rule = mlx5_add_flow_rules(ttc->t, spec, &flow_act, &dest, 1);
935
+
if (IS_ERR(rule)) {
936
+
err = PTR_ERR(rule);
937
+
mlx5_core_err(ttc->mdev, "%s: add rule failed\n", __func__);
938
+
}
939
+
940
+
kvfree(spec);
941
+
return err ? ERR_PTR(err) : rule;
942
+
}
943
+
944
+
int mlx5_ttc_create_ipsec_rules(struct mlx5_ttc_table *ttc,
945
+
struct mlx5_ttc_table *inner_ttc)
946
+
{
947
+
struct mlx5_flow_handle *rule;
948
+
enum mlx5_traffic_types i;
949
+
950
+
if (!mlx5_ttc_has_esp_flow_group(ttc))
951
+
return 0;
952
+
953
+
mutex_lock(&ttc->mutex);
954
+
if (ttc->refcnt)
955
+
goto skip;
956
+
957
+
for (i = MLX5_TT_DECRYPTED_ESP_OUTER_IPV4_TCP;
958
+
i <= MLX5_TT_DECRYPTED_ESP_OUTER_IPV6_UDP; i++) {
959
+
rule = mlx5_ttc_create_ipsec_outer_rule(ttc, i);
960
+
if (IS_ERR(rule))
961
+
goto err_out;
962
+
963
+
ttc->rules[i].rule = rule;
964
+
}
965
+
966
+
for (i = MLX5_TT_DECRYPTED_ESP_INNER_IPV4_TCP;
967
+
i <= MLX5_TT_DECRYPTED_ESP_INNER_IPV6_UDP; i++) {
968
+
rule = mlx5_ttc_create_ipsec_inner_rule(ttc, inner_ttc, i);
969
+
if (IS_ERR(rule))
970
+
goto err_out;
971
+
972
+
ttc->rules[i].rule = rule;
973
+
}
974
+
975
+
skip:
976
+
ttc->refcnt++;
977
+
mutex_unlock(&ttc->mutex);
978
+
return 0;
979
+
980
+
err_out:
981
+
_mlx5_ttc_destroy_ipsec_rules(ttc);
982
+
mutex_unlock(&ttc->mutex);
983
+
return PTR_ERR(rule);
949
984
}
+19
drivers/net/ethernet/mellanox/mlx5/core/lib/fs_ttc.h
+19
drivers/net/ethernet/mellanox/mlx5/core/lib/fs_ttc.h
···
18
18
MLX5_TT_IPV4,
19
19
MLX5_TT_IPV6,
20
20
MLX5_TT_ANY,
21
+
MLX5_TT_DECRYPTED_ESP_OUTER_IPV4_TCP,
22
+
MLX5_TT_DECRYPTED_ESP_OUTER_IPV6_TCP,
23
+
MLX5_TT_DECRYPTED_ESP_OUTER_IPV4_UDP,
24
+
MLX5_TT_DECRYPTED_ESP_OUTER_IPV6_UDP,
25
+
MLX5_TT_DECRYPTED_ESP_INNER_IPV4_TCP,
26
+
MLX5_TT_DECRYPTED_ESP_INNER_IPV6_TCP,
27
+
MLX5_TT_DECRYPTED_ESP_INNER_IPV4_UDP,
28
+
MLX5_TT_DECRYPTED_ESP_INNER_IPV6_UDP,
21
29
MLX5_NUM_TT,
22
30
MLX5_NUM_INDIR_TIRS = MLX5_TT_ANY,
23
31
};
···
55
47
bool inner_ttc;
56
48
DECLARE_BITMAP(ignore_tunnel_dests, MLX5_NUM_TUNNEL_TT);
57
49
struct mlx5_flow_destination tunnel_dests[MLX5_NUM_TUNNEL_TT];
50
+
bool ipsec_rss;
58
51
};
59
52
60
53
const char *mlx5_ttc_get_name(enum mlx5_traffic_types tt);
···
78
69
79
70
bool mlx5_tunnel_inner_ft_supported(struct mlx5_core_dev *mdev);
80
71
u8 mlx5_get_proto_by_tunnel_type(enum mlx5_tunnel_types tt);
72
+
73
+
bool mlx5_ttc_has_esp_flow_group(struct mlx5_ttc_table *ttc);
74
+
int mlx5_ttc_create_ipsec_rules(struct mlx5_ttc_table *ttc,
75
+
struct mlx5_ttc_table *inner_ttc);
76
+
void mlx5_ttc_destroy_ipsec_rules(struct mlx5_ttc_table *ttc);
77
+
static inline bool mlx5_ttc_is_decrypted_esp_tt(enum mlx5_traffic_types tt)
78
+
{
79
+
return tt >= MLX5_TT_DECRYPTED_ESP_OUTER_IPV4_TCP &&
80
+
tt <= MLX5_TT_DECRYPTED_ESP_INNER_IPV6_UDP;
81
+
}
81
82
82
83
#endif /* __MLX5_FS_TTC_H__ */
+4
drivers/net/ethernet/mellanox/mlx5/core/lib/ipsec_fs_roce.c
+4
drivers/net/ethernet/mellanox/mlx5/core/lib/ipsec_fs_roce.c
···
164
164
roce->rule = rule;
165
165
166
166
memset(spec, 0, sizeof(*spec));
167
+
if (default_dst->type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE)
168
+
flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
167
169
rule = mlx5_add_flow_rules(roce->ft, spec, &flow_act, default_dst, 1);
168
170
if (IS_ERR(rule)) {
169
171
err = PTR_ERR(rule);
···
180
178
goto out;
181
179
182
180
flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
181
+
if (default_dst->type == MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE)
182
+
flow_act.flags &= ~FLOW_ACT_IGNORE_FLOW_LEVEL;
183
183
dst.type = MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE;
184
184
dst.ft = roce->ft_rdma;
185
185
rule = mlx5_add_flow_rules(roce->nic_master_ft, NULL, &flow_act, &dst,