Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge branch 'cn10k-ipswec-outbound-inline-support'
Bharat Bhushan says:
====================
cn10k-ipsec: Add outbound inline ipsec support
This patch series adds outbound inline ipsec support on Marvell
cn10k series of platform. One crypto hardware logical function
(cpt-lf) per netdev is required for inline ipsec outbound
functionality. Software prepare and submit crypto hardware
(CPT) instruction for outbound inline ipsec crypto mode offload.
The CPT instruction have details for encryption and authentication
Crypto hardware encrypt, authenticate and provide the ESP packet
to network hardware logic to transmit ipsec packet.
First patch makes dma memory writable for in-place encryption,
Second patch moves code to common file, Third patch disable
backpressure on crypto (CPT) and network (NIX) hardware.
Patch four onwards enables inline outbound ipsec.
v9->v10:
- Removed unlikely() in data-patch and used static_branch when at least
a SA is configured.
- Added missing READ_ONCE() as per comment on previous patch
- Removed "\n" from end of extack messages
- Poll for context write status check reduced to 100ms from 10s
v8->v9:
- Removed mutex lock to use hardware, now using hardware state
- Previous versions were supporting only 64 SAs and a bitmap was
used for same. That limitation is removed from this version.
- Replaced netdev_err with NL_SET_ERR_MSG_MOD in state add flow
as per comment in previous version
v7->v8:
- spell correction in patch 1/8 (s/sdk/skb)
v6->v7:
- skb data was mapped as device writeable but it was not ensured
that skb is writeable. This version calls skb_unshare() to make
skb data writeable (Thanks Jakub Kicinski for pointing out).
v4->v5:
- Fixed un-initialized warning and pointer check
(comment from Kalesh Anakkur Purayil)
v3->v4:
- Few error messages in data-path removed and some moved
under netif_msg_tx_err().
- Added check for crypto offload (XFRM_DEV_OFFLOAD_CRYPTO)
Thanks "Leon Romanovsky" for pointing out
- Fixed codespell error as per comment from Simon Horman
- Added some other cleanup comment from Kalesh Anakkur Purayil
v2->v3:
- Fix smatch and sparse errors (Comment from Simon Horman)
- Fix build error with W=1 (Comment from Simon Horman)
https://patchwork.kernel.org/project/netdevbpf/patch/20240513105446.297451-6-bbhushan2@marvell.com/
- Some other minor cleanup as per comment
https://www.spinics.net/lists/netdev/msg997197.html
v1->v2:
- Fix compilation error to build driver a module
- Use dma_wmb() instead of architecture specific barrier
- Fix couple of other compilation warnings
====================
Signed-off-by: David S. Miller <davem@davemloft.net>
+1581
-54
+1
MAINTAINERS
+1
MAINTAINERS
···
13942
13942
M: Geetha sowjanya <gakula@marvell.com>
13943
13943
M: Subbaraya Sundeep <sbhatta@marvell.com>
13944
13944
M: hariprasad <hkelam@marvell.com>
13945
+
M: Bharat Bhushan <bbhushan2@marvell.com>
13945
13946
L: netdev@vger.kernel.org
13946
13947
S: Supported
13947
13948
F: drivers/net/ethernet/marvell/octeontx2/nic/
+4
drivers/net/ethernet/marvell/octeontx2/af/mbox.h
+4
drivers/net/ethernet/marvell/octeontx2/af/mbox.h
···
313
313
msg_rsp) \
314
314
M(NIX_BANDPROF_GET_HWINFO, 0x801f, nix_bandprof_get_hwinfo, msg_req, \
315
315
nix_bandprof_get_hwinfo_rsp) \
316
+
M(NIX_CPT_BP_ENABLE, 0x8020, nix_cpt_bp_enable, nix_bp_cfg_req, \
317
+
nix_bp_cfg_rsp) \
318
+
M(NIX_CPT_BP_DISABLE, 0x8021, nix_cpt_bp_disable, nix_bp_cfg_req, \
319
+
msg_rsp) \
316
320
M(NIX_READ_INLINE_IPSEC_CFG, 0x8023, nix_read_inline_ipsec_cfg, \
317
321
msg_req, nix_inline_ipsec_cfg) \
318
322
M(NIX_MCAST_GRP_CREATE, 0x802b, nix_mcast_grp_create, nix_mcast_grp_create_req, \
+58
-10
drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c
+58
-10
drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c
···
569
569
mutex_unlock(&rvu->rsrc_lock);
570
570
}
571
571
572
-
int rvu_mbox_handler_nix_bp_disable(struct rvu *rvu,
573
-
struct nix_bp_cfg_req *req,
574
-
struct msg_rsp *rsp)
572
+
static u16 nix_get_channel(u16 chan, bool cpt_link)
573
+
{
574
+
/* CPT channel for a given link channel is always
575
+
* assumed to be BIT(11) set in link channel.
576
+
*/
577
+
return cpt_link ? chan | BIT(11) : chan;
578
+
}
579
+
580
+
static int nix_bp_disable(struct rvu *rvu,
581
+
struct nix_bp_cfg_req *req,
582
+
struct msg_rsp *rsp, bool cpt_link)
575
583
{
576
584
u16 pcifunc = req->hdr.pcifunc;
577
585
int blkaddr, pf, type, err;
···
587
579
struct rvu_pfvf *pfvf;
588
580
struct nix_hw *nix_hw;
589
581
struct nix_bp *bp;
582
+
u16 chan_v;
590
583
u64 cfg;
591
584
592
585
pf = rvu_get_pf(pcifunc);
···
598
589
if (is_sdp_pfvf(pcifunc))
599
590
type = NIX_INTF_TYPE_SDP;
600
591
592
+
if (cpt_link && !rvu->hw->cpt_links)
593
+
return 0;
594
+
601
595
pfvf = rvu_get_pfvf(rvu, pcifunc);
602
596
err = nix_get_struct_ptrs(rvu, pcifunc, &nix_hw, &blkaddr);
603
597
if (err)
···
609
597
bp = &nix_hw->bp;
610
598
chan_base = pfvf->rx_chan_base + req->chan_base;
611
599
for (chan = chan_base; chan < (chan_base + req->chan_cnt); chan++) {
612
-
cfg = rvu_read64(rvu, blkaddr, NIX_AF_RX_CHANX_CFG(chan));
613
-
rvu_write64(rvu, blkaddr, NIX_AF_RX_CHANX_CFG(chan),
600
+
chan_v = nix_get_channel(chan, cpt_link);
601
+
cfg = rvu_read64(rvu, blkaddr, NIX_AF_RX_CHANX_CFG(chan_v));
602
+
rvu_write64(rvu, blkaddr, NIX_AF_RX_CHANX_CFG(chan_v),
614
603
cfg & ~BIT_ULL(16));
615
604
616
605
if (type == NIX_INTF_TYPE_LBK) {
···
628
615
}
629
616
}
630
617
return 0;
618
+
}
619
+
620
+
int rvu_mbox_handler_nix_bp_disable(struct rvu *rvu,
621
+
struct nix_bp_cfg_req *req,
622
+
struct msg_rsp *rsp)
623
+
{
624
+
return nix_bp_disable(rvu, req, rsp, false);
625
+
}
626
+
627
+
int rvu_mbox_handler_nix_cpt_bp_disable(struct rvu *rvu,
628
+
struct nix_bp_cfg_req *req,
629
+
struct msg_rsp *rsp)
630
+
{
631
+
return nix_bp_disable(rvu, req, rsp, true);
631
632
}
632
633
633
634
static int rvu_nix_get_bpid(struct rvu *rvu, struct nix_bp_cfg_req *req,
···
723
696
return bpid;
724
697
}
725
698
726
-
int rvu_mbox_handler_nix_bp_enable(struct rvu *rvu,
727
-
struct nix_bp_cfg_req *req,
728
-
struct nix_bp_cfg_rsp *rsp)
699
+
static int nix_bp_enable(struct rvu *rvu,
700
+
struct nix_bp_cfg_req *req,
701
+
struct nix_bp_cfg_rsp *rsp,
702
+
bool cpt_link)
729
703
{
730
704
int blkaddr, pf, type, chan_id = 0;
731
705
u16 pcifunc = req->hdr.pcifunc;
732
706
struct rvu_pfvf *pfvf;
733
707
u16 chan_base, chan;
734
708
s16 bpid, bpid_base;
709
+
u16 chan_v;
735
710
u64 cfg;
736
711
737
712
pf = rvu_get_pf(pcifunc);
···
744
715
/* Enable backpressure only for CGX mapped PFs and LBK/SDP interface */
745
716
if (!is_pf_cgxmapped(rvu, pf) && type != NIX_INTF_TYPE_LBK &&
746
717
type != NIX_INTF_TYPE_SDP)
718
+
return 0;
719
+
720
+
if (cpt_link && !rvu->hw->cpt_links)
747
721
return 0;
748
722
749
723
pfvf = rvu_get_pfvf(rvu, pcifunc);
···
762
730
return -EINVAL;
763
731
}
764
732
765
-
cfg = rvu_read64(rvu, blkaddr, NIX_AF_RX_CHANX_CFG(chan));
733
+
chan_v = nix_get_channel(chan, cpt_link);
734
+
735
+
cfg = rvu_read64(rvu, blkaddr, NIX_AF_RX_CHANX_CFG(chan_v));
766
736
cfg &= ~GENMASK_ULL(8, 0);
767
-
rvu_write64(rvu, blkaddr, NIX_AF_RX_CHANX_CFG(chan),
737
+
rvu_write64(rvu, blkaddr, NIX_AF_RX_CHANX_CFG(chan_v),
768
738
cfg | (bpid & GENMASK_ULL(8, 0)) | BIT_ULL(16));
769
739
chan_id++;
770
740
bpid = rvu_nix_get_bpid(rvu, req, type, chan_id);
···
782
748
rsp->chan_cnt = req->chan_cnt;
783
749
784
750
return 0;
751
+
}
752
+
753
+
int rvu_mbox_handler_nix_bp_enable(struct rvu *rvu,
754
+
struct nix_bp_cfg_req *req,
755
+
struct nix_bp_cfg_rsp *rsp)
756
+
{
757
+
return nix_bp_enable(rvu, req, rsp, false);
758
+
}
759
+
760
+
int rvu_mbox_handler_nix_cpt_bp_enable(struct rvu *rvu,
761
+
struct nix_bp_cfg_req *req,
762
+
struct nix_bp_cfg_rsp *rsp)
763
+
{
764
+
return nix_bp_enable(rvu, req, rsp, true);
785
765
}
786
766
787
767
static void nix_setup_lso_tso_l3(struct rvu *rvu, int blkaddr,
+1
drivers/net/ethernet/marvell/octeontx2/nic/Makefile
+1
drivers/net/ethernet/marvell/octeontx2/nic/Makefile
+1058
drivers/net/ethernet/marvell/octeontx2/nic/cn10k_ipsec.c
+1058
drivers/net/ethernet/marvell/octeontx2/nic/cn10k_ipsec.c
···
1
+
// SPDX-License-Identifier: GPL-2.0
2
+
/* Marvell IPSEC offload driver
3
+
*
4
+
* Copyright (C) 2024 Marvell.
5
+
*/
6
+
7
+
#include <net/xfrm.h>
8
+
#include <linux/netdevice.h>
9
+
#include <linux/bitfield.h>
10
+
#include <crypto/aead.h>
11
+
#include <crypto/gcm.h>
12
+
13
+
#include "otx2_common.h"
14
+
#include "otx2_struct.h"
15
+
#include "cn10k_ipsec.h"
16
+
17
+
DEFINE_STATIC_KEY_FALSE(cn10k_ipsec_sa_enabled);
18
+
19
+
static bool is_dev_support_ipsec_offload(struct pci_dev *pdev)
20
+
{
21
+
return is_dev_cn10ka_b0(pdev) || is_dev_cn10kb(pdev);
22
+
}
23
+
24
+
static bool cn10k_cpt_device_set_inuse(struct otx2_nic *pf)
25
+
{
26
+
enum cn10k_cpt_hw_state_e state;
27
+
28
+
while (true) {
29
+
state = atomic_cmpxchg(&pf->ipsec.cpt_state,
30
+
CN10K_CPT_HW_AVAILABLE,
31
+
CN10K_CPT_HW_IN_USE);
32
+
if (state == CN10K_CPT_HW_AVAILABLE)
33
+
return true;
34
+
if (state == CN10K_CPT_HW_UNAVAILABLE)
35
+
return false;
36
+
37
+
mdelay(1);
38
+
}
39
+
}
40
+
41
+
static void cn10k_cpt_device_set_available(struct otx2_nic *pf)
42
+
{
43
+
atomic_set(&pf->ipsec.cpt_state, CN10K_CPT_HW_AVAILABLE);
44
+
}
45
+
46
+
static void cn10k_cpt_device_set_unavailable(struct otx2_nic *pf)
47
+
{
48
+
atomic_set(&pf->ipsec.cpt_state, CN10K_CPT_HW_UNAVAILABLE);
49
+
}
50
+
51
+
static int cn10k_outb_cptlf_attach(struct otx2_nic *pf)
52
+
{
53
+
struct rsrc_attach *attach;
54
+
int ret = -ENOMEM;
55
+
56
+
mutex_lock(&pf->mbox.lock);
57
+
/* Get memory to put this msg */
58
+
attach = otx2_mbox_alloc_msg_attach_resources(&pf->mbox);
59
+
if (!attach)
60
+
goto unlock;
61
+
62
+
attach->cptlfs = true;
63
+
attach->modify = true;
64
+
65
+
/* Send attach request to AF */
66
+
ret = otx2_sync_mbox_msg(&pf->mbox);
67
+
68
+
unlock:
69
+
mutex_unlock(&pf->mbox.lock);
70
+
return ret;
71
+
}
72
+
73
+
static int cn10k_outb_cptlf_detach(struct otx2_nic *pf)
74
+
{
75
+
struct rsrc_detach *detach;
76
+
int ret = -ENOMEM;
77
+
78
+
mutex_lock(&pf->mbox.lock);
79
+
detach = otx2_mbox_alloc_msg_detach_resources(&pf->mbox);
80
+
if (!detach)
81
+
goto unlock;
82
+
83
+
detach->partial = true;
84
+
detach->cptlfs = true;
85
+
86
+
/* Send detach request to AF */
87
+
ret = otx2_sync_mbox_msg(&pf->mbox);
88
+
89
+
unlock:
90
+
mutex_unlock(&pf->mbox.lock);
91
+
return ret;
92
+
}
93
+
94
+
static int cn10k_outb_cptlf_alloc(struct otx2_nic *pf)
95
+
{
96
+
struct cpt_lf_alloc_req_msg *req;
97
+
int ret = -ENOMEM;
98
+
99
+
mutex_lock(&pf->mbox.lock);
100
+
req = otx2_mbox_alloc_msg_cpt_lf_alloc(&pf->mbox);
101
+
if (!req)
102
+
goto unlock;
103
+
104
+
/* PF function */
105
+
req->nix_pf_func = pf->pcifunc;
106
+
/* Enable SE-IE Engine Group */
107
+
req->eng_grpmsk = 1 << CN10K_DEF_CPT_IPSEC_EGRP;
108
+
109
+
ret = otx2_sync_mbox_msg(&pf->mbox);
110
+
111
+
unlock:
112
+
mutex_unlock(&pf->mbox.lock);
113
+
return ret;
114
+
}
115
+
116
+
static void cn10k_outb_cptlf_free(struct otx2_nic *pf)
117
+
{
118
+
mutex_lock(&pf->mbox.lock);
119
+
otx2_mbox_alloc_msg_cpt_lf_free(&pf->mbox);
120
+
otx2_sync_mbox_msg(&pf->mbox);
121
+
mutex_unlock(&pf->mbox.lock);
122
+
}
123
+
124
+
static int cn10k_outb_cptlf_config(struct otx2_nic *pf)
125
+
{
126
+
struct cpt_inline_ipsec_cfg_msg *req;
127
+
int ret = -ENOMEM;
128
+
129
+
mutex_lock(&pf->mbox.lock);
130
+
req = otx2_mbox_alloc_msg_cpt_inline_ipsec_cfg(&pf->mbox);
131
+
if (!req)
132
+
goto unlock;
133
+
134
+
req->dir = CPT_INLINE_OUTBOUND;
135
+
req->enable = 1;
136
+
req->nix_pf_func = pf->pcifunc;
137
+
ret = otx2_sync_mbox_msg(&pf->mbox);
138
+
unlock:
139
+
mutex_unlock(&pf->mbox.lock);
140
+
return ret;
141
+
}
142
+
143
+
static void cn10k_outb_cptlf_iq_enable(struct otx2_nic *pf)
144
+
{
145
+
u64 reg_val;
146
+
147
+
/* Set Execution Enable of instruction queue */
148
+
reg_val = otx2_read64(pf, CN10K_CPT_LF_INPROG);
149
+
reg_val |= BIT_ULL(16);
150
+
otx2_write64(pf, CN10K_CPT_LF_INPROG, reg_val);
151
+
152
+
/* Set iqueue's enqueuing */
153
+
reg_val = otx2_read64(pf, CN10K_CPT_LF_CTL);
154
+
reg_val |= BIT_ULL(0);
155
+
otx2_write64(pf, CN10K_CPT_LF_CTL, reg_val);
156
+
}
157
+
158
+
static void cn10k_outb_cptlf_iq_disable(struct otx2_nic *pf)
159
+
{
160
+
u32 inflight, grb_cnt, gwb_cnt;
161
+
u32 nq_ptr, dq_ptr;
162
+
int timeout = 20;
163
+
u64 reg_val;
164
+
int cnt;
165
+
166
+
/* Disable instructions enqueuing */
167
+
otx2_write64(pf, CN10K_CPT_LF_CTL, 0ull);
168
+
169
+
/* Wait for instruction queue to become empty.
170
+
* CPT_LF_INPROG.INFLIGHT count is zero
171
+
*/
172
+
do {
173
+
reg_val = otx2_read64(pf, CN10K_CPT_LF_INPROG);
174
+
inflight = FIELD_GET(CPT_LF_INPROG_INFLIGHT, reg_val);
175
+
if (!inflight)
176
+
break;
177
+
178
+
usleep_range(10000, 20000);
179
+
if (timeout-- < 0) {
180
+
netdev_err(pf->netdev, "Timeout to cleanup CPT IQ\n");
181
+
break;
182
+
}
183
+
} while (1);
184
+
185
+
/* Disable executions in the LF's queue,
186
+
* the queue should be empty at this point
187
+
*/
188
+
reg_val &= ~BIT_ULL(16);
189
+
otx2_write64(pf, CN10K_CPT_LF_INPROG, reg_val);
190
+
191
+
/* Wait for instruction queue to become empty */
192
+
cnt = 0;
193
+
do {
194
+
reg_val = otx2_read64(pf, CN10K_CPT_LF_INPROG);
195
+
if (reg_val & BIT_ULL(31))
196
+
cnt = 0;
197
+
else
198
+
cnt++;
199
+
reg_val = otx2_read64(pf, CN10K_CPT_LF_Q_GRP_PTR);
200
+
nq_ptr = FIELD_GET(CPT_LF_Q_GRP_PTR_DQ_PTR, reg_val);
201
+
dq_ptr = FIELD_GET(CPT_LF_Q_GRP_PTR_DQ_PTR, reg_val);
202
+
} while ((cnt < 10) && (nq_ptr != dq_ptr));
203
+
204
+
cnt = 0;
205
+
do {
206
+
reg_val = otx2_read64(pf, CN10K_CPT_LF_INPROG);
207
+
inflight = FIELD_GET(CPT_LF_INPROG_INFLIGHT, reg_val);
208
+
grb_cnt = FIELD_GET(CPT_LF_INPROG_GRB_CNT, reg_val);
209
+
gwb_cnt = FIELD_GET(CPT_LF_INPROG_GWB_CNT, reg_val);
210
+
if (inflight == 0 && gwb_cnt < 40 &&
211
+
(grb_cnt == 0 || grb_cnt == 40))
212
+
cnt++;
213
+
else
214
+
cnt = 0;
215
+
} while (cnt < 10);
216
+
}
217
+
218
+
/* Allocate memory for CPT outbound Instruction queue.
219
+
* Instruction queue memory format is:
220
+
* -----------------------------
221
+
* | Instruction Group memory |
222
+
* | (CPT_LF_Q_SIZE[SIZE_DIV40] |
223
+
* | x 16 Bytes) |
224
+
* | |
225
+
* ----------------------------- <-- CPT_LF_Q_BASE[ADDR]
226
+
* | Flow Control (128 Bytes) |
227
+
* | |
228
+
* -----------------------------
229
+
* | Instruction Memory |
230
+
* | (CPT_LF_Q_SIZE[SIZE_DIV40] |
231
+
* | × 40 × 64 bytes) |
232
+
* | |
233
+
* -----------------------------
234
+
*/
235
+
static int cn10k_outb_cptlf_iq_alloc(struct otx2_nic *pf)
236
+
{
237
+
struct cn10k_cpt_inst_queue *iq = &pf->ipsec.iq;
238
+
239
+
iq->size = CN10K_CPT_INST_QLEN_BYTES + CN10K_CPT_Q_FC_LEN +
240
+
CN10K_CPT_INST_GRP_QLEN_BYTES + OTX2_ALIGN;
241
+
242
+
iq->real_vaddr = dma_alloc_coherent(pf->dev, iq->size,
243
+
&iq->real_dma_addr, GFP_KERNEL);
244
+
if (!iq->real_vaddr)
245
+
return -ENOMEM;
246
+
247
+
/* iq->vaddr/dma_addr points to Flow Control location */
248
+
iq->vaddr = iq->real_vaddr + CN10K_CPT_INST_GRP_QLEN_BYTES;
249
+
iq->dma_addr = iq->real_dma_addr + CN10K_CPT_INST_GRP_QLEN_BYTES;
250
+
251
+
/* Align pointers */
252
+
iq->vaddr = PTR_ALIGN(iq->vaddr, OTX2_ALIGN);
253
+
iq->dma_addr = PTR_ALIGN(iq->dma_addr, OTX2_ALIGN);
254
+
return 0;
255
+
}
256
+
257
+
static void cn10k_outb_cptlf_iq_free(struct otx2_nic *pf)
258
+
{
259
+
struct cn10k_cpt_inst_queue *iq = &pf->ipsec.iq;
260
+
261
+
if (iq->real_vaddr)
262
+
dma_free_coherent(pf->dev, iq->size, iq->real_vaddr,
263
+
iq->real_dma_addr);
264
+
265
+
iq->real_vaddr = NULL;
266
+
iq->vaddr = NULL;
267
+
}
268
+
269
+
static int cn10k_outb_cptlf_iq_init(struct otx2_nic *pf)
270
+
{
271
+
u64 reg_val;
272
+
int ret;
273
+
274
+
/* Allocate Memory for CPT IQ */
275
+
ret = cn10k_outb_cptlf_iq_alloc(pf);
276
+
if (ret)
277
+
return ret;
278
+
279
+
/* Disable IQ */
280
+
cn10k_outb_cptlf_iq_disable(pf);
281
+
282
+
/* Set IQ base address */
283
+
otx2_write64(pf, CN10K_CPT_LF_Q_BASE, pf->ipsec.iq.dma_addr);
284
+
285
+
/* Set IQ size */
286
+
reg_val = FIELD_PREP(CPT_LF_Q_SIZE_DIV40, CN10K_CPT_SIZE_DIV40 +
287
+
CN10K_CPT_EXTRA_SIZE_DIV40);
288
+
otx2_write64(pf, CN10K_CPT_LF_Q_SIZE, reg_val);
289
+
290
+
return 0;
291
+
}
292
+
293
+
static int cn10k_outb_cptlf_init(struct otx2_nic *pf)
294
+
{
295
+
int ret;
296
+
297
+
/* Initialize CPTLF Instruction Queue (IQ) */
298
+
ret = cn10k_outb_cptlf_iq_init(pf);
299
+
if (ret)
300
+
return ret;
301
+
302
+
/* Configure CPTLF for outbound ipsec offload */
303
+
ret = cn10k_outb_cptlf_config(pf);
304
+
if (ret)
305
+
goto iq_clean;
306
+
307
+
/* Enable CPTLF IQ */
308
+
cn10k_outb_cptlf_iq_enable(pf);
309
+
return 0;
310
+
iq_clean:
311
+
cn10k_outb_cptlf_iq_free(pf);
312
+
return ret;
313
+
}
314
+
315
+
static int cn10k_outb_cpt_init(struct net_device *netdev)
316
+
{
317
+
struct otx2_nic *pf = netdev_priv(netdev);
318
+
int ret;
319
+
320
+
/* Attach a CPT LF for outbound ipsec offload */
321
+
ret = cn10k_outb_cptlf_attach(pf);
322
+
if (ret)
323
+
return ret;
324
+
325
+
/* Allocate a CPT LF for outbound ipsec offload */
326
+
ret = cn10k_outb_cptlf_alloc(pf);
327
+
if (ret)
328
+
goto detach;
329
+
330
+
/* Initialize the CPTLF for outbound ipsec offload */
331
+
ret = cn10k_outb_cptlf_init(pf);
332
+
if (ret)
333
+
goto lf_free;
334
+
335
+
pf->ipsec.io_addr = (__force u64)otx2_get_regaddr(pf,
336
+
CN10K_CPT_LF_NQX(0));
337
+
338
+
/* Set ipsec offload enabled for this device */
339
+
pf->flags |= OTX2_FLAG_IPSEC_OFFLOAD_ENABLED;
340
+
341
+
cn10k_cpt_device_set_available(pf);
342
+
return 0;
343
+
344
+
lf_free:
345
+
cn10k_outb_cptlf_free(pf);
346
+
detach:
347
+
cn10k_outb_cptlf_detach(pf);
348
+
return ret;
349
+
}
350
+
351
+
static int cn10k_outb_cpt_clean(struct otx2_nic *pf)
352
+
{
353
+
int ret;
354
+
355
+
if (!cn10k_cpt_device_set_inuse(pf)) {
356
+
netdev_err(pf->netdev, "CPT LF device unavailable\n");
357
+
return -ENODEV;
358
+
}
359
+
360
+
/* Set ipsec offload disabled for this device */
361
+
pf->flags &= ~OTX2_FLAG_IPSEC_OFFLOAD_ENABLED;
362
+
363
+
/* Disable CPTLF Instruction Queue (IQ) */
364
+
cn10k_outb_cptlf_iq_disable(pf);
365
+
366
+
/* Set IQ base address and size to 0 */
367
+
otx2_write64(pf, CN10K_CPT_LF_Q_BASE, 0);
368
+
otx2_write64(pf, CN10K_CPT_LF_Q_SIZE, 0);
369
+
370
+
/* Free CPTLF IQ */
371
+
cn10k_outb_cptlf_iq_free(pf);
372
+
373
+
/* Free and detach CPT LF */
374
+
cn10k_outb_cptlf_free(pf);
375
+
ret = cn10k_outb_cptlf_detach(pf);
376
+
if (ret)
377
+
netdev_err(pf->netdev, "Failed to detach CPT LF\n");
378
+
379
+
cn10k_cpt_device_set_unavailable(pf);
380
+
return ret;
381
+
}
382
+
383
+
static void cn10k_cpt_inst_flush(struct otx2_nic *pf, struct cpt_inst_s *inst,
384
+
u64 size)
385
+
{
386
+
struct otx2_lmt_info *lmt_info;
387
+
u64 val = 0, tar_addr = 0;
388
+
389
+
lmt_info = per_cpu_ptr(pf->hw.lmt_info, smp_processor_id());
390
+
/* FIXME: val[0:10] LMT_ID.
391
+
* [12:15] no of LMTST - 1 in the burst.
392
+
* [19:63] data size of each LMTST in the burst except first.
393
+
*/
394
+
val = (lmt_info->lmt_id & 0x7FF);
395
+
/* Target address for LMTST flush tells HW how many 128bit
396
+
* words are present.
397
+
* tar_addr[6:4] size of first LMTST - 1 in units of 128b.
398
+
*/
399
+
tar_addr |= pf->ipsec.io_addr | (((size / 16) - 1) & 0x7) << 4;
400
+
dma_wmb();
401
+
memcpy((u64 *)lmt_info->lmt_addr, inst, size);
402
+
cn10k_lmt_flush(val, tar_addr);
403
+
}
404
+
405
+
static int cn10k_wait_for_cpt_respose(struct otx2_nic *pf,
406
+
struct cpt_res_s *res)
407
+
{
408
+
unsigned long timeout = jiffies + msecs_to_jiffies(100);
409
+
u64 *completion_ptr = (u64 *)res;
410
+
411
+
do {
412
+
if (time_after(jiffies, timeout)) {
413
+
netdev_err(pf->netdev, "CPT response timeout\n");
414
+
return -EBUSY;
415
+
}
416
+
} while ((READ_ONCE(*completion_ptr) & CN10K_CPT_COMP_E_MASK) ==
417
+
CN10K_CPT_COMP_E_NOTDONE);
418
+
419
+
if (!(res->compcode == CN10K_CPT_COMP_E_GOOD ||
420
+
res->compcode == CN10K_CPT_COMP_E_WARN) || res->uc_compcode) {
421
+
netdev_err(pf->netdev, "compcode=%x doneint=%x\n",
422
+
res->compcode, res->doneint);
423
+
netdev_err(pf->netdev, "uc_compcode=%x uc_info=%llx esn=%llx\n",
424
+
res->uc_compcode, (u64)res->uc_info, res->esn);
425
+
}
426
+
return 0;
427
+
}
428
+
429
+
static int cn10k_outb_write_sa(struct otx2_nic *pf, struct qmem *sa_info)
430
+
{
431
+
dma_addr_t res_iova, dptr_iova, sa_iova;
432
+
struct cn10k_tx_sa_s *sa_dptr;
433
+
struct cpt_inst_s inst = {};
434
+
struct cpt_res_s *res;
435
+
u32 sa_size, off;
436
+
u64 *sptr, *dptr;
437
+
u64 reg_val;
438
+
int ret;
439
+
440
+
sa_iova = sa_info->iova;
441
+
if (!sa_iova)
442
+
return -EINVAL;
443
+
444
+
res = dma_alloc_coherent(pf->dev, sizeof(struct cpt_res_s),
445
+
&res_iova, GFP_ATOMIC);
446
+
if (!res)
447
+
return -ENOMEM;
448
+
449
+
sa_size = sizeof(struct cn10k_tx_sa_s);
450
+
sa_dptr = dma_alloc_coherent(pf->dev, sa_size, &dptr_iova, GFP_ATOMIC);
451
+
if (!sa_dptr) {
452
+
dma_free_coherent(pf->dev, sizeof(struct cpt_res_s), res,
453
+
res_iova);
454
+
return -ENOMEM;
455
+
}
456
+
457
+
sptr = (__force u64 *)sa_info->base;
458
+
dptr = (__force u64 *)sa_dptr;
459
+
for (off = 0; off < (sa_size / 8); off++)
460
+
*(dptr + off) = (__force u64)cpu_to_be64(*(sptr + off));
461
+
462
+
res->compcode = CN10K_CPT_COMP_E_NOTDONE;
463
+
inst.res_addr = res_iova;
464
+
inst.dptr = (u64)dptr_iova;
465
+
inst.param2 = sa_size >> 3;
466
+
inst.dlen = sa_size;
467
+
inst.opcode_major = CN10K_IPSEC_MAJOR_OP_WRITE_SA;
468
+
inst.opcode_minor = CN10K_IPSEC_MINOR_OP_WRITE_SA;
469
+
inst.cptr = sa_iova;
470
+
inst.ctx_val = 1;
471
+
inst.egrp = CN10K_DEF_CPT_IPSEC_EGRP;
472
+
473
+
/* Check if CPT-LF available */
474
+
if (!cn10k_cpt_device_set_inuse(pf)) {
475
+
ret = -ENODEV;
476
+
goto free_mem;
477
+
}
478
+
479
+
cn10k_cpt_inst_flush(pf, &inst, sizeof(struct cpt_inst_s));
480
+
dma_wmb();
481
+
ret = cn10k_wait_for_cpt_respose(pf, res);
482
+
if (ret)
483
+
goto set_available;
484
+
485
+
/* Trigger CTX flush to write dirty data back to DRAM */
486
+
reg_val = FIELD_PREP(CPT_LF_CTX_FLUSH, sa_iova >> 7);
487
+
otx2_write64(pf, CN10K_CPT_LF_CTX_FLUSH, reg_val);
488
+
489
+
set_available:
490
+
cn10k_cpt_device_set_available(pf);
491
+
free_mem:
492
+
dma_free_coherent(pf->dev, sa_size, sa_dptr, dptr_iova);
493
+
dma_free_coherent(pf->dev, sizeof(struct cpt_res_s), res, res_iova);
494
+
return ret;
495
+
}
496
+
497
+
static int cn10k_ipsec_get_hw_ctx_offset(void)
498
+
{
499
+
/* Offset on Hardware-context offset in word */
500
+
return (offsetof(struct cn10k_tx_sa_s, hw_ctx) / sizeof(u64)) & 0x7F;
501
+
}
502
+
503
+
static int cn10k_ipsec_get_ctx_push_size(void)
504
+
{
505
+
/* Context push size is round up and in multiple of 8 Byte */
506
+
return (roundup(offsetof(struct cn10k_tx_sa_s, hw_ctx), 8) / 8) & 0x7F;
507
+
}
508
+
509
+
static int cn10k_ipsec_get_aes_key_len(int key_len)
510
+
{
511
+
/* key_len is aes key length in bytes */
512
+
switch (key_len) {
513
+
case 16:
514
+
return CN10K_IPSEC_SA_AES_KEY_LEN_128;
515
+
case 24:
516
+
return CN10K_IPSEC_SA_AES_KEY_LEN_192;
517
+
default:
518
+
return CN10K_IPSEC_SA_AES_KEY_LEN_256;
519
+
}
520
+
}
521
+
522
+
static void cn10k_outb_prepare_sa(struct xfrm_state *x,
523
+
struct cn10k_tx_sa_s *sa_entry)
524
+
{
525
+
int key_len = (x->aead->alg_key_len + 7) / 8;
526
+
struct net_device *netdev = x->xso.dev;
527
+
u8 *key = x->aead->alg_key;
528
+
struct otx2_nic *pf;
529
+
u32 *tmp_salt;
530
+
u64 *tmp_key;
531
+
int idx;
532
+
533
+
memset(sa_entry, 0, sizeof(struct cn10k_tx_sa_s));
534
+
535
+
/* context size, 128 Byte aligned up */
536
+
pf = netdev_priv(netdev);
537
+
sa_entry->ctx_size = (pf->ipsec.sa_size / OTX2_ALIGN) & 0xF;
538
+
sa_entry->hw_ctx_off = cn10k_ipsec_get_hw_ctx_offset();
539
+
sa_entry->ctx_push_size = cn10k_ipsec_get_ctx_push_size();
540
+
541
+
/* Ucode to skip two words of CPT_CTX_HW_S */
542
+
sa_entry->ctx_hdr_size = 1;
543
+
544
+
/* Allow Atomic operation (AOP) */
545
+
sa_entry->aop_valid = 1;
546
+
547
+
/* Outbound, ESP TRANSPORT/TUNNEL Mode, AES-GCM with */
548
+
sa_entry->sa_dir = CN10K_IPSEC_SA_DIR_OUTB;
549
+
sa_entry->ipsec_protocol = CN10K_IPSEC_SA_IPSEC_PROTO_ESP;
550
+
sa_entry->enc_type = CN10K_IPSEC_SA_ENCAP_TYPE_AES_GCM;
551
+
sa_entry->iv_src = CN10K_IPSEC_SA_IV_SRC_PACKET;
552
+
if (x->props.mode == XFRM_MODE_TUNNEL)
553
+
sa_entry->ipsec_mode = CN10K_IPSEC_SA_IPSEC_MODE_TUNNEL;
554
+
else
555
+
sa_entry->ipsec_mode = CN10K_IPSEC_SA_IPSEC_MODE_TRANSPORT;
556
+
557
+
/* Last 4 bytes are salt */
558
+
key_len -= 4;
559
+
sa_entry->aes_key_len = cn10k_ipsec_get_aes_key_len(key_len);
560
+
memcpy(sa_entry->cipher_key, key, key_len);
561
+
tmp_key = (u64 *)sa_entry->cipher_key;
562
+
563
+
for (idx = 0; idx < key_len / 8; idx++)
564
+
tmp_key[idx] = (__force u64)cpu_to_be64(tmp_key[idx]);
565
+
566
+
memcpy(&sa_entry->iv_gcm_salt, key + key_len, 4);
567
+
tmp_salt = (u32 *)&sa_entry->iv_gcm_salt;
568
+
*tmp_salt = (__force u32)cpu_to_be32(*tmp_salt);
569
+
570
+
/* Write SA context data to memory before enabling */
571
+
wmb();
572
+
573
+
/* Enable SA */
574
+
sa_entry->sa_valid = 1;
575
+
}
576
+
577
+
static int cn10k_ipsec_validate_state(struct xfrm_state *x,
578
+
struct netlink_ext_ack *extack)
579
+
{
580
+
if (x->props.aalgo != SADB_AALG_NONE) {
581
+
NL_SET_ERR_MSG_MOD(extack,
582
+
"Cannot offload authenticated xfrm states");
583
+
return -EINVAL;
584
+
}
585
+
if (x->props.ealgo != SADB_X_EALG_AES_GCM_ICV16) {
586
+
NL_SET_ERR_MSG_MOD(extack,
587
+
"Only AES-GCM-ICV16 xfrm state may be offloaded");
588
+
return -EINVAL;
589
+
}
590
+
if (x->props.calgo != SADB_X_CALG_NONE) {
591
+
NL_SET_ERR_MSG_MOD(extack,
592
+
"Cannot offload compressed xfrm states");
593
+
return -EINVAL;
594
+
}
595
+
if (x->props.flags & XFRM_STATE_ESN) {
596
+
NL_SET_ERR_MSG_MOD(extack, "Cannot offload ESN xfrm states");
597
+
return -EINVAL;
598
+
}
599
+
if (x->props.family != AF_INET && x->props.family != AF_INET6) {
600
+
NL_SET_ERR_MSG_MOD(extack,
601
+
"Only IPv4/v6 xfrm states may be offloaded");
602
+
return -EINVAL;
603
+
}
604
+
if (x->xso.type != XFRM_DEV_OFFLOAD_CRYPTO) {
605
+
NL_SET_ERR_MSG_MOD(extack,
606
+
"Cannot offload other than crypto-mode");
607
+
return -EINVAL;
608
+
}
609
+
if (x->props.mode != XFRM_MODE_TRANSPORT &&
610
+
x->props.mode != XFRM_MODE_TUNNEL) {
611
+
NL_SET_ERR_MSG_MOD(extack,
612
+
"Only tunnel/transport xfrm states may be offloaded");
613
+
return -EINVAL;
614
+
}
615
+
if (x->id.proto != IPPROTO_ESP) {
616
+
NL_SET_ERR_MSG_MOD(extack,
617
+
"Only ESP xfrm state may be offloaded");
618
+
return -EINVAL;
619
+
}
620
+
if (x->encap) {
621
+
NL_SET_ERR_MSG_MOD(extack,
622
+
"Encapsulated xfrm state may not be offloaded");
623
+
return -EINVAL;
624
+
}
625
+
if (!x->aead) {
626
+
NL_SET_ERR_MSG_MOD(extack,
627
+
"Cannot offload xfrm states without aead");
628
+
return -EINVAL;
629
+
}
630
+
631
+
if (x->aead->alg_icv_len != 128) {
632
+
NL_SET_ERR_MSG_MOD(extack,
633
+
"Cannot offload xfrm states with AEAD ICV length other than 128bit");
634
+
return -EINVAL;
635
+
}
636
+
if (x->aead->alg_key_len != 128 + 32 &&
637
+
x->aead->alg_key_len != 192 + 32 &&
638
+
x->aead->alg_key_len != 256 + 32) {
639
+
NL_SET_ERR_MSG_MOD(extack,
640
+
"Cannot offload xfrm states with AEAD key length other than 128/192/256bit");
641
+
return -EINVAL;
642
+
}
643
+
if (x->tfcpad) {
644
+
NL_SET_ERR_MSG_MOD(extack,
645
+
"Cannot offload xfrm states with tfc padding");
646
+
return -EINVAL;
647
+
}
648
+
if (!x->geniv) {
649
+
NL_SET_ERR_MSG_MOD(extack,
650
+
"Cannot offload xfrm states without geniv");
651
+
return -EINVAL;
652
+
}
653
+
if (strcmp(x->geniv, "seqiv")) {
654
+
NL_SET_ERR_MSG_MOD(extack,
655
+
"Cannot offload xfrm states with geniv other than seqiv");
656
+
return -EINVAL;
657
+
}
658
+
return 0;
659
+
}
660
+
661
+
static int cn10k_ipsec_inb_add_state(struct xfrm_state *x,
662
+
struct netlink_ext_ack *extack)
663
+
{
664
+
NL_SET_ERR_MSG_MOD(extack, "xfrm inbound offload not supported");
665
+
return -EOPNOTSUPP;
666
+
}
667
+
668
+
static int cn10k_ipsec_outb_add_state(struct xfrm_state *x,
669
+
struct netlink_ext_ack *extack)
670
+
{
671
+
struct net_device *netdev = x->xso.dev;
672
+
struct cn10k_tx_sa_s *sa_entry;
673
+
struct qmem *sa_info;
674
+
struct otx2_nic *pf;
675
+
int err;
676
+
677
+
err = cn10k_ipsec_validate_state(x, extack);
678
+
if (err)
679
+
return err;
680
+
681
+
pf = netdev_priv(netdev);
682
+
683
+
err = qmem_alloc(pf->dev, &sa_info, pf->ipsec.sa_size, OTX2_ALIGN);
684
+
if (err)
685
+
return err;
686
+
687
+
sa_entry = (struct cn10k_tx_sa_s *)sa_info->base;
688
+
cn10k_outb_prepare_sa(x, sa_entry);
689
+
690
+
err = cn10k_outb_write_sa(pf, sa_info);
691
+
if (err) {
692
+
NL_SET_ERR_MSG_MOD(extack, "Error writing outbound SA");
693
+
qmem_free(pf->dev, sa_info);
694
+
return err;
695
+
}
696
+
697
+
x->xso.offload_handle = (unsigned long)sa_info;
698
+
/* Enable static branch when first SA setup */
699
+
if (!pf->ipsec.outb_sa_count)
700
+
static_branch_enable(&cn10k_ipsec_sa_enabled);
701
+
pf->ipsec.outb_sa_count++;
702
+
return 0;
703
+
}
704
+
705
+
static int cn10k_ipsec_add_state(struct xfrm_state *x,
706
+
struct netlink_ext_ack *extack)
707
+
{
708
+
if (x->xso.dir == XFRM_DEV_OFFLOAD_IN)
709
+
return cn10k_ipsec_inb_add_state(x, extack);
710
+
else
711
+
return cn10k_ipsec_outb_add_state(x, extack);
712
+
}
713
+
714
+
static void cn10k_ipsec_del_state(struct xfrm_state *x)
715
+
{
716
+
struct net_device *netdev = x->xso.dev;
717
+
struct cn10k_tx_sa_s *sa_entry;
718
+
struct qmem *sa_info;
719
+
struct otx2_nic *pf;
720
+
int err;
721
+
722
+
if (x->xso.dir == XFRM_DEV_OFFLOAD_IN)
723
+
return;
724
+
725
+
pf = netdev_priv(netdev);
726
+
727
+
sa_info = (struct qmem *)x->xso.offload_handle;
728
+
sa_entry = (struct cn10k_tx_sa_s *)sa_info->base;
729
+
memset(sa_entry, 0, sizeof(struct cn10k_tx_sa_s));
730
+
/* Disable SA in CPT h/w */
731
+
sa_entry->ctx_push_size = cn10k_ipsec_get_ctx_push_size();
732
+
sa_entry->ctx_size = (pf->ipsec.sa_size / OTX2_ALIGN) & 0xF;
733
+
sa_entry->aop_valid = 1;
734
+
735
+
err = cn10k_outb_write_sa(pf, sa_info);
736
+
if (err)
737
+
netdev_err(netdev, "Error (%d) deleting SA\n", err);
738
+
739
+
x->xso.offload_handle = 0;
740
+
qmem_free(pf->dev, sa_info);
741
+
742
+
/* If no more SA's then update netdev feature for potential change
743
+
* in NETIF_F_HW_ESP.
744
+
*/
745
+
if (!--pf->ipsec.outb_sa_count)
746
+
queue_work(pf->ipsec.sa_workq, &pf->ipsec.sa_work);
747
+
}
748
+
749
+
static bool cn10k_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *x)
750
+
{
751
+
if (x->props.family == AF_INET) {
752
+
/* Offload with IPv4 options is not supported yet */
753
+
if (ip_hdr(skb)->ihl > 5)
754
+
return false;
755
+
} else {
756
+
/* Offload with IPv6 extension headers is not support yet */
757
+
if (ipv6_ext_hdr(ipv6_hdr(skb)->nexthdr))
758
+
return false;
759
+
}
760
+
return true;
761
+
}
762
+
763
+
static const struct xfrmdev_ops cn10k_ipsec_xfrmdev_ops = {
764
+
.xdo_dev_state_add = cn10k_ipsec_add_state,
765
+
.xdo_dev_state_delete = cn10k_ipsec_del_state,
766
+
.xdo_dev_offload_ok = cn10k_ipsec_offload_ok,
767
+
};
768
+
769
+
static void cn10k_ipsec_sa_wq_handler(struct work_struct *work)
770
+
{
771
+
struct cn10k_ipsec *ipsec = container_of(work, struct cn10k_ipsec,
772
+
sa_work);
773
+
struct otx2_nic *pf = container_of(ipsec, struct otx2_nic, ipsec);
774
+
775
+
/* Disable static branch when no more SA enabled */
776
+
static_branch_disable(&cn10k_ipsec_sa_enabled);
777
+
rtnl_lock();
778
+
netdev_update_features(pf->netdev);
779
+
rtnl_unlock();
780
+
}
781
+
782
+
int cn10k_ipsec_ethtool_init(struct net_device *netdev, bool enable)
783
+
{
784
+
struct otx2_nic *pf = netdev_priv(netdev);
785
+
786
+
/* IPsec offload supported on cn10k */
787
+
if (!is_dev_support_ipsec_offload(pf->pdev))
788
+
return -EOPNOTSUPP;
789
+
790
+
/* Initialize CPT for outbound ipsec offload */
791
+
if (enable)
792
+
return cn10k_outb_cpt_init(netdev);
793
+
794
+
/* Don't do CPT cleanup if SA installed */
795
+
if (pf->ipsec.outb_sa_count) {
796
+
netdev_err(pf->netdev, "SA installed on this device\n");
797
+
return -EBUSY;
798
+
}
799
+
800
+
return cn10k_outb_cpt_clean(pf);
801
+
}
802
+
803
+
int cn10k_ipsec_init(struct net_device *netdev)
804
+
{
805
+
struct otx2_nic *pf = netdev_priv(netdev);
806
+
u32 sa_size;
807
+
808
+
if (!is_dev_support_ipsec_offload(pf->pdev))
809
+
return 0;
810
+
811
+
/* Each SA entry size is 128 Byte round up in size */
812
+
sa_size = sizeof(struct cn10k_tx_sa_s) % OTX2_ALIGN ?
813
+
(sizeof(struct cn10k_tx_sa_s) / OTX2_ALIGN + 1) *
814
+
OTX2_ALIGN : sizeof(struct cn10k_tx_sa_s);
815
+
pf->ipsec.sa_size = sa_size;
816
+
817
+
INIT_WORK(&pf->ipsec.sa_work, cn10k_ipsec_sa_wq_handler);
818
+
pf->ipsec.sa_workq = alloc_workqueue("cn10k_ipsec_sa_workq", 0, 0);
819
+
if (!pf->ipsec.sa_workq) {
820
+
netdev_err(pf->netdev, "SA alloc workqueue failed\n");
821
+
return -ENOMEM;
822
+
}
823
+
824
+
/* Set xfrm device ops */
825
+
netdev->xfrmdev_ops = &cn10k_ipsec_xfrmdev_ops;
826
+
netdev->hw_features |= NETIF_F_HW_ESP;
827
+
netdev->hw_enc_features |= NETIF_F_HW_ESP;
828
+
829
+
cn10k_cpt_device_set_unavailable(pf);
830
+
return 0;
831
+
}
832
+
EXPORT_SYMBOL(cn10k_ipsec_init);
833
+
834
+
void cn10k_ipsec_clean(struct otx2_nic *pf)
835
+
{
836
+
if (!is_dev_support_ipsec_offload(pf->pdev))
837
+
return;
838
+
839
+
if (!(pf->flags & OTX2_FLAG_IPSEC_OFFLOAD_ENABLED))
840
+
return;
841
+
842
+
if (pf->ipsec.sa_workq) {
843
+
destroy_workqueue(pf->ipsec.sa_workq);
844
+
pf->ipsec.sa_workq = NULL;
845
+
}
846
+
847
+
cn10k_outb_cpt_clean(pf);
848
+
}
849
+
EXPORT_SYMBOL(cn10k_ipsec_clean);
850
+
851
+
static u16 cn10k_ipsec_get_ip_data_len(struct xfrm_state *x,
852
+
struct sk_buff *skb)
853
+
{
854
+
struct ipv6hdr *ipv6h;
855
+
struct iphdr *iph;
856
+
u8 *src;
857
+
858
+
src = (u8 *)skb->data + ETH_HLEN;
859
+
860
+
if (x->props.family == AF_INET) {
861
+
iph = (struct iphdr *)src;
862
+
return ntohs(iph->tot_len);
863
+
}
864
+
865
+
ipv6h = (struct ipv6hdr *)src;
866
+
return ntohs(ipv6h->payload_len) + sizeof(struct ipv6hdr);
867
+
}
868
+
869
+
/* Prepare CPT and NIX SQE scatter/gather subdescriptor structure.
870
+
* SG of NIX and CPT are same in size.
871
+
* Layout of a NIX SQE and CPT SG entry:
872
+
* -----------------------------
873
+
* | CPT Scatter Gather |
874
+
* | (SQE SIZE) |
875
+
* | |
876
+
* -----------------------------
877
+
* | NIX SQE |
878
+
* | (SQE SIZE) |
879
+
* | |
880
+
* -----------------------------
881
+
*/
882
+
bool otx2_sqe_add_sg_ipsec(struct otx2_nic *pfvf, struct otx2_snd_queue *sq,
883
+
struct sk_buff *skb, int num_segs, int *offset)
884
+
{
885
+
struct cpt_sg_s *cpt_sg = NULL;
886
+
struct nix_sqe_sg_s *sg = NULL;
887
+
u64 dma_addr, *iova = NULL;
888
+
u64 *cpt_iova = NULL;
889
+
u16 *sg_lens = NULL;
890
+
int seg, len;
891
+
892
+
sq->sg[sq->head].num_segs = 0;
893
+
cpt_sg = (struct cpt_sg_s *)(sq->sqe_base - sq->sqe_size);
894
+
895
+
for (seg = 0; seg < num_segs; seg++) {
896
+
if ((seg % MAX_SEGS_PER_SG) == 0) {
897
+
sg = (struct nix_sqe_sg_s *)(sq->sqe_base + *offset);
898
+
sg->ld_type = NIX_SEND_LDTYPE_LDD;
899
+
sg->subdc = NIX_SUBDC_SG;
900
+
sg->segs = 0;
901
+
sg_lens = (void *)sg;
902
+
iova = (void *)sg + sizeof(*sg);
903
+
/* Next subdc always starts at a 16byte boundary.
904
+
* So if sg->segs is whether 2 or 3, offset += 16bytes.
905
+
*/
906
+
if ((num_segs - seg) >= (MAX_SEGS_PER_SG - 1))
907
+
*offset += sizeof(*sg) + (3 * sizeof(u64));
908
+
else
909
+
*offset += sizeof(*sg) + sizeof(u64);
910
+
911
+
cpt_sg += (seg / MAX_SEGS_PER_SG) * 4;
912
+
cpt_iova = (void *)cpt_sg + sizeof(*cpt_sg);
913
+
}
914
+
dma_addr = otx2_dma_map_skb_frag(pfvf, skb, seg, &len);
915
+
if (dma_mapping_error(pfvf->dev, dma_addr))
916
+
return false;
917
+
918
+
sg_lens[seg % MAX_SEGS_PER_SG] = len;
919
+
sg->segs++;
920
+
*iova++ = dma_addr;
921
+
*cpt_iova++ = dma_addr;
922
+
923
+
/* Save DMA mapping info for later unmapping */
924
+
sq->sg[sq->head].dma_addr[seg] = dma_addr;
925
+
sq->sg[sq->head].size[seg] = len;
926
+
sq->sg[sq->head].num_segs++;
927
+
928
+
*cpt_sg = *(struct cpt_sg_s *)sg;
929
+
cpt_sg->rsvd_63_50 = 0;
930
+
}
931
+
932
+
sq->sg[sq->head].skb = (u64)skb;
933
+
return true;
934
+
}
935
+
936
+
static u16 cn10k_ipsec_get_param1(u8 iv_offset)
937
+
{
938
+
u16 param1_val;
939
+
940
+
/* Set Crypto mode, disable L3/L4 checksum */
941
+
param1_val = CN10K_IPSEC_INST_PARAM1_DIS_L4_CSUM |
942
+
CN10K_IPSEC_INST_PARAM1_DIS_L3_CSUM;
943
+
param1_val |= (u16)iv_offset << CN10K_IPSEC_INST_PARAM1_IV_OFFSET_SHIFT;
944
+
return param1_val;
945
+
}
946
+
947
+
bool cn10k_ipsec_transmit(struct otx2_nic *pf, struct netdev_queue *txq,
948
+
struct otx2_snd_queue *sq, struct sk_buff *skb,
949
+
int num_segs, int size)
950
+
{
951
+
struct cpt_inst_s inst;
952
+
struct cpt_res_s *res;
953
+
struct xfrm_state *x;
954
+
struct qmem *sa_info;
955
+
dma_addr_t dptr_iova;
956
+
struct sec_path *sp;
957
+
u8 encap_offset;
958
+
u8 auth_offset;
959
+
u8 gthr_size;
960
+
u8 iv_offset;
961
+
u16 dlen;
962
+
963
+
/* Check for IPSEC offload enabled */
964
+
if (!(pf->flags & OTX2_FLAG_IPSEC_OFFLOAD_ENABLED))
965
+
goto drop;
966
+
967
+
sp = skb_sec_path(skb);
968
+
if (unlikely(!sp->len))
969
+
goto drop;
970
+
971
+
x = xfrm_input_state(skb);
972
+
if (unlikely(!x))
973
+
goto drop;
974
+
975
+
if (x->props.mode != XFRM_MODE_TRANSPORT &&
976
+
x->props.mode != XFRM_MODE_TUNNEL)
977
+
goto drop;
978
+
979
+
dlen = cn10k_ipsec_get_ip_data_len(x, skb);
980
+
if (dlen == 0 && netif_msg_tx_err(pf)) {
981
+
netdev_err(pf->netdev, "Invalid IP header, ip-length zero\n");
982
+
goto drop;
983
+
}
984
+
985
+
/* Check for valid SA context */
986
+
sa_info = (struct qmem *)x->xso.offload_handle;
987
+
if (!sa_info)
988
+
goto drop;
989
+
990
+
memset(&inst, 0, sizeof(struct cpt_inst_s));
991
+
992
+
/* Get authentication offset */
993
+
if (x->props.family == AF_INET)
994
+
auth_offset = sizeof(struct iphdr);
995
+
else
996
+
auth_offset = sizeof(struct ipv6hdr);
997
+
998
+
/* IV offset is after ESP header */
999
+
iv_offset = auth_offset + sizeof(struct ip_esp_hdr);
1000
+
/* Encap will start after IV */
1001
+
encap_offset = iv_offset + GCM_RFC4106_IV_SIZE;
1002
+
1003
+
/* CPT Instruction word-1 */
1004
+
res = (struct cpt_res_s *)(sq->cpt_resp->base + (64 * sq->head));
1005
+
res->compcode = 0;
1006
+
inst.res_addr = sq->cpt_resp->iova + (64 * sq->head);
1007
+
1008
+
/* CPT Instruction word-2 */
1009
+
inst.rvu_pf_func = pf->pcifunc;
1010
+
1011
+
/* CPT Instruction word-3:
1012
+
* Set QORD to force CPT_RES_S write completion
1013
+
*/
1014
+
inst.qord = 1;
1015
+
1016
+
/* CPT Instruction word-4 */
1017
+
/* inst.dlen should not include ICV length */
1018
+
inst.dlen = dlen + ETH_HLEN - (x->aead->alg_icv_len / 8);
1019
+
inst.opcode_major = CN10K_IPSEC_MAJOR_OP_OUTB_IPSEC;
1020
+
inst.param1 = cn10k_ipsec_get_param1(iv_offset);
1021
+
1022
+
inst.param2 = encap_offset <<
1023
+
CN10K_IPSEC_INST_PARAM2_ENC_DATA_OFFSET_SHIFT;
1024
+
inst.param2 |= (u16)auth_offset <<
1025
+
CN10K_IPSEC_INST_PARAM2_AUTH_DATA_OFFSET_SHIFT;
1026
+
1027
+
/* CPT Instruction word-5 */
1028
+
gthr_size = num_segs / MAX_SEGS_PER_SG;
1029
+
gthr_size = (num_segs % MAX_SEGS_PER_SG) ? gthr_size + 1 : gthr_size;
1030
+
1031
+
gthr_size &= 0xF;
1032
+
dptr_iova = (sq->sqe_ring->iova + (sq->head * (sq->sqe_size * 2)));
1033
+
inst.dptr = dptr_iova | ((u64)gthr_size << 60);
1034
+
1035
+
/* CPT Instruction word-6 */
1036
+
inst.rptr = inst.dptr;
1037
+
1038
+
/* CPT Instruction word-7 */
1039
+
inst.cptr = sa_info->iova;
1040
+
inst.ctx_val = 1;
1041
+
inst.egrp = CN10K_DEF_CPT_IPSEC_EGRP;
1042
+
1043
+
/* CPT Instruction word-0 */
1044
+
inst.nixtxl = (size / 16) - 1;
1045
+
inst.dat_offset = ETH_HLEN;
1046
+
inst.nixtx_offset = sq->sqe_size;
1047
+
1048
+
netdev_tx_sent_queue(txq, skb->len);
1049
+
1050
+
/* Finally Flush the CPT instruction */
1051
+
sq->head++;
1052
+
sq->head &= (sq->sqe_cnt - 1);
1053
+
cn10k_cpt_inst_flush(pf, &inst, sizeof(struct cpt_inst_s));
1054
+
return true;
1055
+
drop:
1056
+
dev_kfree_skb_any(skb);
1057
+
return false;
1058
+
}
+265
drivers/net/ethernet/marvell/octeontx2/nic/cn10k_ipsec.h
+265
drivers/net/ethernet/marvell/octeontx2/nic/cn10k_ipsec.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0 */
2
+
/* Marvell IPSEC offload driver
3
+
*
4
+
* Copyright (C) 2024 Marvell.
5
+
*/
6
+
7
+
#ifndef CN10K_IPSEC_H
8
+
#define CN10K_IPSEC_H
9
+
10
+
#include <linux/types.h>
11
+
12
+
DECLARE_STATIC_KEY_FALSE(cn10k_ipsec_sa_enabled);
13
+
14
+
/* CPT instruction size in bytes */
15
+
#define CN10K_CPT_INST_SIZE 64
16
+
17
+
/* CPT instruction (CPT_INST_S) queue length */
18
+
#define CN10K_CPT_INST_QLEN 8200
19
+
20
+
/* CPT instruction queue size passed to HW is in units of
21
+
* 40*CPT_INST_S messages.
22
+
*/
23
+
#define CN10K_CPT_SIZE_DIV40 (CN10K_CPT_INST_QLEN / 40)
24
+
25
+
/* CPT needs 320 free entries */
26
+
#define CN10K_CPT_INST_QLEN_EXTRA_BYTES (320 * CN10K_CPT_INST_SIZE)
27
+
#define CN10K_CPT_EXTRA_SIZE_DIV40 (320 / 40)
28
+
29
+
/* CPT instruction queue length in bytes */
30
+
#define CN10K_CPT_INST_QLEN_BYTES \
31
+
((CN10K_CPT_SIZE_DIV40 * 40 * CN10K_CPT_INST_SIZE) + \
32
+
CN10K_CPT_INST_QLEN_EXTRA_BYTES)
33
+
34
+
/* CPT instruction group queue length in bytes */
35
+
#define CN10K_CPT_INST_GRP_QLEN_BYTES \
36
+
((CN10K_CPT_SIZE_DIV40 + CN10K_CPT_EXTRA_SIZE_DIV40) * 16)
37
+
38
+
/* CPT FC length in bytes */
39
+
#define CN10K_CPT_Q_FC_LEN 128
40
+
41
+
/* Default CPT engine group for ipsec offload */
42
+
#define CN10K_DEF_CPT_IPSEC_EGRP 1
43
+
44
+
/* CN10K CPT LF registers */
45
+
#define CPT_LFBASE (BLKTYPE_CPT << RVU_FUNC_BLKADDR_SHIFT)
46
+
#define CN10K_CPT_LF_CTL (CPT_LFBASE | 0x10)
47
+
#define CN10K_CPT_LF_INPROG (CPT_LFBASE | 0x40)
48
+
#define CN10K_CPT_LF_Q_BASE (CPT_LFBASE | 0xf0)
49
+
#define CN10K_CPT_LF_Q_SIZE (CPT_LFBASE | 0x100)
50
+
#define CN10K_CPT_LF_Q_INST_PTR (CPT_LFBASE | 0x110)
51
+
#define CN10K_CPT_LF_Q_GRP_PTR (CPT_LFBASE | 0x120)
52
+
#define CN10K_CPT_LF_NQX(a) (CPT_LFBASE | 0x400 | (a) << 3)
53
+
#define CN10K_CPT_LF_CTX_FLUSH (CPT_LFBASE | 0x510)
54
+
55
+
/* IPSEC Instruction opcodes */
56
+
#define CN10K_IPSEC_MAJOR_OP_WRITE_SA 0x01UL
57
+
#define CN10K_IPSEC_MINOR_OP_WRITE_SA 0x09UL
58
+
#define CN10K_IPSEC_MAJOR_OP_OUTB_IPSEC 0x2AUL
59
+
60
+
enum cn10k_cpt_comp_e {
61
+
CN10K_CPT_COMP_E_NOTDONE = 0x00,
62
+
CN10K_CPT_COMP_E_GOOD = 0x01,
63
+
CN10K_CPT_COMP_E_FAULT = 0x02,
64
+
CN10K_CPT_COMP_E_HWERR = 0x04,
65
+
CN10K_CPT_COMP_E_INSTERR = 0x05,
66
+
CN10K_CPT_COMP_E_WARN = 0x06,
67
+
CN10K_CPT_COMP_E_MASK = 0x3F
68
+
};
69
+
70
+
struct cn10k_cpt_inst_queue {
71
+
u8 *vaddr;
72
+
u8 *real_vaddr;
73
+
dma_addr_t dma_addr;
74
+
dma_addr_t real_dma_addr;
75
+
u32 size;
76
+
};
77
+
78
+
enum cn10k_cpt_hw_state_e {
79
+
CN10K_CPT_HW_UNAVAILABLE,
80
+
CN10K_CPT_HW_AVAILABLE,
81
+
CN10K_CPT_HW_IN_USE
82
+
};
83
+
84
+
struct cn10k_ipsec {
85
+
/* Outbound CPT */
86
+
u64 io_addr;
87
+
atomic_t cpt_state;
88
+
struct cn10k_cpt_inst_queue iq;
89
+
90
+
/* SA info */
91
+
u32 sa_size;
92
+
u32 outb_sa_count;
93
+
struct work_struct sa_work;
94
+
struct workqueue_struct *sa_workq;
95
+
};
96
+
97
+
/* CN10K IPSEC Security Association (SA) */
98
+
/* SA direction */
99
+
#define CN10K_IPSEC_SA_DIR_INB 0
100
+
#define CN10K_IPSEC_SA_DIR_OUTB 1
101
+
/* SA protocol */
102
+
#define CN10K_IPSEC_SA_IPSEC_PROTO_AH 0
103
+
#define CN10K_IPSEC_SA_IPSEC_PROTO_ESP 1
104
+
/* SA Encryption Type */
105
+
#define CN10K_IPSEC_SA_ENCAP_TYPE_AES_GCM 5
106
+
/* SA IPSEC mode Transport/Tunnel */
107
+
#define CN10K_IPSEC_SA_IPSEC_MODE_TRANSPORT 0
108
+
#define CN10K_IPSEC_SA_IPSEC_MODE_TUNNEL 1
109
+
/* SA AES Key Length */
110
+
#define CN10K_IPSEC_SA_AES_KEY_LEN_128 1
111
+
#define CN10K_IPSEC_SA_AES_KEY_LEN_192 2
112
+
#define CN10K_IPSEC_SA_AES_KEY_LEN_256 3
113
+
/* IV Source */
114
+
#define CN10K_IPSEC_SA_IV_SRC_COUNTER 0
115
+
#define CN10K_IPSEC_SA_IV_SRC_PACKET 3
116
+
117
+
struct cn10k_tx_sa_s {
118
+
u64 esn_en : 1; /* W0 */
119
+
u64 rsvd_w0_1_8 : 8;
120
+
u64 hw_ctx_off : 7;
121
+
u64 ctx_id : 16;
122
+
u64 rsvd_w0_32_47 : 16;
123
+
u64 ctx_push_size : 7;
124
+
u64 rsvd_w0_55 : 1;
125
+
u64 ctx_hdr_size : 2;
126
+
u64 aop_valid : 1;
127
+
u64 rsvd_w0_59 : 1;
128
+
u64 ctx_size : 4;
129
+
u64 w1; /* W1 */
130
+
u64 sa_valid : 1; /* W2 */
131
+
u64 sa_dir : 1;
132
+
u64 rsvd_w2_2_3 : 2;
133
+
u64 ipsec_mode : 1;
134
+
u64 ipsec_protocol : 1;
135
+
u64 aes_key_len : 2;
136
+
u64 enc_type : 3;
137
+
u64 rsvd_w2_11_19 : 9;
138
+
u64 iv_src : 2;
139
+
u64 rsvd_w2_22_31 : 10;
140
+
u64 rsvd_w2_32_63 : 32;
141
+
u64 w3; /* W3 */
142
+
u8 cipher_key[32]; /* W4 - W7 */
143
+
u32 rsvd_w8_0_31; /* W8 : IV */
144
+
u32 iv_gcm_salt;
145
+
u64 rsvd_w9_w30[22]; /* W9 - W30 */
146
+
u64 hw_ctx[6]; /* W31 - W36 */
147
+
};
148
+
149
+
/* CPT instruction parameter-1 */
150
+
#define CN10K_IPSEC_INST_PARAM1_DIS_L4_CSUM 0x1
151
+
#define CN10K_IPSEC_INST_PARAM1_DIS_L3_CSUM 0x2
152
+
#define CN10K_IPSEC_INST_PARAM1_CRYPTO_MODE 0x20
153
+
#define CN10K_IPSEC_INST_PARAM1_IV_OFFSET_SHIFT 8
154
+
155
+
/* CPT instruction parameter-2 */
156
+
#define CN10K_IPSEC_INST_PARAM2_ENC_DATA_OFFSET_SHIFT 0
157
+
#define CN10K_IPSEC_INST_PARAM2_AUTH_DATA_OFFSET_SHIFT 8
158
+
159
+
/* CPT Instruction Structure */
160
+
struct cpt_inst_s {
161
+
u64 nixtxl : 3; /* W0 */
162
+
u64 doneint : 1;
163
+
u64 rsvd_w0_4_15 : 12;
164
+
u64 dat_offset : 8;
165
+
u64 ext_param1 : 8;
166
+
u64 nixtx_offset : 20;
167
+
u64 rsvd_w0_52_63 : 12;
168
+
u64 res_addr; /* W1 */
169
+
u64 tag : 32; /* W2 */
170
+
u64 tt : 2;
171
+
u64 grp : 10;
172
+
u64 rsvd_w2_44_47 : 4;
173
+
u64 rvu_pf_func : 16;
174
+
u64 qord : 1; /* W3 */
175
+
u64 rsvd_w3_1_2 : 2;
176
+
u64 wqe_ptr : 61;
177
+
u64 dlen : 16; /* W4 */
178
+
u64 param2 : 16;
179
+
u64 param1 : 16;
180
+
u64 opcode_major : 8;
181
+
u64 opcode_minor : 8;
182
+
u64 dptr; /* W5 */
183
+
u64 rptr; /* W6 */
184
+
u64 cptr : 60; /* W7 */
185
+
u64 ctx_val : 1;
186
+
u64 egrp : 3;
187
+
};
188
+
189
+
/* CPT Instruction Result Structure */
190
+
struct cpt_res_s {
191
+
u64 compcode : 7; /* W0 */
192
+
u64 doneint : 1;
193
+
u64 uc_compcode : 8;
194
+
u64 uc_info : 48;
195
+
u64 esn; /* W1 */
196
+
};
197
+
198
+
/* CPT SG structure */
199
+
struct cpt_sg_s {
200
+
u64 seg1_size : 16;
201
+
u64 seg2_size : 16;
202
+
u64 seg3_size : 16;
203
+
u64 segs : 2;
204
+
u64 rsvd_63_50 : 14;
205
+
};
206
+
207
+
/* CPT LF_INPROG Register */
208
+
#define CPT_LF_INPROG_INFLIGHT GENMASK_ULL(8, 0)
209
+
#define CPT_LF_INPROG_GRB_CNT GENMASK_ULL(39, 32)
210
+
#define CPT_LF_INPROG_GWB_CNT GENMASK_ULL(47, 40)
211
+
212
+
/* CPT LF_Q_GRP_PTR Register */
213
+
#define CPT_LF_Q_GRP_PTR_DQ_PTR GENMASK_ULL(14, 0)
214
+
#define CPT_LF_Q_GRP_PTR_NQ_PTR GENMASK_ULL(46, 32)
215
+
216
+
/* CPT LF_Q_SIZE Register */
217
+
#define CPT_LF_Q_BASE_ADDR GENMASK_ULL(52, 7)
218
+
219
+
/* CPT LF_Q_SIZE Register */
220
+
#define CPT_LF_Q_SIZE_DIV40 GENMASK_ULL(14, 0)
221
+
222
+
/* CPT LF CTX Flush Register */
223
+
#define CPT_LF_CTX_FLUSH GENMASK_ULL(45, 0)
224
+
225
+
#ifdef CONFIG_XFRM_OFFLOAD
226
+
int cn10k_ipsec_init(struct net_device *netdev);
227
+
void cn10k_ipsec_clean(struct otx2_nic *pf);
228
+
int cn10k_ipsec_ethtool_init(struct net_device *netdev, bool enable);
229
+
bool otx2_sqe_add_sg_ipsec(struct otx2_nic *pfvf, struct otx2_snd_queue *sq,
230
+
struct sk_buff *skb, int num_segs, int *offset);
231
+
bool cn10k_ipsec_transmit(struct otx2_nic *pf, struct netdev_queue *txq,
232
+
struct otx2_snd_queue *sq, struct sk_buff *skb,
233
+
int num_segs, int size);
234
+
#else
235
+
static inline __maybe_unused int cn10k_ipsec_init(struct net_device *netdev)
236
+
{
237
+
return 0;
238
+
}
239
+
240
+
static inline __maybe_unused void cn10k_ipsec_clean(struct otx2_nic *pf)
241
+
{
242
+
}
243
+
244
+
static inline __maybe_unused
245
+
int cn10k_ipsec_ethtool_init(struct net_device *netdev, bool enable)
246
+
{
247
+
return 0;
248
+
}
249
+
250
+
static inline bool __maybe_unused
251
+
otx2_sqe_add_sg_ipsec(struct otx2_nic *pfvf, struct otx2_snd_queue *sq,
252
+
struct sk_buff *skb, int num_segs, int *offset)
253
+
{
254
+
return true;
255
+
}
256
+
257
+
static inline bool __maybe_unused
258
+
cn10k_ipsec_transmit(struct otx2_nic *pf, struct netdev_queue *txq,
259
+
struct otx2_snd_queue *sq, struct sk_buff *skb,
260
+
int num_segs, int size)
261
+
{
262
+
return true;
263
+
}
264
+
#endif
265
+
#endif // CN10K_IPSEC_H
+106
-7
drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c
+106
-7
drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c
···
10
10
#include <net/page_pool/helpers.h>
11
11
#include <net/tso.h>
12
12
#include <linux/bitfield.h>
13
+
#include <net/xfrm.h>
13
14
14
15
#include "otx2_reg.h"
15
16
#include "otx2_common.h"
16
17
#include "otx2_struct.h"
17
18
#include "cn10k.h"
19
+
20
+
static bool otx2_is_pfc_enabled(struct otx2_nic *pfvf)
21
+
{
22
+
return IS_ENABLED(CONFIG_DCB) && !!pfvf->pfc_en;
23
+
}
18
24
19
25
static void otx2_nix_rq_op_stats(struct queue_stats *stats,
20
26
struct otx2_nic *pfvf, int qidx)
···
970
964
if (err)
971
965
return err;
972
966
967
+
/* Allocate memory for NIX SQE (which includes NIX SG) and CPT SG.
968
+
* SG of NIX and CPT are same in size. Allocate memory for CPT SG
969
+
* same as NIX SQE for base address alignment.
970
+
* Layout of a NIX SQE and CPT SG entry:
971
+
* -----------------------------
972
+
* | CPT Scatter Gather |
973
+
* | (SQE SIZE) |
974
+
* | |
975
+
* -----------------------------
976
+
* | NIX SQE |
977
+
* | (SQE SIZE) |
978
+
* | |
979
+
* -----------------------------
980
+
*/
981
+
err = qmem_alloc(pfvf->dev, &sq->sqe_ring, qset->sqe_cnt,
982
+
sq->sqe_size * 2);
983
+
if (err)
984
+
return err;
985
+
986
+
err = qmem_alloc(pfvf->dev, &sq->cpt_resp, qset->sqe_cnt, 64);
987
+
if (err)
988
+
return err;
989
+
973
990
if (qidx < pfvf->hw.tx_queues) {
974
991
err = qmem_alloc(pfvf->dev, &sq->tso_hdrs, qset->sqe_cnt,
975
992
TSO_HEADER_SIZE);
···
1751
1722
return -ENOMEM;
1752
1723
1753
1724
req->chan_base = 0;
1754
-
#ifdef CONFIG_DCB
1755
-
req->chan_cnt = pfvf->pfc_en ? IEEE_8021QAZ_MAX_TCS : 1;
1756
-
req->bpid_per_chan = pfvf->pfc_en ? 1 : 0;
1757
-
#else
1758
-
req->chan_cnt = 1;
1759
-
req->bpid_per_chan = 0;
1760
-
#endif
1725
+
if (otx2_is_pfc_enabled(pfvf)) {
1726
+
req->chan_cnt = IEEE_8021QAZ_MAX_TCS;
1727
+
req->bpid_per_chan = 1;
1728
+
} else {
1729
+
req->chan_cnt = 1;
1730
+
req->bpid_per_chan = 0;
1731
+
}
1761
1732
1762
1733
return otx2_sync_mbox_msg(&pfvf->mbox);
1763
1734
}
1764
1735
EXPORT_SYMBOL(otx2_nix_config_bp);
1736
+
1737
+
int otx2_nix_cpt_config_bp(struct otx2_nic *pfvf, bool enable)
1738
+
{
1739
+
struct nix_bp_cfg_req *req;
1740
+
1741
+
if (enable)
1742
+
req = otx2_mbox_alloc_msg_nix_cpt_bp_enable(&pfvf->mbox);
1743
+
else
1744
+
req = otx2_mbox_alloc_msg_nix_cpt_bp_disable(&pfvf->mbox);
1745
+
1746
+
if (!req)
1747
+
return -ENOMEM;
1748
+
1749
+
req->chan_base = 0;
1750
+
if (otx2_is_pfc_enabled(pfvf)) {
1751
+
req->chan_cnt = IEEE_8021QAZ_MAX_TCS;
1752
+
req->bpid_per_chan = 1;
1753
+
} else {
1754
+
req->chan_cnt = 1;
1755
+
req->bpid_per_chan = 0;
1756
+
}
1757
+
1758
+
return otx2_sync_mbox_msg(&pfvf->mbox);
1759
+
}
1760
+
EXPORT_SYMBOL(otx2_nix_cpt_config_bp);
1765
1761
1766
1762
/* Mbox message handlers */
1767
1763
void mbox_handler_cgx_stats(struct otx2_nic *pfvf,
···
2001
1947
MBOX_UP_CGX_MESSAGES
2002
1948
MBOX_UP_MCS_MESSAGES
2003
1949
#undef M
1950
+
1951
+
dma_addr_t otx2_dma_map_skb_frag(struct otx2_nic *pfvf,
1952
+
struct sk_buff *skb, int seg, int *len)
1953
+
{
1954
+
enum dma_data_direction dir = DMA_TO_DEVICE;
1955
+
const skb_frag_t *frag;
1956
+
struct page *page;
1957
+
int offset;
1958
+
1959
+
/* Crypto hardware need write permission for ipsec crypto offload */
1960
+
if (unlikely(xfrm_offload(skb))) {
1961
+
dir = DMA_BIDIRECTIONAL;
1962
+
skb = skb_unshare(skb, GFP_ATOMIC);
1963
+
}
1964
+
1965
+
/* First segment is always skb->data */
1966
+
if (!seg) {
1967
+
page = virt_to_page(skb->data);
1968
+
offset = offset_in_page(skb->data);
1969
+
*len = skb_headlen(skb);
1970
+
} else {
1971
+
frag = &skb_shinfo(skb)->frags[seg - 1];
1972
+
page = skb_frag_page(frag);
1973
+
offset = skb_frag_off(frag);
1974
+
*len = skb_frag_size(frag);
1975
+
}
1976
+
return otx2_dma_map_page(pfvf, page, offset, *len, dir);
1977
+
}
1978
+
1979
+
void otx2_dma_unmap_skb_frags(struct otx2_nic *pfvf, struct sg_list *sg)
1980
+
{
1981
+
enum dma_data_direction dir = DMA_TO_DEVICE;
1982
+
struct sk_buff *skb = NULL;
1983
+
int seg;
1984
+
1985
+
skb = (struct sk_buff *)sg->skb;
1986
+
if (unlikely(xfrm_offload(skb)))
1987
+
dir = DMA_BIDIRECTIONAL;
1988
+
1989
+
for (seg = 0; seg < sg->num_segs; seg++) {
1990
+
otx2_dma_unmap_page(pfvf, sg->dma_addr[seg],
1991
+
sg->size[seg], dir);
1992
+
}
1993
+
sg->num_segs = 0;
1994
+
}
+26
drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h
+26
drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h
···
30
30
#include <rvu_trace.h>
31
31
#include "qos.h"
32
32
#include "rep.h"
33
+
#include "cn10k_ipsec.h"
33
34
34
35
/* IPv4 flag more fragment bit */
35
36
#define IPV4_FLAG_MORE 0x20
···
41
40
#define PCI_DEVID_OCTEONTX2_RVU_AFVF 0xA0F8
42
41
43
42
#define PCI_SUBSYS_DEVID_96XX_RVU_PFVF 0xB200
43
+
#define PCI_SUBSYS_DEVID_CN10K_A_RVU_PFVF 0xB900
44
44
#define PCI_SUBSYS_DEVID_CN10K_B_RVU_PFVF 0xBD00
45
45
46
46
#define PCI_DEVID_OCTEONTX2_SDP_REP 0xA0F7
···
56
54
/* Max priority supported for PFC */
57
55
#define NIX_PF_PFC_PRIO_MAX 8
58
56
#endif
57
+
58
+
/* Number of segments per SG structure */
59
+
#define MAX_SEGS_PER_SG 3
59
60
60
61
enum arua_mapped_qtypes {
61
62
AURA_NIX_RQ,
···
453
448
#define OTX2_FLAG_TC_MARK_ENABLED BIT_ULL(17)
454
449
#define OTX2_FLAG_REP_MODE_ENABLED BIT_ULL(18)
455
450
#define OTX2_FLAG_PORT_UP BIT_ULL(19)
451
+
#define OTX2_FLAG_IPSEC_OFFLOAD_ENABLED BIT_ULL(20)
456
452
u64 flags;
457
453
u64 *cq_op_addr;
458
454
···
528
522
u16 rep_pf_map[RVU_MAX_REP];
529
523
u16 esw_mode;
530
524
#endif
525
+
526
+
/* Inline ipsec */
527
+
struct cn10k_ipsec ipsec;
531
528
};
532
529
533
530
static inline bool is_otx2_lbkvf(struct pci_dev *pdev)
···
581
572
return pdev->subsystem_device == PCI_SUBSYS_DEVID_CN10K_B_RVU_PFVF;
582
573
}
583
574
575
+
static inline bool is_dev_cn10ka_b0(struct pci_dev *pdev)
576
+
{
577
+
if (pdev->subsystem_device == PCI_SUBSYS_DEVID_CN10K_A_RVU_PFVF &&
578
+
(pdev->revision & 0xFF) == 0x54)
579
+
return true;
580
+
581
+
return false;
582
+
}
583
+
584
584
static inline void otx2_setup_dev_hw_settings(struct otx2_nic *pfvf)
585
585
{
586
586
struct otx2_hw *hw = &pfvf->hw;
···
638
620
break;
639
621
case BLKTYPE_NPA:
640
622
blkaddr = BLKADDR_NPA;
623
+
break;
624
+
case BLKTYPE_CPT:
625
+
blkaddr = BLKADDR_CPT0;
641
626
break;
642
627
default:
643
628
blkaddr = BLKADDR_RVUM;
···
1006
985
int otx2_rxtx_enable(struct otx2_nic *pfvf, bool enable);
1007
986
void otx2_ctx_disable(struct mbox *mbox, int type, bool npa);
1008
987
int otx2_nix_config_bp(struct otx2_nic *pfvf, bool enable);
988
+
int otx2_nix_cpt_config_bp(struct otx2_nic *pfvf, bool enable);
1009
989
void otx2_cleanup_rx_cqes(struct otx2_nic *pfvf, struct otx2_cq_queue *cq, int qidx);
1010
990
void otx2_cleanup_tx_cqes(struct otx2_nic *pfvf, struct otx2_cq_queue *cq);
1011
991
int otx2_sq_init(struct otx2_nic *pfvf, u16 qidx, u16 sqb_aura);
···
1171
1149
{
1172
1150
return *(u16 *)a - *(u16 *)b;
1173
1151
}
1152
+
1153
+
dma_addr_t otx2_dma_map_skb_frag(struct otx2_nic *pfvf,
1154
+
struct sk_buff *skb, int seg, int *len);
1155
+
void otx2_dma_unmap_skb_frags(struct otx2_nic *pfvf, struct sg_list *sg);
1174
1156
#endif /* OTX2_COMMON_H */
+3
drivers/net/ethernet/marvell/octeontx2/nic/otx2_dcbnl.c
+3
drivers/net/ethernet/marvell/octeontx2/nic/otx2_dcbnl.c
+18
-1
drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c
+18
-1
drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c
···
26
26
#include "cn10k.h"
27
27
#include "qos.h"
28
28
#include <rvu_trace.h>
29
+
#include "cn10k_ipsec.h"
29
30
30
31
#define DRV_NAME "rvu_nicpf"
31
32
#define DRV_STRING "Marvell RVU NIC Physical Function Driver"
···
1485
1484
if (!sq->sqe)
1486
1485
continue;
1487
1486
qmem_free(pf->dev, sq->sqe);
1487
+
qmem_free(pf->dev, sq->sqe_ring);
1488
+
qmem_free(pf->dev, sq->cpt_resp);
1488
1489
qmem_free(pf->dev, sq->tso_hdrs);
1489
1490
kfree(sq->sg);
1490
1491
kfree(sq->sqb_ptrs);
···
1553
1550
err = otx2_config_nix(pf);
1554
1551
if (err)
1555
1552
goto err_free_npa_lf;
1553
+
1554
+
/* Default disable backpressure on NIX-CPT */
1555
+
otx2_nix_cpt_config_bp(pf, false);
1556
1556
1557
1557
/* Enable backpressure for CGX mapped PF/VFs */
1558
1558
if (!is_otx2_lbkvf(pf->pdev))
···
2278
2272
if ((changed & NETIF_F_HW_VLAN_CTAG_RX) && netif_running(netdev))
2279
2273
return otx2_enable_rxvlan(pf,
2280
2274
features & NETIF_F_HW_VLAN_CTAG_RX);
2275
+
2276
+
if (changed & NETIF_F_HW_ESP)
2277
+
return cn10k_ipsec_ethtool_init(netdev,
2278
+
features & NETIF_F_HW_ESP);
2281
2279
2282
2280
return otx2_handle_ntuple_tc_features(netdev, features);
2283
2281
}
···
3172
3162
/* reset CGX/RPM MAC stats */
3173
3163
otx2_reset_mac_stats(pf);
3174
3164
3165
+
err = cn10k_ipsec_init(netdev);
3166
+
if (err)
3167
+
goto err_mcs_free;
3168
+
3175
3169
err = register_netdev(netdev);
3176
3170
if (err) {
3177
3171
dev_err(dev, "Failed to register netdevice\n");
3178
-
goto err_mcs_free;
3172
+
goto err_ipsec_clean;
3179
3173
}
3180
3174
3181
3175
err = otx2_wq_init(pf);
···
3220
3206
otx2_mcam_flow_del(pf);
3221
3207
err_unreg_netdev:
3222
3208
unregister_netdev(netdev);
3209
+
err_ipsec_clean:
3210
+
cn10k_ipsec_clean(pf);
3223
3211
err_mcs_free:
3224
3212
cn10k_mcs_free(pf);
3225
3213
err_del_mcam_entries:
···
3419
3403
3420
3404
otx2_unregister_dl(pf);
3421
3405
unregister_netdev(netdev);
3406
+
cn10k_ipsec_clean(pf);
3422
3407
cn10k_mcs_free(pf);
3423
3408
otx2_sriov_disable(pf->pdev);
3424
3409
otx2_sriov_vfcfg_cleanup(pf);
+29
-35
drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c
+29
-35
drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c
···
11
11
#include <linux/bpf.h>
12
12
#include <linux/bpf_trace.h>
13
13
#include <net/ip6_checksum.h>
14
+
#include <net/xfrm.h>
14
15
15
16
#include "otx2_reg.h"
16
17
#include "otx2_common.h"
···
32
31
struct nix_cqe_rx_s *cqe,
33
32
struct otx2_cq_queue *cq,
34
33
bool *need_xdp_flush);
34
+
35
+
static void otx2_sq_set_sqe_base(struct otx2_snd_queue *sq,
36
+
struct sk_buff *skb)
37
+
{
38
+
if (static_branch_unlikely(&cn10k_ipsec_sa_enabled) &&
39
+
(xfrm_offload(skb)))
40
+
sq->sqe_base = sq->sqe_ring->base + sq->sqe_size +
41
+
(sq->head * (sq->sqe_size * 2));
42
+
else
43
+
sq->sqe_base = sq->sqe->base;
44
+
}
35
45
36
46
static int otx2_nix_cq_op_status(struct otx2_nic *pfvf,
37
47
struct otx2_cq_queue *cq)
···
90
78
#else
91
79
return i;
92
80
#endif
93
-
}
94
-
95
-
static dma_addr_t otx2_dma_map_skb_frag(struct otx2_nic *pfvf,
96
-
struct sk_buff *skb, int seg, int *len)
97
-
{
98
-
const skb_frag_t *frag;
99
-
struct page *page;
100
-
int offset;
101
-
102
-
/* First segment is always skb->data */
103
-
if (!seg) {
104
-
page = virt_to_page(skb->data);
105
-
offset = offset_in_page(skb->data);
106
-
*len = skb_headlen(skb);
107
-
} else {
108
-
frag = &skb_shinfo(skb)->frags[seg - 1];
109
-
page = skb_frag_page(frag);
110
-
offset = skb_frag_off(frag);
111
-
*len = skb_frag_size(frag);
112
-
}
113
-
return otx2_dma_map_page(pfvf, page, offset, *len, DMA_TO_DEVICE);
114
-
}
115
-
116
-
static void otx2_dma_unmap_skb_frags(struct otx2_nic *pfvf, struct sg_list *sg)
117
-
{
118
-
int seg;
119
-
120
-
for (seg = 0; seg < sg->num_segs; seg++) {
121
-
otx2_dma_unmap_page(pfvf, sg->dma_addr[seg],
122
-
sg->size[seg], DMA_TO_DEVICE);
123
-
}
124
-
sg->num_segs = 0;
125
81
}
126
82
127
83
static void otx2_xdp_snd_pkt_handler(struct otx2_nic *pfvf,
···
605
625
sq->head &= (sq->sqe_cnt - 1);
606
626
}
607
627
608
-
#define MAX_SEGS_PER_SG 3
609
628
/* Add SQE scatter/gather subdescriptor structure */
610
629
static bool otx2_sqe_add_sg(struct otx2_nic *pfvf, struct otx2_snd_queue *sq,
611
630
struct sk_buff *skb, int num_segs, int *offset)
···
1140
1161
int offset, num_segs, free_desc;
1141
1162
struct nix_sqe_hdr_s *sqe_hdr;
1142
1163
struct otx2_nic *pfvf = dev;
1164
+
bool ret;
1143
1165
1144
1166
/* Check if there is enough room between producer
1145
1167
* and consumer index.
···
1157
1177
/* If SKB doesn't fit in a single SQE, linearize it.
1158
1178
* TODO: Consider adding JUMP descriptor instead.
1159
1179
*/
1180
+
1160
1181
if (unlikely(num_segs > OTX2_MAX_FRAGS_IN_SQE)) {
1161
1182
if (__skb_linearize(skb)) {
1162
1183
dev_kfree_skb_any(skb);
···
1177
1196
return true;
1178
1197
}
1179
1198
1199
+
/* Set sqe base address */
1200
+
otx2_sq_set_sqe_base(sq, skb);
1201
+
1180
1202
/* Set SQE's SEND_HDR.
1181
1203
* Do not clear the first 64bit as it contains constant info.
1182
1204
*/
···
1192
1208
otx2_sqe_add_ext(pfvf, sq, skb, &offset);
1193
1209
1194
1210
/* Add SG subdesc with data frags */
1195
-
if (!otx2_sqe_add_sg(pfvf, sq, skb, num_segs, &offset)) {
1211
+
if (static_branch_unlikely(&cn10k_ipsec_sa_enabled) &&
1212
+
(xfrm_offload(skb)))
1213
+
ret = otx2_sqe_add_sg_ipsec(pfvf, sq, skb, num_segs, &offset);
1214
+
else
1215
+
ret = otx2_sqe_add_sg(pfvf, sq, skb, num_segs, &offset);
1216
+
1217
+
if (!ret) {
1196
1218
otx2_dma_unmap_skb_frags(pfvf, &sq->sg[sq->head]);
1197
1219
return false;
1198
1220
}
···
1207
1217
1208
1218
sqe_hdr->sizem1 = (offset / 16) - 1;
1209
1219
1220
+
if (static_branch_unlikely(&cn10k_ipsec_sa_enabled) &&
1221
+
(xfrm_offload(skb)))
1222
+
return cn10k_ipsec_transmit(pfvf, txq, sq, skb, num_segs,
1223
+
offset);
1224
+
1210
1225
netdev_tx_sent_queue(txq, skb->len);
1211
1226
1212
1227
/* Flush SQE to HW */
1213
1228
pfvf->hw_ops->sqe_flush(pfvf, sq, offset, qidx);
1214
-
1215
1229
return true;
1216
1230
}
1217
1231
EXPORT_SYMBOL(otx2_sq_append_skb);
+3
drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h
+3
drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h
+9
-1
drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c
+9
-1
drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c
···
14
14
#include "otx2_reg.h"
15
15
#include "otx2_ptp.h"
16
16
#include "cn10k.h"
17
+
#include "cn10k_ipsec.h"
17
18
18
19
#define DRV_NAME "rvu_nicvf"
19
20
#define DRV_STRING "Marvell RVU NIC Virtual Function Driver"
···
694
693
pdev->bus->number, n);
695
694
}
696
695
696
+
err = cn10k_ipsec_init(netdev);
697
+
if (err)
698
+
goto err_ptp_destroy;
699
+
697
700
err = register_netdev(netdev);
698
701
if (err) {
699
702
dev_err(dev, "Failed to register netdevice\n");
700
-
goto err_ptp_destroy;
703
+
goto err_ipsec_clean;
701
704
}
702
705
703
706
err = otx2_vf_wq_init(vf);
···
735
730
otx2_shutdown_tc(vf);
736
731
err_unreg_netdev:
737
732
unregister_netdev(netdev);
733
+
err_ipsec_clean:
734
+
cn10k_ipsec_clean(vf);
738
735
err_ptp_destroy:
739
736
otx2_ptp_destroy(vf);
740
737
err_detach_rsrc:
···
789
782
unregister_netdev(netdev);
790
783
if (vf->otx2_wq)
791
784
destroy_workqueue(vf->otx2_wq);
785
+
cn10k_ipsec_clean(vf);
792
786
otx2_ptp_destroy(vf);
793
787
otx2_mcam_flow_del(vf);
794
788
otx2_shutdown_tc(vf);