Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
crypto: octeontx2 - add ctx_val workaround
HW has a errata that CPT HW may hit an issue, while processing CPT
instructions with CTX_VAL set and CTX_VAL not set. So, this patch
adds the code to always set the CTX_VAL as a workaround.
Signed-off-by: Srujana Challa <schalla@marvell.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
authored by
Srujana Challa
and committed by
Herbert Xu
e9297111
8bb0be9f
+137
-1
+70
drivers/crypto/marvell/octeontx2/cn10k_cpt.c
+70
drivers/crypto/marvell/octeontx2/cn10k_cpt.c
···
96
96
}
97
97
EXPORT_SYMBOL_NS_GPL(cn10k_cptvf_lmtst_init, CRYPTO_DEV_OCTEONTX2_CPT);
98
98
99
+
void cn10k_cpt_hw_ctx_clear(struct pci_dev *pdev,
100
+
struct cn10k_cpt_errata_ctx *er_ctx)
101
+
{
102
+
u64 cptr_dma;
103
+
104
+
if (!is_dev_cn10ka_ax(pdev))
105
+
return;
106
+
107
+
cptr_dma = er_ctx->cptr_dma & ~(BIT_ULL(60));
108
+
cn10k_cpt_ctx_flush(pdev, cptr_dma, true);
109
+
dma_unmap_single(&pdev->dev, cptr_dma, CN10K_CPT_HW_CTX_SIZE,
110
+
DMA_BIDIRECTIONAL);
111
+
kfree(er_ctx->hw_ctx);
112
+
}
113
+
EXPORT_SYMBOL_NS_GPL(cn10k_cpt_hw_ctx_clear, CRYPTO_DEV_OCTEONTX2_CPT);
114
+
115
+
void cn10k_cpt_hw_ctx_set(union cn10k_cpt_hw_ctx *hctx, u16 ctx_sz)
116
+
{
117
+
hctx->w0.aop_valid = 1;
118
+
hctx->w0.ctx_hdr_sz = 0;
119
+
hctx->w0.ctx_sz = ctx_sz;
120
+
hctx->w0.ctx_push_sz = 1;
121
+
}
122
+
EXPORT_SYMBOL_NS_GPL(cn10k_cpt_hw_ctx_set, CRYPTO_DEV_OCTEONTX2_CPT);
123
+
124
+
int cn10k_cpt_hw_ctx_init(struct pci_dev *pdev,
125
+
struct cn10k_cpt_errata_ctx *er_ctx)
126
+
{
127
+
union cn10k_cpt_hw_ctx *hctx;
128
+
u64 cptr_dma;
129
+
130
+
er_ctx->cptr_dma = 0;
131
+
er_ctx->hw_ctx = NULL;
132
+
133
+
if (!is_dev_cn10ka_ax(pdev))
134
+
return 0;
135
+
136
+
hctx = kmalloc(CN10K_CPT_HW_CTX_SIZE, GFP_KERNEL);
137
+
if (unlikely(!hctx))
138
+
return -ENOMEM;
139
+
cptr_dma = dma_map_single(&pdev->dev, hctx, CN10K_CPT_HW_CTX_SIZE,
140
+
DMA_BIDIRECTIONAL);
141
+
142
+
cn10k_cpt_hw_ctx_set(hctx, 1);
143
+
er_ctx->hw_ctx = hctx;
144
+
er_ctx->cptr_dma = cptr_dma | BIT_ULL(60);
145
+
146
+
return 0;
147
+
}
148
+
EXPORT_SYMBOL_NS_GPL(cn10k_cpt_hw_ctx_init, CRYPTO_DEV_OCTEONTX2_CPT);
149
+
150
+
void cn10k_cpt_ctx_flush(struct pci_dev *pdev, u64 cptr, bool inval)
151
+
{
152
+
struct otx2_cptvf_dev *cptvf = pci_get_drvdata(pdev);
153
+
struct otx2_cptlfs_info *lfs = &cptvf->lfs;
154
+
u64 reg;
155
+
156
+
reg = (uintptr_t)cptr >> 7;
157
+
if (inval)
158
+
reg = reg | BIT_ULL(46);
159
+
160
+
otx2_cpt_write64(lfs->reg_base, lfs->blkaddr, lfs->lf[0].slot,
161
+
OTX2_CPT_LF_CTX_FLUSH, reg);
162
+
/* Make sure that the FLUSH operation is complete */
163
+
wmb();
164
+
otx2_cpt_read64(lfs->reg_base, lfs->blkaddr, lfs->lf[0].slot,
165
+
OTX2_CPT_LF_CTX_ERR);
166
+
}
167
+
EXPORT_SYMBOL_NS_GPL(cn10k_cpt_ctx_flush, CRYPTO_DEV_OCTEONTX2_CPT);
168
+
99
169
void cptvf_hw_ops_get(struct otx2_cptvf_dev *cptvf)
100
170
{
101
171
if (test_bit(CN10K_LMTST, &cptvf->cap_flag))
+26
drivers/crypto/marvell/octeontx2/cn10k_cpt.h
+26
drivers/crypto/marvell/octeontx2/cn10k_cpt.h
···
8
8
#include "otx2_cptpf.h"
9
9
#include "otx2_cptvf.h"
10
10
11
+
#define CN10K_CPT_HW_CTX_SIZE 256
12
+
13
+
union cn10k_cpt_hw_ctx {
14
+
u64 u;
15
+
struct {
16
+
u64 reserved_0_47:48;
17
+
u64 ctx_push_sz:7;
18
+
u64 reserved_55:1;
19
+
u64 ctx_hdr_sz:2;
20
+
u64 aop_valid:1;
21
+
u64 reserved_59:1;
22
+
u64 ctx_sz:4;
23
+
} w0;
24
+
};
25
+
26
+
struct cn10k_cpt_errata_ctx {
27
+
union cn10k_cpt_hw_ctx *hw_ctx;
28
+
u64 cptr_dma;
29
+
};
30
+
11
31
static inline u8 cn10k_cpt_get_compcode(union otx2_cpt_res_s *result)
12
32
{
13
33
return ((struct cn10k_cpt_res_s *)result)->compcode;
···
50
30
51
31
int cn10k_cptpf_lmtst_init(struct otx2_cptpf_dev *cptpf);
52
32
int cn10k_cptvf_lmtst_init(struct otx2_cptvf_dev *cptvf);
33
+
void cn10k_cpt_ctx_flush(struct pci_dev *pdev, u64 cptr, bool inval);
34
+
int cn10k_cpt_hw_ctx_init(struct pci_dev *pdev,
35
+
struct cn10k_cpt_errata_ctx *er_ctx);
36
+
void cn10k_cpt_hw_ctx_clear(struct pci_dev *pdev,
37
+
struct cn10k_cpt_errata_ctx *er_ctx);
38
+
void cn10k_cpt_hw_ctx_set(union cn10k_cpt_hw_ctx *hctx, u16 ctx_sz);
53
39
void cptvf_hw_ops_get(struct otx2_cptvf_dev *cptvf);
54
40
55
41
#endif /* __CN10K_CPTLF_H */
+2
drivers/crypto/marvell/octeontx2/otx2_cpt_hw_types.h
+2
drivers/crypto/marvell/octeontx2/otx2_cpt_hw_types.h
···
102
102
#define OTX2_CPT_LF_Q_INST_PTR (0x110)
103
103
#define OTX2_CPT_LF_Q_GRP_PTR (0x120)
104
104
#define OTX2_CPT_LF_NQX(a) (0x400 | (a) << 3)
105
+
#define OTX2_CPT_LF_CTX_FLUSH (0x510)
106
+
#define OTX2_CPT_LF_CTX_ERR (0x520)
105
107
#define OTX2_CPT_RVU_FUNC_BLKADDR_SHIFT 20
106
108
/* LMT LF registers */
107
109
#define OTX2_CPT_LMT_LFBASE BIT_ULL(OTX2_CPT_RVU_FUNC_BLKADDR_SHIFT)
+2
drivers/crypto/marvell/octeontx2/otx2_cpt_reqmgr.h
+2
drivers/crypto/marvell/octeontx2/otx2_cpt_reqmgr.h
+31
drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c
+31
drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c
···
17
17
#include "otx2_cptvf.h"
18
18
#include "otx2_cptvf_algs.h"
19
19
#include "otx2_cpt_reqmgr.h"
20
+
#include "cn10k_cpt.h"
20
21
21
22
/* Size of salt in AES GCM mode */
22
23
#define AES_GCM_SALT_SIZE 4
···
385
384
req_info->is_trunc_hmac = false;
386
385
req_info->ctrl.s.grp = otx2_cpt_get_kcrypto_eng_grp_num(pdev);
387
386
387
+
req_info->req.cptr = ctx->er_ctx.hw_ctx;
388
+
req_info->req.cptr_dma = ctx->er_ctx.cptr_dma;
389
+
388
390
/*
389
391
* We perform an asynchronous send and once
390
392
* the request is completed the driver would
···
534
530
struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm);
535
531
struct crypto_tfm *tfm = crypto_skcipher_tfm(stfm);
536
532
struct crypto_alg *alg = tfm->__crt_alg;
533
+
struct pci_dev *pdev;
534
+
int ret, cpu_num;
537
535
538
536
memset(ctx, 0, sizeof(*ctx));
539
537
/*
···
546
540
crypto_skcipher_set_reqsize_dma(
547
541
stfm, sizeof(struct otx2_cpt_req_ctx) +
548
542
sizeof(struct skcipher_request));
543
+
544
+
ret = get_se_device(&pdev, &cpu_num);
545
+
if (ret)
546
+
return ret;
547
+
548
+
ctx->pdev = pdev;
549
+
ret = cn10k_cpt_hw_ctx_init(pdev, &ctx->er_ctx);
550
+
if (ret)
551
+
return ret;
549
552
550
553
return cpt_skcipher_fallback_init(ctx, alg);
551
554
}
···
567
552
crypto_free_skcipher(ctx->fbk_cipher);
568
553
ctx->fbk_cipher = NULL;
569
554
}
555
+
cn10k_cpt_hw_ctx_clear(ctx->pdev, &ctx->er_ctx);
570
556
}
571
557
572
558
static int cpt_aead_fallback_init(struct otx2_cpt_aead_ctx *ctx,
···
592
576
struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(atfm);
593
577
struct crypto_tfm *tfm = crypto_aead_tfm(atfm);
594
578
struct crypto_alg *alg = tfm->__crt_alg;
579
+
struct pci_dev *pdev;
580
+
int ret, cpu_num;
595
581
596
582
ctx->cipher_type = cipher_type;
597
583
ctx->mac_type = mac_type;
···
649
631
break;
650
632
}
651
633
crypto_aead_set_reqsize_dma(atfm, sizeof(struct otx2_cpt_req_ctx));
634
+
635
+
ret = get_se_device(&pdev, &cpu_num);
636
+
if (ret)
637
+
return ret;
638
+
639
+
ctx->pdev = pdev;
640
+
ret = cn10k_cpt_hw_ctx_init(pdev, &ctx->er_ctx);
641
+
if (ret)
642
+
return ret;
652
643
653
644
return cpt_aead_fallback_init(ctx, alg);
654
645
}
···
721
694
crypto_free_aead(ctx->fbk_cipher);
722
695
ctx->fbk_cipher = NULL;
723
696
}
697
+
cn10k_cpt_hw_ctx_clear(ctx->pdev, &ctx->er_ctx);
724
698
}
725
699
726
700
static int otx2_cpt_aead_gcm_set_authsize(struct crypto_aead *tfm,
···
1326
1298
req_info->req_type = reg_type;
1327
1299
req_info->is_enc = enc;
1328
1300
req_info->is_trunc_hmac = false;
1301
+
1302
+
req_info->req.cptr = ctx->er_ctx.hw_ctx;
1303
+
req_info->req.cptr_dma = ctx->er_ctx.cptr_dma;
1329
1304
1330
1305
switch (reg_type) {
1331
1306
case OTX2_CPT_AEAD_ENC_DEC_REQ:
+5
drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.h
+5
drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.h
···
9
9
#include <crypto/skcipher.h>
10
10
#include <crypto/aead.h>
11
11
#include "otx2_cpt_common.h"
12
+
#include "cn10k_cpt.h"
12
13
13
14
#define OTX2_CPT_MAX_ENC_KEY_SIZE 32
14
15
#define OTX2_CPT_MAX_HASH_KEY_SIZE 64
···
124
123
u8 key_type;
125
124
u8 enc_align_len;
126
125
struct crypto_skcipher *fbk_cipher;
126
+
struct pci_dev *pdev;
127
+
struct cn10k_cpt_errata_ctx er_ctx;
127
128
};
128
129
129
130
union otx2_cpt_offset_ctrl {
···
164
161
struct crypto_shash *hashalg;
165
162
struct otx2_cpt_sdesc *sdesc;
166
163
struct crypto_aead *fbk_cipher;
164
+
struct cn10k_cpt_errata_ctx er_ctx;
165
+
struct pci_dev *pdev;
167
166
u8 *ipad;
168
167
u8 *opad;
169
168
u32 enc_key_len;
+1
-1
drivers/crypto/marvell/octeontx2/otx2_cptvf_reqmgr.c
+1
-1
drivers/crypto/marvell/octeontx2/otx2_cptvf_reqmgr.c
···
159
159
cpu_to_be64s(&iq_cmd.cmd.u);
160
160
iq_cmd.dptr = info->dptr_baddr | info->gthr_sz << 60;
161
161
iq_cmd.rptr = info->rptr_baddr | info->sctr_sz << 60;
162
-
iq_cmd.cptr.u = 0;
162
+
iq_cmd.cptr.s.cptr = cpt_req->cptr_dma;
163
163
iq_cmd.cptr.s.grp = ctrl->s.grp;
164
164
165
165
/* Fill in the CPT_INST_S type command for HW interpretation */