Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'v6.17-p1' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6
Pull crypto update from Herbert Xu:
"API:
- Allow hash drivers without fallbacks (e.g., hardware key)
Algorithms:
- Add hmac hardware key support (phmac) on s390
- Re-enable sha384 in FIPS mode
- Disable sha1 in FIPS mode
- Convert zstd to acomp
Drivers:
- Lower priority of qat skcipher and aead
- Convert aspeed to partial block API
- Add iMX8QXP support in caam
- Add rate limiting support for GEN6 devices in qat
- Enable telemetry for GEN6 devices in qat
- Implement full backlog mode for hisilicon/sec2"
* tag 'v6.17-p1' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6: (116 commits)
crypto: keembay - Use min() to simplify ocs_create_linked_list_from_sg()
crypto: hisilicon/hpre - fix dma unmap sequence
crypto: qat - make adf_dev_autoreset() static
crypto: ccp - reduce stack usage in ccp_run_aes_gcm_cmd
crypto: qat - refactor ring-related debug functions
crypto: qat - fix seq_file position update in adf_ring_next()
crypto: qat - fix DMA direction for compression on GEN2 devices
crypto: jitter - replace ARRAY_SIZE definition with header include
crypto: engine - remove {prepare,unprepare}_crypt_hardware callbacks
crypto: engine - remove request batching support
crypto: qat - flush misc workqueue during device shutdown
crypto: qat - enable rate limiting feature for GEN6 devices
crypto: qat - add compression slice count for rate limiting
crypto: qat - add get_svc_slice_cnt() in device data structure
crypto: qat - add adf_rl_get_num_svc_aes() in rate limiting
crypto: qat - relocate service related functions
crypto: qat - consolidate service enums
crypto: qat - add decompression service for rate limiting
crypto: qat - validate service in rate limiting sysfs api
crypto: hisilicon/sec2 - implement full backlog mode for sec
...
+4108
-2064
+1
-1
Documentation/ABI/testing/debugfs-driver-qat
+1
-1
Documentation/ABI/testing/debugfs-driver-qat
···
67
67
Description: (RO) Read returns power management information specific to the
68
68
QAT device.
69
69
70
-
This attribute is only available for qat_4xxx devices.
70
+
This attribute is only available for qat_4xxx and qat_6xxx devices.
71
71
72
72
What: /sys/kernel/debug/qat_<device>_<BDF>/cnv_errors
73
73
Date: January 2024
+7
-3
Documentation/ABI/testing/debugfs-driver-qat_telemetry
+7
-3
Documentation/ABI/testing/debugfs-driver-qat_telemetry
···
32
32
33
33
echo 0 > /sys/kernel/debug/qat_4xxx_0000:6b:00.0/telemetry/control
34
34
35
-
This attribute is only available for qat_4xxx devices.
35
+
This attribute is only available for qat_4xxx and qat_6xxx devices.
36
36
37
37
What: /sys/kernel/debug/qat_<device>_<BDF>/telemetry/device_data
38
38
Date: March 2024
···
67
67
exec_xlt<N> execution count of Translator slice N
68
68
util_dcpr<N> utilization of Decompression slice N [%]
69
69
exec_dcpr<N> execution count of Decompression slice N
70
+
util_cnv<N> utilization of Compression and verify slice N [%]
71
+
exec_cnv<N> execution count of Compression and verify slice N
72
+
util_dcprz<N> utilization of Decompression slice N [%]
73
+
exec_dcprz<N> execution count of Decompression slice N
70
74
util_pke<N> utilization of PKE N [%]
71
75
exec_pke<N> execution count of PKE N
72
76
util_ucs<N> utilization of UCS slice N [%]
···
104
100
If a device lacks of a specific accelerator, the corresponding
105
101
attribute is not reported.
106
102
107
-
This attribute is only available for qat_4xxx devices.
103
+
This attribute is only available for qat_4xxx and qat_6xxx devices.
108
104
109
105
What: /sys/kernel/debug/qat_<device>_<BDF>/telemetry/rp_<A/B/C/D>_data
110
106
Date: March 2024
···
229
225
``rp2srv`` from sysfs.
230
226
See Documentation/ABI/testing/sysfs-driver-qat for details.
231
227
232
-
This attribute is only available for qat_4xxx devices.
228
+
This attribute is only available for qat_4xxx and qat_6xxx devices.
+27
-23
Documentation/ABI/testing/sysfs-driver-qat
+27
-23
Documentation/ABI/testing/sysfs-driver-qat
···
14
14
It is possible to transition the device from up to down only
15
15
if the device is up and vice versa.
16
16
17
-
This attribute is only available for qat_4xxx devices.
17
+
This attribute is available for qat_4xxx and qat_6xxx devices.
18
18
19
19
What: /sys/bus/pci/devices/<BDF>/qat/cfg_services
20
20
Date: June 2022
···
23
23
Description: (RW) Reports the current configuration of the QAT device.
24
24
Write to the file to change the configured services.
25
25
26
-
The values are:
26
+
One or more services can be enabled per device.
27
+
Certain configurations are restricted to specific device types;
28
+
where applicable this is explicitly indicated, for example
29
+
(qat_6xxx) denotes applicability exclusively to that device series.
27
30
28
-
* sym;asym: the device is configured for running crypto
29
-
services
30
-
* asym;sym: identical to sym;asym
31
-
* dc: the device is configured for running compression services
32
-
* dcc: identical to dc but enables the dc chaining feature,
33
-
hash then compression. If this is not required chose dc
34
-
* sym: the device is configured for running symmetric crypto
35
-
services
36
-
* asym: the device is configured for running asymmetric crypto
37
-
services
38
-
* asym;dc: the device is configured for running asymmetric
39
-
crypto services and compression services
40
-
* dc;asym: identical to asym;dc
41
-
* sym;dc: the device is configured for running symmetric crypto
42
-
services and compression services
43
-
* dc;sym: identical to sym;dc
31
+
The available services include:
32
+
33
+
* sym: Configures the device for symmetric cryptographic operations.
34
+
* asym: Configures the device for asymmetric cryptographic operations.
35
+
* dc: Configures the device for compression and decompression
36
+
operations.
37
+
* dcc: Similar to dc, but with the additional dc chaining feature
38
+
enabled, cipher then compress (qat_6xxx), hash then compression.
39
+
If this is not required choose dc.
40
+
* decomp: Configures the device for decompression operations (qat_6xxx).
41
+
42
+
Service combinations are permitted for all services except dcc.
43
+
On QAT GEN4 devices (qat_4xxx driver) a maximum of two services can be
44
+
combined and on QAT GEN6 devices (qat_6xxx driver ) a maximum of three
45
+
services can be combined.
46
+
The order of services is not significant. For instance, sym;asym is
47
+
functionally equivalent to asym;sym.
44
48
45
49
It is possible to set the configuration only if the device
46
50
is in the `down` state (see /sys/bus/pci/devices/<BDF>/qat/state)
···
63
59
# cat /sys/bus/pci/devices/<BDF>/qat/cfg_services
64
60
dc
65
61
66
-
This attribute is only available for qat_4xxx devices.
62
+
This attribute is available for qat_4xxx and qat_6xxx devices.
67
63
68
64
What: /sys/bus/pci/devices/<BDF>/qat/pm_idle_enabled
69
65
Date: June 2023
···
98
94
# cat /sys/bus/pci/devices/<BDF>/qat/pm_idle_enabled
99
95
0
100
96
101
-
This attribute is only available for qat_4xxx devices.
97
+
This attribute is available for qat_4xxx and qat_6xxx devices.
102
98
103
99
What: /sys/bus/pci/devices/<BDF>/qat/rp2srv
104
100
Date: January 2024
···
130
126
# cat /sys/bus/pci/devices/<BDF>/qat/rp2srv
131
127
sym
132
128
133
-
This attribute is only available for qat_4xxx devices.
129
+
This attribute is available for qat_4xxx and qat_6xxx devices.
134
130
135
131
What: /sys/bus/pci/devices/<BDF>/qat/num_rps
136
132
Date: January 2024
···
144
140
# cat /sys/bus/pci/devices/<BDF>/qat/num_rps
145
141
64
146
142
147
-
This attribute is only available for qat_4xxx devices.
143
+
This attribute is available for qat_4xxx and qat_6xxx devices.
148
144
149
145
What: /sys/bus/pci/devices/<BDF>/qat/auto_reset
150
146
Date: May 2024
···
164
160
* 0/Nn/off: auto reset disabled. If the device encounters an
165
161
unrecoverable error, it will not be reset.
166
162
167
-
This attribute is only available for qat_4xxx devices.
163
+
This attribute is available for qat_4xxx and qat_6xxx devices.
+7
-7
Documentation/ABI/testing/sysfs-driver-qat_rl
+7
-7
Documentation/ABI/testing/sysfs-driver-qat_rl
···
31
31
* rm_all: Removes all the configured SLAs.
32
32
* Inputs: None
33
33
34
-
This attribute is only available for qat_4xxx devices.
34
+
This attribute is only available for qat_4xxx and qat_6xxx devices.
35
35
36
36
What: /sys/bus/pci/devices/<BDF>/qat_rl/rp
37
37
Date: January 2024
···
68
68
## Write
69
69
# echo 0x5 > /sys/bus/pci/devices/<BDF>/qat_rl/rp
70
70
71
-
This attribute is only available for qat_4xxx devices.
71
+
This attribute is only available for qat_4xxx and qat_6xxx devices.
72
72
73
73
What: /sys/bus/pci/devices/<BDF>/qat_rl/id
74
74
Date: January 2024
···
101
101
# cat /sys/bus/pci/devices/<BDF>/qat_rl/rp
102
102
0x5 ## ring pair ID 0 and ring pair ID 2
103
103
104
-
This attribute is only available for qat_4xxx devices.
104
+
This attribute is only available for qat_4xxx and qat_6xxx devices.
105
105
106
106
What: /sys/bus/pci/devices/<BDF>/qat_rl/cir
107
107
Date: January 2024
···
135
135
# cat /sys/bus/pci/devices/<BDF>/qat_rl/cir
136
136
500
137
137
138
-
This attribute is only available for qat_4xxx devices.
138
+
This attribute is only available for qat_4xxx and qat_6xxx devices.
139
139
140
140
What: /sys/bus/pci/devices/<BDF>/qat_rl/pir
141
141
Date: January 2024
···
169
169
# cat /sys/bus/pci/devices/<BDF>/qat_rl/pir
170
170
750
171
171
172
-
This attribute is only available for qat_4xxx devices.
172
+
This attribute is only available for qat_4xxx and qat_6xxx devices.
173
173
174
174
What: /sys/bus/pci/devices/<BDF>/qat_rl/srv
175
175
Date: January 2024
···
202
202
# cat /sys/bus/pci/devices/<BDF>/qat_rl/srv
203
203
dc
204
204
205
-
This attribute is only available for qat_4xxx devices.
205
+
This attribute is only available for qat_4xxx and qat_6xxx devices.
206
206
207
207
What: /sys/bus/pci/devices/<BDF>/qat_rl/cap_rem
208
208
Date: January 2024
···
223
223
# cat /sys/bus/pci/devices/<BDF>/qat_rl/cap_rem
224
224
0
225
225
226
-
This attribute is only available for qat_4xxx devices.
226
+
This attribute is only available for qat_4xxx and qat_6xxx devices.
-6
Documentation/crypto/crypto_engine.rst
-6
Documentation/crypto/crypto_engine.rst
···
36
36
Before transferring any request, you have to fill the context enginectx by
37
37
providing functions for the following:
38
38
39
-
* ``prepare_crypt_hardware``: Called once before any prepare functions are
40
-
called.
41
-
42
-
* ``unprepare_crypt_hardware``: Called once after all unprepare functions have
43
-
been called.
44
-
45
39
* ``prepare_cipher_request``/``prepare_hash_request``: Called before each
46
40
corresponding request is performed. If some processing or other preparatory
47
41
work is required, do it here.
+3
-1
Documentation/devicetree/bindings/crypto/atmel,at91sam9g46-aes.yaml
+3
-1
Documentation/devicetree/bindings/crypto/atmel,at91sam9g46-aes.yaml
+3
-1
Documentation/devicetree/bindings/crypto/atmel,at91sam9g46-sha.yaml
+3
-1
Documentation/devicetree/bindings/crypto/atmel,at91sam9g46-sha.yaml
+3
-1
Documentation/devicetree/bindings/crypto/atmel,at91sam9g46-tdes.yaml
+3
-1
Documentation/devicetree/bindings/crypto/atmel,at91sam9g46-tdes.yaml
+40
-1
Documentation/devicetree/bindings/crypto/fsl,sec-v4.0.yaml
+40
-1
Documentation/devicetree/bindings/crypto/fsl,sec-v4.0.yaml
···
46
46
- items:
47
47
- enum:
48
48
- fsl,imx6ul-caam
49
+
- fsl,imx8qm-caam
50
+
- fsl,imx8qxp-caam
49
51
- fsl,sec-v5.0
50
52
- const: fsl,sec-v4.0
51
53
- const: fsl,sec-v4.0
···
79
77
interrupts:
80
78
maxItems: 1
81
79
80
+
power-domains:
81
+
maxItems: 1
82
+
82
83
fsl,sec-era:
83
84
description: Defines the 'ERA' of the SEC device.
84
85
$ref: /schemas/types.yaml#/definitions/uint32
···
111
106
- const: fsl,sec-v5.0-job-ring
112
107
- const: fsl,sec-v4.0-job-ring
113
108
- items:
114
-
- const: fsl,sec-v5.0-job-ring
109
+
- enum:
110
+
- fsl,imx8qm-job-ring
111
+
- fsl,imx8qxp-job-ring
112
+
- fsl,sec-v5.0-job-ring
115
113
- const: fsl,sec-v4.0-job-ring
116
114
- const: fsl,sec-v4.0-job-ring
117
115
···
122
114
maxItems: 1
123
115
124
116
interrupts:
117
+
maxItems: 1
118
+
119
+
power-domains:
125
120
maxItems: 1
126
121
127
122
fsl,liodn:
···
136
125
$ref: /schemas/types.yaml#/definitions/uint32-array
137
126
items:
138
127
- maximum: 0xfff
128
+
allOf:
129
+
- if:
130
+
properties:
131
+
compatible:
132
+
contains:
133
+
enum:
134
+
- fsl,imx8qm-job-ring
135
+
- fsl,imx8qxp-job-ring
136
+
then:
137
+
required:
138
+
- power-domains
139
+
else:
140
+
properties:
141
+
power-domains: false
139
142
140
143
'^rtic@[0-9a-f]+$':
141
144
type: object
···
236
211
- compatible
237
212
- reg
238
213
- ranges
214
+
215
+
if:
216
+
properties:
217
+
compatible:
218
+
contains:
219
+
enum:
220
+
- fsl,imx8qm-caam
221
+
- fsl,imx8qxp-caam
222
+
then:
223
+
required:
224
+
- power-domains
225
+
else:
226
+
properties:
227
+
power-domains: false
239
228
240
229
additionalProperties: false
241
230
-31
Documentation/devicetree/bindings/crypto/omap-aes.txt
-31
Documentation/devicetree/bindings/crypto/omap-aes.txt
···
1
-
OMAP SoC AES crypto Module
2
-
3
-
Required properties:
4
-
5
-
- compatible : Should contain entries for this and backward compatible
6
-
AES versions:
7
-
- "ti,omap2-aes" for OMAP2.
8
-
- "ti,omap3-aes" for OMAP3.
9
-
- "ti,omap4-aes" for OMAP4 and AM33XX.
10
-
Note that the OMAP2 and 3 versions are compatible (OMAP3 supports
11
-
more algorithms) but they are incompatible with OMAP4.
12
-
- ti,hwmods: Name of the hwmod associated with the AES module
13
-
- reg : Offset and length of the register set for the module
14
-
- interrupts : the interrupt-specifier for the AES module.
15
-
16
-
Optional properties:
17
-
- dmas: DMA specifiers for tx and rx dma. See the DMA client binding,
18
-
Documentation/devicetree/bindings/dma/dma.txt
19
-
- dma-names: DMA request names should include "tx" and "rx" if present.
20
-
21
-
Example:
22
-
/* AM335x */
23
-
aes: aes@53500000 {
24
-
compatible = "ti,omap4-aes";
25
-
ti,hwmods = "aes";
26
-
reg = <0x53500000 0xa0>;
27
-
interrupts = <102>;
28
-
dmas = <&edma 6>,
29
-
<&edma 5>;
30
-
dma-names = "tx", "rx";
31
-
};
-30
Documentation/devicetree/bindings/crypto/omap-des.txt
-30
Documentation/devicetree/bindings/crypto/omap-des.txt
···
1
-
OMAP SoC DES crypto Module
2
-
3
-
Required properties:
4
-
5
-
- compatible : Should contain "ti,omap4-des"
6
-
- ti,hwmods: Name of the hwmod associated with the DES module
7
-
- reg : Offset and length of the register set for the module
8
-
- interrupts : the interrupt-specifier for the DES module
9
-
- clocks : A phandle to the functional clock node of the DES module
10
-
corresponding to each entry in clock-names
11
-
- clock-names : Name of the functional clock, should be "fck"
12
-
13
-
Optional properties:
14
-
- dmas: DMA specifiers for tx and rx dma. See the DMA client binding,
15
-
Documentation/devicetree/bindings/dma/dma.txt
16
-
Each entry corresponds to an entry in dma-names
17
-
- dma-names: DMA request names should include "tx" and "rx" if present
18
-
19
-
Example:
20
-
/* DRA7xx SoC */
21
-
des: des@480a5000 {
22
-
compatible = "ti,omap4-des";
23
-
ti,hwmods = "des";
24
-
reg = <0x480a5000 0xa0>;
25
-
interrupts = <GIC_SPI 82 IRQ_TYPE_LEVEL_HIGH>;
26
-
dmas = <&sdma 117>, <&sdma 116>;
27
-
dma-names = "tx", "rx";
28
-
clocks = <&l3_iclk_div>;
29
-
clock-names = "fck";
30
-
};
+58
Documentation/devicetree/bindings/crypto/ti,omap2-aes.yaml
+58
Documentation/devicetree/bindings/crypto/ti,omap2-aes.yaml
···
1
+
# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2
+
%YAML 1.2
3
+
---
4
+
$id: http://devicetree.org/schemas/ti,omap2-aes.yaml#
5
+
$schema: http://devicetree.org/meta-schemas/core.yaml#
6
+
7
+
title: OMAP SoC AES crypto Module
8
+
9
+
maintainers:
10
+
- Aaro Koskinen <aaro.koskinen@iki.fi>
11
+
- Andreas Kemnade <andreas@kemnade.info>
12
+
- Kevin Hilman <khilman@baylibre.com>
13
+
- Roger Quadros <rogerq@kernel.org>
14
+
- Tony Lindgren <tony@atomide.com>
15
+
16
+
properties:
17
+
compatible:
18
+
enum:
19
+
- ti,omap2-aes
20
+
- ti,omap3-aes
21
+
- ti,omap4-aes
22
+
23
+
reg:
24
+
maxItems: 1
25
+
26
+
interrupts:
27
+
maxItems: 1
28
+
29
+
dmas:
30
+
maxItems: 2
31
+
32
+
dma-names:
33
+
items:
34
+
- const: tx
35
+
- const: rx
36
+
37
+
ti,hwmods:
38
+
description: Name of the hwmod associated with the AES module
39
+
const: aes
40
+
deprecated: true
41
+
42
+
required:
43
+
- compatible
44
+
- reg
45
+
- interrupts
46
+
47
+
additionalProperties: false
48
+
49
+
examples:
50
+
- |
51
+
aes@53500000 {
52
+
compatible = "ti,omap4-aes";
53
+
reg = <0x53500000 0xa0>;
54
+
interrupts = <102>;
55
+
dmas = <&edma 6>,
56
+
<&edma 5>;
57
+
dma-names = "tx", "rx";
58
+
};
+65
Documentation/devicetree/bindings/crypto/ti,omap4-des.yaml
+65
Documentation/devicetree/bindings/crypto/ti,omap4-des.yaml
···
1
+
# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2
+
%YAML 1.2
3
+
---
4
+
$id: http://devicetree.org/schemas/ti,omap4-des.yaml#
5
+
$schema: http://devicetree.org/meta-schemas/core.yaml#
6
+
7
+
title: OMAP4 DES crypto Module
8
+
9
+
maintainers:
10
+
- Aaro Koskinen <aaro.koskinen@iki.fi>
11
+
- Andreas Kemnade <andreas@kemnade.info>
12
+
- Kevin Hilman <khilman@baylibre.com>
13
+
- Roger Quadros <rogerq@kernel.org>
14
+
- Tony Lindgren <tony@atomide.com>
15
+
16
+
properties:
17
+
compatible:
18
+
const: ti,omap4-des
19
+
20
+
reg:
21
+
maxItems: 1
22
+
23
+
interrupts:
24
+
maxItems: 1
25
+
26
+
dmas:
27
+
maxItems: 2
28
+
29
+
dma-names:
30
+
items:
31
+
- const: tx
32
+
- const: rx
33
+
34
+
clocks:
35
+
maxItems: 1
36
+
37
+
clock-names:
38
+
items:
39
+
- const: fck
40
+
41
+
dependencies:
42
+
dmas: [ dma-names ]
43
+
44
+
required:
45
+
- compatible
46
+
- reg
47
+
- interrupts
48
+
- clocks
49
+
- clock-names
50
+
51
+
additionalProperties: false
52
+
53
+
examples:
54
+
- |
55
+
#include <dt-bindings/interrupt-controller/arm-gic.h>
56
+
57
+
des@480a5000 {
58
+
compatible = "ti,omap4-des";
59
+
reg = <0x480a5000 0xa0>;
60
+
interrupts = <GIC_SPI 82 IRQ_TYPE_LEVEL_HIGH>;
61
+
clocks = <&l3_iclk_div>;
62
+
clock-names = "fck";
63
+
dmas = <&sdma 117>, <&sdma 116>;
64
+
dma-names = "tx", "rx";
65
+
};
+1
Documentation/devicetree/bindings/rng/atmel,at91-trng.yaml
+1
Documentation/devicetree/bindings/rng/atmel,at91-trng.yaml
+1
-1
arch/arm/crypto/aes-neonbs-glue.c
+1
-1
arch/arm/crypto/aes-neonbs-glue.c
+1
arch/s390/configs/debug_defconfig
+1
arch/s390/configs/debug_defconfig
+1
arch/s390/configs/defconfig
+1
arch/s390/configs/defconfig
+1
arch/s390/crypto/Makefile
+1
arch/s390/crypto/Makefile
+8
-4
arch/s390/crypto/hmac_s390.c
+8
-4
arch/s390/crypto/hmac_s390.c
···
290
290
struct s390_kmac_sha2_ctx *ctx = shash_desc_ctx(desc);
291
291
unsigned int bs = crypto_shash_blocksize(desc->tfm);
292
292
unsigned int ds = bs / 2;
293
+
u64 lo = ctx->buflen[0];
293
294
union {
294
295
u8 *u8;
295
296
u64 *u64;
···
302
301
else
303
302
memcpy(p.u8, ctx->param, ds);
304
303
p.u8 += ds;
305
-
put_unaligned(ctx->buflen[0], p.u64++);
304
+
lo += bs;
305
+
put_unaligned(lo, p.u64++);
306
306
if (ds == SHA512_DIGEST_SIZE)
307
-
put_unaligned(ctx->buflen[1], p.u64);
307
+
put_unaligned(ctx->buflen[1] + (lo < bs), p.u64);
308
308
return err;
309
309
}
310
310
···
318
316
const u8 *u8;
319
317
const u64 *u64;
320
318
} p = { .u8 = in };
319
+
u64 lo;
321
320
int err;
322
321
323
322
err = s390_hmac_sha2_init(desc);
324
323
memcpy(ctx->param, p.u8, ds);
325
324
p.u8 += ds;
326
-
ctx->buflen[0] = get_unaligned(p.u64++);
325
+
lo = get_unaligned(p.u64++);
326
+
ctx->buflen[0] = lo - bs;
327
327
if (ds == SHA512_DIGEST_SIZE)
328
-
ctx->buflen[1] = get_unaligned(p.u64);
328
+
ctx->buflen[1] = get_unaligned(p.u64) - (lo < bs);
329
329
if (ctx->buflen[0] | ctx->buflen[1])
330
330
ctx->gr0.ikp = 1;
331
331
return err;
+1
-1
arch/s390/crypto/paes_s390.c
+1
-1
arch/s390/crypto/paes_s390.c
···
1633
1633
/* with this pseudo devie alloc and start a crypto engine */
1634
1634
paes_crypto_engine =
1635
1635
crypto_engine_alloc_init_and_set(paes_dev.this_device,
1636
-
true, NULL, false, MAX_QLEN);
1636
+
true, false, MAX_QLEN);
1637
1637
if (!paes_crypto_engine) {
1638
1638
rc = -ENOMEM;
1639
1639
goto out_err;
+1048
arch/s390/crypto/phmac_s390.c
+1048
arch/s390/crypto/phmac_s390.c
···
1
+
// SPDX-License-Identifier: GPL-2.0+
2
+
/*
3
+
* Copyright IBM Corp. 2025
4
+
*
5
+
* s390 specific HMAC support for protected keys.
6
+
*/
7
+
8
+
#define KMSG_COMPONENT "phmac_s390"
9
+
#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
10
+
11
+
#include <asm/cpacf.h>
12
+
#include <asm/pkey.h>
13
+
#include <crypto/engine.h>
14
+
#include <crypto/hash.h>
15
+
#include <crypto/internal/hash.h>
16
+
#include <crypto/sha2.h>
17
+
#include <linux/atomic.h>
18
+
#include <linux/cpufeature.h>
19
+
#include <linux/delay.h>
20
+
#include <linux/miscdevice.h>
21
+
#include <linux/module.h>
22
+
#include <linux/spinlock.h>
23
+
24
+
static struct crypto_engine *phmac_crypto_engine;
25
+
#define MAX_QLEN 10
26
+
27
+
/*
28
+
* A simple hash walk helper
29
+
*/
30
+
31
+
struct hash_walk_helper {
32
+
struct crypto_hash_walk walk;
33
+
const u8 *walkaddr;
34
+
int walkbytes;
35
+
};
36
+
37
+
/*
38
+
* Prepare hash walk helper.
39
+
* Set up the base hash walk, fill walkaddr and walkbytes.
40
+
* Returns 0 on success or negative value on error.
41
+
*/
42
+
static inline int hwh_prepare(struct ahash_request *req,
43
+
struct hash_walk_helper *hwh)
44
+
{
45
+
hwh->walkbytes = crypto_hash_walk_first(req, &hwh->walk);
46
+
if (hwh->walkbytes < 0)
47
+
return hwh->walkbytes;
48
+
hwh->walkaddr = hwh->walk.data;
49
+
return 0;
50
+
}
51
+
52
+
/*
53
+
* Advance hash walk helper by n bytes.
54
+
* Progress the walkbytes and walkaddr fields by n bytes.
55
+
* If walkbytes is then 0, pull next hunk from hash walk
56
+
* and update walkbytes and walkaddr.
57
+
* If n is negative, unmap hash walk and return error.
58
+
* Returns 0 on success or negative value on error.
59
+
*/
60
+
static inline int hwh_advance(struct hash_walk_helper *hwh, int n)
61
+
{
62
+
if (n < 0)
63
+
return crypto_hash_walk_done(&hwh->walk, n);
64
+
65
+
hwh->walkbytes -= n;
66
+
hwh->walkaddr += n;
67
+
if (hwh->walkbytes > 0)
68
+
return 0;
69
+
70
+
hwh->walkbytes = crypto_hash_walk_done(&hwh->walk, 0);
71
+
if (hwh->walkbytes < 0)
72
+
return hwh->walkbytes;
73
+
74
+
hwh->walkaddr = hwh->walk.data;
75
+
return 0;
76
+
}
77
+
78
+
/*
79
+
* KMAC param block layout for sha2 function codes:
80
+
* The layout of the param block for the KMAC instruction depends on the
81
+
* blocksize of the used hashing sha2-algorithm function codes. The param block
82
+
* contains the hash chaining value (cv), the input message bit-length (imbl)
83
+
* and the hmac-secret (key). To prevent code duplication, the sizes of all
84
+
* these are calculated based on the blocksize.
85
+
*
86
+
* param-block:
87
+
* +-------+
88
+
* | cv |
89
+
* +-------+
90
+
* | imbl |
91
+
* +-------+
92
+
* | key |
93
+
* +-------+
94
+
*
95
+
* sizes:
96
+
* part | sh2-alg | calculation | size | type
97
+
* -----+---------+-------------+------+--------
98
+
* cv | 224/256 | blocksize/2 | 32 | u64[8]
99
+
* | 384/512 | | 64 | u128[8]
100
+
* imbl | 224/256 | blocksize/8 | 8 | u64
101
+
* | 384/512 | | 16 | u128
102
+
* key | 224/256 | blocksize | 96 | u8[96]
103
+
* | 384/512 | | 160 | u8[160]
104
+
*/
105
+
106
+
#define MAX_DIGEST_SIZE SHA512_DIGEST_SIZE
107
+
#define MAX_IMBL_SIZE sizeof(u128)
108
+
#define MAX_BLOCK_SIZE SHA512_BLOCK_SIZE
109
+
110
+
#define SHA2_CV_SIZE(bs) ((bs) >> 1)
111
+
#define SHA2_IMBL_SIZE(bs) ((bs) >> 3)
112
+
113
+
#define SHA2_IMBL_OFFSET(bs) (SHA2_CV_SIZE(bs))
114
+
#define SHA2_KEY_OFFSET(bs) (SHA2_CV_SIZE(bs) + SHA2_IMBL_SIZE(bs))
115
+
116
+
#define PHMAC_MAX_KEYSIZE 256
117
+
#define PHMAC_SHA256_PK_SIZE (SHA256_BLOCK_SIZE + 32)
118
+
#define PHMAC_SHA512_PK_SIZE (SHA512_BLOCK_SIZE + 32)
119
+
#define PHMAC_MAX_PK_SIZE PHMAC_SHA512_PK_SIZE
120
+
121
+
/* phmac protected key struct */
122
+
struct phmac_protkey {
123
+
u32 type;
124
+
u32 len;
125
+
u8 protkey[PHMAC_MAX_PK_SIZE];
126
+
};
127
+
128
+
#define PK_STATE_NO_KEY 0
129
+
#define PK_STATE_CONVERT_IN_PROGRESS 1
130
+
#define PK_STATE_VALID 2
131
+
132
+
/* phmac tfm context */
133
+
struct phmac_tfm_ctx {
134
+
/* source key material used to derive a protected key from */
135
+
u8 keybuf[PHMAC_MAX_KEYSIZE];
136
+
unsigned int keylen;
137
+
138
+
/* cpacf function code to use with this protected key type */
139
+
long fc;
140
+
141
+
/* nr of requests enqueued via crypto engine which use this tfm ctx */
142
+
atomic_t via_engine_ctr;
143
+
144
+
/* spinlock to atomic read/update all the following fields */
145
+
spinlock_t pk_lock;
146
+
147
+
/* see PK_STATE* defines above, < 0 holds convert failure rc */
148
+
int pk_state;
149
+
/* if state is valid, pk holds the protected key */
150
+
struct phmac_protkey pk;
151
+
};
152
+
153
+
union kmac_gr0 {
154
+
unsigned long reg;
155
+
struct {
156
+
unsigned long : 48;
157
+
unsigned long ikp : 1;
158
+
unsigned long iimp : 1;
159
+
unsigned long ccup : 1;
160
+
unsigned long : 6;
161
+
unsigned long fc : 7;
162
+
};
163
+
};
164
+
165
+
struct kmac_sha2_ctx {
166
+
u8 param[MAX_DIGEST_SIZE + MAX_IMBL_SIZE + PHMAC_MAX_PK_SIZE];
167
+
union kmac_gr0 gr0;
168
+
u8 buf[MAX_BLOCK_SIZE];
169
+
u64 buflen[2];
170
+
};
171
+
172
+
/* phmac request context */
173
+
struct phmac_req_ctx {
174
+
struct hash_walk_helper hwh;
175
+
struct kmac_sha2_ctx kmac_ctx;
176
+
bool final;
177
+
};
178
+
179
+
/*
180
+
* Pkey 'token' struct used to derive a protected key value from a clear key.
181
+
*/
182
+
struct hmac_clrkey_token {
183
+
u8 type;
184
+
u8 res0[3];
185
+
u8 version;
186
+
u8 res1[3];
187
+
u32 keytype;
188
+
u32 len;
189
+
u8 key[];
190
+
} __packed;
191
+
192
+
static int hash_key(const u8 *in, unsigned int inlen,
193
+
u8 *digest, unsigned int digestsize)
194
+
{
195
+
unsigned long func;
196
+
union {
197
+
struct sha256_paramblock {
198
+
u32 h[8];
199
+
u64 mbl;
200
+
} sha256;
201
+
struct sha512_paramblock {
202
+
u64 h[8];
203
+
u128 mbl;
204
+
} sha512;
205
+
} __packed param;
206
+
207
+
#define PARAM_INIT(x, y, z) \
208
+
param.sha##x.h[0] = SHA##y ## _H0; \
209
+
param.sha##x.h[1] = SHA##y ## _H1; \
210
+
param.sha##x.h[2] = SHA##y ## _H2; \
211
+
param.sha##x.h[3] = SHA##y ## _H3; \
212
+
param.sha##x.h[4] = SHA##y ## _H4; \
213
+
param.sha##x.h[5] = SHA##y ## _H5; \
214
+
param.sha##x.h[6] = SHA##y ## _H6; \
215
+
param.sha##x.h[7] = SHA##y ## _H7; \
216
+
param.sha##x.mbl = (z)
217
+
218
+
switch (digestsize) {
219
+
case SHA224_DIGEST_SIZE:
220
+
func = CPACF_KLMD_SHA_256;
221
+
PARAM_INIT(256, 224, inlen * 8);
222
+
break;
223
+
case SHA256_DIGEST_SIZE:
224
+
func = CPACF_KLMD_SHA_256;
225
+
PARAM_INIT(256, 256, inlen * 8);
226
+
break;
227
+
case SHA384_DIGEST_SIZE:
228
+
func = CPACF_KLMD_SHA_512;
229
+
PARAM_INIT(512, 384, inlen * 8);
230
+
break;
231
+
case SHA512_DIGEST_SIZE:
232
+
func = CPACF_KLMD_SHA_512;
233
+
PARAM_INIT(512, 512, inlen * 8);
234
+
break;
235
+
default:
236
+
return -EINVAL;
237
+
}
238
+
239
+
#undef PARAM_INIT
240
+
241
+
cpacf_klmd(func, ¶m, in, inlen);
242
+
243
+
memcpy(digest, ¶m, digestsize);
244
+
245
+
return 0;
246
+
}
247
+
248
+
/*
249
+
* make_clrkey_token() - wrap the clear key into a pkey clearkey token.
250
+
*/
251
+
static inline int make_clrkey_token(const u8 *clrkey, size_t clrkeylen,
252
+
unsigned int digestsize, u8 *dest)
253
+
{
254
+
struct hmac_clrkey_token *token = (struct hmac_clrkey_token *)dest;
255
+
unsigned int blocksize;
256
+
int rc;
257
+
258
+
token->type = 0x00;
259
+
token->version = 0x02;
260
+
switch (digestsize) {
261
+
case SHA224_DIGEST_SIZE:
262
+
case SHA256_DIGEST_SIZE:
263
+
token->keytype = PKEY_KEYTYPE_HMAC_512;
264
+
blocksize = 64;
265
+
break;
266
+
case SHA384_DIGEST_SIZE:
267
+
case SHA512_DIGEST_SIZE:
268
+
token->keytype = PKEY_KEYTYPE_HMAC_1024;
269
+
blocksize = 128;
270
+
break;
271
+
default:
272
+
return -EINVAL;
273
+
}
274
+
token->len = blocksize;
275
+
276
+
if (clrkeylen > blocksize) {
277
+
rc = hash_key(clrkey, clrkeylen, token->key, digestsize);
278
+
if (rc)
279
+
return rc;
280
+
} else {
281
+
memcpy(token->key, clrkey, clrkeylen);
282
+
}
283
+
284
+
return 0;
285
+
}
286
+
287
+
/*
288
+
* phmac_tfm_ctx_setkey() - Set key value into tfm context, maybe construct
289
+
* a clear key token digestible by pkey from a clear key value.
290
+
*/
291
+
static inline int phmac_tfm_ctx_setkey(struct phmac_tfm_ctx *tfm_ctx,
292
+
const u8 *key, unsigned int keylen)
293
+
{
294
+
if (keylen > sizeof(tfm_ctx->keybuf))
295
+
return -EINVAL;
296
+
297
+
memcpy(tfm_ctx->keybuf, key, keylen);
298
+
tfm_ctx->keylen = keylen;
299
+
300
+
return 0;
301
+
}
302
+
303
+
/*
304
+
* Convert the raw key material into a protected key via PKEY api.
305
+
* This function may sleep - don't call in non-sleeping context.
306
+
*/
307
+
static inline int convert_key(const u8 *key, unsigned int keylen,
308
+
struct phmac_protkey *pk)
309
+
{
310
+
int rc, i;
311
+
312
+
pk->len = sizeof(pk->protkey);
313
+
314
+
/*
315
+
* In case of a busy card retry with increasing delay
316
+
* of 200, 400, 800 and 1600 ms - in total 3 s.
317
+
*/
318
+
for (rc = -EIO, i = 0; rc && i < 5; i++) {
319
+
if (rc == -EBUSY && msleep_interruptible((1 << i) * 100)) {
320
+
rc = -EINTR;
321
+
goto out;
322
+
}
323
+
rc = pkey_key2protkey(key, keylen,
324
+
pk->protkey, &pk->len, &pk->type,
325
+
PKEY_XFLAG_NOMEMALLOC);
326
+
}
327
+
328
+
out:
329
+
pr_debug("rc=%d\n", rc);
330
+
return rc;
331
+
}
332
+
333
+
/*
334
+
* (Re-)Convert the raw key material from the tfm ctx into a protected
335
+
* key via convert_key() function. Update the pk_state, pk_type, pk_len
336
+
* and the protected key in the tfm context.
337
+
* Please note this function may be invoked concurrently with the very
338
+
* same tfm context. The pk_lock spinlock in the context ensures an
339
+
* atomic update of the pk and the pk state but does not guarantee any
340
+
* order of update. So a fresh converted valid protected key may get
341
+
* updated with an 'old' expired key value. As the cpacf instructions
342
+
* detect this, refuse to operate with an invalid key and the calling
343
+
* code triggers a (re-)conversion this does no harm. This may lead to
344
+
* unnecessary additional conversion but never to invalid data on the
345
+
* hash operation.
346
+
*/
347
+
static int phmac_convert_key(struct phmac_tfm_ctx *tfm_ctx)
348
+
{
349
+
struct phmac_protkey pk;
350
+
int rc;
351
+
352
+
spin_lock_bh(&tfm_ctx->pk_lock);
353
+
tfm_ctx->pk_state = PK_STATE_CONVERT_IN_PROGRESS;
354
+
spin_unlock_bh(&tfm_ctx->pk_lock);
355
+
356
+
rc = convert_key(tfm_ctx->keybuf, tfm_ctx->keylen, &pk);
357
+
358
+
/* update context */
359
+
spin_lock_bh(&tfm_ctx->pk_lock);
360
+
if (rc) {
361
+
tfm_ctx->pk_state = rc;
362
+
} else {
363
+
tfm_ctx->pk_state = PK_STATE_VALID;
364
+
tfm_ctx->pk = pk;
365
+
}
366
+
spin_unlock_bh(&tfm_ctx->pk_lock);
367
+
368
+
memzero_explicit(&pk, sizeof(pk));
369
+
pr_debug("rc=%d\n", rc);
370
+
return rc;
371
+
}
372
+
373
+
/*
374
+
* kmac_sha2_set_imbl - sets the input message bit-length based on the blocksize
375
+
*/
376
+
static inline void kmac_sha2_set_imbl(u8 *param, u64 buflen_lo,
377
+
u64 buflen_hi, unsigned int blocksize)
378
+
{
379
+
u8 *imbl = param + SHA2_IMBL_OFFSET(blocksize);
380
+
381
+
switch (blocksize) {
382
+
case SHA256_BLOCK_SIZE:
383
+
*(u64 *)imbl = buflen_lo * BITS_PER_BYTE;
384
+
break;
385
+
case SHA512_BLOCK_SIZE:
386
+
*(u128 *)imbl = (((u128)buflen_hi << 64) + buflen_lo) << 3;
387
+
break;
388
+
default:
389
+
break;
390
+
}
391
+
}
392
+
393
+
static int phmac_kmac_update(struct ahash_request *req, bool maysleep)
394
+
{
395
+
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
396
+
struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm);
397
+
struct phmac_req_ctx *req_ctx = ahash_request_ctx(req);
398
+
struct kmac_sha2_ctx *ctx = &req_ctx->kmac_ctx;
399
+
struct hash_walk_helper *hwh = &req_ctx->hwh;
400
+
unsigned int bs = crypto_ahash_blocksize(tfm);
401
+
unsigned int offset, k, n;
402
+
int rc = 0;
403
+
404
+
/*
405
+
* The walk is always mapped when this function is called.
406
+
* Note that in case of partial processing or failure the walk
407
+
* is NOT unmapped here. So a follow up task may reuse the walk
408
+
* or in case of unrecoverable failure needs to unmap it.
409
+
*/
410
+
411
+
while (hwh->walkbytes > 0) {
412
+
/* check sha2 context buffer */
413
+
offset = ctx->buflen[0] % bs;
414
+
if (offset + hwh->walkbytes < bs)
415
+
goto store;
416
+
417
+
if (offset) {
418
+
/* fill ctx buffer up to blocksize and process this block */
419
+
n = bs - offset;
420
+
memcpy(ctx->buf + offset, hwh->walkaddr, n);
421
+
ctx->gr0.iimp = 1;
422
+
for (;;) {
423
+
k = _cpacf_kmac(&ctx->gr0.reg, ctx->param, ctx->buf, bs);
424
+
if (likely(k == bs))
425
+
break;
426
+
if (unlikely(k > 0)) {
427
+
/*
428
+
* Can't deal with hunks smaller than blocksize.
429
+
* And kmac should always return the nr of
430
+
* processed bytes as 0 or a multiple of the
431
+
* blocksize.
432
+
*/
433
+
rc = -EIO;
434
+
goto out;
435
+
}
436
+
/* protected key is invalid and needs re-conversion */
437
+
if (!maysleep) {
438
+
rc = -EKEYEXPIRED;
439
+
goto out;
440
+
}
441
+
rc = phmac_convert_key(tfm_ctx);
442
+
if (rc)
443
+
goto out;
444
+
spin_lock_bh(&tfm_ctx->pk_lock);
445
+
memcpy(ctx->param + SHA2_KEY_OFFSET(bs),
446
+
tfm_ctx->pk.protkey, tfm_ctx->pk.len);
447
+
spin_unlock_bh(&tfm_ctx->pk_lock);
448
+
}
449
+
ctx->buflen[0] += n;
450
+
if (ctx->buflen[0] < n)
451
+
ctx->buflen[1]++;
452
+
rc = hwh_advance(hwh, n);
453
+
if (unlikely(rc))
454
+
goto out;
455
+
offset = 0;
456
+
}
457
+
458
+
/* process as many blocks as possible from the walk */
459
+
while (hwh->walkbytes >= bs) {
460
+
n = (hwh->walkbytes / bs) * bs;
461
+
ctx->gr0.iimp = 1;
462
+
k = _cpacf_kmac(&ctx->gr0.reg, ctx->param, hwh->walkaddr, n);
463
+
if (likely(k > 0)) {
464
+
ctx->buflen[0] += k;
465
+
if (ctx->buflen[0] < k)
466
+
ctx->buflen[1]++;
467
+
rc = hwh_advance(hwh, k);
468
+
if (unlikely(rc))
469
+
goto out;
470
+
}
471
+
if (unlikely(k < n)) {
472
+
/* protected key is invalid and needs re-conversion */
473
+
if (!maysleep) {
474
+
rc = -EKEYEXPIRED;
475
+
goto out;
476
+
}
477
+
rc = phmac_convert_key(tfm_ctx);
478
+
if (rc)
479
+
goto out;
480
+
spin_lock_bh(&tfm_ctx->pk_lock);
481
+
memcpy(ctx->param + SHA2_KEY_OFFSET(bs),
482
+
tfm_ctx->pk.protkey, tfm_ctx->pk.len);
483
+
spin_unlock_bh(&tfm_ctx->pk_lock);
484
+
}
485
+
}
486
+
487
+
store:
488
+
/* store incomplete block in context buffer */
489
+
if (hwh->walkbytes) {
490
+
memcpy(ctx->buf + offset, hwh->walkaddr, hwh->walkbytes);
491
+
ctx->buflen[0] += hwh->walkbytes;
492
+
if (ctx->buflen[0] < hwh->walkbytes)
493
+
ctx->buflen[1]++;
494
+
rc = hwh_advance(hwh, hwh->walkbytes);
495
+
if (unlikely(rc))
496
+
goto out;
497
+
}
498
+
499
+
} /* end of while (hwh->walkbytes > 0) */
500
+
501
+
out:
502
+
pr_debug("rc=%d\n", rc);
503
+
return rc;
504
+
}
505
+
506
+
static int phmac_kmac_final(struct ahash_request *req, bool maysleep)
507
+
{
508
+
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
509
+
struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm);
510
+
struct phmac_req_ctx *req_ctx = ahash_request_ctx(req);
511
+
struct kmac_sha2_ctx *ctx = &req_ctx->kmac_ctx;
512
+
unsigned int ds = crypto_ahash_digestsize(tfm);
513
+
unsigned int bs = crypto_ahash_blocksize(tfm);
514
+
unsigned int k, n;
515
+
int rc = 0;
516
+
517
+
n = ctx->buflen[0] % bs;
518
+
ctx->gr0.iimp = 0;
519
+
kmac_sha2_set_imbl(ctx->param, ctx->buflen[0], ctx->buflen[1], bs);
520
+
for (;;) {
521
+
k = _cpacf_kmac(&ctx->gr0.reg, ctx->param, ctx->buf, n);
522
+
if (likely(k == n))
523
+
break;
524
+
if (unlikely(k > 0)) {
525
+
/* Can't deal with hunks smaller than blocksize. */
526
+
rc = -EIO;
527
+
goto out;
528
+
}
529
+
/* protected key is invalid and needs re-conversion */
530
+
if (!maysleep) {
531
+
rc = -EKEYEXPIRED;
532
+
goto out;
533
+
}
534
+
rc = phmac_convert_key(tfm_ctx);
535
+
if (rc)
536
+
goto out;
537
+
spin_lock_bh(&tfm_ctx->pk_lock);
538
+
memcpy(ctx->param + SHA2_KEY_OFFSET(bs),
539
+
tfm_ctx->pk.protkey, tfm_ctx->pk.len);
540
+
spin_unlock_bh(&tfm_ctx->pk_lock);
541
+
}
542
+
543
+
memcpy(req->result, ctx->param, ds);
544
+
545
+
out:
546
+
pr_debug("rc=%d\n", rc);
547
+
return rc;
548
+
}
549
+
550
+
static int phmac_init(struct ahash_request *req)
551
+
{
552
+
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
553
+
struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm);
554
+
struct phmac_req_ctx *req_ctx = ahash_request_ctx(req);
555
+
struct kmac_sha2_ctx *kmac_ctx = &req_ctx->kmac_ctx;
556
+
unsigned int bs = crypto_ahash_blocksize(tfm);
557
+
int rc = 0;
558
+
559
+
/* zero request context (includes the kmac sha2 context) */
560
+
memset(req_ctx, 0, sizeof(*req_ctx));
561
+
562
+
/*
563
+
* setkey() should have set a valid fc into the tfm context.
564
+
* Copy this function code into the gr0 field of the kmac context.
565
+
*/
566
+
if (!tfm_ctx->fc) {
567
+
rc = -ENOKEY;
568
+
goto out;
569
+
}
570
+
kmac_ctx->gr0.fc = tfm_ctx->fc;
571
+
572
+
/*
573
+
* Copy the pk from tfm ctx into kmac ctx. The protected key
574
+
* may be outdated but update() and final() will handle this.
575
+
*/
576
+
spin_lock_bh(&tfm_ctx->pk_lock);
577
+
memcpy(kmac_ctx->param + SHA2_KEY_OFFSET(bs),
578
+
tfm_ctx->pk.protkey, tfm_ctx->pk.len);
579
+
spin_unlock_bh(&tfm_ctx->pk_lock);
580
+
581
+
out:
582
+
pr_debug("rc=%d\n", rc);
583
+
return rc;
584
+
}
585
+
586
+
static int phmac_update(struct ahash_request *req)
587
+
{
588
+
struct phmac_req_ctx *req_ctx = ahash_request_ctx(req);
589
+
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
590
+
struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm);
591
+
struct kmac_sha2_ctx *kmac_ctx = &req_ctx->kmac_ctx;
592
+
struct hash_walk_helper *hwh = &req_ctx->hwh;
593
+
int rc;
594
+
595
+
/* prep the walk in the request context */
596
+
rc = hwh_prepare(req, hwh);
597
+
if (rc)
598
+
goto out;
599
+
600
+
/* Try synchronous operation if no active engine usage */
601
+
if (!atomic_read(&tfm_ctx->via_engine_ctr)) {
602
+
rc = phmac_kmac_update(req, false);
603
+
if (rc == 0)
604
+
goto out;
605
+
}
606
+
607
+
/*
608
+
* If sync operation failed or key expired or there are already
609
+
* requests enqueued via engine, fallback to async. Mark tfm as
610
+
* using engine to serialize requests.
611
+
*/
612
+
if (rc == 0 || rc == -EKEYEXPIRED) {
613
+
atomic_inc(&tfm_ctx->via_engine_ctr);
614
+
rc = crypto_transfer_hash_request_to_engine(phmac_crypto_engine, req);
615
+
if (rc != -EINPROGRESS)
616
+
atomic_dec(&tfm_ctx->via_engine_ctr);
617
+
}
618
+
619
+
if (rc != -EINPROGRESS) {
620
+
hwh_advance(hwh, rc);
621
+
memzero_explicit(kmac_ctx, sizeof(*kmac_ctx));
622
+
}
623
+
624
+
out:
625
+
pr_debug("rc=%d\n", rc);
626
+
return rc;
627
+
}
628
+
629
+
static int phmac_final(struct ahash_request *req)
630
+
{
631
+
struct phmac_req_ctx *req_ctx = ahash_request_ctx(req);
632
+
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
633
+
struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm);
634
+
struct kmac_sha2_ctx *kmac_ctx = &req_ctx->kmac_ctx;
635
+
int rc = 0;
636
+
637
+
/* Try synchronous operation if no active engine usage */
638
+
if (!atomic_read(&tfm_ctx->via_engine_ctr)) {
639
+
rc = phmac_kmac_final(req, false);
640
+
if (rc == 0)
641
+
goto out;
642
+
}
643
+
644
+
/*
645
+
* If sync operation failed or key expired or there are already
646
+
* requests enqueued via engine, fallback to async. Mark tfm as
647
+
* using engine to serialize requests.
648
+
*/
649
+
if (rc == 0 || rc == -EKEYEXPIRED) {
650
+
req->nbytes = 0;
651
+
req_ctx->final = true;
652
+
atomic_inc(&tfm_ctx->via_engine_ctr);
653
+
rc = crypto_transfer_hash_request_to_engine(phmac_crypto_engine, req);
654
+
if (rc != -EINPROGRESS)
655
+
atomic_dec(&tfm_ctx->via_engine_ctr);
656
+
}
657
+
658
+
out:
659
+
if (rc != -EINPROGRESS)
660
+
memzero_explicit(kmac_ctx, sizeof(*kmac_ctx));
661
+
pr_debug("rc=%d\n", rc);
662
+
return rc;
663
+
}
664
+
665
+
static int phmac_finup(struct ahash_request *req)
666
+
{
667
+
struct phmac_req_ctx *req_ctx = ahash_request_ctx(req);
668
+
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
669
+
struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm);
670
+
struct kmac_sha2_ctx *kmac_ctx = &req_ctx->kmac_ctx;
671
+
struct hash_walk_helper *hwh = &req_ctx->hwh;
672
+
int rc;
673
+
674
+
/* prep the walk in the request context */
675
+
rc = hwh_prepare(req, hwh);
676
+
if (rc)
677
+
goto out;
678
+
679
+
/* Try synchronous operations if no active engine usage */
680
+
if (!atomic_read(&tfm_ctx->via_engine_ctr)) {
681
+
rc = phmac_kmac_update(req, false);
682
+
if (rc == 0)
683
+
req->nbytes = 0;
684
+
}
685
+
if (!rc && !req->nbytes && !atomic_read(&tfm_ctx->via_engine_ctr)) {
686
+
rc = phmac_kmac_final(req, false);
687
+
if (rc == 0)
688
+
goto out;
689
+
}
690
+
691
+
/*
692
+
* If sync operation failed or key expired or there are already
693
+
* requests enqueued via engine, fallback to async. Mark tfm as
694
+
* using engine to serialize requests.
695
+
*/
696
+
if (rc == 0 || rc == -EKEYEXPIRED) {
697
+
req_ctx->final = true;
698
+
atomic_inc(&tfm_ctx->via_engine_ctr);
699
+
rc = crypto_transfer_hash_request_to_engine(phmac_crypto_engine, req);
700
+
if (rc != -EINPROGRESS)
701
+
atomic_dec(&tfm_ctx->via_engine_ctr);
702
+
}
703
+
704
+
if (rc != -EINPROGRESS)
705
+
hwh_advance(hwh, rc);
706
+
707
+
out:
708
+
if (rc != -EINPROGRESS)
709
+
memzero_explicit(kmac_ctx, sizeof(*kmac_ctx));
710
+
pr_debug("rc=%d\n", rc);
711
+
return rc;
712
+
}
713
+
714
+
static int phmac_digest(struct ahash_request *req)
715
+
{
716
+
int rc;
717
+
718
+
rc = phmac_init(req);
719
+
if (rc)
720
+
goto out;
721
+
722
+
rc = phmac_finup(req);
723
+
724
+
out:
725
+
pr_debug("rc=%d\n", rc);
726
+
return rc;
727
+
}
728
+
729
+
static int phmac_setkey(struct crypto_ahash *tfm,
730
+
const u8 *key, unsigned int keylen)
731
+
{
732
+
struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm);
733
+
unsigned int ds = crypto_ahash_digestsize(tfm);
734
+
unsigned int bs = crypto_ahash_blocksize(tfm);
735
+
unsigned int tmpkeylen;
736
+
u8 *tmpkey = NULL;
737
+
int rc = 0;
738
+
739
+
if (!crypto_ahash_tested(tfm)) {
740
+
/*
741
+
* selftest running: key is a raw hmac clear key and needs
742
+
* to get embedded into a 'clear key token' in order to have
743
+
* it correctly processed by the pkey module.
744
+
*/
745
+
tmpkeylen = sizeof(struct hmac_clrkey_token) + bs;
746
+
tmpkey = kzalloc(tmpkeylen, GFP_KERNEL);
747
+
if (!tmpkey) {
748
+
rc = -ENOMEM;
749
+
goto out;
750
+
}
751
+
rc = make_clrkey_token(key, keylen, ds, tmpkey);
752
+
if (rc)
753
+
goto out;
754
+
keylen = tmpkeylen;
755
+
key = tmpkey;
756
+
}
757
+
758
+
/* copy raw key into tfm context */
759
+
rc = phmac_tfm_ctx_setkey(tfm_ctx, key, keylen);
760
+
if (rc)
761
+
goto out;
762
+
763
+
/* convert raw key into protected key */
764
+
rc = phmac_convert_key(tfm_ctx);
765
+
if (rc)
766
+
goto out;
767
+
768
+
/* set function code in tfm context, check for valid pk type */
769
+
switch (ds) {
770
+
case SHA224_DIGEST_SIZE:
771
+
if (tfm_ctx->pk.type != PKEY_KEYTYPE_HMAC_512)
772
+
rc = -EINVAL;
773
+
else
774
+
tfm_ctx->fc = CPACF_KMAC_PHMAC_SHA_224;
775
+
break;
776
+
case SHA256_DIGEST_SIZE:
777
+
if (tfm_ctx->pk.type != PKEY_KEYTYPE_HMAC_512)
778
+
rc = -EINVAL;
779
+
else
780
+
tfm_ctx->fc = CPACF_KMAC_PHMAC_SHA_256;
781
+
break;
782
+
case SHA384_DIGEST_SIZE:
783
+
if (tfm_ctx->pk.type != PKEY_KEYTYPE_HMAC_1024)
784
+
rc = -EINVAL;
785
+
else
786
+
tfm_ctx->fc = CPACF_KMAC_PHMAC_SHA_384;
787
+
break;
788
+
case SHA512_DIGEST_SIZE:
789
+
if (tfm_ctx->pk.type != PKEY_KEYTYPE_HMAC_1024)
790
+
rc = -EINVAL;
791
+
else
792
+
tfm_ctx->fc = CPACF_KMAC_PHMAC_SHA_512;
793
+
break;
794
+
default:
795
+
tfm_ctx->fc = 0;
796
+
rc = -EINVAL;
797
+
}
798
+
799
+
out:
800
+
kfree(tmpkey);
801
+
pr_debug("rc=%d\n", rc);
802
+
return rc;
803
+
}
804
+
805
+
static int phmac_export(struct ahash_request *req, void *out)
806
+
{
807
+
struct phmac_req_ctx *req_ctx = ahash_request_ctx(req);
808
+
struct kmac_sha2_ctx *ctx = &req_ctx->kmac_ctx;
809
+
810
+
memcpy(out, ctx, sizeof(*ctx));
811
+
812
+
return 0;
813
+
}
814
+
815
+
static int phmac_import(struct ahash_request *req, const void *in)
816
+
{
817
+
struct phmac_req_ctx *req_ctx = ahash_request_ctx(req);
818
+
struct kmac_sha2_ctx *ctx = &req_ctx->kmac_ctx;
819
+
820
+
memset(req_ctx, 0, sizeof(*req_ctx));
821
+
memcpy(ctx, in, sizeof(*ctx));
822
+
823
+
return 0;
824
+
}
825
+
826
+
static int phmac_init_tfm(struct crypto_ahash *tfm)
827
+
{
828
+
struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm);
829
+
830
+
memset(tfm_ctx, 0, sizeof(*tfm_ctx));
831
+
spin_lock_init(&tfm_ctx->pk_lock);
832
+
833
+
crypto_ahash_set_reqsize(tfm, sizeof(struct phmac_req_ctx));
834
+
835
+
return 0;
836
+
}
837
+
838
+
static void phmac_exit_tfm(struct crypto_ahash *tfm)
839
+
{
840
+
struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm);
841
+
842
+
memzero_explicit(tfm_ctx->keybuf, sizeof(tfm_ctx->keybuf));
843
+
memzero_explicit(&tfm_ctx->pk, sizeof(tfm_ctx->pk));
844
+
}
845
+
846
+
static int phmac_do_one_request(struct crypto_engine *engine, void *areq)
847
+
{
848
+
struct ahash_request *req = ahash_request_cast(areq);
849
+
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
850
+
struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm);
851
+
struct phmac_req_ctx *req_ctx = ahash_request_ctx(req);
852
+
struct kmac_sha2_ctx *kmac_ctx = &req_ctx->kmac_ctx;
853
+
struct hash_walk_helper *hwh = &req_ctx->hwh;
854
+
int rc = -EINVAL;
855
+
856
+
/*
857
+
* Three kinds of requests come in here:
858
+
* update when req->nbytes > 0 and req_ctx->final is false
859
+
* final when req->nbytes = 0 and req_ctx->final is true
860
+
* finup when req->nbytes > 0 and req_ctx->final is true
861
+
* For update and finup the hwh walk needs to be prepared and
862
+
* up to date but the actual nr of bytes in req->nbytes may be
863
+
* any non zero number. For final there is no hwh walk needed.
864
+
*/
865
+
866
+
if (req->nbytes) {
867
+
rc = phmac_kmac_update(req, true);
868
+
if (rc == -EKEYEXPIRED) {
869
+
/*
870
+
* Protected key expired, conversion is in process.
871
+
* Trigger a re-schedule of this request by returning
872
+
* -ENOSPC ("hardware queue full") to the crypto engine.
873
+
* To avoid immediately re-invocation of this callback,
874
+
* tell scheduler to voluntarily give up the CPU here.
875
+
*/
876
+
pr_debug("rescheduling request\n");
877
+
cond_resched();
878
+
return -ENOSPC;
879
+
} else if (rc) {
880
+
hwh_advance(hwh, rc);
881
+
goto out;
882
+
}
883
+
req->nbytes = 0;
884
+
}
885
+
886
+
if (req_ctx->final) {
887
+
rc = phmac_kmac_final(req, true);
888
+
if (rc == -EKEYEXPIRED) {
889
+
/*
890
+
* Protected key expired, conversion is in process.
891
+
* Trigger a re-schedule of this request by returning
892
+
* -ENOSPC ("hardware queue full") to the crypto engine.
893
+
* To avoid immediately re-invocation of this callback,
894
+
* tell scheduler to voluntarily give up the CPU here.
895
+
*/
896
+
pr_debug("rescheduling request\n");
897
+
cond_resched();
898
+
return -ENOSPC;
899
+
}
900
+
}
901
+
902
+
out:
903
+
if (rc || req_ctx->final)
904
+
memzero_explicit(kmac_ctx, sizeof(*kmac_ctx));
905
+
pr_debug("request complete with rc=%d\n", rc);
906
+
local_bh_disable();
907
+
atomic_dec(&tfm_ctx->via_engine_ctr);
908
+
crypto_finalize_hash_request(engine, req, rc);
909
+
local_bh_enable();
910
+
return rc;
911
+
}
912
+
913
+
#define S390_ASYNC_PHMAC_ALG(x) \
914
+
{ \
915
+
.base = { \
916
+
.init = phmac_init, \
917
+
.update = phmac_update, \
918
+
.final = phmac_final, \
919
+
.finup = phmac_finup, \
920
+
.digest = phmac_digest, \
921
+
.setkey = phmac_setkey, \
922
+
.import = phmac_import, \
923
+
.export = phmac_export, \
924
+
.init_tfm = phmac_init_tfm, \
925
+
.exit_tfm = phmac_exit_tfm, \
926
+
.halg = { \
927
+
.digestsize = SHA##x##_DIGEST_SIZE, \
928
+
.statesize = sizeof(struct kmac_sha2_ctx), \
929
+
.base = { \
930
+
.cra_name = "phmac(sha" #x ")", \
931
+
.cra_driver_name = "phmac_s390_sha" #x, \
932
+
.cra_blocksize = SHA##x##_BLOCK_SIZE, \
933
+
.cra_priority = 400, \
934
+
.cra_flags = CRYPTO_ALG_ASYNC | \
935
+
CRYPTO_ALG_NO_FALLBACK, \
936
+
.cra_ctxsize = sizeof(struct phmac_tfm_ctx), \
937
+
.cra_module = THIS_MODULE, \
938
+
}, \
939
+
}, \
940
+
}, \
941
+
.op = { \
942
+
.do_one_request = phmac_do_one_request, \
943
+
}, \
944
+
}
945
+
946
+
static struct phmac_alg {
947
+
unsigned int fc;
948
+
struct ahash_engine_alg alg;
949
+
bool registered;
950
+
} phmac_algs[] = {
951
+
{
952
+
.fc = CPACF_KMAC_PHMAC_SHA_224,
953
+
.alg = S390_ASYNC_PHMAC_ALG(224),
954
+
}, {
955
+
.fc = CPACF_KMAC_PHMAC_SHA_256,
956
+
.alg = S390_ASYNC_PHMAC_ALG(256),
957
+
}, {
958
+
.fc = CPACF_KMAC_PHMAC_SHA_384,
959
+
.alg = S390_ASYNC_PHMAC_ALG(384),
960
+
}, {
961
+
.fc = CPACF_KMAC_PHMAC_SHA_512,
962
+
.alg = S390_ASYNC_PHMAC_ALG(512),
963
+
}
964
+
};
965
+
966
+
static struct miscdevice phmac_dev = {
967
+
.name = "phmac",
968
+
.minor = MISC_DYNAMIC_MINOR,
969
+
};
970
+
971
+
static void s390_phmac_exit(void)
972
+
{
973
+
struct phmac_alg *phmac;
974
+
int i;
975
+
976
+
if (phmac_crypto_engine) {
977
+
crypto_engine_stop(phmac_crypto_engine);
978
+
crypto_engine_exit(phmac_crypto_engine);
979
+
}
980
+
981
+
for (i = ARRAY_SIZE(phmac_algs) - 1; i >= 0; i--) {
982
+
phmac = &phmac_algs[i];
983
+
if (phmac->registered)
984
+
crypto_engine_unregister_ahash(&phmac->alg);
985
+
}
986
+
987
+
misc_deregister(&phmac_dev);
988
+
}
989
+
990
+
static int __init s390_phmac_init(void)
991
+
{
992
+
struct phmac_alg *phmac;
993
+
int i, rc;
994
+
995
+
/* for selftest cpacf klmd subfunction is needed */
996
+
if (!cpacf_query_func(CPACF_KLMD, CPACF_KLMD_SHA_256))
997
+
return -ENODEV;
998
+
if (!cpacf_query_func(CPACF_KLMD, CPACF_KLMD_SHA_512))
999
+
return -ENODEV;
1000
+
1001
+
/* register a simple phmac pseudo misc device */
1002
+
rc = misc_register(&phmac_dev);
1003
+
if (rc)
1004
+
return rc;
1005
+
1006
+
/* with this pseudo device alloc and start a crypto engine */
1007
+
phmac_crypto_engine =
1008
+
crypto_engine_alloc_init_and_set(phmac_dev.this_device,
1009
+
true, false, MAX_QLEN);
1010
+
if (!phmac_crypto_engine) {
1011
+
rc = -ENOMEM;
1012
+
goto out_err;
1013
+
}
1014
+
rc = crypto_engine_start(phmac_crypto_engine);
1015
+
if (rc) {
1016
+
crypto_engine_exit(phmac_crypto_engine);
1017
+
phmac_crypto_engine = NULL;
1018
+
goto out_err;
1019
+
}
1020
+
1021
+
for (i = 0; i < ARRAY_SIZE(phmac_algs); i++) {
1022
+
phmac = &phmac_algs[i];
1023
+
if (!cpacf_query_func(CPACF_KMAC, phmac->fc))
1024
+
continue;
1025
+
rc = crypto_engine_register_ahash(&phmac->alg);
1026
+
if (rc)
1027
+
goto out_err;
1028
+
phmac->registered = true;
1029
+
pr_debug("%s registered\n", phmac->alg.base.halg.base.cra_name);
1030
+
}
1031
+
1032
+
return 0;
1033
+
1034
+
out_err:
1035
+
s390_phmac_exit();
1036
+
return rc;
1037
+
}
1038
+
1039
+
module_init(s390_phmac_init);
1040
+
module_exit(s390_phmac_exit);
1041
+
1042
+
MODULE_ALIAS_CRYPTO("phmac(sha224)");
1043
+
MODULE_ALIAS_CRYPTO("phmac(sha256)");
1044
+
MODULE_ALIAS_CRYPTO("phmac(sha384)");
1045
+
MODULE_ALIAS_CRYPTO("phmac(sha512)");
1046
+
1047
+
MODULE_DESCRIPTION("S390 HMAC driver for protected keys");
1048
+
MODULE_LICENSE("GPL");
+3
arch/s390/crypto/sha.h
+3
arch/s390/crypto/sha.h
+16
-6
arch/s390/crypto/sha3_256_s390.c
+16
-6
arch/s390/crypto/sha3_256_s390.c
···
35
35
static int sha3_256_export(struct shash_desc *desc, void *out)
36
36
{
37
37
struct s390_sha_ctx *sctx = shash_desc_ctx(desc);
38
-
struct sha3_state *octx = out;
38
+
union {
39
+
u8 *u8;
40
+
u64 *u64;
41
+
} p = { .u8 = out };
42
+
int i;
39
43
40
44
if (sctx->first_message_part) {
41
-
memset(sctx->state, 0, sizeof(sctx->state));
42
-
sctx->first_message_part = 0;
45
+
memset(out, 0, SHA3_STATE_SIZE);
46
+
return 0;
43
47
}
44
-
memcpy(octx->st, sctx->state, sizeof(octx->st));
48
+
for (i = 0; i < SHA3_STATE_SIZE / 8; i++)
49
+
put_unaligned(le64_to_cpu(sctx->sha3.state[i]), p.u64++);
45
50
return 0;
46
51
}
47
52
48
53
static int sha3_256_import(struct shash_desc *desc, const void *in)
49
54
{
50
55
struct s390_sha_ctx *sctx = shash_desc_ctx(desc);
51
-
const struct sha3_state *ictx = in;
56
+
union {
57
+
const u8 *u8;
58
+
const u64 *u64;
59
+
} p = { .u8 = in };
60
+
int i;
52
61
62
+
for (i = 0; i < SHA3_STATE_SIZE / 8; i++)
63
+
sctx->sha3.state[i] = cpu_to_le64(get_unaligned(p.u64++));
53
64
sctx->count = 0;
54
-
memcpy(sctx->state, ictx->st, sizeof(ictx->st));
55
65
sctx->first_message_part = 0;
56
66
sctx->func = CPACF_KIMD_SHA3_256;
57
67
+16
-7
arch/s390/crypto/sha3_512_s390.c
+16
-7
arch/s390/crypto/sha3_512_s390.c
···
34
34
static int sha3_512_export(struct shash_desc *desc, void *out)
35
35
{
36
36
struct s390_sha_ctx *sctx = shash_desc_ctx(desc);
37
-
struct sha3_state *octx = out;
38
-
37
+
union {
38
+
u8 *u8;
39
+
u64 *u64;
40
+
} p = { .u8 = out };
41
+
int i;
39
42
40
43
if (sctx->first_message_part) {
41
-
memset(sctx->state, 0, sizeof(sctx->state));
42
-
sctx->first_message_part = 0;
44
+
memset(out, 0, SHA3_STATE_SIZE);
45
+
return 0;
43
46
}
44
-
memcpy(octx->st, sctx->state, sizeof(octx->st));
47
+
for (i = 0; i < SHA3_STATE_SIZE / 8; i++)
48
+
put_unaligned(le64_to_cpu(sctx->sha3.state[i]), p.u64++);
45
49
return 0;
46
50
}
47
51
48
52
static int sha3_512_import(struct shash_desc *desc, const void *in)
49
53
{
50
54
struct s390_sha_ctx *sctx = shash_desc_ctx(desc);
51
-
const struct sha3_state *ictx = in;
55
+
union {
56
+
const u8 *u8;
57
+
const u64 *u64;
58
+
} p = { .u8 = in };
59
+
int i;
52
60
61
+
for (i = 0; i < SHA3_STATE_SIZE / 8; i++)
62
+
sctx->sha3.state[i] = cpu_to_le64(get_unaligned(p.u64++));
53
63
sctx->count = 0;
54
-
memcpy(sctx->state, ictx->st, sizeof(ictx->st));
55
64
sctx->first_message_part = 0;
56
65
sctx->func = CPACF_KIMD_SHA3_512;
57
66
+4
arch/s390/include/asm/cpacf.h
+4
arch/s390/include/asm/cpacf.h
···
129
129
#define CPACF_KMAC_HMAC_SHA_256 0x71
130
130
#define CPACF_KMAC_HMAC_SHA_384 0x72
131
131
#define CPACF_KMAC_HMAC_SHA_512 0x73
132
+
#define CPACF_KMAC_PHMAC_SHA_224 0x78
133
+
#define CPACF_KMAC_PHMAC_SHA_256 0x79
134
+
#define CPACF_KMAC_PHMAC_SHA_384 0x7a
135
+
#define CPACF_KMAC_PHMAC_SHA_512 0x7b
132
136
133
137
/*
134
138
* Function codes for the PCKMO (PERFORM CRYPTOGRAPHIC KEY MANAGEMENT)
+29
-11
arch/x86/crypto/aegis128-aesni-glue.c
+29
-11
arch/x86/crypto/aegis128-aesni-glue.c
···
104
104
}
105
105
}
106
106
107
-
static __always_inline void
107
+
static __always_inline int
108
108
crypto_aegis128_aesni_process_crypt(struct aegis_state *state,
109
109
struct skcipher_walk *walk, bool enc)
110
110
{
111
+
int err = 0;
112
+
111
113
while (walk->nbytes >= AEGIS128_BLOCK_SIZE) {
112
114
if (enc)
113
115
aegis128_aesni_enc(state, walk->src.virt.addr,
···
121
119
walk->dst.virt.addr,
122
120
round_down(walk->nbytes,
123
121
AEGIS128_BLOCK_SIZE));
124
-
skcipher_walk_done(walk, walk->nbytes % AEGIS128_BLOCK_SIZE);
122
+
kernel_fpu_end();
123
+
err = skcipher_walk_done(walk,
124
+
walk->nbytes % AEGIS128_BLOCK_SIZE);
125
+
kernel_fpu_begin();
125
126
}
126
127
127
128
if (walk->nbytes) {
···
136
131
aegis128_aesni_dec_tail(state, walk->src.virt.addr,
137
132
walk->dst.virt.addr,
138
133
walk->nbytes);
139
-
skcipher_walk_done(walk, 0);
134
+
kernel_fpu_end();
135
+
err = skcipher_walk_done(walk, 0);
136
+
kernel_fpu_begin();
140
137
}
138
+
return err;
141
139
}
142
140
143
141
static struct aegis_ctx *crypto_aegis128_aesni_ctx(struct crypto_aead *aead)
···
173
165
return 0;
174
166
}
175
167
176
-
static __always_inline void
168
+
static __always_inline int
177
169
crypto_aegis128_aesni_crypt(struct aead_request *req,
178
170
struct aegis_block *tag_xor,
179
171
unsigned int cryptlen, bool enc)
···
182
174
struct aegis_ctx *ctx = crypto_aegis128_aesni_ctx(tfm);
183
175
struct skcipher_walk walk;
184
176
struct aegis_state state;
177
+
int err;
185
178
186
179
if (enc)
187
-
skcipher_walk_aead_encrypt(&walk, req, true);
180
+
err = skcipher_walk_aead_encrypt(&walk, req, false);
188
181
else
189
-
skcipher_walk_aead_decrypt(&walk, req, true);
182
+
err = skcipher_walk_aead_decrypt(&walk, req, false);
183
+
if (err)
184
+
return err;
190
185
191
186
kernel_fpu_begin();
192
187
193
188
aegis128_aesni_init(&state, &ctx->key, req->iv);
194
189
crypto_aegis128_aesni_process_ad(&state, req->src, req->assoclen);
195
-
crypto_aegis128_aesni_process_crypt(&state, &walk, enc);
196
-
aegis128_aesni_final(&state, tag_xor, req->assoclen, cryptlen);
197
-
190
+
err = crypto_aegis128_aesni_process_crypt(&state, &walk, enc);
191
+
if (err == 0)
192
+
aegis128_aesni_final(&state, tag_xor, req->assoclen, cryptlen);
198
193
kernel_fpu_end();
194
+
return err;
199
195
}
200
196
201
197
static int crypto_aegis128_aesni_encrypt(struct aead_request *req)
···
208
196
struct aegis_block tag = {};
209
197
unsigned int authsize = crypto_aead_authsize(tfm);
210
198
unsigned int cryptlen = req->cryptlen;
199
+
int err;
211
200
212
-
crypto_aegis128_aesni_crypt(req, &tag, cryptlen, true);
201
+
err = crypto_aegis128_aesni_crypt(req, &tag, cryptlen, true);
202
+
if (err)
203
+
return err;
213
204
214
205
scatterwalk_map_and_copy(tag.bytes, req->dst,
215
206
req->assoclen + cryptlen, authsize, 1);
···
227
212
struct aegis_block tag;
228
213
unsigned int authsize = crypto_aead_authsize(tfm);
229
214
unsigned int cryptlen = req->cryptlen - authsize;
215
+
int err;
230
216
231
217
scatterwalk_map_and_copy(tag.bytes, req->src,
232
218
req->assoclen + cryptlen, authsize, 0);
233
219
234
-
crypto_aegis128_aesni_crypt(req, &tag, cryptlen, false);
220
+
err = crypto_aegis128_aesni_crypt(req, &tag, cryptlen, false);
221
+
if (err)
222
+
return err;
235
223
236
224
return crypto_memneq(tag.bytes, zeros.bytes, authsize) ? -EBADMSG : 0;
237
225
}
+1
arch/x86/crypto/aria_aesni_avx2_glue.c
+1
arch/x86/crypto/aria_aesni_avx2_glue.c
+1
arch/x86/crypto/aria_aesni_avx_glue.c
+1
arch/x86/crypto/aria_aesni_avx_glue.c
+1
arch/x86/crypto/camellia_aesni_avx_glue.c
+1
arch/x86/crypto/camellia_aesni_avx_glue.c
+1
arch/x86/crypto/camellia_glue.c
+1
arch/x86/crypto/camellia_glue.c
+1
arch/x86/crypto/curve25519-x86_64.c
+1
arch/x86/crypto/curve25519-x86_64.c
+1
arch/x86/crypto/serpent_avx_glue.c
+1
arch/x86/crypto/serpent_avx_glue.c
+1
arch/x86/crypto/sm4_aesni_avx_glue.c
+1
arch/x86/crypto/sm4_aesni_avx_glue.c
+1
arch/x86/crypto/twofish_glue.c
+1
arch/x86/crypto/twofish_glue.c
+1
arch/x86/crypto/twofish_glue_3way.c
+1
arch/x86/crypto/twofish_glue_3way.c
+17
-22
crypto/ahash.c
+17
-22
crypto/ahash.c
···
29
29
30
30
#define CRYPTO_ALG_TYPE_AHASH_MASK 0x0000000e
31
31
32
-
struct crypto_hash_walk {
33
-
const char *data;
34
-
35
-
unsigned int offset;
36
-
unsigned int flags;
37
-
38
-
struct page *pg;
39
-
unsigned int entrylen;
40
-
41
-
unsigned int total;
42
-
struct scatterlist *sg;
43
-
};
44
-
45
32
static int ahash_def_finup(struct ahash_request *req);
46
33
47
34
static inline bool crypto_ahash_block_only(struct crypto_ahash *tfm)
···
99
112
return hash_walk_next(walk);
100
113
}
101
114
102
-
static int crypto_hash_walk_first(struct ahash_request *req,
103
-
struct crypto_hash_walk *walk)
115
+
int crypto_hash_walk_first(struct ahash_request *req,
116
+
struct crypto_hash_walk *walk)
104
117
{
105
118
walk->total = req->nbytes;
106
119
walk->entrylen = 0;
···
120
133
121
134
return hash_walk_new_entry(walk);
122
135
}
136
+
EXPORT_SYMBOL_GPL(crypto_hash_walk_first);
123
137
124
-
static int crypto_hash_walk_done(struct crypto_hash_walk *walk, int err)
138
+
int crypto_hash_walk_done(struct crypto_hash_walk *walk, int err)
125
139
{
126
140
if ((walk->flags & CRYPTO_AHASH_REQ_VIRT))
127
141
return err;
···
148
160
149
161
return hash_walk_new_entry(walk);
150
162
}
151
-
152
-
static inline int crypto_hash_walk_last(struct crypto_hash_walk *walk)
153
-
{
154
-
return !(walk->entrylen | walk->total);
155
-
}
163
+
EXPORT_SYMBOL_GPL(crypto_hash_walk_done);
156
164
157
165
/*
158
166
* For an ahash tfm that is using an shash algorithm (instead of an ahash
···
329
345
return (*op)(req);
330
346
331
347
if (crypto_ahash_statesize(tfm) > HASH_MAX_STATESIZE)
348
+
return -ENOSYS;
349
+
350
+
if (!crypto_ahash_need_fallback(tfm))
351
+
return -ENOSYS;
352
+
353
+
if (crypto_hash_no_export_core(tfm))
332
354
return -ENOSYS;
333
355
334
356
{
···
944
954
base->cra_reqsize > MAX_SYNC_HASH_REQSIZE)
945
955
return -EINVAL;
946
956
957
+
if (base->cra_flags & CRYPTO_ALG_NEED_FALLBACK &&
958
+
base->cra_flags & CRYPTO_ALG_NO_FALLBACK)
959
+
return -EINVAL;
960
+
947
961
err = hash_prepare_alg(&alg->halg);
948
962
if (err)
949
963
return err;
···
956
962
base->cra_flags |= CRYPTO_ALG_TYPE_AHASH;
957
963
958
964
if ((base->cra_flags ^ CRYPTO_ALG_REQ_VIRT) &
959
-
(CRYPTO_ALG_ASYNC | CRYPTO_ALG_REQ_VIRT))
965
+
(CRYPTO_ALG_ASYNC | CRYPTO_ALG_REQ_VIRT) &&
966
+
!(base->cra_flags & CRYPTO_ALG_NO_FALLBACK))
960
967
base->cra_flags |= CRYPTO_ALG_NEED_FALLBACK;
961
968
962
969
if (!alg->setkey)
+6
crypto/cryptd.c
+6
crypto/cryptd.c
···
34
34
static struct workqueue_struct *cryptd_wq;
35
35
36
36
struct cryptd_cpu_queue {
37
+
local_lock_t bh_lock;
37
38
struct crypto_queue queue;
38
39
struct work_struct work;
39
40
};
···
111
110
cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu);
112
111
crypto_init_queue(&cpu_queue->queue, max_cpu_qlen);
113
112
INIT_WORK(&cpu_queue->work, cryptd_queue_worker);
113
+
local_lock_init(&cpu_queue->bh_lock);
114
114
}
115
115
pr_info("cryptd: max_cpu_qlen set to %d\n", max_cpu_qlen);
116
116
return 0;
···
137
135
refcount_t *refcnt;
138
136
139
137
local_bh_disable();
138
+
local_lock_nested_bh(&queue->cpu_queue->bh_lock);
140
139
cpu_queue = this_cpu_ptr(queue->cpu_queue);
141
140
err = crypto_enqueue_request(&cpu_queue->queue, request);
142
141
···
154
151
refcount_inc(refcnt);
155
152
156
153
out:
154
+
local_unlock_nested_bh(&queue->cpu_queue->bh_lock);
157
155
local_bh_enable();
158
156
159
157
return err;
···
173
169
* Only handle one request at a time to avoid hogging crypto workqueue.
174
170
*/
175
171
local_bh_disable();
172
+
__local_lock_nested_bh(&cpu_queue->bh_lock);
176
173
backlog = crypto_get_backlog(&cpu_queue->queue);
177
174
req = crypto_dequeue_request(&cpu_queue->queue);
175
+
__local_unlock_nested_bh(&cpu_queue->bh_lock);
178
176
local_bh_enable();
179
177
180
178
if (!req)
+2
-53
crypto/crypto_engine.c
+2
-53
crypto/crypto_engine.c
···
74
74
struct crypto_engine_alg *alg;
75
75
struct crypto_engine_op *op;
76
76
unsigned long flags;
77
-
bool was_busy = false;
78
77
int ret;
79
78
80
79
spin_lock_irqsave(&engine->queue_lock, flags);
···
81
82
/* Make sure we are not already running a request */
82
83
if (!engine->retry_support && engine->cur_req)
83
84
goto out;
84
-
85
-
/* If another context is idling then defer */
86
-
if (engine->idling) {
87
-
kthread_queue_work(engine->kworker, &engine->pump_requests);
88
-
goto out;
89
-
}
90
85
91
86
/* Check if the engine queue is idle */
92
87
if (!crypto_queue_len(&engine->queue) || !engine->running) {
···
95
102
}
96
103
97
104
engine->busy = false;
98
-
engine->idling = true;
99
-
spin_unlock_irqrestore(&engine->queue_lock, flags);
100
-
101
-
if (engine->unprepare_crypt_hardware &&
102
-
engine->unprepare_crypt_hardware(engine))
103
-
dev_err(engine->dev, "failed to unprepare crypt hardware\n");
104
-
105
-
spin_lock_irqsave(&engine->queue_lock, flags);
106
-
engine->idling = false;
107
105
goto out;
108
106
}
109
107
···
113
129
if (!engine->retry_support)
114
130
engine->cur_req = async_req;
115
131
116
-
if (engine->busy)
117
-
was_busy = true;
118
-
else
132
+
if (!engine->busy)
119
133
engine->busy = true;
120
134
121
135
spin_unlock_irqrestore(&engine->queue_lock, flags);
122
-
123
-
/* Until here we get the request need to be encrypted successfully */
124
-
if (!was_busy && engine->prepare_crypt_hardware) {
125
-
ret = engine->prepare_crypt_hardware(engine);
126
-
if (ret) {
127
-
dev_err(engine->dev, "failed to prepare crypt hardware\n");
128
-
goto req_err_1;
129
-
}
130
-
}
131
136
132
137
alg = container_of(async_req->tfm->__crt_alg,
133
138
struct crypto_engine_alg, base);
···
167
194
168
195
out:
169
196
spin_unlock_irqrestore(&engine->queue_lock, flags);
170
-
171
-
/*
172
-
* Batch requests is possible only if
173
-
* hardware can enqueue multiple requests
174
-
*/
175
-
if (engine->do_batch_requests) {
176
-
ret = engine->do_batch_requests(engine);
177
-
if (ret)
178
-
dev_err(engine->dev, "failed to do batch requests: %d\n",
179
-
ret);
180
-
}
181
197
182
198
return;
183
199
}
···
424
462
* crypto-engine queue.
425
463
* @dev: the device attached with one hardware engine
426
464
* @retry_support: whether hardware has support for retry mechanism
427
-
* @cbk_do_batch: pointer to a callback function to be invoked when executing
428
-
* a batch of requests.
429
-
* This has the form:
430
-
* callback(struct crypto_engine *engine)
431
-
* where:
432
-
* engine: the crypto engine structure.
433
465
* @rt: whether this queue is set to run as a realtime task
434
466
* @qlen: maximum size of the crypto-engine queue
435
467
*
···
432
476
*/
433
477
struct crypto_engine *crypto_engine_alloc_init_and_set(struct device *dev,
434
478
bool retry_support,
435
-
int (*cbk_do_batch)(struct crypto_engine *engine),
436
479
bool rt, int qlen)
437
480
{
438
481
struct crypto_engine *engine;
···
447
492
engine->rt = rt;
448
493
engine->running = false;
449
494
engine->busy = false;
450
-
engine->idling = false;
451
495
engine->retry_support = retry_support;
452
496
engine->priv_data = dev;
453
-
/*
454
-
* Batch requests is possible only if
455
-
* hardware has support for retry mechanism.
456
-
*/
457
-
engine->do_batch_requests = retry_support ? cbk_do_batch : NULL;
458
497
459
498
snprintf(engine->name, sizeof(engine->name),
460
499
"%s-engine", dev_name(dev));
···
483
534
*/
484
535
struct crypto_engine *crypto_engine_alloc_init(struct device *dev, bool rt)
485
536
{
486
-
return crypto_engine_alloc_init_and_set(dev, false, NULL, rt,
537
+
return crypto_engine_alloc_init_and_set(dev, false, rt,
487
538
CRYPTO_ENGINE_MAX_QLEN);
488
539
}
489
540
EXPORT_SYMBOL_GPL(crypto_engine_alloc_init);
+6
-1
crypto/deflate.c
+6
-1
crypto/deflate.c
···
48
48
return ctx;
49
49
}
50
50
51
+
static void deflate_free_stream(void *ctx)
52
+
{
53
+
kvfree(ctx);
54
+
}
55
+
51
56
static struct crypto_acomp_streams deflate_streams = {
52
57
.alloc_ctx = deflate_alloc_stream,
53
-
.cfree_ctx = kvfree,
58
+
.free_ctx = deflate_free_stream,
54
59
};
55
60
56
61
static int deflate_compress_one(struct acomp_req *req,
+5
-4
crypto/jitterentropy-kcapi.c
+5
-4
crypto/jitterentropy-kcapi.c
···
144
144
* Inject the data from the previous loop into the pool. This data is
145
145
* not considered to contain any entropy, but it stirs the pool a bit.
146
146
*/
147
-
ret = crypto_shash_update(desc, intermediary, sizeof(intermediary));
147
+
ret = crypto_shash_update(hash_state_desc, intermediary, sizeof(intermediary));
148
148
if (ret)
149
149
goto err;
150
150
···
157
157
* conditioning operation to have an identical amount of input data
158
158
* according to section 3.1.5.
159
159
*/
160
-
if (!stuck) {
161
-
ret = crypto_shash_update(hash_state_desc, (u8 *)&time,
162
-
sizeof(__u64));
160
+
if (stuck) {
161
+
time = 0;
163
162
}
163
+
164
+
ret = crypto_shash_update(hash_state_desc, (u8 *)&time, sizeof(__u64));
164
165
165
166
err:
166
167
shash_desc_zero(desc);
+1
-1
crypto/jitterentropy.c
+1
-1
crypto/jitterentropy.c
···
145
145
*/
146
146
#define JENT_ENTROPY_SAFETY_FACTOR 64
147
147
148
+
#include <linux/array_size.h>
148
149
#include <linux/fips.h>
149
150
#include <linux/minmax.h>
150
151
#include "jitterentropy.h"
···
179
178
static const unsigned int jent_apt_cutoff_permanent_lookup[15] = {
180
179
355, 447, 479, 494, 502, 507, 510, 512,
181
180
512, 512, 512, 512, 512, 512, 512 };
182
-
#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
183
181
184
182
static void jent_apt_init(struct rand_data *ec, unsigned int osr)
185
183
{
+1
crypto/krb5/selftest.c
+1
crypto/krb5/selftest.c
+2
-5
crypto/pcrypt.c
+2
-5
crypto/pcrypt.c
···
178
178
179
179
static int pcrypt_aead_init_tfm(struct crypto_aead *tfm)
180
180
{
181
-
int cpu, cpu_index;
181
+
int cpu_index;
182
182
struct aead_instance *inst = aead_alg_instance(tfm);
183
183
struct pcrypt_instance_ctx *ictx = aead_instance_ctx(inst);
184
184
struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(tfm);
···
187
187
cpu_index = (unsigned int)atomic_inc_return(&ictx->tfm_count) %
188
188
cpumask_weight(cpu_online_mask);
189
189
190
-
ctx->cb_cpu = cpumask_first(cpu_online_mask);
191
-
for (cpu = 0; cpu < cpu_index; cpu++)
192
-
ctx->cb_cpu = cpumask_next(ctx->cb_cpu, cpu_online_mask);
193
-
190
+
ctx->cb_cpu = cpumask_nth(cpu_index, cpu_online_mask);
194
191
cipher = crypto_spawn_aead(&ictx->spawn);
195
192
196
193
if (IS_ERR(cipher))
+34
-5
crypto/testmgr.c
+34
-5
crypto/testmgr.c
···
4186
4186
.alg = "authenc(hmac(sha1),cbc(aes))",
4187
4187
.generic_driver = "authenc(hmac-sha1-lib,cbc(aes-generic))",
4188
4188
.test = alg_test_aead,
4189
-
.fips_allowed = 1,
4190
4189
.suite = {
4191
4190
.aead = __VECS(hmac_sha1_aes_cbc_tv_temp)
4192
4191
}
···
4206
4207
}, {
4207
4208
.alg = "authenc(hmac(sha1),ctr(aes))",
4208
4209
.test = alg_test_null,
4209
-
.fips_allowed = 1,
4210
4210
}, {
4211
4211
.alg = "authenc(hmac(sha1),ecb(cipher_null))",
4212
4212
.generic_driver = "authenc(hmac-sha1-lib,ecb-cipher_null)",
···
4216
4218
}, {
4217
4219
.alg = "authenc(hmac(sha1),rfc3686(ctr(aes)))",
4218
4220
.test = alg_test_null,
4219
-
.fips_allowed = 1,
4220
4221
}, {
4221
4222
.alg = "authenc(hmac(sha224),cbc(des))",
4222
4223
.generic_driver = "authenc(hmac-sha224-lib,cbc(des-generic))",
···
4709
4712
*/
4710
4713
.alg = "drbg_nopr_hmac_sha384",
4711
4714
.test = alg_test_null,
4715
+
.fips_allowed = 1
4712
4716
}, {
4713
4717
.alg = "drbg_nopr_hmac_sha512",
4714
4718
.test = alg_test_drbg,
···
4728
4730
/* covered by drbg_nopr_sha256 test */
4729
4731
.alg = "drbg_nopr_sha384",
4730
4732
.test = alg_test_null,
4733
+
.fips_allowed = 1
4731
4734
}, {
4732
4735
.alg = "drbg_nopr_sha512",
4733
4736
.fips_allowed = 1,
···
4760
4761
/* covered by drbg_pr_hmac_sha256 test */
4761
4762
.alg = "drbg_pr_hmac_sha384",
4762
4763
.test = alg_test_null,
4764
+
.fips_allowed = 1
4763
4765
}, {
4764
4766
.alg = "drbg_pr_hmac_sha512",
4765
4767
.test = alg_test_null,
···
4776
4776
/* covered by drbg_pr_sha256 test */
4777
4777
.alg = "drbg_pr_sha384",
4778
4778
.test = alg_test_null,
4779
+
.fips_allowed = 1
4779
4780
}, {
4780
4781
.alg = "drbg_pr_sha512",
4781
4782
.fips_allowed = 1,
···
5078
5077
.alg = "hmac(sha1)",
5079
5078
.generic_driver = "hmac-sha1-lib",
5080
5079
.test = alg_test_hash,
5081
-
.fips_allowed = 1,
5082
5080
.suite = {
5083
5081
.hash = __VECS(hmac_sha1_tv_template)
5084
5082
}
···
5291
5291
.cipher = __VECS(fcrypt_pcbc_tv_template)
5292
5292
}
5293
5293
}, {
5294
+
#if IS_ENABLED(CONFIG_CRYPTO_PHMAC_S390)
5295
+
.alg = "phmac(sha224)",
5296
+
.test = alg_test_hash,
5297
+
.fips_allowed = 1,
5298
+
.suite = {
5299
+
.hash = __VECS(hmac_sha224_tv_template)
5300
+
}
5301
+
}, {
5302
+
.alg = "phmac(sha256)",
5303
+
.test = alg_test_hash,
5304
+
.fips_allowed = 1,
5305
+
.suite = {
5306
+
.hash = __VECS(hmac_sha256_tv_template)
5307
+
}
5308
+
}, {
5309
+
.alg = "phmac(sha384)",
5310
+
.test = alg_test_hash,
5311
+
.fips_allowed = 1,
5312
+
.suite = {
5313
+
.hash = __VECS(hmac_sha384_tv_template)
5314
+
}
5315
+
}, {
5316
+
.alg = "phmac(sha512)",
5317
+
.test = alg_test_hash,
5318
+
.fips_allowed = 1,
5319
+
.suite = {
5320
+
.hash = __VECS(hmac_sha512_tv_template)
5321
+
}
5322
+
}, {
5323
+
#endif
5294
5324
.alg = "pkcs1(rsa,none)",
5295
5325
.test = alg_test_sig,
5296
5326
.suite = {
···
5448
5418
.alg = "sha1",
5449
5419
.generic_driver = "sha1-lib",
5450
5420
.test = alg_test_hash,
5451
-
.fips_allowed = 1,
5452
5421
.suite = {
5453
5422
.hash = __VECS(sha1_tv_template)
5454
5423
}
+253
-137
crypto/zstd.c
+253
-137
crypto/zstd.c
···
12
12
#include <linux/net.h>
13
13
#include <linux/vmalloc.h>
14
14
#include <linux/zstd.h>
15
-
#include <crypto/internal/scompress.h>
15
+
#include <crypto/internal/acompress.h>
16
+
#include <crypto/scatterwalk.h>
16
17
17
18
18
-
#define ZSTD_DEF_LEVEL 3
19
+
#define ZSTD_DEF_LEVEL 3
20
+
#define ZSTD_MAX_WINDOWLOG 18
21
+
#define ZSTD_MAX_SIZE BIT(ZSTD_MAX_WINDOWLOG)
19
22
20
23
struct zstd_ctx {
21
24
zstd_cctx *cctx;
22
25
zstd_dctx *dctx;
23
-
void *cwksp;
24
-
void *dwksp;
26
+
size_t wksp_size;
27
+
zstd_parameters params;
28
+
u8 wksp[] __aligned(8);
25
29
};
26
30
27
-
static zstd_parameters zstd_params(void)
31
+
static DEFINE_MUTEX(zstd_stream_lock);
32
+
33
+
static void *zstd_alloc_stream(void)
28
34
{
29
-
return zstd_get_params(ZSTD_DEF_LEVEL, 0);
30
-
}
31
-
32
-
static int zstd_comp_init(struct zstd_ctx *ctx)
33
-
{
34
-
int ret = 0;
35
-
const zstd_parameters params = zstd_params();
36
-
const size_t wksp_size = zstd_cctx_workspace_bound(¶ms.cParams);
37
-
38
-
ctx->cwksp = vzalloc(wksp_size);
39
-
if (!ctx->cwksp) {
40
-
ret = -ENOMEM;
41
-
goto out;
42
-
}
43
-
44
-
ctx->cctx = zstd_init_cctx(ctx->cwksp, wksp_size);
45
-
if (!ctx->cctx) {
46
-
ret = -EINVAL;
47
-
goto out_free;
48
-
}
49
-
out:
50
-
return ret;
51
-
out_free:
52
-
vfree(ctx->cwksp);
53
-
goto out;
54
-
}
55
-
56
-
static int zstd_decomp_init(struct zstd_ctx *ctx)
57
-
{
58
-
int ret = 0;
59
-
const size_t wksp_size = zstd_dctx_workspace_bound();
60
-
61
-
ctx->dwksp = vzalloc(wksp_size);
62
-
if (!ctx->dwksp) {
63
-
ret = -ENOMEM;
64
-
goto out;
65
-
}
66
-
67
-
ctx->dctx = zstd_init_dctx(ctx->dwksp, wksp_size);
68
-
if (!ctx->dctx) {
69
-
ret = -EINVAL;
70
-
goto out_free;
71
-
}
72
-
out:
73
-
return ret;
74
-
out_free:
75
-
vfree(ctx->dwksp);
76
-
goto out;
77
-
}
78
-
79
-
static void zstd_comp_exit(struct zstd_ctx *ctx)
80
-
{
81
-
vfree(ctx->cwksp);
82
-
ctx->cwksp = NULL;
83
-
ctx->cctx = NULL;
84
-
}
85
-
86
-
static void zstd_decomp_exit(struct zstd_ctx *ctx)
87
-
{
88
-
vfree(ctx->dwksp);
89
-
ctx->dwksp = NULL;
90
-
ctx->dctx = NULL;
91
-
}
92
-
93
-
static int __zstd_init(void *ctx)
94
-
{
95
-
int ret;
96
-
97
-
ret = zstd_comp_init(ctx);
98
-
if (ret)
99
-
return ret;
100
-
ret = zstd_decomp_init(ctx);
101
-
if (ret)
102
-
zstd_comp_exit(ctx);
103
-
return ret;
104
-
}
105
-
106
-
static void *zstd_alloc_ctx(void)
107
-
{
108
-
int ret;
35
+
zstd_parameters params;
109
36
struct zstd_ctx *ctx;
37
+
size_t wksp_size;
110
38
111
-
ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
39
+
params = zstd_get_params(ZSTD_DEF_LEVEL, ZSTD_MAX_SIZE);
40
+
41
+
wksp_size = max_t(size_t,
42
+
zstd_cstream_workspace_bound(¶ms.cParams),
43
+
zstd_dstream_workspace_bound(ZSTD_MAX_SIZE));
44
+
if (!wksp_size)
45
+
return ERR_PTR(-EINVAL);
46
+
47
+
ctx = kvmalloc(sizeof(*ctx) + wksp_size, GFP_KERNEL);
112
48
if (!ctx)
113
49
return ERR_PTR(-ENOMEM);
114
50
115
-
ret = __zstd_init(ctx);
116
-
if (ret) {
117
-
kfree(ctx);
118
-
return ERR_PTR(ret);
119
-
}
51
+
ctx->params = params;
52
+
ctx->wksp_size = wksp_size;
120
53
121
54
return ctx;
122
55
}
123
56
124
-
static void __zstd_exit(void *ctx)
57
+
static void zstd_free_stream(void *ctx)
125
58
{
126
-
zstd_comp_exit(ctx);
127
-
zstd_decomp_exit(ctx);
59
+
kvfree(ctx);
128
60
}
129
61
130
-
static void zstd_free_ctx(void *ctx)
62
+
static struct crypto_acomp_streams zstd_streams = {
63
+
.alloc_ctx = zstd_alloc_stream,
64
+
.free_ctx = zstd_free_stream,
65
+
};
66
+
67
+
static int zstd_init(struct crypto_acomp *acomp_tfm)
131
68
{
132
-
__zstd_exit(ctx);
133
-
kfree_sensitive(ctx);
69
+
int ret = 0;
70
+
71
+
mutex_lock(&zstd_stream_lock);
72
+
ret = crypto_acomp_alloc_streams(&zstd_streams);
73
+
mutex_unlock(&zstd_stream_lock);
74
+
75
+
return ret;
134
76
}
135
77
136
-
static int __zstd_compress(const u8 *src, unsigned int slen,
137
-
u8 *dst, unsigned int *dlen, void *ctx)
78
+
static void zstd_exit(struct crypto_acomp *acomp_tfm)
138
79
{
139
-
size_t out_len;
140
-
struct zstd_ctx *zctx = ctx;
141
-
const zstd_parameters params = zstd_params();
80
+
crypto_acomp_free_streams(&zstd_streams);
81
+
}
142
82
143
-
out_len = zstd_compress_cctx(zctx->cctx, dst, *dlen, src, slen, ¶ms);
83
+
static int zstd_compress_one(struct acomp_req *req, struct zstd_ctx *ctx,
84
+
const void *src, void *dst, unsigned int *dlen)
85
+
{
86
+
unsigned int out_len;
87
+
88
+
ctx->cctx = zstd_init_cctx(ctx->wksp, ctx->wksp_size);
89
+
if (!ctx->cctx)
90
+
return -EINVAL;
91
+
92
+
out_len = zstd_compress_cctx(ctx->cctx, dst, req->dlen, src, req->slen,
93
+
&ctx->params);
144
94
if (zstd_is_error(out_len))
145
95
return -EINVAL;
96
+
146
97
*dlen = out_len;
98
+
147
99
return 0;
148
100
}
149
101
150
-
static int zstd_scompress(struct crypto_scomp *tfm, const u8 *src,
151
-
unsigned int slen, u8 *dst, unsigned int *dlen,
152
-
void *ctx)
102
+
static int zstd_compress(struct acomp_req *req)
153
103
{
154
-
return __zstd_compress(src, slen, dst, dlen, ctx);
155
-
}
104
+
struct crypto_acomp_stream *s;
105
+
unsigned int pos, scur, dcur;
106
+
unsigned int total_out = 0;
107
+
bool data_available = true;
108
+
zstd_out_buffer outbuf;
109
+
struct acomp_walk walk;
110
+
zstd_in_buffer inbuf;
111
+
struct zstd_ctx *ctx;
112
+
size_t pending_bytes;
113
+
size_t num_bytes;
114
+
int ret;
156
115
157
-
static int __zstd_decompress(const u8 *src, unsigned int slen,
158
-
u8 *dst, unsigned int *dlen, void *ctx)
159
-
{
160
-
size_t out_len;
161
-
struct zstd_ctx *zctx = ctx;
116
+
s = crypto_acomp_lock_stream_bh(&zstd_streams);
117
+
ctx = s->ctx;
162
118
163
-
out_len = zstd_decompress_dctx(zctx->dctx, dst, *dlen, src, slen);
164
-
if (zstd_is_error(out_len))
165
-
return -EINVAL;
166
-
*dlen = out_len;
167
-
return 0;
168
-
}
119
+
ret = acomp_walk_virt(&walk, req, true);
120
+
if (ret)
121
+
goto out;
169
122
170
-
static int zstd_sdecompress(struct crypto_scomp *tfm, const u8 *src,
171
-
unsigned int slen, u8 *dst, unsigned int *dlen,
172
-
void *ctx)
173
-
{
174
-
return __zstd_decompress(src, slen, dst, dlen, ctx);
175
-
}
176
-
177
-
static struct scomp_alg scomp = {
178
-
.alloc_ctx = zstd_alloc_ctx,
179
-
.free_ctx = zstd_free_ctx,
180
-
.compress = zstd_scompress,
181
-
.decompress = zstd_sdecompress,
182
-
.base = {
183
-
.cra_name = "zstd",
184
-
.cra_driver_name = "zstd-scomp",
185
-
.cra_module = THIS_MODULE,
123
+
ctx->cctx = zstd_init_cstream(&ctx->params, 0, ctx->wksp, ctx->wksp_size);
124
+
if (!ctx->cctx) {
125
+
ret = -EINVAL;
126
+
goto out;
186
127
}
128
+
129
+
do {
130
+
dcur = acomp_walk_next_dst(&walk);
131
+
if (!dcur) {
132
+
ret = -ENOSPC;
133
+
goto out;
134
+
}
135
+
136
+
outbuf.pos = 0;
137
+
outbuf.dst = (u8 *)walk.dst.virt.addr;
138
+
outbuf.size = dcur;
139
+
140
+
do {
141
+
scur = acomp_walk_next_src(&walk);
142
+
if (dcur == req->dlen && scur == req->slen) {
143
+
ret = zstd_compress_one(req, ctx, walk.src.virt.addr,
144
+
walk.dst.virt.addr, &total_out);
145
+
acomp_walk_done_src(&walk, scur);
146
+
acomp_walk_done_dst(&walk, dcur);
147
+
goto out;
148
+
}
149
+
150
+
if (scur) {
151
+
inbuf.pos = 0;
152
+
inbuf.src = walk.src.virt.addr;
153
+
inbuf.size = scur;
154
+
} else {
155
+
data_available = false;
156
+
break;
157
+
}
158
+
159
+
num_bytes = zstd_compress_stream(ctx->cctx, &outbuf, &inbuf);
160
+
if (ZSTD_isError(num_bytes)) {
161
+
ret = -EIO;
162
+
goto out;
163
+
}
164
+
165
+
pending_bytes = zstd_flush_stream(ctx->cctx, &outbuf);
166
+
if (ZSTD_isError(pending_bytes)) {
167
+
ret = -EIO;
168
+
goto out;
169
+
}
170
+
acomp_walk_done_src(&walk, inbuf.pos);
171
+
} while (dcur != outbuf.pos);
172
+
173
+
total_out += outbuf.pos;
174
+
acomp_walk_done_dst(&walk, dcur);
175
+
} while (data_available);
176
+
177
+
pos = outbuf.pos;
178
+
num_bytes = zstd_end_stream(ctx->cctx, &outbuf);
179
+
if (ZSTD_isError(num_bytes))
180
+
ret = -EIO;
181
+
else
182
+
total_out += (outbuf.pos - pos);
183
+
184
+
out:
185
+
if (ret)
186
+
req->dlen = 0;
187
+
else
188
+
req->dlen = total_out;
189
+
190
+
crypto_acomp_unlock_stream_bh(s);
191
+
192
+
return ret;
193
+
}
194
+
195
+
static int zstd_decompress_one(struct acomp_req *req, struct zstd_ctx *ctx,
196
+
const void *src, void *dst, unsigned int *dlen)
197
+
{
198
+
size_t out_len;
199
+
200
+
ctx->dctx = zstd_init_dctx(ctx->wksp, ctx->wksp_size);
201
+
if (!ctx->dctx)
202
+
return -EINVAL;
203
+
204
+
out_len = zstd_decompress_dctx(ctx->dctx, dst, req->dlen, src, req->slen);
205
+
if (zstd_is_error(out_len))
206
+
return -EINVAL;
207
+
208
+
*dlen = out_len;
209
+
210
+
return 0;
211
+
}
212
+
213
+
static int zstd_decompress(struct acomp_req *req)
214
+
{
215
+
struct crypto_acomp_stream *s;
216
+
unsigned int total_out = 0;
217
+
unsigned int scur, dcur;
218
+
zstd_out_buffer outbuf;
219
+
struct acomp_walk walk;
220
+
zstd_in_buffer inbuf;
221
+
struct zstd_ctx *ctx;
222
+
size_t pending_bytes;
223
+
int ret;
224
+
225
+
s = crypto_acomp_lock_stream_bh(&zstd_streams);
226
+
ctx = s->ctx;
227
+
228
+
ret = acomp_walk_virt(&walk, req, true);
229
+
if (ret)
230
+
goto out;
231
+
232
+
ctx->dctx = zstd_init_dstream(ZSTD_MAX_SIZE, ctx->wksp, ctx->wksp_size);
233
+
if (!ctx->dctx) {
234
+
ret = -EINVAL;
235
+
goto out;
236
+
}
237
+
238
+
do {
239
+
scur = acomp_walk_next_src(&walk);
240
+
if (scur) {
241
+
inbuf.pos = 0;
242
+
inbuf.size = scur;
243
+
inbuf.src = walk.src.virt.addr;
244
+
} else {
245
+
break;
246
+
}
247
+
248
+
do {
249
+
dcur = acomp_walk_next_dst(&walk);
250
+
if (dcur == req->dlen && scur == req->slen) {
251
+
ret = zstd_decompress_one(req, ctx, walk.src.virt.addr,
252
+
walk.dst.virt.addr, &total_out);
253
+
acomp_walk_done_dst(&walk, dcur);
254
+
acomp_walk_done_src(&walk, scur);
255
+
goto out;
256
+
}
257
+
258
+
if (!dcur) {
259
+
ret = -ENOSPC;
260
+
goto out;
261
+
}
262
+
263
+
outbuf.pos = 0;
264
+
outbuf.dst = (u8 *)walk.dst.virt.addr;
265
+
outbuf.size = dcur;
266
+
267
+
pending_bytes = zstd_decompress_stream(ctx->dctx, &outbuf, &inbuf);
268
+
if (ZSTD_isError(pending_bytes)) {
269
+
ret = -EIO;
270
+
goto out;
271
+
}
272
+
273
+
total_out += outbuf.pos;
274
+
275
+
acomp_walk_done_dst(&walk, outbuf.pos);
276
+
} while (inbuf.pos != scur);
277
+
278
+
acomp_walk_done_src(&walk, scur);
279
+
} while (ret == 0);
280
+
281
+
out:
282
+
if (ret)
283
+
req->dlen = 0;
284
+
else
285
+
req->dlen = total_out;
286
+
287
+
crypto_acomp_unlock_stream_bh(s);
288
+
289
+
return ret;
290
+
}
291
+
292
+
static struct acomp_alg zstd_acomp = {
293
+
.base = {
294
+
.cra_name = "zstd",
295
+
.cra_driver_name = "zstd-generic",
296
+
.cra_flags = CRYPTO_ALG_REQ_VIRT,
297
+
.cra_module = THIS_MODULE,
298
+
},
299
+
.init = zstd_init,
300
+
.exit = zstd_exit,
301
+
.compress = zstd_compress,
302
+
.decompress = zstd_decompress,
187
303
};
188
304
189
305
static int __init zstd_mod_init(void)
190
306
{
191
-
return crypto_register_scomp(&scomp);
307
+
return crypto_register_acomp(&zstd_acomp);
192
308
}
193
309
194
310
static void __exit zstd_mod_fini(void)
195
311
{
196
-
crypto_unregister_scomp(&scomp);
312
+
crypto_unregister_acomp(&zstd_acomp);
197
313
}
198
314
199
315
module_init(zstd_mod_init);
-1
drivers/char/hw_random/atmel-rng.c
-1
drivers/char/hw_random/atmel-rng.c
-1
drivers/char/hw_random/cctrng.c
-1
drivers/char/hw_random/cctrng.c
+3
-2
drivers/char/hw_random/mtk-rng.c
+3
-2
drivers/char/hw_random/mtk-rng.c
···
98
98
max -= sizeof(u32);
99
99
}
100
100
101
-
pm_runtime_mark_last_busy(priv->dev);
102
101
pm_runtime_put_sync_autosuspend(priv->dev);
103
102
104
103
return retval || !wait ? retval : -EIO;
···
142
143
dev_set_drvdata(&pdev->dev, priv);
143
144
pm_runtime_set_autosuspend_delay(&pdev->dev, RNG_AUTOSUSPEND_TIMEOUT);
144
145
pm_runtime_use_autosuspend(&pdev->dev);
145
-
devm_pm_runtime_enable(&pdev->dev);
146
+
ret = devm_pm_runtime_enable(&pdev->dev);
147
+
if (ret)
148
+
return ret;
146
149
147
150
dev_info(&pdev->dev, "registered RNG driver\n");
148
151
-1
drivers/char/hw_random/npcm-rng.c
-1
drivers/char/hw_random/npcm-rng.c
-1
drivers/char/hw_random/omap3-rom-rng.c
-1
drivers/char/hw_random/omap3-rom-rng.c
-3
drivers/char/hw_random/rockchip-rng.c
-3
drivers/char/hw_random/rockchip-rng.c
···
223
223
/* Read random data stored in the registers */
224
224
memcpy_fromio(buf, rk_rng->base + TRNG_RNG_DOUT, to_read);
225
225
out:
226
-
pm_runtime_mark_last_busy(rk_rng->dev);
227
226
pm_runtime_put_sync_autosuspend(rk_rng->dev);
228
227
229
228
return (ret < 0) ? ret : to_read;
···
262
263
memcpy_fromio(buf, rk_rng->base + RKRNG_TRNG_DATA0, to_read);
263
264
264
265
out:
265
-
pm_runtime_mark_last_busy(rk_rng->dev);
266
266
pm_runtime_put_sync_autosuspend(rk_rng->dev);
267
267
268
268
return (ret < 0) ? ret : to_read;
···
353
355
/* close the TRNG */
354
356
rk_rng_writel(rk_rng, TRNG_V1_CTRL_NOP, TRNG_V1_CTRL);
355
357
356
-
pm_runtime_mark_last_busy(rk_rng->dev);
357
358
pm_runtime_put_sync_autosuspend(rk_rng->dev);
358
359
359
360
return (ret < 0) ? ret : to_read;
-1
drivers/char/hw_random/stm32-rng.c
-1
drivers/char/hw_random/stm32-rng.c
+13
drivers/crypto/Kconfig
+13
drivers/crypto/Kconfig
···
188
188
Select this option if you want to use the paes cipher
189
189
for example to use protected key encrypted devices.
190
190
191
+
config CRYPTO_PHMAC_S390
192
+
tristate "PHMAC cipher algorithms"
193
+
depends on S390
194
+
depends on PKEY
195
+
select CRYPTO_HASH
196
+
select CRYPTO_ENGINE
197
+
help
198
+
This is the s390 hardware accelerated implementation of the
199
+
protected key HMAC support for SHA224, SHA256, SHA384 and SHA512.
200
+
201
+
Select this option if you want to use the phmac digests
202
+
for example to use dm-integrity with secure/protected keys.
203
+
191
204
config S390_PRNG
192
205
tristate "Pseudo random number generator device driver"
193
206
depends on S390
+8
-7
drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c
+8
-7
drivers/crypto/allwinner/sun8i-ce/sun8i-ce-cipher.c
···
206
206
cet->t_key = desc_addr_val_le32(ce, rctx->addr_key);
207
207
208
208
ivsize = crypto_skcipher_ivsize(tfm);
209
-
if (areq->iv && crypto_skcipher_ivsize(tfm) > 0) {
210
-
rctx->ivlen = ivsize;
209
+
if (areq->iv && ivsize > 0) {
211
210
if (rctx->op_dir & CE_DECRYPTION) {
212
211
offset = areq->cryptlen - ivsize;
213
212
scatterwalk_map_and_copy(chan->backup_iv, areq->src,
214
213
offset, ivsize, 0);
215
214
}
216
215
memcpy(chan->bounce_iv, areq->iv, ivsize);
217
-
rctx->addr_iv = dma_map_single(ce->dev, chan->bounce_iv, rctx->ivlen,
216
+
rctx->addr_iv = dma_map_single(ce->dev, chan->bounce_iv, ivsize,
218
217
DMA_TO_DEVICE);
219
218
if (dma_mapping_error(ce->dev, rctx->addr_iv)) {
220
219
dev_err(ce->dev, "Cannot DMA MAP IV\n");
···
277
278
}
278
279
279
280
chan->timeout = areq->cryptlen;
280
-
rctx->nr_sgs = nr_sgs;
281
-
rctx->nr_sgd = nr_sgd;
281
+
rctx->nr_sgs = ns;
282
+
rctx->nr_sgd = nd;
282
283
return 0;
283
284
284
285
theend_sgs:
···
295
296
theend_iv:
296
297
if (areq->iv && ivsize > 0) {
297
298
if (!dma_mapping_error(ce->dev, rctx->addr_iv))
298
-
dma_unmap_single(ce->dev, rctx->addr_iv, rctx->ivlen, DMA_TO_DEVICE);
299
+
dma_unmap_single(ce->dev, rctx->addr_iv, ivsize,
300
+
DMA_TO_DEVICE);
299
301
300
302
offset = areq->cryptlen - ivsize;
301
303
if (rctx->op_dir & CE_DECRYPTION) {
···
345
345
346
346
if (areq->iv && ivsize > 0) {
347
347
if (cet->t_iv)
348
-
dma_unmap_single(ce->dev, rctx->addr_iv, rctx->ivlen, DMA_TO_DEVICE);
348
+
dma_unmap_single(ce->dev, rctx->addr_iv, ivsize,
349
+
DMA_TO_DEVICE);
349
350
offset = areq->cryptlen - ivsize;
350
351
if (rctx->op_dir & CE_DECRYPTION) {
351
352
memcpy(areq->iv, chan->backup_iv, ivsize);
+3
-3
drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c
+3
-3
drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c
···
342
342
algt = container_of(alg, struct sun8i_ce_alg_template, alg.hash.base);
343
343
ce = algt->ce;
344
344
345
-
bs = algt->alg.hash.base.halg.base.cra_blocksize;
346
-
digestsize = algt->alg.hash.base.halg.digestsize;
345
+
bs = crypto_ahash_blocksize(tfm);
346
+
digestsize = crypto_ahash_digestsize(tfm);
347
347
if (digestsize == SHA224_DIGEST_SIZE)
348
348
digestsize = SHA256_DIGEST_SIZE;
349
349
if (digestsize == SHA384_DIGEST_SIZE)
···
455
455
err_unmap_result:
456
456
dma_unmap_single(ce->dev, addr_res, digestsize, DMA_FROM_DEVICE);
457
457
if (!err)
458
-
memcpy(areq->result, result, algt->alg.hash.base.halg.digestsize);
458
+
memcpy(areq->result, result, crypto_ahash_digestsize(tfm));
459
459
460
460
err_unmap_src:
461
461
dma_unmap_sg(ce->dev, areq->src, ns, DMA_TO_DEVICE);
-2
drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h
-2
drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h
···
260
260
* struct sun8i_cipher_req_ctx - context for a skcipher request
261
261
* @op_dir: direction (encrypt vs decrypt) for this request
262
262
* @flow: the flow to use for this request
263
-
* @ivlen: size of bounce_iv
264
263
* @nr_sgs: The number of source SG (as given by dma_map_sg())
265
264
* @nr_sgd: The number of destination SG (as given by dma_map_sg())
266
265
* @addr_iv: The IV addr returned by dma_map_single, need to unmap later
···
269
270
struct sun8i_cipher_req_ctx {
270
271
u32 op_dir;
271
272
int flow;
272
-
unsigned int ivlen;
273
273
int nr_sgs;
274
274
int nr_sgd;
275
275
dma_addr_t addr_iv;
+198
-604
drivers/crypto/aspeed/aspeed-hace-hash.c
+198
-604
drivers/crypto/aspeed/aspeed-hace-hash.c
···
5
5
6
6
#include "aspeed-hace.h"
7
7
#include <crypto/engine.h>
8
-
#include <crypto/hmac.h>
9
8
#include <crypto/internal/hash.h>
10
9
#include <crypto/scatterwalk.h>
11
10
#include <crypto/sha1.h>
···
13
14
#include <linux/err.h>
14
15
#include <linux/io.h>
15
16
#include <linux/kernel.h>
17
+
#include <linux/scatterlist.h>
16
18
#include <linux/string.h>
17
19
18
20
#ifdef CONFIG_CRYPTO_DEV_ASPEED_DEBUG
···
59
59
cpu_to_be64(SHA512_H6), cpu_to_be64(SHA512_H7)
60
60
};
61
61
62
+
static int aspeed_sham_init(struct ahash_request *req);
63
+
static int aspeed_ahash_req_update(struct aspeed_hace_dev *hace_dev);
64
+
65
+
static int aspeed_sham_export(struct ahash_request *req, void *out)
66
+
{
67
+
struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req);
68
+
union {
69
+
u8 *u8;
70
+
u64 *u64;
71
+
} p = { .u8 = out };
72
+
73
+
memcpy(out, rctx->digest, rctx->ivsize);
74
+
p.u8 += rctx->ivsize;
75
+
put_unaligned(rctx->digcnt[0], p.u64++);
76
+
if (rctx->ivsize == 64)
77
+
put_unaligned(rctx->digcnt[1], p.u64);
78
+
return 0;
79
+
}
80
+
81
+
static int aspeed_sham_import(struct ahash_request *req, const void *in)
82
+
{
83
+
struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req);
84
+
union {
85
+
const u8 *u8;
86
+
const u64 *u64;
87
+
} p = { .u8 = in };
88
+
int err;
89
+
90
+
err = aspeed_sham_init(req);
91
+
if (err)
92
+
return err;
93
+
94
+
memcpy(rctx->digest, in, rctx->ivsize);
95
+
p.u8 += rctx->ivsize;
96
+
rctx->digcnt[0] = get_unaligned(p.u64++);
97
+
if (rctx->ivsize == 64)
98
+
rctx->digcnt[1] = get_unaligned(p.u64);
99
+
return 0;
100
+
}
101
+
62
102
/* The purpose of this padding is to ensure that the padded message is a
63
103
* multiple of 512 bits (SHA1/SHA224/SHA256) or 1024 bits (SHA384/SHA512).
64
104
* The bit "1" is appended at the end of the message followed by
···
114
74
* - if message length < 112 bytes then padlen = 112 - message length
115
75
* - else padlen = 128 + 112 - message length
116
76
*/
117
-
static void aspeed_ahash_fill_padding(struct aspeed_hace_dev *hace_dev,
118
-
struct aspeed_sham_reqctx *rctx)
77
+
static int aspeed_ahash_fill_padding(struct aspeed_hace_dev *hace_dev,
78
+
struct aspeed_sham_reqctx *rctx, u8 *buf)
119
79
{
120
-
unsigned int index, padlen;
80
+
unsigned int index, padlen, bitslen;
121
81
__be64 bits[2];
122
82
123
83
AHASH_DBG(hace_dev, "rctx flags:0x%x\n", (u32)rctx->flags);
···
127
87
case SHA_FLAGS_SHA224:
128
88
case SHA_FLAGS_SHA256:
129
89
bits[0] = cpu_to_be64(rctx->digcnt[0] << 3);
130
-
index = rctx->bufcnt & 0x3f;
90
+
index = rctx->digcnt[0] & 0x3f;
131
91
padlen = (index < 56) ? (56 - index) : ((64 + 56) - index);
132
-
*(rctx->buffer + rctx->bufcnt) = 0x80;
133
-
memset(rctx->buffer + rctx->bufcnt + 1, 0, padlen - 1);
134
-
memcpy(rctx->buffer + rctx->bufcnt + padlen, bits, 8);
135
-
rctx->bufcnt += padlen + 8;
92
+
bitslen = 8;
136
93
break;
137
94
default:
138
95
bits[1] = cpu_to_be64(rctx->digcnt[0] << 3);
139
96
bits[0] = cpu_to_be64(rctx->digcnt[1] << 3 |
140
97
rctx->digcnt[0] >> 61);
141
-
index = rctx->bufcnt & 0x7f;
98
+
index = rctx->digcnt[0] & 0x7f;
142
99
padlen = (index < 112) ? (112 - index) : ((128 + 112) - index);
143
-
*(rctx->buffer + rctx->bufcnt) = 0x80;
144
-
memset(rctx->buffer + rctx->bufcnt + 1, 0, padlen - 1);
145
-
memcpy(rctx->buffer + rctx->bufcnt + padlen, bits, 16);
146
-
rctx->bufcnt += padlen + 16;
100
+
bitslen = 16;
147
101
break;
148
102
}
103
+
buf[0] = 0x80;
104
+
memset(buf + 1, 0, padlen - 1);
105
+
memcpy(buf + padlen, bits, bitslen);
106
+
return padlen + bitslen;
107
+
}
108
+
109
+
static void aspeed_ahash_update_counter(struct aspeed_sham_reqctx *rctx,
110
+
unsigned int len)
111
+
{
112
+
rctx->offset += len;
113
+
rctx->digcnt[0] += len;
114
+
if (rctx->digcnt[0] < len)
115
+
rctx->digcnt[1]++;
149
116
}
150
117
151
118
/*
···
164
117
struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine;
165
118
struct ahash_request *req = hash_engine->req;
166
119
struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req);
167
-
int length, remain;
120
+
unsigned int length, remain;
121
+
bool final = false;
168
122
169
-
length = rctx->total + rctx->bufcnt;
170
-
remain = length % rctx->block_size;
123
+
length = rctx->total - rctx->offset;
124
+
remain = length - round_down(length, rctx->block_size);
171
125
172
126
AHASH_DBG(hace_dev, "length:0x%x, remain:0x%x\n", length, remain);
173
127
174
-
if (rctx->bufcnt)
175
-
memcpy(hash_engine->ahash_src_addr, rctx->buffer, rctx->bufcnt);
128
+
if (length > ASPEED_HASH_SRC_DMA_BUF_LEN)
129
+
length = ASPEED_HASH_SRC_DMA_BUF_LEN;
130
+
else if (rctx->flags & SHA_FLAGS_FINUP) {
131
+
if (round_up(length, rctx->block_size) + rctx->block_size >
132
+
ASPEED_CRYPTO_SRC_DMA_BUF_LEN)
133
+
length = round_down(length - 1, rctx->block_size);
134
+
else
135
+
final = true;
136
+
} else
137
+
length -= remain;
138
+
scatterwalk_map_and_copy(hash_engine->ahash_src_addr, rctx->src_sg,
139
+
rctx->offset, length, 0);
140
+
aspeed_ahash_update_counter(rctx, length);
141
+
if (final)
142
+
length += aspeed_ahash_fill_padding(
143
+
hace_dev, rctx, hash_engine->ahash_src_addr + length);
176
144
177
-
if (rctx->total + rctx->bufcnt < ASPEED_CRYPTO_SRC_DMA_BUF_LEN) {
178
-
scatterwalk_map_and_copy(hash_engine->ahash_src_addr +
179
-
rctx->bufcnt, rctx->src_sg,
180
-
rctx->offset, rctx->total - remain, 0);
181
-
rctx->offset += rctx->total - remain;
182
-
183
-
} else {
184
-
dev_warn(hace_dev->dev, "Hash data length is too large\n");
185
-
return -EINVAL;
186
-
}
187
-
188
-
scatterwalk_map_and_copy(rctx->buffer, rctx->src_sg,
189
-
rctx->offset, remain, 0);
190
-
191
-
rctx->bufcnt = remain;
192
145
rctx->digest_dma_addr = dma_map_single(hace_dev->dev, rctx->digest,
193
146
SHA512_DIGEST_SIZE,
194
147
DMA_BIDIRECTIONAL);
···
197
150
return -ENOMEM;
198
151
}
199
152
200
-
hash_engine->src_length = length - remain;
153
+
hash_engine->src_length = length;
201
154
hash_engine->src_dma = hash_engine->ahash_src_dma_addr;
202
155
hash_engine->digest_dma = rctx->digest_dma_addr;
203
156
···
213
166
struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine;
214
167
struct ahash_request *req = hash_engine->req;
215
168
struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req);
169
+
bool final = rctx->flags & SHA_FLAGS_FINUP;
170
+
int remain, sg_len, i, max_sg_nents;
171
+
unsigned int length, offset, total;
216
172
struct aspeed_sg_list *src_list;
217
173
struct scatterlist *s;
218
-
int length, remain, sg_len, i;
219
174
int rc = 0;
220
175
221
-
remain = (rctx->total + rctx->bufcnt) % rctx->block_size;
222
-
length = rctx->total + rctx->bufcnt - remain;
176
+
offset = rctx->offset;
177
+
length = rctx->total - offset;
178
+
remain = final ? 0 : length - round_down(length, rctx->block_size);
179
+
length -= remain;
223
180
224
-
AHASH_DBG(hace_dev, "%s:0x%x, %s:%zu, %s:0x%x, %s:0x%x\n",
225
-
"rctx total", rctx->total, "bufcnt", rctx->bufcnt,
181
+
AHASH_DBG(hace_dev, "%s:0x%x, %s:0x%x, %s:0x%x\n",
182
+
"rctx total", rctx->total,
226
183
"length", length, "remain", remain);
227
184
228
185
sg_len = dma_map_sg(hace_dev->dev, rctx->src_sg, rctx->src_nents,
···
237
186
goto end;
238
187
}
239
188
189
+
max_sg_nents = ASPEED_HASH_SRC_DMA_BUF_LEN / sizeof(*src_list) - final;
190
+
sg_len = min(sg_len, max_sg_nents);
240
191
src_list = (struct aspeed_sg_list *)hash_engine->ahash_src_addr;
241
192
rctx->digest_dma_addr = dma_map_single(hace_dev->dev, rctx->digest,
242
193
SHA512_DIGEST_SIZE,
···
249
196
goto free_src_sg;
250
197
}
251
198
252
-
if (rctx->bufcnt != 0) {
253
-
u32 phy_addr;
254
-
u32 len;
199
+
total = 0;
200
+
for_each_sg(rctx->src_sg, s, sg_len, i) {
201
+
u32 phy_addr = sg_dma_address(s);
202
+
u32 len = sg_dma_len(s);
255
203
204
+
if (len <= offset) {
205
+
offset -= len;
206
+
continue;
207
+
}
208
+
209
+
len -= offset;
210
+
phy_addr += offset;
211
+
offset = 0;
212
+
213
+
if (length > len)
214
+
length -= len;
215
+
else {
216
+
/* Last sg list */
217
+
len = length;
218
+
length = 0;
219
+
}
220
+
221
+
total += len;
222
+
src_list[i].phy_addr = cpu_to_le32(phy_addr);
223
+
src_list[i].len = cpu_to_le32(len);
224
+
}
225
+
226
+
if (length != 0) {
227
+
total = round_down(total, rctx->block_size);
228
+
final = false;
229
+
}
230
+
231
+
aspeed_ahash_update_counter(rctx, total);
232
+
if (final) {
233
+
int len = aspeed_ahash_fill_padding(hace_dev, rctx,
234
+
rctx->buffer);
235
+
236
+
total += len;
256
237
rctx->buffer_dma_addr = dma_map_single(hace_dev->dev,
257
238
rctx->buffer,
258
-
rctx->block_size * 2,
239
+
sizeof(rctx->buffer),
259
240
DMA_TO_DEVICE);
260
241
if (dma_mapping_error(hace_dev->dev, rctx->buffer_dma_addr)) {
261
242
dev_warn(hace_dev->dev, "dma_map() rctx buffer error\n");
···
297
210
goto free_rctx_digest;
298
211
}
299
212
300
-
phy_addr = rctx->buffer_dma_addr;
301
-
len = rctx->bufcnt;
302
-
length -= len;
303
-
304
-
/* Last sg list */
305
-
if (length == 0)
306
-
len |= HASH_SG_LAST_LIST;
307
-
308
-
src_list[0].phy_addr = cpu_to_le32(phy_addr);
309
-
src_list[0].len = cpu_to_le32(len);
310
-
src_list++;
213
+
src_list[i].phy_addr = cpu_to_le32(rctx->buffer_dma_addr);
214
+
src_list[i].len = cpu_to_le32(len);
215
+
i++;
311
216
}
217
+
src_list[i - 1].len |= cpu_to_le32(HASH_SG_LAST_LIST);
312
218
313
-
if (length != 0) {
314
-
for_each_sg(rctx->src_sg, s, sg_len, i) {
315
-
u32 phy_addr = sg_dma_address(s);
316
-
u32 len = sg_dma_len(s);
317
-
318
-
if (length > len)
319
-
length -= len;
320
-
else {
321
-
/* Last sg list */
322
-
len = length;
323
-
len |= HASH_SG_LAST_LIST;
324
-
length = 0;
325
-
}
326
-
327
-
src_list[i].phy_addr = cpu_to_le32(phy_addr);
328
-
src_list[i].len = cpu_to_le32(len);
329
-
}
330
-
}
331
-
332
-
if (length != 0) {
333
-
rc = -EINVAL;
334
-
goto free_rctx_buffer;
335
-
}
336
-
337
-
rctx->offset = rctx->total - remain;
338
-
hash_engine->src_length = rctx->total + rctx->bufcnt - remain;
219
+
hash_engine->src_length = total;
339
220
hash_engine->src_dma = hash_engine->ahash_src_dma_addr;
340
221
hash_engine->digest_dma = rctx->digest_dma_addr;
341
222
342
223
return 0;
343
224
344
-
free_rctx_buffer:
345
-
if (rctx->bufcnt != 0)
346
-
dma_unmap_single(hace_dev->dev, rctx->buffer_dma_addr,
347
-
rctx->block_size * 2, DMA_TO_DEVICE);
348
225
free_rctx_digest:
349
226
dma_unmap_single(hace_dev->dev, rctx->digest_dma_addr,
350
227
SHA512_DIGEST_SIZE, DMA_BIDIRECTIONAL);
···
323
272
{
324
273
struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine;
325
274
struct ahash_request *req = hash_engine->req;
326
-
327
-
AHASH_DBG(hace_dev, "\n");
328
-
329
-
hash_engine->flags &= ~CRYPTO_FLAGS_BUSY;
330
-
331
-
crypto_finalize_hash_request(hace_dev->crypt_engine_hash, req, 0);
332
-
333
-
return 0;
334
-
}
335
-
336
-
/*
337
-
* Copy digest to the corresponding request result.
338
-
* This function will be called at final() stage.
339
-
*/
340
-
static int aspeed_ahash_transfer(struct aspeed_hace_dev *hace_dev)
341
-
{
342
-
struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine;
343
-
struct ahash_request *req = hash_engine->req;
344
275
struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req);
345
276
346
277
AHASH_DBG(hace_dev, "\n");
···
330
297
dma_unmap_single(hace_dev->dev, rctx->digest_dma_addr,
331
298
SHA512_DIGEST_SIZE, DMA_BIDIRECTIONAL);
332
299
333
-
dma_unmap_single(hace_dev->dev, rctx->buffer_dma_addr,
334
-
rctx->block_size * 2, DMA_TO_DEVICE);
300
+
if (rctx->total - rctx->offset >= rctx->block_size ||
301
+
(rctx->total != rctx->offset && rctx->flags & SHA_FLAGS_FINUP))
302
+
return aspeed_ahash_req_update(hace_dev);
335
303
336
-
memcpy(req->result, rctx->digest, rctx->digsize);
304
+
hash_engine->flags &= ~CRYPTO_FLAGS_BUSY;
337
305
338
-
return aspeed_ahash_complete(hace_dev);
306
+
if (rctx->flags & SHA_FLAGS_FINUP)
307
+
memcpy(req->result, rctx->digest, rctx->digsize);
308
+
309
+
crypto_finalize_hash_request(hace_dev->crypt_engine_hash, req,
310
+
rctx->total - rctx->offset);
311
+
312
+
return 0;
339
313
}
340
314
341
315
/*
···
378
338
return -EINPROGRESS;
379
339
}
380
340
381
-
/*
382
-
* HMAC resume aims to do the second pass produces
383
-
* the final HMAC code derived from the inner hash
384
-
* result and the outer key.
385
-
*/
386
-
static int aspeed_ahash_hmac_resume(struct aspeed_hace_dev *hace_dev)
387
-
{
388
-
struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine;
389
-
struct ahash_request *req = hash_engine->req;
390
-
struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req);
391
-
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
392
-
struct aspeed_sham_ctx *tctx = crypto_ahash_ctx(tfm);
393
-
struct aspeed_sha_hmac_ctx *bctx = tctx->base;
394
-
int rc = 0;
395
-
396
-
AHASH_DBG(hace_dev, "\n");
397
-
398
-
dma_unmap_single(hace_dev->dev, rctx->digest_dma_addr,
399
-
SHA512_DIGEST_SIZE, DMA_BIDIRECTIONAL);
400
-
401
-
dma_unmap_single(hace_dev->dev, rctx->buffer_dma_addr,
402
-
rctx->block_size * 2, DMA_TO_DEVICE);
403
-
404
-
/* o key pad + hash sum 1 */
405
-
memcpy(rctx->buffer, bctx->opad, rctx->block_size);
406
-
memcpy(rctx->buffer + rctx->block_size, rctx->digest, rctx->digsize);
407
-
408
-
rctx->bufcnt = rctx->block_size + rctx->digsize;
409
-
rctx->digcnt[0] = rctx->block_size + rctx->digsize;
410
-
411
-
aspeed_ahash_fill_padding(hace_dev, rctx);
412
-
memcpy(rctx->digest, rctx->sha_iv, rctx->ivsize);
413
-
414
-
rctx->digest_dma_addr = dma_map_single(hace_dev->dev, rctx->digest,
415
-
SHA512_DIGEST_SIZE,
416
-
DMA_BIDIRECTIONAL);
417
-
if (dma_mapping_error(hace_dev->dev, rctx->digest_dma_addr)) {
418
-
dev_warn(hace_dev->dev, "dma_map() rctx digest error\n");
419
-
rc = -ENOMEM;
420
-
goto end;
421
-
}
422
-
423
-
rctx->buffer_dma_addr = dma_map_single(hace_dev->dev, rctx->buffer,
424
-
rctx->block_size * 2,
425
-
DMA_TO_DEVICE);
426
-
if (dma_mapping_error(hace_dev->dev, rctx->buffer_dma_addr)) {
427
-
dev_warn(hace_dev->dev, "dma_map() rctx buffer error\n");
428
-
rc = -ENOMEM;
429
-
goto free_rctx_digest;
430
-
}
431
-
432
-
hash_engine->src_dma = rctx->buffer_dma_addr;
433
-
hash_engine->src_length = rctx->bufcnt;
434
-
hash_engine->digest_dma = rctx->digest_dma_addr;
435
-
436
-
return aspeed_hace_ahash_trigger(hace_dev, aspeed_ahash_transfer);
437
-
438
-
free_rctx_digest:
439
-
dma_unmap_single(hace_dev->dev, rctx->digest_dma_addr,
440
-
SHA512_DIGEST_SIZE, DMA_BIDIRECTIONAL);
441
-
end:
442
-
return rc;
443
-
}
444
-
445
-
static int aspeed_ahash_req_final(struct aspeed_hace_dev *hace_dev)
446
-
{
447
-
struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine;
448
-
struct ahash_request *req = hash_engine->req;
449
-
struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req);
450
-
int rc = 0;
451
-
452
-
AHASH_DBG(hace_dev, "\n");
453
-
454
-
aspeed_ahash_fill_padding(hace_dev, rctx);
455
-
456
-
rctx->digest_dma_addr = dma_map_single(hace_dev->dev,
457
-
rctx->digest,
458
-
SHA512_DIGEST_SIZE,
459
-
DMA_BIDIRECTIONAL);
460
-
if (dma_mapping_error(hace_dev->dev, rctx->digest_dma_addr)) {
461
-
dev_warn(hace_dev->dev, "dma_map() rctx digest error\n");
462
-
rc = -ENOMEM;
463
-
goto end;
464
-
}
465
-
466
-
rctx->buffer_dma_addr = dma_map_single(hace_dev->dev,
467
-
rctx->buffer,
468
-
rctx->block_size * 2,
469
-
DMA_TO_DEVICE);
470
-
if (dma_mapping_error(hace_dev->dev, rctx->buffer_dma_addr)) {
471
-
dev_warn(hace_dev->dev, "dma_map() rctx buffer error\n");
472
-
rc = -ENOMEM;
473
-
goto free_rctx_digest;
474
-
}
475
-
476
-
hash_engine->src_dma = rctx->buffer_dma_addr;
477
-
hash_engine->src_length = rctx->bufcnt;
478
-
hash_engine->digest_dma = rctx->digest_dma_addr;
479
-
480
-
if (rctx->flags & SHA_FLAGS_HMAC)
481
-
return aspeed_hace_ahash_trigger(hace_dev,
482
-
aspeed_ahash_hmac_resume);
483
-
484
-
return aspeed_hace_ahash_trigger(hace_dev, aspeed_ahash_transfer);
485
-
486
-
free_rctx_digest:
487
-
dma_unmap_single(hace_dev->dev, rctx->digest_dma_addr,
488
-
SHA512_DIGEST_SIZE, DMA_BIDIRECTIONAL);
489
-
end:
490
-
return rc;
491
-
}
492
-
493
341
static int aspeed_ahash_update_resume_sg(struct aspeed_hace_dev *hace_dev)
494
342
{
495
343
struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine;
···
389
461
dma_unmap_sg(hace_dev->dev, rctx->src_sg, rctx->src_nents,
390
462
DMA_TO_DEVICE);
391
463
392
-
if (rctx->bufcnt != 0)
464
+
if (rctx->flags & SHA_FLAGS_FINUP && rctx->total == rctx->offset)
393
465
dma_unmap_single(hace_dev->dev, rctx->buffer_dma_addr,
394
-
rctx->block_size * 2,
395
-
DMA_TO_DEVICE);
466
+
sizeof(rctx->buffer), DMA_TO_DEVICE);
396
467
397
-
dma_unmap_single(hace_dev->dev, rctx->digest_dma_addr,
398
-
SHA512_DIGEST_SIZE, DMA_BIDIRECTIONAL);
399
-
400
-
scatterwalk_map_and_copy(rctx->buffer, rctx->src_sg, rctx->offset,
401
-
rctx->total - rctx->offset, 0);
402
-
403
-
rctx->bufcnt = rctx->total - rctx->offset;
404
468
rctx->cmd &= ~HASH_CMD_HASH_SRC_SG_CTRL;
405
-
406
-
if (rctx->flags & SHA_FLAGS_FINUP)
407
-
return aspeed_ahash_req_final(hace_dev);
408
-
409
-
return aspeed_ahash_complete(hace_dev);
410
-
}
411
-
412
-
static int aspeed_ahash_update_resume(struct aspeed_hace_dev *hace_dev)
413
-
{
414
-
struct aspeed_engine_hash *hash_engine = &hace_dev->hash_engine;
415
-
struct ahash_request *req = hash_engine->req;
416
-
struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req);
417
-
418
-
AHASH_DBG(hace_dev, "\n");
419
-
420
-
dma_unmap_single(hace_dev->dev, rctx->digest_dma_addr,
421
-
SHA512_DIGEST_SIZE, DMA_BIDIRECTIONAL);
422
-
423
-
if (rctx->flags & SHA_FLAGS_FINUP)
424
-
return aspeed_ahash_req_final(hace_dev);
425
469
426
470
return aspeed_ahash_complete(hace_dev);
427
471
}
···
413
513
resume = aspeed_ahash_update_resume_sg;
414
514
415
515
} else {
416
-
resume = aspeed_ahash_update_resume;
516
+
resume = aspeed_ahash_complete;
417
517
}
418
518
419
519
ret = hash_engine->dma_prepare(hace_dev);
···
430
530
hace_dev->crypt_engine_hash, req);
431
531
}
432
532
533
+
static noinline int aspeed_ahash_fallback(struct ahash_request *req)
534
+
{
535
+
struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req);
536
+
HASH_FBREQ_ON_STACK(fbreq, req);
537
+
u8 *state = rctx->buffer;
538
+
struct scatterlist sg[2];
539
+
struct scatterlist *ssg;
540
+
int ret;
541
+
542
+
ssg = scatterwalk_ffwd(sg, req->src, rctx->offset);
543
+
ahash_request_set_crypt(fbreq, ssg, req->result,
544
+
rctx->total - rctx->offset);
545
+
546
+
ret = aspeed_sham_export(req, state) ?:
547
+
crypto_ahash_import_core(fbreq, state);
548
+
549
+
if (rctx->flags & SHA_FLAGS_FINUP)
550
+
ret = ret ?: crypto_ahash_finup(fbreq);
551
+
else
552
+
ret = ret ?: crypto_ahash_update(fbreq) ?:
553
+
crypto_ahash_export_core(fbreq, state) ?:
554
+
aspeed_sham_import(req, state);
555
+
HASH_REQUEST_ZERO(fbreq);
556
+
return ret;
557
+
}
558
+
433
559
static int aspeed_ahash_do_request(struct crypto_engine *engine, void *areq)
434
560
{
435
561
struct ahash_request *req = ahash_request_cast(areq);
436
-
struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req);
437
562
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
438
563
struct aspeed_sham_ctx *tctx = crypto_ahash_ctx(tfm);
439
564
struct aspeed_hace_dev *hace_dev = tctx->hace_dev;
440
565
struct aspeed_engine_hash *hash_engine;
441
-
int ret = 0;
566
+
int ret;
442
567
443
568
hash_engine = &hace_dev->hash_engine;
444
569
hash_engine->flags |= CRYPTO_FLAGS_BUSY;
445
570
446
-
if (rctx->op == SHA_OP_UPDATE)
447
-
ret = aspeed_ahash_req_update(hace_dev);
448
-
else if (rctx->op == SHA_OP_FINAL)
449
-
ret = aspeed_ahash_req_final(hace_dev);
450
-
571
+
ret = aspeed_ahash_req_update(hace_dev);
451
572
if (ret != -EINPROGRESS)
452
-
return ret;
573
+
return aspeed_ahash_fallback(req);
453
574
454
575
return 0;
455
576
}
···
511
590
rctx->total = req->nbytes;
512
591
rctx->src_sg = req->src;
513
592
rctx->offset = 0;
514
-
rctx->src_nents = sg_nents(req->src);
515
-
rctx->op = SHA_OP_UPDATE;
516
-
517
-
rctx->digcnt[0] += rctx->total;
518
-
if (rctx->digcnt[0] < rctx->total)
519
-
rctx->digcnt[1]++;
520
-
521
-
if (rctx->bufcnt + rctx->total < rctx->block_size) {
522
-
scatterwalk_map_and_copy(rctx->buffer + rctx->bufcnt,
523
-
rctx->src_sg, rctx->offset,
524
-
rctx->total, 0);
525
-
rctx->bufcnt += rctx->total;
526
-
527
-
return 0;
528
-
}
529
-
530
-
return aspeed_hace_hash_handle_queue(hace_dev, req);
531
-
}
532
-
533
-
static int aspeed_sham_shash_digest(struct crypto_shash *tfm, u32 flags,
534
-
const u8 *data, unsigned int len, u8 *out)
535
-
{
536
-
SHASH_DESC_ON_STACK(shash, tfm);
537
-
538
-
shash->tfm = tfm;
539
-
540
-
return crypto_shash_digest(shash, data, len, out);
541
-
}
542
-
543
-
static int aspeed_sham_final(struct ahash_request *req)
544
-
{
545
-
struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req);
546
-
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
547
-
struct aspeed_sham_ctx *tctx = crypto_ahash_ctx(tfm);
548
-
struct aspeed_hace_dev *hace_dev = tctx->hace_dev;
549
-
550
-
AHASH_DBG(hace_dev, "req->nbytes:%d, rctx->total:%d\n",
551
-
req->nbytes, rctx->total);
552
-
rctx->op = SHA_OP_FINAL;
593
+
rctx->src_nents = sg_nents_for_len(req->src, req->nbytes);
553
594
554
595
return aspeed_hace_hash_handle_queue(hace_dev, req);
555
596
}
···
522
639
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
523
640
struct aspeed_sham_ctx *tctx = crypto_ahash_ctx(tfm);
524
641
struct aspeed_hace_dev *hace_dev = tctx->hace_dev;
525
-
int rc1, rc2;
526
642
527
643
AHASH_DBG(hace_dev, "req->nbytes: %d\n", req->nbytes);
528
644
529
645
rctx->flags |= SHA_FLAGS_FINUP;
530
646
531
-
rc1 = aspeed_sham_update(req);
532
-
if (rc1 == -EINPROGRESS || rc1 == -EBUSY)
533
-
return rc1;
534
-
535
-
/*
536
-
* final() has to be always called to cleanup resources
537
-
* even if update() failed, except EINPROGRESS
538
-
*/
539
-
rc2 = aspeed_sham_final(req);
540
-
541
-
return rc1 ? : rc2;
647
+
return aspeed_sham_update(req);
542
648
}
543
649
544
650
static int aspeed_sham_init(struct ahash_request *req)
···
536
664
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
537
665
struct aspeed_sham_ctx *tctx = crypto_ahash_ctx(tfm);
538
666
struct aspeed_hace_dev *hace_dev = tctx->hace_dev;
539
-
struct aspeed_sha_hmac_ctx *bctx = tctx->base;
540
667
541
668
AHASH_DBG(hace_dev, "%s: digest size:%d\n",
542
669
crypto_tfm_alg_name(&tfm->base),
···
550
679
rctx->flags |= SHA_FLAGS_SHA1;
551
680
rctx->digsize = SHA1_DIGEST_SIZE;
552
681
rctx->block_size = SHA1_BLOCK_SIZE;
553
-
rctx->sha_iv = sha1_iv;
554
682
rctx->ivsize = 32;
555
683
memcpy(rctx->digest, sha1_iv, rctx->ivsize);
556
684
break;
···
558
688
rctx->flags |= SHA_FLAGS_SHA224;
559
689
rctx->digsize = SHA224_DIGEST_SIZE;
560
690
rctx->block_size = SHA224_BLOCK_SIZE;
561
-
rctx->sha_iv = sha224_iv;
562
691
rctx->ivsize = 32;
563
692
memcpy(rctx->digest, sha224_iv, rctx->ivsize);
564
693
break;
···
566
697
rctx->flags |= SHA_FLAGS_SHA256;
567
698
rctx->digsize = SHA256_DIGEST_SIZE;
568
699
rctx->block_size = SHA256_BLOCK_SIZE;
569
-
rctx->sha_iv = sha256_iv;
570
700
rctx->ivsize = 32;
571
701
memcpy(rctx->digest, sha256_iv, rctx->ivsize);
572
702
break;
···
575
707
rctx->flags |= SHA_FLAGS_SHA384;
576
708
rctx->digsize = SHA384_DIGEST_SIZE;
577
709
rctx->block_size = SHA384_BLOCK_SIZE;
578
-
rctx->sha_iv = (const __be32 *)sha384_iv;
579
710
rctx->ivsize = 64;
580
711
memcpy(rctx->digest, sha384_iv, rctx->ivsize);
581
712
break;
···
584
717
rctx->flags |= SHA_FLAGS_SHA512;
585
718
rctx->digsize = SHA512_DIGEST_SIZE;
586
719
rctx->block_size = SHA512_BLOCK_SIZE;
587
-
rctx->sha_iv = (const __be32 *)sha512_iv;
588
720
rctx->ivsize = 64;
589
721
memcpy(rctx->digest, sha512_iv, rctx->ivsize);
590
722
break;
···
593
727
return -EINVAL;
594
728
}
595
729
596
-
rctx->bufcnt = 0;
597
730
rctx->total = 0;
598
731
rctx->digcnt[0] = 0;
599
732
rctx->digcnt[1] = 0;
600
-
601
-
/* HMAC init */
602
-
if (tctx->flags & SHA_FLAGS_HMAC) {
603
-
rctx->digcnt[0] = rctx->block_size;
604
-
rctx->bufcnt = rctx->block_size;
605
-
memcpy(rctx->buffer, bctx->ipad, rctx->block_size);
606
-
rctx->flags |= SHA_FLAGS_HMAC;
607
-
}
608
733
609
734
return 0;
610
735
}
···
605
748
return aspeed_sham_init(req) ? : aspeed_sham_finup(req);
606
749
}
607
750
608
-
static int aspeed_sham_setkey(struct crypto_ahash *tfm, const u8 *key,
609
-
unsigned int keylen)
751
+
static int aspeed_sham_cra_init(struct crypto_ahash *tfm)
610
752
{
753
+
struct ahash_alg *alg = crypto_ahash_alg(tfm);
611
754
struct aspeed_sham_ctx *tctx = crypto_ahash_ctx(tfm);
612
-
struct aspeed_hace_dev *hace_dev = tctx->hace_dev;
613
-
struct aspeed_sha_hmac_ctx *bctx = tctx->base;
614
-
int ds = crypto_shash_digestsize(bctx->shash);
615
-
int bs = crypto_shash_blocksize(bctx->shash);
616
-
int err = 0;
617
-
int i;
618
-
619
-
AHASH_DBG(hace_dev, "%s: keylen:%d\n", crypto_tfm_alg_name(&tfm->base),
620
-
keylen);
621
-
622
-
if (keylen > bs) {
623
-
err = aspeed_sham_shash_digest(bctx->shash,
624
-
crypto_shash_get_flags(bctx->shash),
625
-
key, keylen, bctx->ipad);
626
-
if (err)
627
-
return err;
628
-
keylen = ds;
629
-
630
-
} else {
631
-
memcpy(bctx->ipad, key, keylen);
632
-
}
633
-
634
-
memset(bctx->ipad + keylen, 0, bs - keylen);
635
-
memcpy(bctx->opad, bctx->ipad, bs);
636
-
637
-
for (i = 0; i < bs; i++) {
638
-
bctx->ipad[i] ^= HMAC_IPAD_VALUE;
639
-
bctx->opad[i] ^= HMAC_OPAD_VALUE;
640
-
}
641
-
642
-
return err;
643
-
}
644
-
645
-
static int aspeed_sham_cra_init(struct crypto_tfm *tfm)
646
-
{
647
-
struct ahash_alg *alg = __crypto_ahash_alg(tfm->__crt_alg);
648
-
struct aspeed_sham_ctx *tctx = crypto_tfm_ctx(tfm);
649
755
struct aspeed_hace_alg *ast_alg;
650
756
651
757
ast_alg = container_of(alg, struct aspeed_hace_alg, alg.ahash.base);
652
758
tctx->hace_dev = ast_alg->hace_dev;
653
-
tctx->flags = 0;
654
-
655
-
crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
656
-
sizeof(struct aspeed_sham_reqctx));
657
-
658
-
if (ast_alg->alg_base) {
659
-
/* hmac related */
660
-
struct aspeed_sha_hmac_ctx *bctx = tctx->base;
661
-
662
-
tctx->flags |= SHA_FLAGS_HMAC;
663
-
bctx->shash = crypto_alloc_shash(ast_alg->alg_base, 0,
664
-
CRYPTO_ALG_NEED_FALLBACK);
665
-
if (IS_ERR(bctx->shash)) {
666
-
dev_warn(ast_alg->hace_dev->dev,
667
-
"base driver '%s' could not be loaded.\n",
668
-
ast_alg->alg_base);
669
-
return PTR_ERR(bctx->shash);
670
-
}
671
-
}
672
-
673
-
return 0;
674
-
}
675
-
676
-
static void aspeed_sham_cra_exit(struct crypto_tfm *tfm)
677
-
{
678
-
struct aspeed_sham_ctx *tctx = crypto_tfm_ctx(tfm);
679
-
struct aspeed_hace_dev *hace_dev = tctx->hace_dev;
680
-
681
-
AHASH_DBG(hace_dev, "%s\n", crypto_tfm_alg_name(tfm));
682
-
683
-
if (tctx->flags & SHA_FLAGS_HMAC) {
684
-
struct aspeed_sha_hmac_ctx *bctx = tctx->base;
685
-
686
-
crypto_free_shash(bctx->shash);
687
-
}
688
-
}
689
-
690
-
static int aspeed_sham_export(struct ahash_request *req, void *out)
691
-
{
692
-
struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req);
693
-
694
-
memcpy(out, rctx, sizeof(*rctx));
695
-
696
-
return 0;
697
-
}
698
-
699
-
static int aspeed_sham_import(struct ahash_request *req, const void *in)
700
-
{
701
-
struct aspeed_sham_reqctx *rctx = ahash_request_ctx(req);
702
-
703
-
memcpy(rctx, in, sizeof(*rctx));
704
759
705
760
return 0;
706
761
}
···
622
853
.alg.ahash.base = {
623
854
.init = aspeed_sham_init,
624
855
.update = aspeed_sham_update,
625
-
.final = aspeed_sham_final,
626
856
.finup = aspeed_sham_finup,
627
857
.digest = aspeed_sham_digest,
628
858
.export = aspeed_sham_export,
629
859
.import = aspeed_sham_import,
860
+
.init_tfm = aspeed_sham_cra_init,
630
861
.halg = {
631
862
.digestsize = SHA1_DIGEST_SIZE,
632
863
.statesize = sizeof(struct aspeed_sham_reqctx),
···
636
867
.cra_priority = 300,
637
868
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
638
869
CRYPTO_ALG_ASYNC |
870
+
CRYPTO_AHASH_ALG_BLOCK_ONLY |
639
871
CRYPTO_ALG_KERN_DRIVER_ONLY,
640
872
.cra_blocksize = SHA1_BLOCK_SIZE,
641
873
.cra_ctxsize = sizeof(struct aspeed_sham_ctx),
874
+
.cra_reqsize = sizeof(struct aspeed_sham_reqctx),
642
875
.cra_alignmask = 0,
643
876
.cra_module = THIS_MODULE,
644
-
.cra_init = aspeed_sham_cra_init,
645
-
.cra_exit = aspeed_sham_cra_exit,
646
877
}
647
878
}
648
879
},
···
654
885
.alg.ahash.base = {
655
886
.init = aspeed_sham_init,
656
887
.update = aspeed_sham_update,
657
-
.final = aspeed_sham_final,
658
888
.finup = aspeed_sham_finup,
659
889
.digest = aspeed_sham_digest,
660
890
.export = aspeed_sham_export,
661
891
.import = aspeed_sham_import,
892
+
.init_tfm = aspeed_sham_cra_init,
662
893
.halg = {
663
894
.digestsize = SHA256_DIGEST_SIZE,
664
895
.statesize = sizeof(struct aspeed_sham_reqctx),
···
668
899
.cra_priority = 300,
669
900
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
670
901
CRYPTO_ALG_ASYNC |
902
+
CRYPTO_AHASH_ALG_BLOCK_ONLY |
671
903
CRYPTO_ALG_KERN_DRIVER_ONLY,
672
904
.cra_blocksize = SHA256_BLOCK_SIZE,
673
905
.cra_ctxsize = sizeof(struct aspeed_sham_ctx),
906
+
.cra_reqsize = sizeof(struct aspeed_sham_reqctx),
674
907
.cra_alignmask = 0,
675
908
.cra_module = THIS_MODULE,
676
-
.cra_init = aspeed_sham_cra_init,
677
-
.cra_exit = aspeed_sham_cra_exit,
678
909
}
679
910
}
680
911
},
···
686
917
.alg.ahash.base = {
687
918
.init = aspeed_sham_init,
688
919
.update = aspeed_sham_update,
689
-
.final = aspeed_sham_final,
690
920
.finup = aspeed_sham_finup,
691
921
.digest = aspeed_sham_digest,
692
922
.export = aspeed_sham_export,
693
923
.import = aspeed_sham_import,
924
+
.init_tfm = aspeed_sham_cra_init,
694
925
.halg = {
695
926
.digestsize = SHA224_DIGEST_SIZE,
696
927
.statesize = sizeof(struct aspeed_sham_reqctx),
···
700
931
.cra_priority = 300,
701
932
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
702
933
CRYPTO_ALG_ASYNC |
934
+
CRYPTO_AHASH_ALG_BLOCK_ONLY |
703
935
CRYPTO_ALG_KERN_DRIVER_ONLY,
704
936
.cra_blocksize = SHA224_BLOCK_SIZE,
705
937
.cra_ctxsize = sizeof(struct aspeed_sham_ctx),
938
+
.cra_reqsize = sizeof(struct aspeed_sham_reqctx),
706
939
.cra_alignmask = 0,
707
940
.cra_module = THIS_MODULE,
708
-
.cra_init = aspeed_sham_cra_init,
709
-
.cra_exit = aspeed_sham_cra_exit,
710
-
}
711
-
}
712
-
},
713
-
.alg.ahash.op = {
714
-
.do_one_request = aspeed_ahash_do_one,
715
-
},
716
-
},
717
-
{
718
-
.alg_base = "sha1",
719
-
.alg.ahash.base = {
720
-
.init = aspeed_sham_init,
721
-
.update = aspeed_sham_update,
722
-
.final = aspeed_sham_final,
723
-
.finup = aspeed_sham_finup,
724
-
.digest = aspeed_sham_digest,
725
-
.setkey = aspeed_sham_setkey,
726
-
.export = aspeed_sham_export,
727
-
.import = aspeed_sham_import,
728
-
.halg = {
729
-
.digestsize = SHA1_DIGEST_SIZE,
730
-
.statesize = sizeof(struct aspeed_sham_reqctx),
731
-
.base = {
732
-
.cra_name = "hmac(sha1)",
733
-
.cra_driver_name = "aspeed-hmac-sha1",
734
-
.cra_priority = 300,
735
-
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
736
-
CRYPTO_ALG_ASYNC |
737
-
CRYPTO_ALG_KERN_DRIVER_ONLY,
738
-
.cra_blocksize = SHA1_BLOCK_SIZE,
739
-
.cra_ctxsize = sizeof(struct aspeed_sham_ctx) +
740
-
sizeof(struct aspeed_sha_hmac_ctx),
741
-
.cra_alignmask = 0,
742
-
.cra_module = THIS_MODULE,
743
-
.cra_init = aspeed_sham_cra_init,
744
-
.cra_exit = aspeed_sham_cra_exit,
745
-
}
746
-
}
747
-
},
748
-
.alg.ahash.op = {
749
-
.do_one_request = aspeed_ahash_do_one,
750
-
},
751
-
},
752
-
{
753
-
.alg_base = "sha224",
754
-
.alg.ahash.base = {
755
-
.init = aspeed_sham_init,
756
-
.update = aspeed_sham_update,
757
-
.final = aspeed_sham_final,
758
-
.finup = aspeed_sham_finup,
759
-
.digest = aspeed_sham_digest,
760
-
.setkey = aspeed_sham_setkey,
761
-
.export = aspeed_sham_export,
762
-
.import = aspeed_sham_import,
763
-
.halg = {
764
-
.digestsize = SHA224_DIGEST_SIZE,
765
-
.statesize = sizeof(struct aspeed_sham_reqctx),
766
-
.base = {
767
-
.cra_name = "hmac(sha224)",
768
-
.cra_driver_name = "aspeed-hmac-sha224",
769
-
.cra_priority = 300,
770
-
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
771
-
CRYPTO_ALG_ASYNC |
772
-
CRYPTO_ALG_KERN_DRIVER_ONLY,
773
-
.cra_blocksize = SHA224_BLOCK_SIZE,
774
-
.cra_ctxsize = sizeof(struct aspeed_sham_ctx) +
775
-
sizeof(struct aspeed_sha_hmac_ctx),
776
-
.cra_alignmask = 0,
777
-
.cra_module = THIS_MODULE,
778
-
.cra_init = aspeed_sham_cra_init,
779
-
.cra_exit = aspeed_sham_cra_exit,
780
-
}
781
-
}
782
-
},
783
-
.alg.ahash.op = {
784
-
.do_one_request = aspeed_ahash_do_one,
785
-
},
786
-
},
787
-
{
788
-
.alg_base = "sha256",
789
-
.alg.ahash.base = {
790
-
.init = aspeed_sham_init,
791
-
.update = aspeed_sham_update,
792
-
.final = aspeed_sham_final,
793
-
.finup = aspeed_sham_finup,
794
-
.digest = aspeed_sham_digest,
795
-
.setkey = aspeed_sham_setkey,
796
-
.export = aspeed_sham_export,
797
-
.import = aspeed_sham_import,
798
-
.halg = {
799
-
.digestsize = SHA256_DIGEST_SIZE,
800
-
.statesize = sizeof(struct aspeed_sham_reqctx),
801
-
.base = {
802
-
.cra_name = "hmac(sha256)",
803
-
.cra_driver_name = "aspeed-hmac-sha256",
804
-
.cra_priority = 300,
805
-
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
806
-
CRYPTO_ALG_ASYNC |
807
-
CRYPTO_ALG_KERN_DRIVER_ONLY,
808
-
.cra_blocksize = SHA256_BLOCK_SIZE,
809
-
.cra_ctxsize = sizeof(struct aspeed_sham_ctx) +
810
-
sizeof(struct aspeed_sha_hmac_ctx),
811
-
.cra_alignmask = 0,
812
-
.cra_module = THIS_MODULE,
813
-
.cra_init = aspeed_sham_cra_init,
814
-
.cra_exit = aspeed_sham_cra_exit,
815
941
}
816
942
}
817
943
},
···
721
1057
.alg.ahash.base = {
722
1058
.init = aspeed_sham_init,
723
1059
.update = aspeed_sham_update,
724
-
.final = aspeed_sham_final,
725
1060
.finup = aspeed_sham_finup,
726
1061
.digest = aspeed_sham_digest,
727
1062
.export = aspeed_sham_export,
728
1063
.import = aspeed_sham_import,
1064
+
.init_tfm = aspeed_sham_cra_init,
729
1065
.halg = {
730
1066
.digestsize = SHA384_DIGEST_SIZE,
731
1067
.statesize = sizeof(struct aspeed_sham_reqctx),
···
735
1071
.cra_priority = 300,
736
1072
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
737
1073
CRYPTO_ALG_ASYNC |
1074
+
CRYPTO_AHASH_ALG_BLOCK_ONLY |
738
1075
CRYPTO_ALG_KERN_DRIVER_ONLY,
739
1076
.cra_blocksize = SHA384_BLOCK_SIZE,
740
1077
.cra_ctxsize = sizeof(struct aspeed_sham_ctx),
1078
+
.cra_reqsize = sizeof(struct aspeed_sham_reqctx),
741
1079
.cra_alignmask = 0,
742
1080
.cra_module = THIS_MODULE,
743
-
.cra_init = aspeed_sham_cra_init,
744
-
.cra_exit = aspeed_sham_cra_exit,
745
1081
}
746
1082
}
747
1083
},
···
753
1089
.alg.ahash.base = {
754
1090
.init = aspeed_sham_init,
755
1091
.update = aspeed_sham_update,
756
-
.final = aspeed_sham_final,
757
1092
.finup = aspeed_sham_finup,
758
1093
.digest = aspeed_sham_digest,
759
1094
.export = aspeed_sham_export,
760
1095
.import = aspeed_sham_import,
1096
+
.init_tfm = aspeed_sham_cra_init,
761
1097
.halg = {
762
1098
.digestsize = SHA512_DIGEST_SIZE,
763
1099
.statesize = sizeof(struct aspeed_sham_reqctx),
···
767
1103
.cra_priority = 300,
768
1104
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
769
1105
CRYPTO_ALG_ASYNC |
1106
+
CRYPTO_AHASH_ALG_BLOCK_ONLY |
770
1107
CRYPTO_ALG_KERN_DRIVER_ONLY,
771
1108
.cra_blocksize = SHA512_BLOCK_SIZE,
772
1109
.cra_ctxsize = sizeof(struct aspeed_sham_ctx),
1110
+
.cra_reqsize = sizeof(struct aspeed_sham_reqctx),
773
1111
.cra_alignmask = 0,
774
1112
.cra_module = THIS_MODULE,
775
-
.cra_init = aspeed_sham_cra_init,
776
-
.cra_exit = aspeed_sham_cra_exit,
777
-
}
778
-
}
779
-
},
780
-
.alg.ahash.op = {
781
-
.do_one_request = aspeed_ahash_do_one,
782
-
},
783
-
},
784
-
{
785
-
.alg_base = "sha384",
786
-
.alg.ahash.base = {
787
-
.init = aspeed_sham_init,
788
-
.update = aspeed_sham_update,
789
-
.final = aspeed_sham_final,
790
-
.finup = aspeed_sham_finup,
791
-
.digest = aspeed_sham_digest,
792
-
.setkey = aspeed_sham_setkey,
793
-
.export = aspeed_sham_export,
794
-
.import = aspeed_sham_import,
795
-
.halg = {
796
-
.digestsize = SHA384_DIGEST_SIZE,
797
-
.statesize = sizeof(struct aspeed_sham_reqctx),
798
-
.base = {
799
-
.cra_name = "hmac(sha384)",
800
-
.cra_driver_name = "aspeed-hmac-sha384",
801
-
.cra_priority = 300,
802
-
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
803
-
CRYPTO_ALG_ASYNC |
804
-
CRYPTO_ALG_KERN_DRIVER_ONLY,
805
-
.cra_blocksize = SHA384_BLOCK_SIZE,
806
-
.cra_ctxsize = sizeof(struct aspeed_sham_ctx) +
807
-
sizeof(struct aspeed_sha_hmac_ctx),
808
-
.cra_alignmask = 0,
809
-
.cra_module = THIS_MODULE,
810
-
.cra_init = aspeed_sham_cra_init,
811
-
.cra_exit = aspeed_sham_cra_exit,
812
-
}
813
-
}
814
-
},
815
-
.alg.ahash.op = {
816
-
.do_one_request = aspeed_ahash_do_one,
817
-
},
818
-
},
819
-
{
820
-
.alg_base = "sha512",
821
-
.alg.ahash.base = {
822
-
.init = aspeed_sham_init,
823
-
.update = aspeed_sham_update,
824
-
.final = aspeed_sham_final,
825
-
.finup = aspeed_sham_finup,
826
-
.digest = aspeed_sham_digest,
827
-
.setkey = aspeed_sham_setkey,
828
-
.export = aspeed_sham_export,
829
-
.import = aspeed_sham_import,
830
-
.halg = {
831
-
.digestsize = SHA512_DIGEST_SIZE,
832
-
.statesize = sizeof(struct aspeed_sham_reqctx),
833
-
.base = {
834
-
.cra_name = "hmac(sha512)",
835
-
.cra_driver_name = "aspeed-hmac-sha512",
836
-
.cra_priority = 300,
837
-
.cra_flags = CRYPTO_ALG_TYPE_AHASH |
838
-
CRYPTO_ALG_ASYNC |
839
-
CRYPTO_ALG_KERN_DRIVER_ONLY,
840
-
.cra_blocksize = SHA512_BLOCK_SIZE,
841
-
.cra_ctxsize = sizeof(struct aspeed_sham_ctx) +
842
-
sizeof(struct aspeed_sha_hmac_ctx),
843
-
.cra_alignmask = 0,
844
-
.cra_module = THIS_MODULE,
845
-
.cra_init = aspeed_sham_cra_init,
846
-
.cra_exit = aspeed_sham_cra_exit,
847
1113
}
848
1114
}
849
1115
},
+9
-19
drivers/crypto/aspeed/aspeed-hace.h
+9
-19
drivers/crypto/aspeed/aspeed-hace.h
···
119
119
#define SHA_FLAGS_SHA512 BIT(4)
120
120
#define SHA_FLAGS_SHA512_224 BIT(5)
121
121
#define SHA_FLAGS_SHA512_256 BIT(6)
122
-
#define SHA_FLAGS_HMAC BIT(8)
123
122
#define SHA_FLAGS_FINUP BIT(9)
124
123
#define SHA_FLAGS_MASK (0xff)
125
124
···
160
161
aspeed_hace_fn_t dma_prepare;
161
162
};
162
163
163
-
struct aspeed_sha_hmac_ctx {
164
-
struct crypto_shash *shash;
165
-
u8 ipad[SHA512_BLOCK_SIZE];
166
-
u8 opad[SHA512_BLOCK_SIZE];
167
-
};
168
-
169
164
struct aspeed_sham_ctx {
170
165
struct aspeed_hace_dev *hace_dev;
171
-
unsigned long flags; /* hmac flag */
172
-
173
-
struct aspeed_sha_hmac_ctx base[];
174
166
};
175
167
176
168
struct aspeed_sham_reqctx {
169
+
/* DMA buffer written by hardware */
170
+
u8 digest[SHA512_DIGEST_SIZE] __aligned(64);
171
+
172
+
/* Software state sorted by size. */
173
+
u64 digcnt[2];
174
+
177
175
unsigned long flags; /* final update flag should no use*/
178
-
unsigned long op; /* final or update */
179
176
u32 cmd; /* trigger cmd */
180
177
181
178
/* walk state */
···
183
188
size_t digsize;
184
189
size_t block_size;
185
190
size_t ivsize;
186
-
const __be32 *sha_iv;
187
191
188
-
/* remain data buffer */
189
-
u8 buffer[SHA512_BLOCK_SIZE * 2];
190
192
dma_addr_t buffer_dma_addr;
191
-
size_t bufcnt; /* buffer counter */
192
-
193
-
/* output buffer */
194
-
u8 digest[SHA512_DIGEST_SIZE] __aligned(64);
195
193
dma_addr_t digest_dma_addr;
196
-
u64 digcnt[2];
194
+
195
+
/* This is DMA too but read-only for hardware. */
196
+
u8 buffer[SHA512_BLOCK_SIZE + 16];
197
197
};
198
198
199
199
struct aspeed_engine_crypto {
+1
drivers/crypto/atmel-aes.c
+1
drivers/crypto/atmel-aes.c
+1
drivers/crypto/atmel-sha.c
+1
drivers/crypto/atmel-sha.c
-4
drivers/crypto/caam/Makefile
-4
drivers/crypto/caam/Makefile
···
25
25
caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_BLOB_GEN) += blob_gen.o
26
26
27
27
caam-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI) += qi.o
28
-
ifneq ($(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI),)
29
-
ccflags-y += -DCONFIG_CAAM_QI
30
-
endif
31
-
32
28
caam-$(CONFIG_DEBUG_FS) += debugfs.o
33
29
34
30
obj-$(CONFIG_CRYPTO_DEV_FSL_DPAA2_CAAM) += dpaa2_caam.o
+7
-6
drivers/crypto/caam/ctrl.c
+7
-6
drivers/crypto/caam/ctrl.c
···
24
24
bool caam_dpaa2;
25
25
EXPORT_SYMBOL(caam_dpaa2);
26
26
27
-
#ifdef CONFIG_CAAM_QI
27
+
#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI
28
28
#include "qi.h"
29
29
#endif
30
30
···
573
573
{ .soc_id = "i.MX7*", .data = &caam_imx7_data },
574
574
{ .soc_id = "i.MX8M*", .data = &caam_imx7_data },
575
575
{ .soc_id = "i.MX8ULP", .data = &caam_imx8ulp_data },
576
-
{ .soc_id = "i.MX8QM", .data = &caam_imx8ulp_data },
576
+
{ .soc_id = "i.MX8Q*", .data = &caam_imx8ulp_data },
577
577
{ .soc_id = "VF*", .data = &caam_vf610_data },
578
578
{ .family = "Freescale i.MX" },
579
579
{ /* sentinel */ }
···
831
831
{
832
832
const struct caam_drv_private *ctrlpriv = dev_get_drvdata(dev);
833
833
834
-
if (ctrlpriv->caam_off_during_pm && !ctrlpriv->optee_en)
834
+
if (ctrlpriv->caam_off_during_pm && !ctrlpriv->no_page0)
835
835
caam_state_save(dev);
836
836
837
837
return 0;
···
842
842
struct caam_drv_private *ctrlpriv = dev_get_drvdata(dev);
843
843
int ret = 0;
844
844
845
-
if (ctrlpriv->caam_off_during_pm && !ctrlpriv->optee_en) {
845
+
if (ctrlpriv->caam_off_during_pm && !ctrlpriv->no_page0) {
846
846
caam_state_restore(dev);
847
847
848
848
/* HW and rng will be reset so deinstantiation can be removed */
···
908
908
909
909
imx_soc_data = imx_soc_match->data;
910
910
reg_access = reg_access && imx_soc_data->page0_access;
911
+
ctrlpriv->no_page0 = !reg_access;
911
912
/*
912
913
* CAAM clocks cannot be controlled from kernel.
913
914
*/
···
968
967
caam_dpaa2 = !!(comp_params & CTPR_MS_DPAA2);
969
968
ctrlpriv->qi_present = !!(comp_params & CTPR_MS_QI_MASK);
970
969
971
-
#ifdef CONFIG_CAAM_QI
970
+
#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI
972
971
/* If (DPAA 1.x) QI present, check whether dependencies are available */
973
972
if (ctrlpriv->qi_present && !caam_dpaa2) {
974
973
ret = qman_is_probed();
···
1099
1098
wr_reg32(&ctrlpriv->qi->qi_control_lo, QICTL_DQEN);
1100
1099
1101
1100
/* If QMAN driver is present, init CAAM-QI backend */
1102
-
#ifdef CONFIG_CAAM_QI
1101
+
#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI
1103
1102
ret = caam_qi_init(pdev);
1104
1103
if (ret)
1105
1104
dev_err(dev, "caam qi i/f init failed: %d\n", ret);
+1
-1
drivers/crypto/caam/debugfs.c
+1
-1
drivers/crypto/caam/debugfs.c
···
22
22
DEFINE_DEBUGFS_ATTRIBUTE(caam_fops_u32_ro, caam_debugfs_u32_get, NULL, "%llu\n");
23
23
DEFINE_DEBUGFS_ATTRIBUTE(caam_fops_u64_ro, caam_debugfs_u64_get, NULL, "%llu\n");
24
24
25
-
#ifdef CONFIG_CAAM_QI
25
+
#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI
26
26
/*
27
27
* This is a counter for the number of times the congestion group (where all
28
28
* the request and response queueus are) reached congestion. Incremented
+1
-1
drivers/crypto/caam/debugfs.h
+1
-1
drivers/crypto/caam/debugfs.h
···
18
18
{}
19
19
#endif
20
20
21
-
#if defined(CONFIG_DEBUG_FS) && defined(CONFIG_CAAM_QI)
21
+
#if defined(CONFIG_DEBUG_FS) && defined(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI)
22
22
void caam_debugfs_qi_congested(void);
23
23
void caam_debugfs_qi_init(struct caam_drv_private *ctrlpriv);
24
24
#else
+3
-2
drivers/crypto/caam/intern.h
+3
-2
drivers/crypto/caam/intern.h
···
115
115
u8 blob_present; /* Nonzero if BLOB support present in device */
116
116
u8 mc_en; /* Nonzero if MC f/w is active */
117
117
u8 optee_en; /* Nonzero if OP-TEE f/w is active */
118
+
u8 no_page0; /* Nonzero if register page 0 is not controlled by Linux */
118
119
bool pr_support; /* RNG prediction resistance available */
119
120
int secvio_irq; /* Security violation interrupt number */
120
121
int virt_en; /* Virtualization enabled in CAAM */
···
227
226
static inline void caam_prng_unregister(void *data) {}
228
227
#endif /* CONFIG_CRYPTO_DEV_FSL_CAAM_PRNG_API */
229
228
230
-
#ifdef CONFIG_CAAM_QI
229
+
#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI
231
230
232
231
int caam_qi_algapi_init(struct device *dev);
233
232
void caam_qi_algapi_exit(void);
···
243
242
{
244
243
}
245
244
246
-
#endif /* CONFIG_CAAM_QI */
245
+
#endif /* CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI */
247
246
248
247
static inline u64 caam_get_dma_mask(struct device *dev)
249
248
{
+1
-2
drivers/crypto/caam/jr.c
+1
-2
drivers/crypto/caam/jr.c
···
629
629
}
630
630
631
631
/* Initialize crypto engine */
632
-
jrpriv->engine = crypto_engine_alloc_init_and_set(jrdev, true, NULL,
633
-
false,
632
+
jrpriv->engine = crypto_engine_alloc_init_and_set(jrdev, true, false,
634
633
CRYPTO_ENGINE_MAX_QLEN);
635
634
if (!jrpriv->engine) {
636
635
dev_err(jrdev, "Could not init crypto-engine\n");
+1
-4
drivers/crypto/caam/qi.c
+1
-4
drivers/crypto/caam/qi.c
···
442
442
if (!cpumask_test_cpu(*cpu, cpus)) {
443
443
int *pcpu = &get_cpu_var(last_cpu);
444
444
445
-
*pcpu = cpumask_next(*pcpu, cpus);
446
-
if (*pcpu >= nr_cpu_ids)
447
-
*pcpu = cpumask_first(cpus);
445
+
*pcpu = cpumask_next_wrap(*pcpu, cpus);
448
446
*cpu = *pcpu;
449
-
450
447
put_cpu_var(last_cpu);
451
448
}
452
449
drv_ctx->cpu = *cpu;
+3
drivers/crypto/ccp/ccp-debugfs.c
+3
drivers/crypto/ccp/ccp-debugfs.c
+86
-77
drivers/crypto/ccp/ccp-ops.c
+86
-77
drivers/crypto/ccp/ccp-ops.c
···
633
633
ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
634
634
{
635
635
struct ccp_aes_engine *aes = &cmd->u.aes;
636
-
struct ccp_dm_workarea key, ctx, final_wa, tag;
637
-
struct ccp_data src, dst;
638
-
struct ccp_data aad;
639
-
struct ccp_op op;
636
+
struct {
637
+
struct ccp_dm_workarea key;
638
+
struct ccp_dm_workarea ctx;
639
+
struct ccp_dm_workarea final;
640
+
struct ccp_dm_workarea tag;
641
+
struct ccp_data src;
642
+
struct ccp_data dst;
643
+
struct ccp_data aad;
644
+
struct ccp_op op;
645
+
} *wa __cleanup(kfree) = kzalloc(sizeof *wa, GFP_KERNEL);
640
646
unsigned int dm_offset;
641
647
unsigned int authsize;
642
648
unsigned int jobid;
···
655
649
struct scatterlist *p_tag, sg_tag[2];
656
650
struct scatterlist *p_outp, sg_outp[2];
657
651
struct scatterlist *p_aad;
652
+
653
+
if (!wa)
654
+
return -ENOMEM;
658
655
659
656
if (!aes->iv)
660
657
return -EINVAL;
···
705
696
706
697
jobid = CCP_NEW_JOBID(cmd_q->ccp);
707
698
708
-
memset(&op, 0, sizeof(op));
709
-
op.cmd_q = cmd_q;
710
-
op.jobid = jobid;
711
-
op.sb_key = cmd_q->sb_key; /* Pre-allocated */
712
-
op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
713
-
op.init = 1;
714
-
op.u.aes.type = aes->type;
699
+
memset(&wa->op, 0, sizeof(wa->op));
700
+
wa->op.cmd_q = cmd_q;
701
+
wa->op.jobid = jobid;
702
+
wa->op.sb_key = cmd_q->sb_key; /* Pre-allocated */
703
+
wa->op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
704
+
wa->op.init = 1;
705
+
wa->op.u.aes.type = aes->type;
715
706
716
707
/* Copy the key to the LSB */
717
-
ret = ccp_init_dm_workarea(&key, cmd_q,
708
+
ret = ccp_init_dm_workarea(&wa->key, cmd_q,
718
709
CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
719
710
DMA_TO_DEVICE);
720
711
if (ret)
721
712
return ret;
722
713
723
714
dm_offset = CCP_SB_BYTES - aes->key_len;
724
-
ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
715
+
ret = ccp_set_dm_area(&wa->key, dm_offset, aes->key, 0, aes->key_len);
725
716
if (ret)
726
717
goto e_key;
727
-
ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
718
+
ret = ccp_copy_to_sb(cmd_q, &wa->key, wa->op.jobid, wa->op.sb_key,
728
719
CCP_PASSTHRU_BYTESWAP_256BIT);
729
720
if (ret) {
730
721
cmd->engine_error = cmd_q->cmd_error;
···
735
726
* There is an assumption here that the IV is 96 bits in length, plus
736
727
* a nonce of 32 bits. If no IV is present, use a zeroed buffer.
737
728
*/
738
-
ret = ccp_init_dm_workarea(&ctx, cmd_q,
729
+
ret = ccp_init_dm_workarea(&wa->ctx, cmd_q,
739
730
CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
740
731
DMA_BIDIRECTIONAL);
741
732
if (ret)
742
733
goto e_key;
743
734
744
735
dm_offset = CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES - aes->iv_len;
745
-
ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
736
+
ret = ccp_set_dm_area(&wa->ctx, dm_offset, aes->iv, 0, aes->iv_len);
746
737
if (ret)
747
738
goto e_ctx;
748
739
749
-
ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
740
+
ret = ccp_copy_to_sb(cmd_q, &wa->ctx, wa->op.jobid, wa->op.sb_ctx,
750
741
CCP_PASSTHRU_BYTESWAP_256BIT);
751
742
if (ret) {
752
743
cmd->engine_error = cmd_q->cmd_error;
753
744
goto e_ctx;
754
745
}
755
746
756
-
op.init = 1;
747
+
wa->op.init = 1;
757
748
if (aes->aad_len > 0) {
758
749
/* Step 1: Run a GHASH over the Additional Authenticated Data */
759
-
ret = ccp_init_data(&aad, cmd_q, p_aad, aes->aad_len,
750
+
ret = ccp_init_data(&wa->aad, cmd_q, p_aad, aes->aad_len,
760
751
AES_BLOCK_SIZE,
761
752
DMA_TO_DEVICE);
762
753
if (ret)
763
754
goto e_ctx;
764
755
765
-
op.u.aes.mode = CCP_AES_MODE_GHASH;
766
-
op.u.aes.action = CCP_AES_GHASHAAD;
756
+
wa->op.u.aes.mode = CCP_AES_MODE_GHASH;
757
+
wa->op.u.aes.action = CCP_AES_GHASHAAD;
767
758
768
-
while (aad.sg_wa.bytes_left) {
769
-
ccp_prepare_data(&aad, NULL, &op, AES_BLOCK_SIZE, true);
759
+
while (wa->aad.sg_wa.bytes_left) {
760
+
ccp_prepare_data(&wa->aad, NULL, &wa->op, AES_BLOCK_SIZE, true);
770
761
771
-
ret = cmd_q->ccp->vdata->perform->aes(&op);
762
+
ret = cmd_q->ccp->vdata->perform->aes(&wa->op);
772
763
if (ret) {
773
764
cmd->engine_error = cmd_q->cmd_error;
774
765
goto e_aad;
775
766
}
776
767
777
-
ccp_process_data(&aad, NULL, &op);
778
-
op.init = 0;
768
+
ccp_process_data(&wa->aad, NULL, &wa->op);
769
+
wa->op.init = 0;
779
770
}
780
771
}
781
772
782
-
op.u.aes.mode = CCP_AES_MODE_GCTR;
783
-
op.u.aes.action = aes->action;
773
+
wa->op.u.aes.mode = CCP_AES_MODE_GCTR;
774
+
wa->op.u.aes.action = aes->action;
784
775
785
776
if (ilen > 0) {
786
777
/* Step 2: Run a GCTR over the plaintext */
787
778
in_place = (sg_virt(p_inp) == sg_virt(p_outp)) ? true : false;
788
779
789
-
ret = ccp_init_data(&src, cmd_q, p_inp, ilen,
780
+
ret = ccp_init_data(&wa->src, cmd_q, p_inp, ilen,
790
781
AES_BLOCK_SIZE,
791
782
in_place ? DMA_BIDIRECTIONAL
792
783
: DMA_TO_DEVICE);
···
794
785
goto e_aad;
795
786
796
787
if (in_place) {
797
-
dst = src;
788
+
wa->dst = wa->src;
798
789
} else {
799
-
ret = ccp_init_data(&dst, cmd_q, p_outp, ilen,
790
+
ret = ccp_init_data(&wa->dst, cmd_q, p_outp, ilen,
800
791
AES_BLOCK_SIZE, DMA_FROM_DEVICE);
801
792
if (ret)
802
793
goto e_src;
803
794
}
804
795
805
-
op.soc = 0;
806
-
op.eom = 0;
807
-
op.init = 1;
808
-
while (src.sg_wa.bytes_left) {
809
-
ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
810
-
if (!src.sg_wa.bytes_left) {
796
+
wa->op.soc = 0;
797
+
wa->op.eom = 0;
798
+
wa->op.init = 1;
799
+
while (wa->src.sg_wa.bytes_left) {
800
+
ccp_prepare_data(&wa->src, &wa->dst, &wa->op, AES_BLOCK_SIZE, true);
801
+
if (!wa->src.sg_wa.bytes_left) {
811
802
unsigned int nbytes = ilen % AES_BLOCK_SIZE;
812
803
813
804
if (nbytes) {
814
-
op.eom = 1;
815
-
op.u.aes.size = (nbytes * 8) - 1;
805
+
wa->op.eom = 1;
806
+
wa->op.u.aes.size = (nbytes * 8) - 1;
816
807
}
817
808
}
818
809
819
-
ret = cmd_q->ccp->vdata->perform->aes(&op);
810
+
ret = cmd_q->ccp->vdata->perform->aes(&wa->op);
820
811
if (ret) {
821
812
cmd->engine_error = cmd_q->cmd_error;
822
813
goto e_dst;
823
814
}
824
815
825
-
ccp_process_data(&src, &dst, &op);
826
-
op.init = 0;
816
+
ccp_process_data(&wa->src, &wa->dst, &wa->op);
817
+
wa->op.init = 0;
827
818
}
828
819
}
829
820
830
821
/* Step 3: Update the IV portion of the context with the original IV */
831
-
ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
822
+
ret = ccp_copy_from_sb(cmd_q, &wa->ctx, wa->op.jobid, wa->op.sb_ctx,
832
823
CCP_PASSTHRU_BYTESWAP_256BIT);
833
824
if (ret) {
834
825
cmd->engine_error = cmd_q->cmd_error;
835
826
goto e_dst;
836
827
}
837
828
838
-
ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
829
+
ret = ccp_set_dm_area(&wa->ctx, dm_offset, aes->iv, 0, aes->iv_len);
839
830
if (ret)
840
831
goto e_dst;
841
832
842
-
ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
833
+
ret = ccp_copy_to_sb(cmd_q, &wa->ctx, wa->op.jobid, wa->op.sb_ctx,
843
834
CCP_PASSTHRU_BYTESWAP_256BIT);
844
835
if (ret) {
845
836
cmd->engine_error = cmd_q->cmd_error;
···
849
840
/* Step 4: Concatenate the lengths of the AAD and source, and
850
841
* hash that 16 byte buffer.
851
842
*/
852
-
ret = ccp_init_dm_workarea(&final_wa, cmd_q, AES_BLOCK_SIZE,
843
+
ret = ccp_init_dm_workarea(&wa->final, cmd_q, AES_BLOCK_SIZE,
853
844
DMA_BIDIRECTIONAL);
854
845
if (ret)
855
846
goto e_dst;
856
-
final = (__be64 *)final_wa.address;
847
+
final = (__be64 *)wa->final.address;
857
848
final[0] = cpu_to_be64(aes->aad_len * 8);
858
849
final[1] = cpu_to_be64(ilen * 8);
859
850
860
-
memset(&op, 0, sizeof(op));
861
-
op.cmd_q = cmd_q;
862
-
op.jobid = jobid;
863
-
op.sb_key = cmd_q->sb_key; /* Pre-allocated */
864
-
op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
865
-
op.init = 1;
866
-
op.u.aes.type = aes->type;
867
-
op.u.aes.mode = CCP_AES_MODE_GHASH;
868
-
op.u.aes.action = CCP_AES_GHASHFINAL;
869
-
op.src.type = CCP_MEMTYPE_SYSTEM;
870
-
op.src.u.dma.address = final_wa.dma.address;
871
-
op.src.u.dma.length = AES_BLOCK_SIZE;
872
-
op.dst.type = CCP_MEMTYPE_SYSTEM;
873
-
op.dst.u.dma.address = final_wa.dma.address;
874
-
op.dst.u.dma.length = AES_BLOCK_SIZE;
875
-
op.eom = 1;
876
-
op.u.aes.size = 0;
877
-
ret = cmd_q->ccp->vdata->perform->aes(&op);
851
+
memset(&wa->op, 0, sizeof(wa->op));
852
+
wa->op.cmd_q = cmd_q;
853
+
wa->op.jobid = jobid;
854
+
wa->op.sb_key = cmd_q->sb_key; /* Pre-allocated */
855
+
wa->op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
856
+
wa->op.init = 1;
857
+
wa->op.u.aes.type = aes->type;
858
+
wa->op.u.aes.mode = CCP_AES_MODE_GHASH;
859
+
wa->op.u.aes.action = CCP_AES_GHASHFINAL;
860
+
wa->op.src.type = CCP_MEMTYPE_SYSTEM;
861
+
wa->op.src.u.dma.address = wa->final.dma.address;
862
+
wa->op.src.u.dma.length = AES_BLOCK_SIZE;
863
+
wa->op.dst.type = CCP_MEMTYPE_SYSTEM;
864
+
wa->op.dst.u.dma.address = wa->final.dma.address;
865
+
wa->op.dst.u.dma.length = AES_BLOCK_SIZE;
866
+
wa->op.eom = 1;
867
+
wa->op.u.aes.size = 0;
868
+
ret = cmd_q->ccp->vdata->perform->aes(&wa->op);
878
869
if (ret)
879
870
goto e_final_wa;
880
871
881
872
if (aes->action == CCP_AES_ACTION_ENCRYPT) {
882
873
/* Put the ciphered tag after the ciphertext. */
883
-
ccp_get_dm_area(&final_wa, 0, p_tag, 0, authsize);
874
+
ccp_get_dm_area(&wa->final, 0, p_tag, 0, authsize);
884
875
} else {
885
876
/* Does this ciphered tag match the input? */
886
-
ret = ccp_init_dm_workarea(&tag, cmd_q, authsize,
877
+
ret = ccp_init_dm_workarea(&wa->tag, cmd_q, authsize,
887
878
DMA_BIDIRECTIONAL);
888
879
if (ret)
889
880
goto e_final_wa;
890
-
ret = ccp_set_dm_area(&tag, 0, p_tag, 0, authsize);
881
+
ret = ccp_set_dm_area(&wa->tag, 0, p_tag, 0, authsize);
891
882
if (ret) {
892
-
ccp_dm_free(&tag);
883
+
ccp_dm_free(&wa->tag);
893
884
goto e_final_wa;
894
885
}
895
886
896
-
ret = crypto_memneq(tag.address, final_wa.address,
887
+
ret = crypto_memneq(wa->tag.address, wa->final.address,
897
888
authsize) ? -EBADMSG : 0;
898
-
ccp_dm_free(&tag);
889
+
ccp_dm_free(&wa->tag);
899
890
}
900
891
901
892
e_final_wa:
902
-
ccp_dm_free(&final_wa);
893
+
ccp_dm_free(&wa->final);
903
894
904
895
e_dst:
905
896
if (ilen > 0 && !in_place)
906
-
ccp_free_data(&dst, cmd_q);
897
+
ccp_free_data(&wa->dst, cmd_q);
907
898
908
899
e_src:
909
900
if (ilen > 0)
910
-
ccp_free_data(&src, cmd_q);
901
+
ccp_free_data(&wa->src, cmd_q);
911
902
912
903
e_aad:
913
904
if (aes->aad_len)
914
-
ccp_free_data(&aad, cmd_q);
905
+
ccp_free_data(&wa->aad, cmd_q);
915
906
916
907
e_ctx:
917
-
ccp_dm_free(&ctx);
908
+
ccp_dm_free(&wa->ctx);
918
909
919
910
e_key:
920
-
ccp_dm_free(&key);
911
+
ccp_dm_free(&wa->key);
921
912
922
913
return ret;
923
914
}
+17
-9
drivers/crypto/ccp/sev-dev.c
+17
-9
drivers/crypto/ccp/sev-dev.c
···
434
434
return rc;
435
435
}
436
436
437
-
static struct page *__snp_alloc_firmware_pages(gfp_t gfp_mask, int order)
437
+
static struct page *__snp_alloc_firmware_pages(gfp_t gfp_mask, int order, bool locked)
438
438
{
439
439
unsigned long npages = 1ul << order, paddr;
440
440
struct sev_device *sev;
···
453
453
return page;
454
454
455
455
paddr = __pa((unsigned long)page_address(page));
456
-
if (rmp_mark_pages_firmware(paddr, npages, false))
456
+
if (rmp_mark_pages_firmware(paddr, npages, locked))
457
457
return NULL;
458
458
459
459
return page;
···
463
463
{
464
464
struct page *page;
465
465
466
-
page = __snp_alloc_firmware_pages(gfp_mask, 0);
466
+
page = __snp_alloc_firmware_pages(gfp_mask, 0, false);
467
467
468
468
return page ? page_address(page) : NULL;
469
469
}
···
498
498
{
499
499
struct page *page;
500
500
501
-
page = __snp_alloc_firmware_pages(GFP_KERNEL, get_order(len));
501
+
page = __snp_alloc_firmware_pages(GFP_KERNEL, get_order(len), true);
502
502
if (!page)
503
503
return NULL;
504
504
···
1276
1276
1277
1277
static int __sev_platform_init_locked(int *error)
1278
1278
{
1279
-
int rc, psp_ret = SEV_RET_NO_FW_CALL;
1279
+
int rc, psp_ret, dfflush_error;
1280
1280
struct sev_device *sev;
1281
+
1282
+
psp_ret = dfflush_error = SEV_RET_NO_FW_CALL;
1281
1283
1282
1284
if (!psp_master || !psp_master->sev_data)
1283
1285
return -ENODEV;
···
1322
1320
1323
1321
/* Prepare for first SEV guest launch after INIT */
1324
1322
wbinvd_on_all_cpus();
1325
-
rc = __sev_do_cmd_locked(SEV_CMD_DF_FLUSH, NULL, error);
1323
+
rc = __sev_do_cmd_locked(SEV_CMD_DF_FLUSH, NULL, &dfflush_error);
1326
1324
if (rc) {
1327
1325
dev_err(sev->dev, "SEV: DF_FLUSH failed %#x, rc %d\n",
1328
-
*error, rc);
1326
+
dfflush_error, rc);
1329
1327
return rc;
1330
1328
}
1331
1329
···
1787
1785
sev->snp_initialized = false;
1788
1786
dev_dbg(sev->dev, "SEV-SNP firmware shutdown\n");
1789
1787
1790
-
atomic_notifier_chain_unregister(&panic_notifier_list,
1791
-
&snp_panic_notifier);
1788
+
/*
1789
+
* __sev_snp_shutdown_locked() deadlocks when it tries to unregister
1790
+
* itself during panic as the panic notifier is called with RCU read
1791
+
* lock held and notifier unregistration does RCU synchronization.
1792
+
*/
1793
+
if (!panic)
1794
+
atomic_notifier_chain_unregister(&panic_notifier_list,
1795
+
&snp_panic_notifier);
1792
1796
1793
1797
/* Reset TMR size back to default */
1794
1798
sev_es_tmr_size = SEV_TMR_SIZE;
+1
drivers/crypto/ccp/sp-pci.c
+1
drivers/crypto/ccp/sp-pci.c
···
453
453
.cmdresp_reg = 0x10944, /* C2PMSG_17 */
454
454
.cmdbuff_addr_lo_reg = 0x10948, /* C2PMSG_18 */
455
455
.cmdbuff_addr_hi_reg = 0x1094c, /* C2PMSG_19 */
456
+
.bootloader_info_reg = 0x109ec, /* C2PMSG_59 */
456
457
.feature_reg = 0x109fc, /* C2PMSG_63 */
457
458
.inten_reg = 0x10510, /* P2CMSG_INTEN */
458
459
.intsts_reg = 0x10514, /* P2CMSG_INTSTS */
+27
-27
drivers/crypto/ccree/cc_buffer_mgr.c
+27
-27
drivers/crypto/ccree/cc_buffer_mgr.c
···
224
224
/* Set MLLI size for the bypass operation */
225
225
mlli_params->mlli_len = (total_nents * LLI_ENTRY_BYTE_SIZE);
226
226
227
-
dev_dbg(dev, "MLLI params: virt_addr=%pK dma_addr=%pad mlli_len=0x%X\n",
227
+
dev_dbg(dev, "MLLI params: virt_addr=%p dma_addr=%pad mlli_len=0x%X\n",
228
228
mlli_params->mlli_virt_addr, &mlli_params->mlli_dma_addr,
229
229
mlli_params->mlli_len);
230
230
···
239
239
{
240
240
unsigned int index = sgl_data->num_of_buffers;
241
241
242
-
dev_dbg(dev, "index=%u nents=%u sgl=%pK data_len=0x%08X is_last=%d\n",
242
+
dev_dbg(dev, "index=%u nents=%u sgl=%p data_len=0x%08X is_last=%d\n",
243
243
index, nents, sgl, data_len, is_last_table);
244
244
sgl_data->nents[index] = nents;
245
245
sgl_data->entry[index].sgl = sgl;
···
298
298
dev_err(dev, "dma_map_sg() config buffer failed\n");
299
299
return -ENOMEM;
300
300
}
301
-
dev_dbg(dev, "Mapped curr_buff: dma_address=%pad page=%p addr=%pK offset=%u length=%u\n",
301
+
dev_dbg(dev, "Mapped curr_buff: dma_address=%pad page=%p addr=%p offset=%u length=%u\n",
302
302
&sg_dma_address(&areq_ctx->ccm_adata_sg),
303
303
sg_page(&areq_ctx->ccm_adata_sg),
304
304
sg_virt(&areq_ctx->ccm_adata_sg),
···
323
323
dev_err(dev, "dma_map_sg() src buffer failed\n");
324
324
return -ENOMEM;
325
325
}
326
-
dev_dbg(dev, "Mapped curr_buff: dma_address=%pad page=%p addr=%pK offset=%u length=%u\n",
326
+
dev_dbg(dev, "Mapped curr_buff: dma_address=%pad page=%p addr=%p offset=%u length=%u\n",
327
327
&sg_dma_address(areq_ctx->buff_sg), sg_page(areq_ctx->buff_sg),
328
328
sg_virt(areq_ctx->buff_sg), areq_ctx->buff_sg->offset,
329
329
areq_ctx->buff_sg->length);
···
359
359
if (src != dst) {
360
360
dma_unmap_sg(dev, src, req_ctx->in_nents, DMA_TO_DEVICE);
361
361
dma_unmap_sg(dev, dst, req_ctx->out_nents, DMA_FROM_DEVICE);
362
-
dev_dbg(dev, "Unmapped req->dst=%pK\n", sg_virt(dst));
363
-
dev_dbg(dev, "Unmapped req->src=%pK\n", sg_virt(src));
362
+
dev_dbg(dev, "Unmapped req->dst=%p\n", sg_virt(dst));
363
+
dev_dbg(dev, "Unmapped req->src=%p\n", sg_virt(src));
364
364
} else {
365
365
dma_unmap_sg(dev, src, req_ctx->in_nents, DMA_BIDIRECTIONAL);
366
-
dev_dbg(dev, "Unmapped req->src=%pK\n", sg_virt(src));
366
+
dev_dbg(dev, "Unmapped req->src=%p\n", sg_virt(src));
367
367
}
368
368
}
369
369
···
391
391
req_ctx->gen_ctx.iv_dma_addr =
392
392
dma_map_single(dev, info, ivsize, DMA_BIDIRECTIONAL);
393
393
if (dma_mapping_error(dev, req_ctx->gen_ctx.iv_dma_addr)) {
394
-
dev_err(dev, "Mapping iv %u B at va=%pK for DMA failed\n",
394
+
dev_err(dev, "Mapping iv %u B at va=%p for DMA failed\n",
395
395
ivsize, info);
396
396
return -ENOMEM;
397
397
}
398
-
dev_dbg(dev, "Mapped iv %u B at va=%pK to dma=%pad\n",
398
+
dev_dbg(dev, "Mapped iv %u B at va=%p to dma=%pad\n",
399
399
ivsize, info, &req_ctx->gen_ctx.iv_dma_addr);
400
400
} else {
401
401
req_ctx->gen_ctx.iv_dma_addr = 0;
···
506
506
if ((areq_ctx->assoc_buff_type == CC_DMA_BUF_MLLI ||
507
507
areq_ctx->data_buff_type == CC_DMA_BUF_MLLI) &&
508
508
(areq_ctx->mlli_params.mlli_virt_addr)) {
509
-
dev_dbg(dev, "free MLLI buffer: dma=%pad virt=%pK\n",
509
+
dev_dbg(dev, "free MLLI buffer: dma=%pad virt=%p\n",
510
510
&areq_ctx->mlli_params.mlli_dma_addr,
511
511
areq_ctx->mlli_params.mlli_virt_addr);
512
512
dma_pool_free(areq_ctx->mlli_params.curr_pool,
···
514
514
areq_ctx->mlli_params.mlli_dma_addr);
515
515
}
516
516
517
-
dev_dbg(dev, "Unmapping src sgl: req->src=%pK areq_ctx->src.nents=%u areq_ctx->assoc.nents=%u assoclen:%u cryptlen=%u\n",
517
+
dev_dbg(dev, "Unmapping src sgl: req->src=%p areq_ctx->src.nents=%u areq_ctx->assoc.nents=%u assoclen:%u cryptlen=%u\n",
518
518
sg_virt(req->src), areq_ctx->src.nents, areq_ctx->assoc.nents,
519
519
areq_ctx->assoclen, req->cryptlen);
520
520
521
521
dma_unmap_sg(dev, req->src, areq_ctx->src.mapped_nents, src_direction);
522
522
if (req->src != req->dst) {
523
-
dev_dbg(dev, "Unmapping dst sgl: req->dst=%pK\n",
523
+
dev_dbg(dev, "Unmapping dst sgl: req->dst=%p\n",
524
524
sg_virt(req->dst));
525
525
dma_unmap_sg(dev, req->dst, areq_ctx->dst.mapped_nents, DMA_FROM_DEVICE);
526
526
}
···
566
566
dma_map_single(dev, areq_ctx->gen_ctx.iv, hw_iv_size,
567
567
DMA_BIDIRECTIONAL);
568
568
if (dma_mapping_error(dev, areq_ctx->gen_ctx.iv_dma_addr)) {
569
-
dev_err(dev, "Mapping iv %u B at va=%pK for DMA failed\n",
569
+
dev_err(dev, "Mapping iv %u B at va=%p for DMA failed\n",
570
570
hw_iv_size, req->iv);
571
571
kfree_sensitive(areq_ctx->gen_ctx.iv);
572
572
areq_ctx->gen_ctx.iv = NULL;
···
574
574
goto chain_iv_exit;
575
575
}
576
576
577
-
dev_dbg(dev, "Mapped iv %u B at va=%pK to dma=%pad\n",
577
+
dev_dbg(dev, "Mapped iv %u B at va=%p to dma=%pad\n",
578
578
hw_iv_size, req->iv, &areq_ctx->gen_ctx.iv_dma_addr);
579
579
580
580
chain_iv_exit:
···
977
977
dma_addr = dma_map_single(dev, areq_ctx->mac_buf, MAX_MAC_SIZE,
978
978
DMA_BIDIRECTIONAL);
979
979
if (dma_mapping_error(dev, dma_addr)) {
980
-
dev_err(dev, "Mapping mac_buf %u B at va=%pK for DMA failed\n",
980
+
dev_err(dev, "Mapping mac_buf %u B at va=%p for DMA failed\n",
981
981
MAX_MAC_SIZE, areq_ctx->mac_buf);
982
982
rc = -ENOMEM;
983
983
goto aead_map_failure;
···
991
991
DMA_TO_DEVICE);
992
992
993
993
if (dma_mapping_error(dev, dma_addr)) {
994
-
dev_err(dev, "Mapping mac_buf %u B at va=%pK for DMA failed\n",
994
+
dev_err(dev, "Mapping mac_buf %u B at va=%p for DMA failed\n",
995
995
AES_BLOCK_SIZE, addr);
996
996
areq_ctx->ccm_iv0_dma_addr = 0;
997
997
rc = -ENOMEM;
···
1009
1009
dma_addr = dma_map_single(dev, areq_ctx->hkey, AES_BLOCK_SIZE,
1010
1010
DMA_BIDIRECTIONAL);
1011
1011
if (dma_mapping_error(dev, dma_addr)) {
1012
-
dev_err(dev, "Mapping hkey %u B at va=%pK for DMA failed\n",
1012
+
dev_err(dev, "Mapping hkey %u B at va=%p for DMA failed\n",
1013
1013
AES_BLOCK_SIZE, areq_ctx->hkey);
1014
1014
rc = -ENOMEM;
1015
1015
goto aead_map_failure;
···
1019
1019
dma_addr = dma_map_single(dev, &areq_ctx->gcm_len_block,
1020
1020
AES_BLOCK_SIZE, DMA_TO_DEVICE);
1021
1021
if (dma_mapping_error(dev, dma_addr)) {
1022
-
dev_err(dev, "Mapping gcm_len_block %u B at va=%pK for DMA failed\n",
1022
+
dev_err(dev, "Mapping gcm_len_block %u B at va=%p for DMA failed\n",
1023
1023
AES_BLOCK_SIZE, &areq_ctx->gcm_len_block);
1024
1024
rc = -ENOMEM;
1025
1025
goto aead_map_failure;
···
1030
1030
AES_BLOCK_SIZE, DMA_TO_DEVICE);
1031
1031
1032
1032
if (dma_mapping_error(dev, dma_addr)) {
1033
-
dev_err(dev, "Mapping gcm_iv_inc1 %u B at va=%pK for DMA failed\n",
1033
+
dev_err(dev, "Mapping gcm_iv_inc1 %u B at va=%p for DMA failed\n",
1034
1034
AES_BLOCK_SIZE, (areq_ctx->gcm_iv_inc1));
1035
1035
areq_ctx->gcm_iv_inc1_dma_addr = 0;
1036
1036
rc = -ENOMEM;
···
1042
1042
AES_BLOCK_SIZE, DMA_TO_DEVICE);
1043
1043
1044
1044
if (dma_mapping_error(dev, dma_addr)) {
1045
-
dev_err(dev, "Mapping gcm_iv_inc2 %u B at va=%pK for DMA failed\n",
1045
+
dev_err(dev, "Mapping gcm_iv_inc2 %u B at va=%p for DMA failed\n",
1046
1046
AES_BLOCK_SIZE, (areq_ctx->gcm_iv_inc2));
1047
1047
areq_ctx->gcm_iv_inc2_dma_addr = 0;
1048
1048
rc = -ENOMEM;
···
1152
1152
u32 dummy = 0;
1153
1153
u32 mapped_nents = 0;
1154
1154
1155
-
dev_dbg(dev, "final params : curr_buff=%pK curr_buff_cnt=0x%X nbytes = 0x%X src=%pK curr_index=%u\n",
1155
+
dev_dbg(dev, "final params : curr_buff=%p curr_buff_cnt=0x%X nbytes = 0x%X src=%p curr_index=%u\n",
1156
1156
curr_buff, *curr_buff_cnt, nbytes, src, areq_ctx->buff_index);
1157
1157
/* Init the type of the dma buffer */
1158
1158
areq_ctx->data_dma_buf_type = CC_DMA_BUF_NULL;
···
1236
1236
u32 dummy = 0;
1237
1237
u32 mapped_nents = 0;
1238
1238
1239
-
dev_dbg(dev, " update params : curr_buff=%pK curr_buff_cnt=0x%X nbytes=0x%X src=%pK curr_index=%u\n",
1239
+
dev_dbg(dev, " update params : curr_buff=%p curr_buff_cnt=0x%X nbytes=0x%X src=%p curr_index=%u\n",
1240
1240
curr_buff, *curr_buff_cnt, nbytes, src, areq_ctx->buff_index);
1241
1241
/* Init the type of the dma buffer */
1242
1242
areq_ctx->data_dma_buf_type = CC_DMA_BUF_NULL;
···
1246
1246
areq_ctx->in_nents = 0;
1247
1247
1248
1248
if (total_in_len < block_size) {
1249
-
dev_dbg(dev, " less than one block: curr_buff=%pK *curr_buff_cnt=0x%X copy_to=%pK\n",
1249
+
dev_dbg(dev, " less than one block: curr_buff=%p *curr_buff_cnt=0x%X copy_to=%p\n",
1250
1250
curr_buff, *curr_buff_cnt, &curr_buff[*curr_buff_cnt]);
1251
1251
areq_ctx->in_nents = sg_nents_for_len(src, nbytes);
1252
1252
sg_copy_to_buffer(src, areq_ctx->in_nents,
···
1265
1265
1266
1266
/* Copy the new residue to next buffer */
1267
1267
if (*next_buff_cnt) {
1268
-
dev_dbg(dev, " handle residue: next buff %pK skip data %u residue %u\n",
1268
+
dev_dbg(dev, " handle residue: next buff %p skip data %u residue %u\n",
1269
1269
next_buff, (update_data_len - *curr_buff_cnt),
1270
1270
*next_buff_cnt);
1271
1271
cc_copy_sg_portion(dev, next_buff, src,
···
1338
1338
*allocated and should be released
1339
1339
*/
1340
1340
if (areq_ctx->mlli_params.curr_pool) {
1341
-
dev_dbg(dev, "free MLLI buffer: dma=%pad virt=%pK\n",
1341
+
dev_dbg(dev, "free MLLI buffer: dma=%pad virt=%p\n",
1342
1342
&areq_ctx->mlli_params.mlli_dma_addr,
1343
1343
areq_ctx->mlli_params.mlli_virt_addr);
1344
1344
dma_pool_free(areq_ctx->mlli_params.curr_pool,
···
1347
1347
}
1348
1348
1349
1349
if (src && areq_ctx->in_nents) {
1350
-
dev_dbg(dev, "Unmapped sg src: virt=%pK dma=%pad len=0x%X\n",
1350
+
dev_dbg(dev, "Unmapped sg src: virt=%p dma=%pad len=0x%X\n",
1351
1351
sg_virt(src), &sg_dma_address(src), sg_dma_len(src));
1352
1352
dma_unmap_sg(dev, src,
1353
1353
areq_ctx->in_nents, DMA_TO_DEVICE);
1354
1354
}
1355
1355
1356
1356
if (*prev_len) {
1357
-
dev_dbg(dev, "Unmapped buffer: areq_ctx->buff_sg=%pK dma=%pad len 0x%X\n",
1357
+
dev_dbg(dev, "Unmapped buffer: areq_ctx->buff_sg=%p dma=%pad len 0x%X\n",
1358
1358
sg_virt(areq_ctx->buff_sg),
1359
1359
&sg_dma_address(areq_ctx->buff_sg),
1360
1360
sg_dma_len(areq_ctx->buff_sg));
+2
-2
drivers/crypto/ccree/cc_cipher.c
+2
-2
drivers/crypto/ccree/cc_cipher.c
···
211
211
max_key_buf_size,
212
212
DMA_TO_DEVICE);
213
213
if (dma_mapping_error(dev, ctx_p->user.key_dma_addr)) {
214
-
dev_err(dev, "Mapping Key %u B at va=%pK for DMA failed\n",
214
+
dev_err(dev, "Mapping Key %u B at va=%p for DMA failed\n",
215
215
max_key_buf_size, ctx_p->user.key);
216
216
goto free_key;
217
217
}
218
-
dev_dbg(dev, "Mapped key %u B at va=%pK to dma=%pad\n",
218
+
dev_dbg(dev, "Mapped key %u B at va=%p to dma=%pad\n",
219
219
max_key_buf_size, ctx_p->user.key, &ctx_p->user.key_dma_addr);
220
220
221
221
return 0;
+15
-15
drivers/crypto/ccree/cc_hash.c
+15
-15
drivers/crypto/ccree/cc_hash.c
···
125
125
digestsize);
126
126
return -ENOMEM;
127
127
}
128
-
dev_dbg(dev, "Mapped digest result buffer %u B at va=%pK to dma=%pad\n",
128
+
dev_dbg(dev, "Mapped digest result buffer %u B at va=%p to dma=%pad\n",
129
129
digestsize, state->digest_result_buff,
130
130
&state->digest_result_dma_addr);
131
131
···
184
184
dma_map_single(dev, state->digest_buff,
185
185
ctx->inter_digestsize, DMA_BIDIRECTIONAL);
186
186
if (dma_mapping_error(dev, state->digest_buff_dma_addr)) {
187
-
dev_err(dev, "Mapping digest len %d B at va=%pK for DMA failed\n",
187
+
dev_err(dev, "Mapping digest len %d B at va=%p for DMA failed\n",
188
188
ctx->inter_digestsize, state->digest_buff);
189
189
return -EINVAL;
190
190
}
191
-
dev_dbg(dev, "Mapped digest %d B at va=%pK to dma=%pad\n",
191
+
dev_dbg(dev, "Mapped digest %d B at va=%p to dma=%pad\n",
192
192
ctx->inter_digestsize, state->digest_buff,
193
193
&state->digest_buff_dma_addr);
194
194
···
197
197
dma_map_single(dev, state->digest_bytes_len,
198
198
HASH_MAX_LEN_SIZE, DMA_BIDIRECTIONAL);
199
199
if (dma_mapping_error(dev, state->digest_bytes_len_dma_addr)) {
200
-
dev_err(dev, "Mapping digest len %u B at va=%pK for DMA failed\n",
200
+
dev_err(dev, "Mapping digest len %u B at va=%p for DMA failed\n",
201
201
HASH_MAX_LEN_SIZE, state->digest_bytes_len);
202
202
goto unmap_digest_buf;
203
203
}
204
-
dev_dbg(dev, "Mapped digest len %u B at va=%pK to dma=%pad\n",
204
+
dev_dbg(dev, "Mapped digest len %u B at va=%p to dma=%pad\n",
205
205
HASH_MAX_LEN_SIZE, state->digest_bytes_len,
206
206
&state->digest_bytes_len_dma_addr);
207
207
}
···
212
212
ctx->inter_digestsize,
213
213
DMA_BIDIRECTIONAL);
214
214
if (dma_mapping_error(dev, state->opad_digest_dma_addr)) {
215
-
dev_err(dev, "Mapping opad digest %d B at va=%pK for DMA failed\n",
215
+
dev_err(dev, "Mapping opad digest %d B at va=%p for DMA failed\n",
216
216
ctx->inter_digestsize,
217
217
state->opad_digest_buff);
218
218
goto unmap_digest_len;
219
219
}
220
-
dev_dbg(dev, "Mapped opad digest %d B at va=%pK to dma=%pad\n",
220
+
dev_dbg(dev, "Mapped opad digest %d B at va=%p to dma=%pad\n",
221
221
ctx->inter_digestsize, state->opad_digest_buff,
222
222
&state->opad_digest_dma_addr);
223
223
}
···
272
272
if (state->digest_result_dma_addr) {
273
273
dma_unmap_single(dev, state->digest_result_dma_addr, digestsize,
274
274
DMA_BIDIRECTIONAL);
275
-
dev_dbg(dev, "unmpa digest result buffer va (%pK) pa (%pad) len %u\n",
275
+
dev_dbg(dev, "unmpa digest result buffer va (%p) pa (%pad) len %u\n",
276
276
state->digest_result_buff,
277
277
&state->digest_result_dma_addr, digestsize);
278
278
memcpy(result, state->digest_result_buff, digestsize);
···
287
287
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
288
288
struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);
289
289
290
-
dev_dbg(dev, "req=%pK\n", req);
290
+
dev_dbg(dev, "req=%p\n", req);
291
291
292
292
if (err != -EINPROGRESS) {
293
293
/* Not a BACKLOG notification */
···
306
306
struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);
307
307
u32 digestsize = crypto_ahash_digestsize(tfm);
308
308
309
-
dev_dbg(dev, "req=%pK\n", req);
309
+
dev_dbg(dev, "req=%p\n", req);
310
310
311
311
if (err != -EINPROGRESS) {
312
312
/* Not a BACKLOG notification */
···
326
326
struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);
327
327
u32 digestsize = crypto_ahash_digestsize(tfm);
328
328
329
-
dev_dbg(dev, "req=%pK\n", req);
329
+
dev_dbg(dev, "req=%p\n", req);
330
330
331
331
if (err != -EINPROGRESS) {
332
332
/* Not a BACKLOG notification */
···
1077
1077
dma_map_single(dev, ctx->digest_buff, sizeof(ctx->digest_buff),
1078
1078
DMA_BIDIRECTIONAL);
1079
1079
if (dma_mapping_error(dev, ctx->digest_buff_dma_addr)) {
1080
-
dev_err(dev, "Mapping digest len %zu B at va=%pK for DMA failed\n",
1080
+
dev_err(dev, "Mapping digest len %zu B at va=%p for DMA failed\n",
1081
1081
sizeof(ctx->digest_buff), ctx->digest_buff);
1082
1082
goto fail;
1083
1083
}
1084
-
dev_dbg(dev, "Mapped digest %zu B at va=%pK to dma=%pad\n",
1084
+
dev_dbg(dev, "Mapped digest %zu B at va=%p to dma=%pad\n",
1085
1085
sizeof(ctx->digest_buff), ctx->digest_buff,
1086
1086
&ctx->digest_buff_dma_addr);
1087
1087
···
1090
1090
sizeof(ctx->opad_tmp_keys_buff),
1091
1091
DMA_BIDIRECTIONAL);
1092
1092
if (dma_mapping_error(dev, ctx->opad_tmp_keys_dma_addr)) {
1093
-
dev_err(dev, "Mapping opad digest %zu B at va=%pK for DMA failed\n",
1093
+
dev_err(dev, "Mapping opad digest %zu B at va=%p for DMA failed\n",
1094
1094
sizeof(ctx->opad_tmp_keys_buff),
1095
1095
ctx->opad_tmp_keys_buff);
1096
1096
goto fail;
1097
1097
}
1098
-
dev_dbg(dev, "Mapped opad_tmp_keys %zu B at va=%pK to dma=%pad\n",
1098
+
dev_dbg(dev, "Mapped opad_tmp_keys %zu B at va=%p to dma=%pad\n",
1099
1099
sizeof(ctx->opad_tmp_keys_buff), ctx->opad_tmp_keys_buff,
1100
1100
&ctx->opad_tmp_keys_dma_addr);
1101
1101
-1
drivers/crypto/ccree/cc_pm.c
-1
drivers/crypto/ccree/cc_pm.c
+6
-2
drivers/crypto/hisilicon/hpre/hpre_crypto.c
+6
-2
drivers/crypto/hisilicon/hpre/hpre_crypto.c
···
1491
1491
if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld))
1492
1492
atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value);
1493
1493
1494
+
/* Do unmap before data processing */
1495
+
hpre_ecdh_hw_data_clr_all(ctx, req, areq->dst, areq->src);
1496
+
1494
1497
p = sg_virt(areq->dst);
1495
1498
memmove(p, p + ctx->key_sz - curve_sz, curve_sz);
1496
1499
memmove(p + curve_sz, p + areq->dst_len - curve_sz, curve_sz);
1497
1500
1498
-
hpre_ecdh_hw_data_clr_all(ctx, req, areq->dst, areq->src);
1499
1501
kpp_request_complete(areq, ret);
1500
1502
1501
1503
atomic64_inc(&dfx[HPRE_RECV_CNT].value);
···
1810
1808
if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld))
1811
1809
atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value);
1812
1810
1811
+
/* Do unmap before data processing */
1812
+
hpre_curve25519_hw_data_clr_all(ctx, req, areq->dst, areq->src);
1813
+
1813
1814
hpre_key_to_big_end(sg_virt(areq->dst), CURVE25519_KEY_SIZE);
1814
1815
1815
-
hpre_curve25519_hw_data_clr_all(ctx, req, areq->dst, areq->src);
1816
1816
kpp_request_complete(areq, ret);
1817
1817
1818
1818
atomic64_inc(&dfx[HPRE_RECV_CNT].value);
-1
drivers/crypto/hisilicon/qm.c
-1
drivers/crypto/hisilicon/qm.c
+56
-7
drivers/crypto/hisilicon/sec2/sec.h
+56
-7
drivers/crypto/hisilicon/sec2/sec.h
···
7
7
#include <linux/hisi_acc_qm.h>
8
8
#include "sec_crypto.h"
9
9
10
+
#define SEC_PBUF_SZ 512
11
+
#define SEC_MAX_MAC_LEN 64
12
+
#define SEC_IV_SIZE 24
13
+
#define SEC_SGE_NR_NUM 4
14
+
#define SEC_SGL_ALIGN_SIZE 64
15
+
10
16
/* Algorithm resource per hardware SEC queue */
11
17
struct sec_alg_res {
12
18
u8 *pbuf;
···
26
20
u16 depth;
27
21
};
28
22
23
+
struct sec_hw_sge {
24
+
dma_addr_t buf;
25
+
void *page_ctrl;
26
+
__le32 len;
27
+
__le32 pad;
28
+
__le32 pad0;
29
+
__le32 pad1;
30
+
};
31
+
32
+
struct sec_hw_sgl {
33
+
dma_addr_t next_dma;
34
+
__le16 entry_sum_in_chain;
35
+
__le16 entry_sum_in_sgl;
36
+
__le16 entry_length_in_sgl;
37
+
__le16 pad0;
38
+
__le64 pad1[5];
39
+
struct sec_hw_sgl *next;
40
+
struct sec_hw_sge sge_entries[SEC_SGE_NR_NUM];
41
+
} __aligned(SEC_SGL_ALIGN_SIZE);
42
+
43
+
struct sec_src_dst_buf {
44
+
struct sec_hw_sgl in;
45
+
struct sec_hw_sgl out;
46
+
};
47
+
48
+
struct sec_request_buf {
49
+
union {
50
+
struct sec_src_dst_buf data_buf;
51
+
__u8 pbuf[SEC_PBUF_SZ];
52
+
};
53
+
dma_addr_t in_dma;
54
+
dma_addr_t out_dma;
55
+
};
56
+
29
57
/* Cipher request of SEC private */
30
58
struct sec_cipher_req {
31
59
struct hisi_acc_hw_sgl *c_out;
···
69
29
struct skcipher_request *sk_req;
70
30
u32 c_len;
71
31
bool encrypt;
32
+
__u8 c_ivin_buf[SEC_IV_SIZE];
72
33
};
73
34
74
35
struct sec_aead_req {
···
78
37
u8 *a_ivin;
79
38
dma_addr_t a_ivin_dma;
80
39
struct aead_request *aead_req;
40
+
__u8 a_ivin_buf[SEC_IV_SIZE];
41
+
__u8 out_mac_buf[SEC_MAX_MAC_LEN];
42
+
};
43
+
44
+
struct sec_instance_backlog {
45
+
struct list_head list;
46
+
spinlock_t lock;
81
47
};
82
48
83
49
/* SEC request of Crypto */
···
103
55
dma_addr_t in_dma;
104
56
struct sec_cipher_req c_req;
105
57
struct sec_aead_req aead_req;
106
-
struct list_head backlog_head;
58
+
struct crypto_async_request *base;
107
59
108
60
int err_type;
109
61
int req_id;
110
62
u32 flag;
111
63
112
-
/* Status of the SEC request */
113
-
bool fake_busy;
114
64
bool use_pbuf;
65
+
66
+
struct list_head list;
67
+
struct sec_instance_backlog *backlog;
68
+
struct sec_request_buf buf;
115
69
};
116
70
117
71
/**
···
169
119
struct sec_alg_res *res;
170
120
struct sec_ctx *ctx;
171
121
spinlock_t req_lock;
172
-
struct list_head backlog;
122
+
spinlock_t id_lock;
173
123
struct hisi_acc_sgl_pool *c_in_pool;
174
124
struct hisi_acc_sgl_pool *c_out_pool;
125
+
struct sec_instance_backlog backlog;
126
+
u16 send_head;
175
127
};
176
128
177
129
enum sec_alg_type {
···
190
138
191
139
/* Half queues for encipher, and half for decipher */
192
140
u32 hlf_q_num;
193
-
194
-
/* Threshold for fake busy, trigger to return -EBUSY to user */
195
-
u32 fake_req_limit;
196
141
197
142
/* Current cyclic index to select a queue for encipher */
198
143
atomic_t enc_qcyclic;
+414
-181
drivers/crypto/hisilicon/sec2/sec_crypto.c
+414
-181
drivers/crypto/hisilicon/sec2/sec_crypto.c
···
67
67
#define SEC_MAX_CCM_AAD_LEN 65279
68
68
#define SEC_TOTAL_MAC_SZ(depth) (SEC_MAX_MAC_LEN * (depth))
69
69
70
-
#define SEC_PBUF_SZ 512
71
70
#define SEC_PBUF_IV_OFFSET SEC_PBUF_SZ
72
71
#define SEC_PBUF_MAC_OFFSET (SEC_PBUF_SZ + SEC_IV_SIZE)
73
72
#define SEC_PBUF_PKG (SEC_PBUF_SZ + SEC_IV_SIZE + \
···
101
102
#define IV_LAST_BYTE_MASK 0xFF
102
103
#define IV_CTR_INIT 0x1
103
104
#define IV_BYTE_OFFSET 0x8
105
+
#define SEC_GCM_MIN_AUTH_SZ 0x8
106
+
#define SEC_RETRY_MAX_CNT 5U
104
107
105
108
static DEFINE_MUTEX(sec_algs_lock);
106
109
static unsigned int sec_available_devs;
···
117
116
struct aead_alg alg;
118
117
};
119
118
120
-
/* Get an en/de-cipher queue cyclically to balance load over queues of TFM */
121
-
static inline u32 sec_alloc_queue_id(struct sec_ctx *ctx, struct sec_req *req)
122
-
{
123
-
if (req->c_req.encrypt)
124
-
return (u32)atomic_inc_return(&ctx->enc_qcyclic) %
125
-
ctx->hlf_q_num;
126
-
127
-
return (u32)atomic_inc_return(&ctx->dec_qcyclic) % ctx->hlf_q_num +
128
-
ctx->hlf_q_num;
129
-
}
130
-
131
-
static inline void sec_free_queue_id(struct sec_ctx *ctx, struct sec_req *req)
132
-
{
133
-
if (req->c_req.encrypt)
134
-
atomic_dec(&ctx->enc_qcyclic);
135
-
else
136
-
atomic_dec(&ctx->dec_qcyclic);
137
-
}
119
+
static int sec_aead_soft_crypto(struct sec_ctx *ctx,
120
+
struct aead_request *aead_req,
121
+
bool encrypt);
122
+
static int sec_skcipher_soft_crypto(struct sec_ctx *ctx,
123
+
struct skcipher_request *sreq, bool encrypt);
138
124
139
125
static int sec_alloc_req_id(struct sec_req *req, struct sec_qp_ctx *qp_ctx)
140
126
{
141
127
int req_id;
142
128
143
-
spin_lock_bh(&qp_ctx->req_lock);
129
+
spin_lock_bh(&qp_ctx->id_lock);
144
130
req_id = idr_alloc_cyclic(&qp_ctx->req_idr, NULL, 0, qp_ctx->qp->sq_depth, GFP_ATOMIC);
145
-
spin_unlock_bh(&qp_ctx->req_lock);
146
-
if (unlikely(req_id < 0)) {
147
-
dev_err(req->ctx->dev, "alloc req id fail!\n");
148
-
return req_id;
149
-
}
150
-
151
-
req->qp_ctx = qp_ctx;
152
-
qp_ctx->req_list[req_id] = req;
153
-
131
+
spin_unlock_bh(&qp_ctx->id_lock);
154
132
return req_id;
155
133
}
156
134
···
143
163
return;
144
164
}
145
165
146
-
qp_ctx->req_list[req_id] = NULL;
147
-
req->qp_ctx = NULL;
148
-
149
-
spin_lock_bh(&qp_ctx->req_lock);
166
+
spin_lock_bh(&qp_ctx->id_lock);
150
167
idr_remove(&qp_ctx->req_idr, req_id);
151
-
spin_unlock_bh(&qp_ctx->req_lock);
168
+
spin_unlock_bh(&qp_ctx->id_lock);
152
169
}
153
170
154
171
static u8 pre_parse_finished_bd(struct bd_status *status, void *resp)
···
206
229
return 0;
207
230
}
208
231
232
+
static int qp_send_message(struct sec_req *req)
233
+
{
234
+
struct sec_qp_ctx *qp_ctx = req->qp_ctx;
235
+
int ret;
236
+
237
+
if (atomic_read(&qp_ctx->qp->qp_status.used) == qp_ctx->qp->sq_depth - 1)
238
+
return -EBUSY;
239
+
240
+
spin_lock_bh(&qp_ctx->req_lock);
241
+
if (atomic_read(&qp_ctx->qp->qp_status.used) == qp_ctx->qp->sq_depth - 1) {
242
+
spin_unlock_bh(&qp_ctx->req_lock);
243
+
return -EBUSY;
244
+
}
245
+
246
+
if (qp_ctx->ctx->type_supported == SEC_BD_TYPE2) {
247
+
req->sec_sqe.type2.tag = cpu_to_le16((u16)qp_ctx->send_head);
248
+
qp_ctx->req_list[qp_ctx->send_head] = req;
249
+
}
250
+
251
+
ret = hisi_qp_send(qp_ctx->qp, &req->sec_sqe);
252
+
if (ret) {
253
+
spin_unlock_bh(&qp_ctx->req_lock);
254
+
return ret;
255
+
}
256
+
if (qp_ctx->ctx->type_supported == SEC_BD_TYPE2)
257
+
qp_ctx->send_head = (qp_ctx->send_head + 1) % qp_ctx->qp->sq_depth;
258
+
259
+
spin_unlock_bh(&qp_ctx->req_lock);
260
+
261
+
atomic64_inc(&req->ctx->sec->debug.dfx.send_cnt);
262
+
return -EINPROGRESS;
263
+
}
264
+
265
+
static void sec_alg_send_backlog_soft(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx)
266
+
{
267
+
struct sec_req *req, *tmp;
268
+
int ret;
269
+
270
+
list_for_each_entry_safe(req, tmp, &qp_ctx->backlog.list, list) {
271
+
list_del(&req->list);
272
+
ctx->req_op->buf_unmap(ctx, req);
273
+
if (req->req_id >= 0)
274
+
sec_free_req_id(req);
275
+
276
+
if (ctx->alg_type == SEC_AEAD)
277
+
ret = sec_aead_soft_crypto(ctx, req->aead_req.aead_req,
278
+
req->c_req.encrypt);
279
+
else
280
+
ret = sec_skcipher_soft_crypto(ctx, req->c_req.sk_req,
281
+
req->c_req.encrypt);
282
+
283
+
/* Wake up the busy thread first, then return the errno. */
284
+
crypto_request_complete(req->base, -EINPROGRESS);
285
+
crypto_request_complete(req->base, ret);
286
+
}
287
+
}
288
+
289
+
static void sec_alg_send_backlog(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx)
290
+
{
291
+
struct sec_req *req, *tmp;
292
+
int ret;
293
+
294
+
spin_lock_bh(&qp_ctx->backlog.lock);
295
+
list_for_each_entry_safe(req, tmp, &qp_ctx->backlog.list, list) {
296
+
ret = qp_send_message(req);
297
+
switch (ret) {
298
+
case -EINPROGRESS:
299
+
list_del(&req->list);
300
+
crypto_request_complete(req->base, -EINPROGRESS);
301
+
break;
302
+
case -EBUSY:
303
+
/* Device is busy and stop send any request. */
304
+
goto unlock;
305
+
default:
306
+
/* Release memory resources and send all requests through software. */
307
+
sec_alg_send_backlog_soft(ctx, qp_ctx);
308
+
goto unlock;
309
+
}
310
+
}
311
+
312
+
unlock:
313
+
spin_unlock_bh(&qp_ctx->backlog.lock);
314
+
}
315
+
209
316
static void sec_req_cb(struct hisi_qp *qp, void *resp)
210
317
{
211
318
struct sec_qp_ctx *qp_ctx = qp->qp_ctx;
···
334
273
ctx->req_op->callback(ctx, req, err);
335
274
}
336
275
337
-
static int sec_bd_send(struct sec_ctx *ctx, struct sec_req *req)
276
+
static int sec_alg_send_message_retry(struct sec_req *req)
338
277
{
339
-
struct sec_qp_ctx *qp_ctx = req->qp_ctx;
278
+
int ctr = 0;
340
279
int ret;
341
280
342
-
if (ctx->fake_req_limit <=
343
-
atomic_read(&qp_ctx->qp->qp_status.used) &&
344
-
!(req->flag & CRYPTO_TFM_REQ_MAY_BACKLOG))
345
-
return -EBUSY;
346
-
347
-
spin_lock_bh(&qp_ctx->req_lock);
348
-
ret = hisi_qp_send(qp_ctx->qp, &req->sec_sqe);
349
-
if (ctx->fake_req_limit <=
350
-
atomic_read(&qp_ctx->qp->qp_status.used) && !ret) {
351
-
list_add_tail(&req->backlog_head, &qp_ctx->backlog);
352
-
atomic64_inc(&ctx->sec->debug.dfx.send_cnt);
353
-
atomic64_inc(&ctx->sec->debug.dfx.send_busy_cnt);
354
-
spin_unlock_bh(&qp_ctx->req_lock);
355
-
return -EBUSY;
356
-
}
357
-
spin_unlock_bh(&qp_ctx->req_lock);
358
-
359
-
if (unlikely(ret == -EBUSY))
360
-
return -ENOBUFS;
361
-
362
-
if (likely(!ret)) {
363
-
ret = -EINPROGRESS;
364
-
atomic64_inc(&ctx->sec->debug.dfx.send_cnt);
365
-
}
281
+
do {
282
+
ret = qp_send_message(req);
283
+
} while (ret == -EBUSY && ctr++ < SEC_RETRY_MAX_CNT);
366
284
367
285
return ret;
368
286
}
369
287
370
-
/* Get DMA memory resources */
288
+
static int sec_alg_try_enqueue(struct sec_req *req)
289
+
{
290
+
/* Check if any request is already backlogged */
291
+
if (!list_empty(&req->backlog->list))
292
+
return -EBUSY;
293
+
294
+
/* Try to enqueue to HW ring */
295
+
return qp_send_message(req);
296
+
}
297
+
298
+
299
+
static int sec_alg_send_message_maybacklog(struct sec_req *req)
300
+
{
301
+
int ret;
302
+
303
+
ret = sec_alg_try_enqueue(req);
304
+
if (ret != -EBUSY)
305
+
return ret;
306
+
307
+
spin_lock_bh(&req->backlog->lock);
308
+
ret = sec_alg_try_enqueue(req);
309
+
if (ret == -EBUSY)
310
+
list_add_tail(&req->list, &req->backlog->list);
311
+
spin_unlock_bh(&req->backlog->lock);
312
+
313
+
return ret;
314
+
}
315
+
316
+
static int sec_bd_send(struct sec_ctx *ctx, struct sec_req *req)
317
+
{
318
+
if (req->flag & CRYPTO_TFM_REQ_MAY_BACKLOG)
319
+
return sec_alg_send_message_maybacklog(req);
320
+
321
+
return sec_alg_send_message_retry(req);
322
+
}
323
+
371
324
static int sec_alloc_civ_resource(struct device *dev, struct sec_alg_res *res)
372
325
{
373
326
u16 q_depth = res->depth;
···
633
558
634
559
spin_lock_init(&qp_ctx->req_lock);
635
560
idr_init(&qp_ctx->req_idr);
636
-
INIT_LIST_HEAD(&qp_ctx->backlog);
561
+
spin_lock_init(&qp_ctx->backlog.lock);
562
+
spin_lock_init(&qp_ctx->id_lock);
563
+
INIT_LIST_HEAD(&qp_ctx->backlog.list);
564
+
qp_ctx->send_head = 0;
637
565
638
566
ret = sec_alloc_qp_ctx_resource(ctx, qp_ctx);
639
567
if (ret)
···
680
602
ctx->hlf_q_num = sec->ctx_q_num >> 1;
681
603
682
604
ctx->pbuf_supported = ctx->sec->iommu_used;
683
-
684
-
/* Half of queue depth is taken as fake requests limit in the queue. */
685
-
ctx->fake_req_limit = ctx->qps[0]->sq_depth >> 1;
686
605
ctx->qp_ctx = kcalloc(sec->ctx_q_num, sizeof(struct sec_qp_ctx),
687
606
GFP_KERNEL);
688
607
if (!ctx->qp_ctx) {
···
781
706
int ret;
782
707
783
708
ctx->alg_type = SEC_SKCIPHER;
784
-
crypto_skcipher_set_reqsize(tfm, sizeof(struct sec_req));
709
+
crypto_skcipher_set_reqsize_dma(tfm, sizeof(struct sec_req));
785
710
ctx->c_ctx.ivsize = crypto_skcipher_ivsize(tfm);
786
711
if (ctx->c_ctx.ivsize > SEC_IV_SIZE) {
787
712
pr_err("get error skcipher iv size!\n");
···
958
883
static int sec_cipher_pbuf_map(struct sec_ctx *ctx, struct sec_req *req,
959
884
struct scatterlist *src)
960
885
{
961
-
struct sec_aead_req *a_req = &req->aead_req;
962
-
struct aead_request *aead_req = a_req->aead_req;
886
+
struct aead_request *aead_req = req->aead_req.aead_req;
963
887
struct sec_cipher_req *c_req = &req->c_req;
964
888
struct sec_qp_ctx *qp_ctx = req->qp_ctx;
889
+
struct sec_request_buf *buf = &req->buf;
965
890
struct device *dev = ctx->dev;
966
891
int copy_size, pbuf_length;
967
892
int req_id = req->req_id;
968
893
struct crypto_aead *tfm;
894
+
u8 *mac_offset, *pbuf;
969
895
size_t authsize;
970
-
u8 *mac_offset;
971
896
972
897
if (ctx->alg_type == SEC_AEAD)
973
898
copy_size = aead_req->cryptlen + aead_req->assoclen;
974
899
else
975
900
copy_size = c_req->c_len;
976
901
977
-
pbuf_length = sg_copy_to_buffer(src, sg_nents(src),
978
-
qp_ctx->res[req_id].pbuf, copy_size);
902
+
903
+
pbuf = req->req_id < 0 ? buf->pbuf : qp_ctx->res[req_id].pbuf;
904
+
pbuf_length = sg_copy_to_buffer(src, sg_nents(src), pbuf, copy_size);
979
905
if (unlikely(pbuf_length != copy_size)) {
980
906
dev_err(dev, "copy src data to pbuf error!\n");
981
907
return -EINVAL;
···
984
908
if (!c_req->encrypt && ctx->alg_type == SEC_AEAD) {
985
909
tfm = crypto_aead_reqtfm(aead_req);
986
910
authsize = crypto_aead_authsize(tfm);
987
-
mac_offset = qp_ctx->res[req_id].pbuf + copy_size - authsize;
988
-
memcpy(a_req->out_mac, mac_offset, authsize);
911
+
mac_offset = pbuf + copy_size - authsize;
912
+
memcpy(req->aead_req.out_mac, mac_offset, authsize);
913
+
}
914
+
915
+
if (req->req_id < 0) {
916
+
buf->in_dma = dma_map_single(dev, buf->pbuf, SEC_PBUF_SZ, DMA_BIDIRECTIONAL);
917
+
if (unlikely(dma_mapping_error(dev, buf->in_dma)))
918
+
return -ENOMEM;
919
+
920
+
buf->out_dma = buf->in_dma;
921
+
return 0;
989
922
}
990
923
991
924
req->in_dma = qp_ctx->res[req_id].pbuf_dma;
···
1009
924
struct aead_request *aead_req = req->aead_req.aead_req;
1010
925
struct sec_cipher_req *c_req = &req->c_req;
1011
926
struct sec_qp_ctx *qp_ctx = req->qp_ctx;
927
+
struct sec_request_buf *buf = &req->buf;
1012
928
int copy_size, pbuf_length;
1013
929
int req_id = req->req_id;
1014
930
···
1018
932
else
1019
933
copy_size = c_req->c_len;
1020
934
1021
-
pbuf_length = sg_copy_from_buffer(dst, sg_nents(dst),
1022
-
qp_ctx->res[req_id].pbuf, copy_size);
935
+
if (req->req_id < 0)
936
+
pbuf_length = sg_copy_from_buffer(dst, sg_nents(dst), buf->pbuf, copy_size);
937
+
else
938
+
pbuf_length = sg_copy_from_buffer(dst, sg_nents(dst), qp_ctx->res[req_id].pbuf,
939
+
copy_size);
1023
940
if (unlikely(pbuf_length != copy_size))
1024
941
dev_err(ctx->dev, "copy pbuf data to dst error!\n");
942
+
943
+
if (req->req_id < 0)
944
+
dma_unmap_single(ctx->dev, buf->in_dma, SEC_PBUF_SZ, DMA_BIDIRECTIONAL);
1025
945
}
1026
946
1027
947
static int sec_aead_mac_init(struct sec_aead_req *req)
···
1049
957
return 0;
1050
958
}
1051
959
1052
-
static int sec_cipher_map(struct sec_ctx *ctx, struct sec_req *req,
1053
-
struct scatterlist *src, struct scatterlist *dst)
960
+
static void fill_sg_to_hw_sge(struct scatterlist *sgl, struct sec_hw_sge *hw_sge)
961
+
{
962
+
hw_sge->buf = sg_dma_address(sgl);
963
+
hw_sge->len = cpu_to_le32(sg_dma_len(sgl));
964
+
hw_sge->page_ctrl = sg_virt(sgl);
965
+
}
966
+
967
+
static int sec_cipher_to_hw_sgl(struct device *dev, struct scatterlist *src,
968
+
struct sec_hw_sgl *src_in, dma_addr_t *hw_sgl_dma,
969
+
int dma_dir)
970
+
{
971
+
struct sec_hw_sge *curr_hw_sge = src_in->sge_entries;
972
+
u32 i, sg_n, sg_n_mapped;
973
+
struct scatterlist *sg;
974
+
u32 sge_var = 0;
975
+
976
+
sg_n = sg_nents(src);
977
+
sg_n_mapped = dma_map_sg(dev, src, sg_n, dma_dir);
978
+
if (unlikely(!sg_n_mapped)) {
979
+
dev_err(dev, "dma mapping for SG error!\n");
980
+
return -EINVAL;
981
+
} else if (unlikely(sg_n_mapped > SEC_SGE_NR_NUM)) {
982
+
dev_err(dev, "the number of entries in input scatterlist error!\n");
983
+
dma_unmap_sg(dev, src, sg_n, dma_dir);
984
+
return -EINVAL;
985
+
}
986
+
987
+
for_each_sg(src, sg, sg_n_mapped, i) {
988
+
fill_sg_to_hw_sge(sg, curr_hw_sge);
989
+
curr_hw_sge++;
990
+
sge_var++;
991
+
}
992
+
993
+
src_in->entry_sum_in_sgl = cpu_to_le16(sge_var);
994
+
src_in->entry_sum_in_chain = cpu_to_le16(SEC_SGE_NR_NUM);
995
+
src_in->entry_length_in_sgl = cpu_to_le16(SEC_SGE_NR_NUM);
996
+
*hw_sgl_dma = dma_map_single(dev, src_in, sizeof(struct sec_hw_sgl), dma_dir);
997
+
if (unlikely(dma_mapping_error(dev, *hw_sgl_dma))) {
998
+
dma_unmap_sg(dev, src, sg_n, dma_dir);
999
+
return -ENOMEM;
1000
+
}
1001
+
1002
+
return 0;
1003
+
}
1004
+
1005
+
static void sec_cipher_put_hw_sgl(struct device *dev, struct scatterlist *src,
1006
+
dma_addr_t src_in, int dma_dir)
1007
+
{
1008
+
dma_unmap_single(dev, src_in, sizeof(struct sec_hw_sgl), dma_dir);
1009
+
dma_unmap_sg(dev, src, sg_nents(src), dma_dir);
1010
+
}
1011
+
1012
+
static int sec_cipher_map_sgl(struct device *dev, struct sec_req *req,
1013
+
struct scatterlist *src, struct scatterlist *dst)
1014
+
{
1015
+
struct sec_hw_sgl *src_in = &req->buf.data_buf.in;
1016
+
struct sec_hw_sgl *dst_out = &req->buf.data_buf.out;
1017
+
int ret;
1018
+
1019
+
if (dst == src) {
1020
+
ret = sec_cipher_to_hw_sgl(dev, src, src_in, &req->buf.in_dma,
1021
+
DMA_BIDIRECTIONAL);
1022
+
req->buf.out_dma = req->buf.in_dma;
1023
+
return ret;
1024
+
}
1025
+
1026
+
ret = sec_cipher_to_hw_sgl(dev, src, src_in, &req->buf.in_dma, DMA_TO_DEVICE);
1027
+
if (unlikely(ret))
1028
+
return ret;
1029
+
1030
+
ret = sec_cipher_to_hw_sgl(dev, dst, dst_out, &req->buf.out_dma,
1031
+
DMA_FROM_DEVICE);
1032
+
if (unlikely(ret)) {
1033
+
sec_cipher_put_hw_sgl(dev, src, req->buf.in_dma, DMA_TO_DEVICE);
1034
+
return ret;
1035
+
}
1036
+
1037
+
return 0;
1038
+
}
1039
+
1040
+
static int sec_cipher_map_inner(struct sec_ctx *ctx, struct sec_req *req,
1041
+
struct scatterlist *src, struct scatterlist *dst)
1054
1042
{
1055
1043
struct sec_cipher_req *c_req = &req->c_req;
1056
1044
struct sec_aead_req *a_req = &req->aead_req;
1057
1045
struct sec_qp_ctx *qp_ctx = req->qp_ctx;
1058
1046
struct sec_alg_res *res = &qp_ctx->res[req->req_id];
1059
1047
struct device *dev = ctx->dev;
1048
+
enum dma_data_direction src_direction;
1060
1049
int ret;
1061
1050
1062
1051
if (req->use_pbuf) {
···
1150
977
a_req->out_mac_dma = res->pbuf_dma +
1151
978
SEC_PBUF_MAC_OFFSET;
1152
979
}
1153
-
ret = sec_cipher_pbuf_map(ctx, req, src);
1154
-
1155
-
return ret;
980
+
return sec_cipher_pbuf_map(ctx, req, src);
1156
981
}
982
+
1157
983
c_req->c_ivin = res->c_ivin;
1158
984
c_req->c_ivin_dma = res->c_ivin_dma;
1159
985
if (ctx->alg_type == SEC_AEAD) {
···
1162
990
a_req->out_mac_dma = res->out_mac_dma;
1163
991
}
1164
992
993
+
src_direction = dst == src ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
1165
994
req->in = hisi_acc_sg_buf_map_to_hw_sgl(dev, src,
1166
995
qp_ctx->c_in_pool,
1167
996
req->req_id,
1168
-
&req->in_dma);
997
+
&req->in_dma, src_direction);
1169
998
if (IS_ERR(req->in)) {
1170
999
dev_err(dev, "fail to dma map input sgl buffers!\n");
1171
1000
return PTR_ERR(req->in);
···
1176
1003
ret = sec_aead_mac_init(a_req);
1177
1004
if (unlikely(ret)) {
1178
1005
dev_err(dev, "fail to init mac data for ICV!\n");
1179
-
hisi_acc_sg_buf_unmap(dev, src, req->in);
1006
+
hisi_acc_sg_buf_unmap(dev, src, req->in, src_direction);
1180
1007
return ret;
1181
1008
}
1182
1009
}
···
1188
1015
c_req->c_out = hisi_acc_sg_buf_map_to_hw_sgl(dev, dst,
1189
1016
qp_ctx->c_out_pool,
1190
1017
req->req_id,
1191
-
&c_req->c_out_dma);
1018
+
&c_req->c_out_dma,
1019
+
DMA_FROM_DEVICE);
1192
1020
1193
1021
if (IS_ERR(c_req->c_out)) {
1194
1022
dev_err(dev, "fail to dma map output sgl buffers!\n");
1195
-
hisi_acc_sg_buf_unmap(dev, src, req->in);
1023
+
hisi_acc_sg_buf_unmap(dev, src, req->in, src_direction);
1196
1024
return PTR_ERR(c_req->c_out);
1197
1025
}
1198
1026
}
···
1201
1027
return 0;
1202
1028
}
1203
1029
1030
+
static int sec_cipher_map(struct sec_ctx *ctx, struct sec_req *req,
1031
+
struct scatterlist *src, struct scatterlist *dst)
1032
+
{
1033
+
struct sec_aead_req *a_req = &req->aead_req;
1034
+
struct sec_cipher_req *c_req = &req->c_req;
1035
+
bool is_aead = (ctx->alg_type == SEC_AEAD);
1036
+
struct device *dev = ctx->dev;
1037
+
int ret = -ENOMEM;
1038
+
1039
+
if (req->req_id >= 0)
1040
+
return sec_cipher_map_inner(ctx, req, src, dst);
1041
+
1042
+
c_req->c_ivin = c_req->c_ivin_buf;
1043
+
c_req->c_ivin_dma = dma_map_single(dev, c_req->c_ivin,
1044
+
SEC_IV_SIZE, DMA_TO_DEVICE);
1045
+
if (unlikely(dma_mapping_error(dev, c_req->c_ivin_dma)))
1046
+
return -ENOMEM;
1047
+
1048
+
if (is_aead) {
1049
+
a_req->a_ivin = a_req->a_ivin_buf;
1050
+
a_req->out_mac = a_req->out_mac_buf;
1051
+
a_req->a_ivin_dma = dma_map_single(dev, a_req->a_ivin,
1052
+
SEC_IV_SIZE, DMA_TO_DEVICE);
1053
+
if (unlikely(dma_mapping_error(dev, a_req->a_ivin_dma)))
1054
+
goto free_c_ivin_dma;
1055
+
1056
+
a_req->out_mac_dma = dma_map_single(dev, a_req->out_mac,
1057
+
SEC_MAX_MAC_LEN, DMA_BIDIRECTIONAL);
1058
+
if (unlikely(dma_mapping_error(dev, a_req->out_mac_dma)))
1059
+
goto free_a_ivin_dma;
1060
+
}
1061
+
if (req->use_pbuf) {
1062
+
ret = sec_cipher_pbuf_map(ctx, req, src);
1063
+
if (unlikely(ret))
1064
+
goto free_out_mac_dma;
1065
+
1066
+
return 0;
1067
+
}
1068
+
1069
+
if (!c_req->encrypt && is_aead) {
1070
+
ret = sec_aead_mac_init(a_req);
1071
+
if (unlikely(ret)) {
1072
+
dev_err(dev, "fail to init mac data for ICV!\n");
1073
+
goto free_out_mac_dma;
1074
+
}
1075
+
}
1076
+
1077
+
ret = sec_cipher_map_sgl(dev, req, src, dst);
1078
+
if (unlikely(ret)) {
1079
+
dev_err(dev, "fail to dma map input sgl buffers!\n");
1080
+
goto free_out_mac_dma;
1081
+
}
1082
+
1083
+
return 0;
1084
+
1085
+
free_out_mac_dma:
1086
+
if (is_aead)
1087
+
dma_unmap_single(dev, a_req->out_mac_dma, SEC_MAX_MAC_LEN, DMA_BIDIRECTIONAL);
1088
+
free_a_ivin_dma:
1089
+
if (is_aead)
1090
+
dma_unmap_single(dev, a_req->a_ivin_dma, SEC_IV_SIZE, DMA_TO_DEVICE);
1091
+
free_c_ivin_dma:
1092
+
dma_unmap_single(dev, c_req->c_ivin_dma, SEC_IV_SIZE, DMA_TO_DEVICE);
1093
+
return ret;
1094
+
}
1095
+
1204
1096
static void sec_cipher_unmap(struct sec_ctx *ctx, struct sec_req *req,
1205
1097
struct scatterlist *src, struct scatterlist *dst)
1206
1098
{
1099
+
struct sec_aead_req *a_req = &req->aead_req;
1207
1100
struct sec_cipher_req *c_req = &req->c_req;
1208
1101
struct device *dev = ctx->dev;
1102
+
1103
+
if (req->req_id >= 0) {
1104
+
if (req->use_pbuf) {
1105
+
sec_cipher_pbuf_unmap(ctx, req, dst);
1106
+
} else {
1107
+
if (dst != src) {
1108
+
hisi_acc_sg_buf_unmap(dev, dst, c_req->c_out, DMA_FROM_DEVICE);
1109
+
hisi_acc_sg_buf_unmap(dev, src, req->in, DMA_TO_DEVICE);
1110
+
} else {
1111
+
hisi_acc_sg_buf_unmap(dev, src, req->in, DMA_BIDIRECTIONAL);
1112
+
}
1113
+
}
1114
+
return;
1115
+
}
1209
1116
1210
1117
if (req->use_pbuf) {
1211
1118
sec_cipher_pbuf_unmap(ctx, req, dst);
1212
1119
} else {
1213
-
if (dst != src)
1214
-
hisi_acc_sg_buf_unmap(dev, src, req->in);
1120
+
if (dst != src) {
1121
+
sec_cipher_put_hw_sgl(dev, dst, req->buf.out_dma, DMA_FROM_DEVICE);
1122
+
sec_cipher_put_hw_sgl(dev, src, req->buf.in_dma, DMA_TO_DEVICE);
1123
+
} else {
1124
+
sec_cipher_put_hw_sgl(dev, src, req->buf.in_dma, DMA_BIDIRECTIONAL);
1125
+
}
1126
+
}
1215
1127
1216
-
hisi_acc_sg_buf_unmap(dev, dst, c_req->c_out);
1128
+
dma_unmap_single(dev, c_req->c_ivin_dma, SEC_IV_SIZE, DMA_TO_DEVICE);
1129
+
if (ctx->alg_type == SEC_AEAD) {
1130
+
dma_unmap_single(dev, a_req->a_ivin_dma, SEC_IV_SIZE, DMA_TO_DEVICE);
1131
+
dma_unmap_single(dev, a_req->out_mac_dma, SEC_MAX_MAC_LEN, DMA_BIDIRECTIONAL);
1217
1132
}
1218
1133
}
1219
1134
···
1520
1257
1521
1258
sec_sqe->type2.c_key_addr = cpu_to_le64(c_ctx->c_key_dma);
1522
1259
sec_sqe->type2.c_ivin_addr = cpu_to_le64(c_req->c_ivin_dma);
1523
-
sec_sqe->type2.data_src_addr = cpu_to_le64(req->in_dma);
1524
-
sec_sqe->type2.data_dst_addr = cpu_to_le64(c_req->c_out_dma);
1260
+
if (req->req_id < 0) {
1261
+
sec_sqe->type2.data_src_addr = cpu_to_le64(req->buf.in_dma);
1262
+
sec_sqe->type2.data_dst_addr = cpu_to_le64(req->buf.out_dma);
1263
+
} else {
1264
+
sec_sqe->type2.data_src_addr = cpu_to_le64(req->in_dma);
1265
+
sec_sqe->type2.data_dst_addr = cpu_to_le64(c_req->c_out_dma);
1266
+
}
1267
+
if (sec_sqe->type2.data_src_addr != sec_sqe->type2.data_dst_addr)
1268
+
de = 0x1 << SEC_DE_OFFSET;
1525
1269
1526
1270
sec_sqe->type2.icvw_kmode |= cpu_to_le16(((u16)c_ctx->c_mode) <<
1527
1271
SEC_CMODE_OFFSET);
···
1554
1284
1555
1285
sec_sqe->sdm_addr_type |= da_type;
1556
1286
scene = SEC_COMM_SCENE << SEC_SCENE_OFFSET;
1557
-
if (req->in_dma != c_req->c_out_dma)
1558
-
de = 0x1 << SEC_DE_OFFSET;
1559
1287
1560
1288
sec_sqe->sds_sa_type = (de | scene | sa_type);
1561
1289
1562
1290
sec_sqe->type2.clen_ivhlen |= cpu_to_le32(c_req->c_len);
1563
-
sec_sqe->type2.tag = cpu_to_le16((u16)req->req_id);
1564
1291
1565
1292
return 0;
1566
1293
}
···
1574
1307
1575
1308
sec_sqe3->c_key_addr = cpu_to_le64(c_ctx->c_key_dma);
1576
1309
sec_sqe3->no_scene.c_ivin_addr = cpu_to_le64(c_req->c_ivin_dma);
1577
-
sec_sqe3->data_src_addr = cpu_to_le64(req->in_dma);
1578
-
sec_sqe3->data_dst_addr = cpu_to_le64(c_req->c_out_dma);
1310
+
if (req->req_id < 0) {
1311
+
sec_sqe3->data_src_addr = cpu_to_le64(req->buf.in_dma);
1312
+
sec_sqe3->data_dst_addr = cpu_to_le64(req->buf.out_dma);
1313
+
} else {
1314
+
sec_sqe3->data_src_addr = cpu_to_le64(req->in_dma);
1315
+
sec_sqe3->data_dst_addr = cpu_to_le64(c_req->c_out_dma);
1316
+
}
1317
+
if (sec_sqe3->data_src_addr != sec_sqe3->data_dst_addr)
1318
+
bd_param |= 0x1 << SEC_DE_OFFSET_V3;
1579
1319
1580
1320
sec_sqe3->c_mode_alg = ((u8)c_ctx->c_alg << SEC_CALG_OFFSET_V3) |
1581
1321
c_ctx->c_mode;
···
1608
1334
}
1609
1335
1610
1336
bd_param |= SEC_COMM_SCENE << SEC_SCENE_OFFSET_V3;
1611
-
if (req->in_dma != c_req->c_out_dma)
1612
-
bd_param |= 0x1 << SEC_DE_OFFSET_V3;
1613
1337
1614
1338
bd_param |= SEC_BD_TYPE3;
1615
1339
sec_sqe3->bd_param = cpu_to_le32(bd_param);
···
1639
1367
size_t sz;
1640
1368
u8 *iv;
1641
1369
1642
-
if (req->c_req.encrypt)
1643
-
sgl = alg_type == SEC_SKCIPHER ? sk_req->dst : aead_req->dst;
1644
-
else
1645
-
sgl = alg_type == SEC_SKCIPHER ? sk_req->src : aead_req->src;
1646
-
1647
1370
if (alg_type == SEC_SKCIPHER) {
1371
+
sgl = req->c_req.encrypt ? sk_req->dst : sk_req->src;
1648
1372
iv = sk_req->iv;
1649
1373
cryptlen = sk_req->cryptlen;
1650
1374
} else {
1375
+
sgl = req->c_req.encrypt ? aead_req->dst : aead_req->src;
1651
1376
iv = aead_req->iv;
1652
1377
cryptlen = aead_req->cryptlen;
1653
1378
}
···
1655
1386
if (unlikely(sz != iv_size))
1656
1387
dev_err(req->ctx->dev, "copy output iv error!\n");
1657
1388
} else {
1658
-
sz = cryptlen / iv_size;
1659
-
if (cryptlen % iv_size)
1660
-
sz += 1;
1389
+
sz = (cryptlen + iv_size - 1) / iv_size;
1661
1390
ctr_iv_inc(iv, iv_size, sz);
1662
1391
}
1663
-
}
1664
-
1665
-
static struct sec_req *sec_back_req_clear(struct sec_ctx *ctx,
1666
-
struct sec_qp_ctx *qp_ctx)
1667
-
{
1668
-
struct sec_req *backlog_req = NULL;
1669
-
1670
-
spin_lock_bh(&qp_ctx->req_lock);
1671
-
if (ctx->fake_req_limit >=
1672
-
atomic_read(&qp_ctx->qp->qp_status.used) &&
1673
-
!list_empty(&qp_ctx->backlog)) {
1674
-
backlog_req = list_first_entry(&qp_ctx->backlog,
1675
-
typeof(*backlog_req), backlog_head);
1676
-
list_del(&backlog_req->backlog_head);
1677
-
}
1678
-
spin_unlock_bh(&qp_ctx->req_lock);
1679
-
1680
-
return backlog_req;
1681
1392
}
1682
1393
1683
1394
static void sec_skcipher_callback(struct sec_ctx *ctx, struct sec_req *req,
1684
1395
int err)
1685
1396
{
1686
-
struct skcipher_request *sk_req = req->c_req.sk_req;
1687
1397
struct sec_qp_ctx *qp_ctx = req->qp_ctx;
1688
-
struct skcipher_request *backlog_sk_req;
1689
-
struct sec_req *backlog_req;
1690
1398
1691
-
sec_free_req_id(req);
1399
+
if (req->req_id >= 0)
1400
+
sec_free_req_id(req);
1692
1401
1693
1402
/* IV output at encrypto of CBC/CTR mode */
1694
1403
if (!err && (ctx->c_ctx.c_mode == SEC_CMODE_CBC ||
1695
1404
ctx->c_ctx.c_mode == SEC_CMODE_CTR) && req->c_req.encrypt)
1696
1405
sec_update_iv(req, SEC_SKCIPHER);
1697
1406
1698
-
while (1) {
1699
-
backlog_req = sec_back_req_clear(ctx, qp_ctx);
1700
-
if (!backlog_req)
1701
-
break;
1702
-
1703
-
backlog_sk_req = backlog_req->c_req.sk_req;
1704
-
skcipher_request_complete(backlog_sk_req, -EINPROGRESS);
1705
-
atomic64_inc(&ctx->sec->debug.dfx.recv_busy_cnt);
1706
-
}
1707
-
1708
-
skcipher_request_complete(sk_req, err);
1407
+
crypto_request_complete(req->base, err);
1408
+
sec_alg_send_backlog(ctx, qp_ctx);
1709
1409
}
1710
1410
1711
1411
static void set_aead_auth_iv(struct sec_ctx *ctx, struct sec_req *req)
···
1913
1675
struct aead_request *a_req = req->aead_req.aead_req;
1914
1676
struct crypto_aead *tfm = crypto_aead_reqtfm(a_req);
1915
1677
size_t authsize = crypto_aead_authsize(tfm);
1916
-
struct sec_aead_req *aead_req = &req->aead_req;
1917
-
struct sec_cipher_req *c_req = &req->c_req;
1918
1678
struct sec_qp_ctx *qp_ctx = req->qp_ctx;
1919
-
struct aead_request *backlog_aead_req;
1920
-
struct sec_req *backlog_req;
1921
1679
size_t sz;
1922
1680
1923
-
if (!err && c->c_ctx.c_mode == SEC_CMODE_CBC && c_req->encrypt)
1924
-
sec_update_iv(req, SEC_AEAD);
1681
+
if (!err && req->c_req.encrypt) {
1682
+
if (c->c_ctx.c_mode == SEC_CMODE_CBC)
1683
+
sec_update_iv(req, SEC_AEAD);
1925
1684
1926
-
/* Copy output mac */
1927
-
if (!err && c_req->encrypt) {
1928
-
struct scatterlist *sgl = a_req->dst;
1929
-
1930
-
sz = sg_pcopy_from_buffer(sgl, sg_nents(sgl), aead_req->out_mac,
1685
+
sz = sg_pcopy_from_buffer(a_req->dst, sg_nents(a_req->dst), req->aead_req.out_mac,
1931
1686
authsize, a_req->cryptlen + a_req->assoclen);
1932
1687
if (unlikely(sz != authsize)) {
1933
1688
dev_err(c->dev, "copy out mac err!\n");
···
1928
1697
}
1929
1698
}
1930
1699
1931
-
sec_free_req_id(req);
1700
+
if (req->req_id >= 0)
1701
+
sec_free_req_id(req);
1932
1702
1933
-
while (1) {
1934
-
backlog_req = sec_back_req_clear(c, qp_ctx);
1935
-
if (!backlog_req)
1936
-
break;
1937
-
1938
-
backlog_aead_req = backlog_req->aead_req.aead_req;
1939
-
aead_request_complete(backlog_aead_req, -EINPROGRESS);
1940
-
atomic64_inc(&c->sec->debug.dfx.recv_busy_cnt);
1941
-
}
1942
-
1943
-
aead_request_complete(a_req, err);
1703
+
crypto_request_complete(req->base, err);
1704
+
sec_alg_send_backlog(c, qp_ctx);
1944
1705
}
1945
1706
1946
-
static void sec_request_uninit(struct sec_ctx *ctx, struct sec_req *req)
1707
+
static void sec_request_uninit(struct sec_req *req)
1947
1708
{
1948
-
sec_free_req_id(req);
1949
-
sec_free_queue_id(ctx, req);
1709
+
if (req->req_id >= 0)
1710
+
sec_free_req_id(req);
1950
1711
}
1951
1712
1952
1713
static int sec_request_init(struct sec_ctx *ctx, struct sec_req *req)
1953
1714
{
1954
1715
struct sec_qp_ctx *qp_ctx;
1955
-
int queue_id;
1716
+
int i;
1956
1717
1957
-
/* To load balance */
1958
-
queue_id = sec_alloc_queue_id(ctx, req);
1959
-
qp_ctx = &ctx->qp_ctx[queue_id];
1960
-
1961
-
req->req_id = sec_alloc_req_id(req, qp_ctx);
1962
-
if (unlikely(req->req_id < 0)) {
1963
-
sec_free_queue_id(ctx, req);
1964
-
return req->req_id;
1718
+
for (i = 0; i < ctx->sec->ctx_q_num; i++) {
1719
+
qp_ctx = &ctx->qp_ctx[i];
1720
+
req->req_id = sec_alloc_req_id(req, qp_ctx);
1721
+
if (req->req_id >= 0)
1722
+
break;
1965
1723
}
1724
+
1725
+
req->qp_ctx = qp_ctx;
1726
+
req->backlog = &qp_ctx->backlog;
1966
1727
1967
1728
return 0;
1968
1729
}
1969
1730
1970
1731
static int sec_process(struct sec_ctx *ctx, struct sec_req *req)
1971
1732
{
1972
-
struct sec_cipher_req *c_req = &req->c_req;
1973
1733
int ret;
1974
1734
1975
1735
ret = sec_request_init(ctx, req);
···
1977
1755
sec_update_iv(req, ctx->alg_type);
1978
1756
1979
1757
ret = ctx->req_op->bd_send(ctx, req);
1980
-
if (unlikely((ret != -EBUSY && ret != -EINPROGRESS) ||
1981
-
(ret == -EBUSY && !(req->flag & CRYPTO_TFM_REQ_MAY_BACKLOG)))) {
1758
+
if (unlikely((ret != -EBUSY && ret != -EINPROGRESS))) {
1982
1759
dev_err_ratelimited(ctx->dev, "send sec request failed!\n");
1983
1760
goto err_send_req;
1984
1761
}
···
1988
1767
/* As failing, restore the IV from user */
1989
1768
if (ctx->c_ctx.c_mode == SEC_CMODE_CBC && !req->c_req.encrypt) {
1990
1769
if (ctx->alg_type == SEC_SKCIPHER)
1991
-
memcpy(req->c_req.sk_req->iv, c_req->c_ivin,
1770
+
memcpy(req->c_req.sk_req->iv, req->c_req.c_ivin,
1992
1771
ctx->c_ctx.ivsize);
1993
1772
else
1994
-
memcpy(req->aead_req.aead_req->iv, c_req->c_ivin,
1773
+
memcpy(req->aead_req.aead_req->iv, req->c_req.c_ivin,
1995
1774
ctx->c_ctx.ivsize);
1996
1775
}
1997
1776
1998
1777
sec_request_untransfer(ctx, req);
1778
+
1999
1779
err_uninit_req:
2000
-
sec_request_uninit(ctx, req);
1780
+
sec_request_uninit(req);
1781
+
if (ctx->alg_type == SEC_AEAD)
1782
+
ret = sec_aead_soft_crypto(ctx, req->aead_req.aead_req,
1783
+
req->c_req.encrypt);
1784
+
else
1785
+
ret = sec_skcipher_soft_crypto(ctx, req->c_req.sk_req,
1786
+
req->c_req.encrypt);
2001
1787
return ret;
2002
1788
}
2003
1789
···
2078
1850
struct sec_ctx *ctx = crypto_aead_ctx(tfm);
2079
1851
int ret;
2080
1852
2081
-
crypto_aead_set_reqsize(tfm, sizeof(struct sec_req));
1853
+
crypto_aead_set_reqsize_dma(tfm, sizeof(struct sec_req));
2082
1854
ctx->alg_type = SEC_AEAD;
2083
1855
ctx->c_ctx.ivsize = crypto_aead_ivsize(tfm);
2084
1856
if (ctx->c_ctx.ivsize < SEC_AIV_SIZE ||
···
2315
2087
static int sec_skcipher_crypto(struct skcipher_request *sk_req, bool encrypt)
2316
2088
{
2317
2089
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(sk_req);
2318
-
struct sec_req *req = skcipher_request_ctx(sk_req);
2090
+
struct sec_req *req = skcipher_request_ctx_dma(sk_req);
2319
2091
struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);
2320
2092
bool need_fallback = false;
2321
2093
int ret;
···
2330
2102
req->c_req.sk_req = sk_req;
2331
2103
req->c_req.encrypt = encrypt;
2332
2104
req->ctx = ctx;
2105
+
req->base = &sk_req->base;
2333
2106
2334
2107
ret = sec_skcipher_param_check(ctx, req, &need_fallback);
2335
2108
if (unlikely(ret))
···
2465
2236
return -EINVAL;
2466
2237
if (unlikely(ctx->a_ctx.a_key_len & WORD_MASK))
2467
2238
return -EINVAL;
2239
+
} else if (c_mode == SEC_CMODE_GCM) {
2240
+
if (unlikely(sz < SEC_GCM_MIN_AUTH_SZ))
2241
+
return -EINVAL;
2468
2242
}
2469
2243
2470
2244
return 0;
···
2541
2309
static int sec_aead_crypto(struct aead_request *a_req, bool encrypt)
2542
2310
{
2543
2311
struct crypto_aead *tfm = crypto_aead_reqtfm(a_req);
2544
-
struct sec_req *req = aead_request_ctx(a_req);
2312
+
struct sec_req *req = aead_request_ctx_dma(a_req);
2545
2313
struct sec_ctx *ctx = crypto_aead_ctx(tfm);
2546
2314
size_t sz = crypto_aead_authsize(tfm);
2547
2315
bool need_fallback = false;
···
2551
2319
req->aead_req.aead_req = a_req;
2552
2320
req->c_req.encrypt = encrypt;
2553
2321
req->ctx = ctx;
2322
+
req->base = &a_req->base;
2554
2323
req->c_req.c_len = a_req->cryptlen - (req->c_req.encrypt ? 0 : sz);
2555
2324
2556
2325
ret = sec_aead_param_check(ctx, req, &need_fallback);
+8
-7
drivers/crypto/hisilicon/sgl.c
+8
-7
drivers/crypto/hisilicon/sgl.c
···
210
210
* @pool: Pool which hw sgl memory will be allocated in.
211
211
* @index: Index of hisi_acc_hw_sgl in pool.
212
212
* @hw_sgl_dma: The dma address of allocated hw sgl.
213
+
* @dir: DMA direction.
213
214
*
214
215
* This function builds hw sgl according input sgl, user can use hw_sgl_dma
215
216
* as src/dst in its BD. Only support single hw sgl currently.
216
217
*/
217
218
struct hisi_acc_hw_sgl *
218
-
hisi_acc_sg_buf_map_to_hw_sgl(struct device *dev,
219
-
struct scatterlist *sgl,
220
-
struct hisi_acc_sgl_pool *pool,
221
-
u32 index, dma_addr_t *hw_sgl_dma)
219
+
hisi_acc_sg_buf_map_to_hw_sgl(struct device *dev, struct scatterlist *sgl,
220
+
struct hisi_acc_sgl_pool *pool, u32 index,
221
+
dma_addr_t *hw_sgl_dma, enum dma_data_direction dir)
222
222
{
223
223
struct hisi_acc_hw_sgl *curr_hw_sgl;
224
224
unsigned int i, sg_n_mapped;
···
232
232
233
233
sg_n = sg_nents(sgl);
234
234
235
-
sg_n_mapped = dma_map_sg(dev, sgl, sg_n, DMA_BIDIRECTIONAL);
235
+
sg_n_mapped = dma_map_sg(dev, sgl, sg_n, dir);
236
236
if (!sg_n_mapped) {
237
237
dev_err(dev, "DMA mapping for SG error!\n");
238
238
return ERR_PTR(-EINVAL);
···
276
276
* @dev: The device which hw sgl belongs to.
277
277
* @sgl: Related scatterlist.
278
278
* @hw_sgl: Virtual address of hw sgl.
279
+
* @dir: DMA direction.
279
280
*
280
281
* This function unmaps allocated hw sgl.
281
282
*/
282
283
void hisi_acc_sg_buf_unmap(struct device *dev, struct scatterlist *sgl,
283
-
struct hisi_acc_hw_sgl *hw_sgl)
284
+
struct hisi_acc_hw_sgl *hw_sgl, enum dma_data_direction dir)
284
285
{
285
286
if (!dev || !sgl || !hw_sgl)
286
287
return;
287
288
288
-
dma_unmap_sg(dev, sgl, sg_nents(sgl), DMA_BIDIRECTIONAL);
289
+
dma_unmap_sg(dev, sgl, sg_nents(sgl), dir);
289
290
clear_hw_sgl_sge(hw_sgl);
290
291
hw_sgl->entry_sum_in_chain = 0;
291
292
hw_sgl->entry_sum_in_sgl = 0;
+7
-6
drivers/crypto/hisilicon/zip/zip_crypto.c
+7
-6
drivers/crypto/hisilicon/zip/zip_crypto.c
···
224
224
return -EINVAL;
225
225
226
226
req->hw_src = hisi_acc_sg_buf_map_to_hw_sgl(dev, a_req->src, pool,
227
-
req->req_id << 1, &req->dma_src);
227
+
req->req_id << 1, &req->dma_src,
228
+
DMA_TO_DEVICE);
228
229
if (IS_ERR(req->hw_src)) {
229
230
dev_err(dev, "failed to map the src buffer to hw sgl (%ld)!\n",
230
231
PTR_ERR(req->hw_src));
···
234
233
235
234
req->hw_dst = hisi_acc_sg_buf_map_to_hw_sgl(dev, a_req->dst, pool,
236
235
(req->req_id << 1) + 1,
237
-
&req->dma_dst);
236
+
&req->dma_dst, DMA_FROM_DEVICE);
238
237
if (IS_ERR(req->hw_dst)) {
239
238
ret = PTR_ERR(req->hw_dst);
240
239
dev_err(dev, "failed to map the dst buffer to hw slg (%d)!\n",
···
259
258
return -EINPROGRESS;
260
259
261
260
err_unmap_output:
262
-
hisi_acc_sg_buf_unmap(dev, a_req->dst, req->hw_dst);
261
+
hisi_acc_sg_buf_unmap(dev, a_req->dst, req->hw_dst, DMA_FROM_DEVICE);
263
262
err_unmap_input:
264
-
hisi_acc_sg_buf_unmap(dev, a_req->src, req->hw_src);
263
+
hisi_acc_sg_buf_unmap(dev, a_req->src, req->hw_src, DMA_TO_DEVICE);
265
264
return ret;
266
265
}
267
266
···
304
303
err = -EIO;
305
304
}
306
305
307
-
hisi_acc_sg_buf_unmap(dev, acomp_req->src, req->hw_src);
308
-
hisi_acc_sg_buf_unmap(dev, acomp_req->dst, req->hw_dst);
306
+
hisi_acc_sg_buf_unmap(dev, acomp_req->dst, req->hw_dst, DMA_FROM_DEVICE);
307
+
hisi_acc_sg_buf_unmap(dev, acomp_req->src, req->hw_src, DMA_TO_DEVICE);
309
308
310
309
acomp_req->dlen = ops->get_dstlen(sqe);
311
310
+1
-1
drivers/crypto/img-hash.c
+1
-1
drivers/crypto/img-hash.c
+6
-2
drivers/crypto/inside-secure/safexcel_hash.c
+6
-2
drivers/crypto/inside-secure/safexcel_hash.c
···
249
249
safexcel_complete(priv, ring);
250
250
251
251
if (sreq->nents) {
252
-
dma_unmap_sg(priv->dev, areq->src, sreq->nents, DMA_TO_DEVICE);
252
+
dma_unmap_sg(priv->dev, areq->src,
253
+
sg_nents_for_len(areq->src, areq->nbytes),
254
+
DMA_TO_DEVICE);
253
255
sreq->nents = 0;
254
256
}
255
257
···
493
491
DMA_FROM_DEVICE);
494
492
unmap_sg:
495
493
if (req->nents) {
496
-
dma_unmap_sg(priv->dev, areq->src, req->nents, DMA_TO_DEVICE);
494
+
dma_unmap_sg(priv->dev, areq->src,
495
+
sg_nents_for_len(areq->src, areq->nbytes),
496
+
DMA_TO_DEVICE);
497
497
req->nents = 0;
498
498
}
499
499
cdesc_rollback:
+7
-1
drivers/crypto/intel/keembay/keembay-ocs-hcu-core.c
+7
-1
drivers/crypto/intel/keembay/keembay-ocs-hcu-core.c
···
68
68
* @sg_data_total: Total data in the SG list at any time.
69
69
* @sg_data_offset: Offset into the data of the current individual SG node.
70
70
* @sg_dma_nents: Number of sg entries mapped in dma_list.
71
+
* @nents: Number of entries in the scatterlist.
71
72
*/
72
73
struct ocs_hcu_rctx {
73
74
struct ocs_hcu_dev *hcu_dev;
···
92
91
unsigned int sg_data_total;
93
92
unsigned int sg_data_offset;
94
93
unsigned int sg_dma_nents;
94
+
unsigned int nents;
95
95
};
96
96
97
97
/**
···
201
199
202
200
/* Unmap req->src (if mapped). */
203
201
if (rctx->sg_dma_nents) {
204
-
dma_unmap_sg(dev, req->src, rctx->sg_dma_nents, DMA_TO_DEVICE);
202
+
dma_unmap_sg(dev, req->src, rctx->nents, DMA_TO_DEVICE);
205
203
rctx->sg_dma_nents = 0;
206
204
}
207
205
···
262
260
rc = -ENOMEM;
263
261
goto cleanup;
264
262
}
263
+
264
+
/* Save the value of nents to pass to dma_unmap_sg. */
265
+
rctx->nents = nents;
266
+
265
267
/*
266
268
* The value returned by dma_map_sg() can be < nents; so update
267
269
* nents accordingly.
+2
-2
drivers/crypto/intel/keembay/ocs-aes.c
+2
-2
drivers/crypto/intel/keembay/ocs-aes.c
···
7
7
8
8
#include <linux/dma-mapping.h>
9
9
#include <linux/interrupt.h>
10
+
#include <linux/kernel.h>
10
11
#include <linux/platform_device.h>
11
12
#include <linux/slab.h>
12
13
#include <linux/swab.h>
···
1474
1473
ll = dll_desc->vaddr;
1475
1474
for (i = 0; i < dma_nents; i++, sg = sg_next(sg)) {
1476
1475
ll[i].src_addr = sg_dma_address(sg) + data_offset;
1477
-
ll[i].src_len = (sg_dma_len(sg) - data_offset) < data_size ?
1478
-
(sg_dma_len(sg) - data_offset) : data_size;
1476
+
ll[i].src_len = min(sg_dma_len(sg) - data_offset, data_size);
1479
1477
data_offset = 0;
1480
1478
data_size -= ll[i].src_len;
1481
1479
/* Current element points to the DMA address of the next one. */
+7
-11
drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c
+7
-11
drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c
···
191
191
ICP_ACCEL_CAPABILITIES_SM4 |
192
192
ICP_ACCEL_CAPABILITIES_AES_V2 |
193
193
ICP_ACCEL_CAPABILITIES_ZUC |
194
-
ICP_ACCEL_CAPABILITIES_ZUC_256 |
195
194
ICP_ACCEL_CAPABILITIES_WIRELESS_CRYPTO_EXT |
196
195
ICP_ACCEL_CAPABILITIES_EXT_ALGCHAIN;
197
196
···
222
223
223
224
if (fusectl1 & ICP_ACCEL_GEN4_MASK_WCP_WAT_SLICE) {
224
225
capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_ZUC;
225
-
capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_ZUC_256;
226
226
capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_WIRELESS_CRYPTO_EXT;
227
227
}
228
228
229
-
if (fusectl1 & ICP_ACCEL_GEN4_MASK_EIA3_SLICE) {
229
+
if (fusectl1 & ICP_ACCEL_GEN4_MASK_EIA3_SLICE)
230
230
capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_ZUC;
231
-
capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_ZUC_256;
232
-
}
233
-
234
-
if (fusectl1 & ICP_ACCEL_GEN4_MASK_ZUC_256_SLICE)
235
-
capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_ZUC_256;
236
231
237
232
capabilities_asym = ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC |
238
233
ICP_ACCEL_CAPABILITIES_SM2 |
···
296
303
rl_data->pcie_scale_div = ADF_420XX_RL_PCIE_SCALE_FACTOR_DIV;
297
304
rl_data->pcie_scale_mul = ADF_420XX_RL_PCIE_SCALE_FACTOR_MUL;
298
305
rl_data->dcpr_correction = ADF_420XX_RL_DCPR_CORRECTION;
299
-
rl_data->max_tp[ADF_SVC_ASYM] = ADF_420XX_RL_MAX_TP_ASYM;
300
-
rl_data->max_tp[ADF_SVC_SYM] = ADF_420XX_RL_MAX_TP_SYM;
301
-
rl_data->max_tp[ADF_SVC_DC] = ADF_420XX_RL_MAX_TP_DC;
306
+
rl_data->max_tp[SVC_ASYM] = ADF_420XX_RL_MAX_TP_ASYM;
307
+
rl_data->max_tp[SVC_SYM] = ADF_420XX_RL_MAX_TP_SYM;
308
+
rl_data->max_tp[SVC_DC] = ADF_420XX_RL_MAX_TP_DC;
302
309
rl_data->scan_interval = ADF_420XX_RL_SCANS_PER_SEC;
303
310
rl_data->scale_ref = ADF_420XX_RL_SLICE_REF;
311
+
312
+
adf_gen4_init_num_svc_aes(rl_data);
304
313
}
305
314
306
315
static int get_rp_group(struct adf_accel_dev *accel_dev, u32 ae_mask)
···
468
473
hw_data->num_hb_ctrs = ADF_NUM_HB_CNT_PER_AE;
469
474
hw_data->clock_frequency = ADF_420XX_AE_FREQ;
470
475
hw_data->services_supported = adf_gen4_services_supported;
476
+
hw_data->get_svc_slice_cnt = adf_gen4_get_svc_slice_cnt;
471
477
472
478
adf_gen4_set_err_mask(&hw_data->dev_err_mask);
473
479
adf_gen4_init_hw_csr_ops(&hw_data->csr_ops);
+9
-5
drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c
+9
-5
drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c
···
3
3
#include <linux/iopoll.h>
4
4
#include <adf_accel_devices.h>
5
5
#include <adf_admin.h>
6
+
#include <adf_bank_state.h>
6
7
#include <adf_cfg.h>
7
8
#include <adf_cfg_services.h>
8
9
#include <adf_clock.h>
···
222
221
rl_data->pcie_scale_div = ADF_4XXX_RL_PCIE_SCALE_FACTOR_DIV;
223
222
rl_data->pcie_scale_mul = ADF_4XXX_RL_PCIE_SCALE_FACTOR_MUL;
224
223
rl_data->dcpr_correction = ADF_4XXX_RL_DCPR_CORRECTION;
225
-
rl_data->max_tp[ADF_SVC_ASYM] = ADF_4XXX_RL_MAX_TP_ASYM;
226
-
rl_data->max_tp[ADF_SVC_SYM] = ADF_4XXX_RL_MAX_TP_SYM;
227
-
rl_data->max_tp[ADF_SVC_DC] = ADF_4XXX_RL_MAX_TP_DC;
224
+
rl_data->max_tp[SVC_ASYM] = ADF_4XXX_RL_MAX_TP_ASYM;
225
+
rl_data->max_tp[SVC_SYM] = ADF_4XXX_RL_MAX_TP_SYM;
226
+
rl_data->max_tp[SVC_DC] = ADF_4XXX_RL_MAX_TP_DC;
228
227
rl_data->scan_interval = ADF_4XXX_RL_SCANS_PER_SEC;
229
228
rl_data->scale_ref = ADF_4XXX_RL_SLICE_REF;
229
+
230
+
adf_gen4_init_num_svc_aes(rl_data);
230
231
}
231
232
232
233
static u32 uof_get_num_objs(struct adf_accel_dev *accel_dev)
···
451
448
hw_data->get_ring_to_svc_map = adf_gen4_get_ring_to_svc_map;
452
449
hw_data->disable_iov = adf_disable_sriov;
453
450
hw_data->ring_pair_reset = adf_gen4_ring_pair_reset;
454
-
hw_data->bank_state_save = adf_gen4_bank_state_save;
455
-
hw_data->bank_state_restore = adf_gen4_bank_state_restore;
451
+
hw_data->bank_state_save = adf_bank_state_save;
452
+
hw_data->bank_state_restore = adf_bank_state_restore;
456
453
hw_data->enable_pm = adf_gen4_enable_pm;
457
454
hw_data->handle_pm_interrupt = adf_gen4_handle_pm_interrupt;
458
455
hw_data->dev_config = adf_gen4_dev_config;
···
462
459
hw_data->num_hb_ctrs = ADF_NUM_HB_CNT_PER_AE;
463
460
hw_data->clock_frequency = ADF_4XXX_AE_FREQ;
464
461
hw_data->services_supported = adf_gen4_services_supported;
462
+
hw_data->get_svc_slice_cnt = adf_gen4_get_svc_slice_cnt;
465
463
466
464
adf_gen4_set_err_mask(&hw_data->dev_err_mask);
467
465
adf_gen4_init_hw_csr_ops(&hw_data->csr_ops);
+117
-12
drivers/crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.c
+117
-12
drivers/crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.c
···
10
10
11
11
#include <adf_accel_devices.h>
12
12
#include <adf_admin.h>
13
+
#include <adf_bank_state.h>
13
14
#include <adf_cfg.h>
14
15
#include <adf_cfg_services.h>
15
16
#include <adf_clock.h>
···
19
18
#include <adf_gen6_pm.h>
20
19
#include <adf_gen6_ras.h>
21
20
#include <adf_gen6_shared.h>
21
+
#include <adf_gen6_tl.h>
22
22
#include <adf_timer.h>
23
23
#include "adf_6xxx_hw_data.h"
24
24
#include "icp_qat_fw_comp.h"
···
78
76
0x00, 0x00, 0x00, 0x00, 0x07, 0x07, 0x03, 0x03, 0x00
79
77
};
80
78
79
+
static const unsigned long thrd_mask_dcpr[ADF_6XXX_MAX_ACCELENGINES] = {
80
+
0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x00
81
+
};
82
+
81
83
static const char *const adf_6xxx_fw_objs[] = {
82
84
[ADF_FW_CY_OBJ] = ADF_6XXX_CY_OBJ,
83
85
[ADF_FW_DC_OBJ] = ADF_6XXX_DC_OBJ,
···
103
97
{
104
98
int num_svc;
105
99
106
-
if (mask >= BIT(SVC_BASE_COUNT))
100
+
if (mask >= BIT(SVC_COUNT))
107
101
return false;
108
102
109
103
num_svc = hweight_long(mask);
···
132
126
if (test_and_clear_bit(SVC_DCC, mask))
133
127
return SVC_DCC;
134
128
129
+
if (test_and_clear_bit(SVC_DECOMP, mask))
130
+
return SVC_DECOMP;
131
+
135
132
return -EINVAL;
136
133
}
137
134
138
-
static enum adf_cfg_service_type get_ring_type(enum adf_services service)
135
+
static enum adf_cfg_service_type get_ring_type(unsigned int service)
139
136
{
140
137
switch (service) {
141
138
case SVC_SYM:
···
148
139
case SVC_DC:
149
140
case SVC_DCC:
150
141
return COMP;
142
+
case SVC_DECOMP:
143
+
return DECOMP;
151
144
default:
152
145
return UNUSED;
153
146
}
154
147
}
155
148
156
-
static const unsigned long *get_thrd_mask(enum adf_services service)
149
+
static const unsigned long *get_thrd_mask(unsigned int service)
157
150
{
158
151
switch (service) {
159
152
case SVC_SYM:
···
166
155
return thrd_mask_cpr;
167
156
case SVC_DCC:
168
157
return thrd_mask_dcc;
158
+
case SVC_DECOMP:
159
+
return thrd_mask_dcpr;
169
160
default:
170
161
return NULL;
171
162
}
···
524
511
return 0;
525
512
}
526
513
514
+
static void init_num_svc_aes(struct adf_rl_hw_data *device_data)
515
+
{
516
+
enum adf_fw_objs obj_type, obj_iter;
517
+
unsigned int svc, i, num_grp;
518
+
u32 ae_mask;
519
+
520
+
for (svc = 0; svc < SVC_BASE_COUNT; svc++) {
521
+
switch (svc) {
522
+
case SVC_SYM:
523
+
case SVC_ASYM:
524
+
obj_type = ADF_FW_CY_OBJ;
525
+
break;
526
+
case SVC_DC:
527
+
case SVC_DECOMP:
528
+
obj_type = ADF_FW_DC_OBJ;
529
+
break;
530
+
}
531
+
532
+
num_grp = ARRAY_SIZE(adf_default_fw_config);
533
+
for (i = 0; i < num_grp; i++) {
534
+
obj_iter = adf_default_fw_config[i].obj;
535
+
if (obj_iter == obj_type) {
536
+
ae_mask = adf_default_fw_config[i].ae_mask;
537
+
device_data->svc_ae_mask[svc] = hweight32(ae_mask);
538
+
break;
539
+
}
540
+
}
541
+
}
542
+
}
543
+
544
+
static u32 adf_gen6_get_svc_slice_cnt(struct adf_accel_dev *accel_dev,
545
+
enum adf_base_services svc)
546
+
{
547
+
struct adf_rl_hw_data *device_data = &accel_dev->hw_device->rl_data;
548
+
549
+
switch (svc) {
550
+
case SVC_SYM:
551
+
return device_data->slices.cph_cnt;
552
+
case SVC_ASYM:
553
+
return device_data->slices.pke_cnt;
554
+
case SVC_DC:
555
+
return device_data->slices.cpr_cnt + device_data->slices.dcpr_cnt;
556
+
case SVC_DECOMP:
557
+
return device_data->slices.dcpr_cnt;
558
+
default:
559
+
return 0;
560
+
}
561
+
}
562
+
527
563
static void set_vc_csr_for_bank(void __iomem *csr, u32 bank_number)
528
564
{
529
565
u32 value;
···
582
520
* driver must program the ringmodectl CSRs.
583
521
*/
584
522
value = ADF_CSR_RD(csr, ADF_GEN6_CSR_RINGMODECTL(bank_number));
585
-
value |= FIELD_PREP(ADF_GEN6_RINGMODECTL_TC_MASK, ADF_GEN6_RINGMODECTL_TC_DEFAULT);
586
-
value |= FIELD_PREP(ADF_GEN6_RINGMODECTL_TC_EN_MASK, ADF_GEN6_RINGMODECTL_TC_EN_OP1);
523
+
FIELD_MODIFY(ADF_GEN6_RINGMODECTL_TC_MASK, &value, ADF_GEN6_RINGMODECTL_TC_DEFAULT);
524
+
FIELD_MODIFY(ADF_GEN6_RINGMODECTL_TC_EN_MASK, &value, ADF_GEN6_RINGMODECTL_TC_EN_OP1);
587
525
ADF_CSR_WR(csr, ADF_GEN6_CSR_RINGMODECTL(bank_number), value);
588
526
}
589
527
···
599
537
* Read PVC0CTL then write the masked values.
600
538
*/
601
539
pci_read_config_dword(pdev, ADF_GEN6_PVC0CTL_OFFSET, &value);
602
-
value |= FIELD_PREP(ADF_GEN6_PVC0CTL_TCVCMAP_MASK, ADF_GEN6_PVC0CTL_TCVCMAP_DEFAULT);
540
+
FIELD_MODIFY(ADF_GEN6_PVC0CTL_TCVCMAP_MASK, &value, ADF_GEN6_PVC0CTL_TCVCMAP_DEFAULT);
603
541
err = pci_write_config_dword(pdev, ADF_GEN6_PVC0CTL_OFFSET, value);
604
542
if (err) {
605
543
dev_err(&GET_DEV(accel_dev), "pci write to PVC0CTL failed\n");
···
608
546
609
547
/* Read PVC1CTL then write masked values */
610
548
pci_read_config_dword(pdev, ADF_GEN6_PVC1CTL_OFFSET, &value);
611
-
value |= FIELD_PREP(ADF_GEN6_PVC1CTL_TCVCMAP_MASK, ADF_GEN6_PVC1CTL_TCVCMAP_DEFAULT);
612
-
value |= FIELD_PREP(ADF_GEN6_PVC1CTL_VCEN_MASK, ADF_GEN6_PVC1CTL_VCEN_ON);
549
+
FIELD_MODIFY(ADF_GEN6_PVC1CTL_TCVCMAP_MASK, &value, ADF_GEN6_PVC1CTL_TCVCMAP_DEFAULT);
550
+
FIELD_MODIFY(ADF_GEN6_PVC1CTL_VCEN_MASK, &value, ADF_GEN6_PVC1CTL_VCEN_ON);
613
551
err = pci_write_config_dword(pdev, ADF_GEN6_PVC1CTL_OFFSET, value);
614
552
if (err)
615
553
dev_err(&GET_DEV(accel_dev), "pci write to PVC1CTL failed\n");
···
680
618
capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CHACHA_POLY;
681
619
capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_AESGCM_SPC;
682
620
capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_AES_V2;
683
-
capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CIPHER;
684
621
}
685
622
if (fusectl1 & ICP_ACCEL_GEN6_MASK_AUTH_SLICE) {
686
623
capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_AUTHENTICATION;
···
688
627
capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CIPHER;
689
628
}
690
629
691
-
capabilities_asym = 0;
630
+
capabilities_asym = ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC |
631
+
ICP_ACCEL_CAPABILITIES_SM2 |
632
+
ICP_ACCEL_CAPABILITIES_ECEDMONT;
633
+
634
+
if (fusectl1 & ICP_ACCEL_GEN6_MASK_PKE_SLICE) {
635
+
capabilities_asym &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC;
636
+
capabilities_asym &= ~ICP_ACCEL_CAPABILITIES_SM2;
637
+
capabilities_asym &= ~ICP_ACCEL_CAPABILITIES_ECEDMONT;
638
+
}
692
639
693
640
capabilities_dc = ICP_ACCEL_CAPABILITIES_COMPRESSION |
694
641
ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION |
···
717
648
caps |= capabilities_asym;
718
649
if (test_bit(SVC_SYM, &mask))
719
650
caps |= capabilities_sym;
720
-
if (test_bit(SVC_DC, &mask))
651
+
if (test_bit(SVC_DC, &mask) || test_bit(SVC_DECOMP, &mask))
721
652
caps |= capabilities_dc;
722
653
if (test_bit(SVC_DCC, &mask)) {
723
654
/*
···
813
744
814
745
static int enable_pm(struct adf_accel_dev *accel_dev)
815
746
{
816
-
return adf_init_admin_pm(accel_dev, ADF_GEN6_PM_DEFAULT_IDLE_FILTER);
747
+
int ret;
748
+
749
+
ret = adf_init_admin_pm(accel_dev, ADF_GEN6_PM_DEFAULT_IDLE_FILTER);
750
+
if (ret)
751
+
return ret;
752
+
753
+
/* Initialize PM internal data */
754
+
adf_gen6_init_dev_pm_data(accel_dev);
755
+
756
+
return 0;
817
757
}
818
758
819
759
static int dev_config(struct adf_accel_dev *accel_dev)
···
852
774
__set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status);
853
775
854
776
return ret;
777
+
}
778
+
779
+
static void adf_gen6_init_rl_data(struct adf_rl_hw_data *rl_data)
780
+
{
781
+
rl_data->pciout_tb_offset = ADF_GEN6_RL_TOKEN_PCIEOUT_BUCKET_OFFSET;
782
+
rl_data->pciin_tb_offset = ADF_GEN6_RL_TOKEN_PCIEIN_BUCKET_OFFSET;
783
+
rl_data->r2l_offset = ADF_GEN6_RL_R2L_OFFSET;
784
+
rl_data->l2c_offset = ADF_GEN6_RL_L2C_OFFSET;
785
+
rl_data->c2s_offset = ADF_GEN6_RL_C2S_OFFSET;
786
+
rl_data->pcie_scale_div = ADF_6XXX_RL_PCIE_SCALE_FACTOR_DIV;
787
+
rl_data->pcie_scale_mul = ADF_6XXX_RL_PCIE_SCALE_FACTOR_MUL;
788
+
rl_data->max_tp[SVC_ASYM] = ADF_6XXX_RL_MAX_TP_ASYM;
789
+
rl_data->max_tp[SVC_SYM] = ADF_6XXX_RL_MAX_TP_SYM;
790
+
rl_data->max_tp[SVC_DC] = ADF_6XXX_RL_MAX_TP_DC;
791
+
rl_data->max_tp[SVC_DECOMP] = ADF_6XXX_RL_MAX_TP_DECOMP;
792
+
rl_data->scan_interval = ADF_6XXX_RL_SCANS_PER_SEC;
793
+
rl_data->scale_ref = ADF_6XXX_RL_SLICE_REF;
794
+
795
+
init_num_svc_aes(rl_data);
855
796
}
856
797
857
798
void adf_init_hw_data_6xxx(struct adf_hw_device_data *hw_data)
···
921
824
hw_data->disable_iov = adf_disable_sriov;
922
825
hw_data->ring_pair_reset = ring_pair_reset;
923
826
hw_data->dev_config = dev_config;
827
+
hw_data->bank_state_save = adf_bank_state_save;
828
+
hw_data->bank_state_restore = adf_bank_state_restore;
924
829
hw_data->get_hb_clock = get_heartbeat_clock;
925
830
hw_data->num_hb_ctrs = ADF_NUM_HB_CNT_PER_AE;
926
831
hw_data->start_timer = adf_timer_start;
···
930
831
hw_data->init_device = adf_init_device;
931
832
hw_data->enable_pm = enable_pm;
932
833
hw_data->services_supported = services_supported;
834
+
hw_data->num_rps = ADF_GEN6_ETR_MAX_BANKS;
835
+
hw_data->clock_frequency = ADF_6XXX_AE_FREQ;
836
+
hw_data->get_svc_slice_cnt = adf_gen6_get_svc_slice_cnt;
933
837
934
838
adf_gen6_init_hw_csr_ops(&hw_data->csr_ops);
935
839
adf_gen6_init_pf_pfvf_ops(&hw_data->pfvf_ops);
936
840
adf_gen6_init_dc_ops(&hw_data->dc_ops);
841
+
adf_gen6_init_vf_mig_ops(&hw_data->vfmig_ops);
937
842
adf_gen6_init_ras_ops(&hw_data->ras_ops);
843
+
adf_gen6_init_tl_data(&hw_data->tl_data);
844
+
adf_gen6_init_rl_data(&hw_data->rl_data);
938
845
}
939
846
940
847
void adf_clean_hw_data_6xxx(struct adf_hw_device_data *hw_data)
+21
-1
drivers/crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.h
+21
-1
drivers/crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.h
···
99
99
#define ADF_GEN6_PVC0CTL_OFFSET 0x204
100
100
#define ADF_GEN6_PVC0CTL_TCVCMAP_OFFSET 1
101
101
#define ADF_GEN6_PVC0CTL_TCVCMAP_MASK GENMASK(7, 1)
102
-
#define ADF_GEN6_PVC0CTL_TCVCMAP_DEFAULT 0x7F
102
+
#define ADF_GEN6_PVC0CTL_TCVCMAP_DEFAULT 0x3F
103
103
104
104
/* VC1 Resource Control Register */
105
105
#define ADF_GEN6_PVC1CTL_OFFSET 0x210
···
122
122
/* Number of heartbeat counter pairs */
123
123
#define ADF_NUM_HB_CNT_PER_AE ADF_NUM_THREADS_PER_AE
124
124
125
+
/* Rate Limiting */
126
+
#define ADF_GEN6_RL_R2L_OFFSET 0x508000
127
+
#define ADF_GEN6_RL_L2C_OFFSET 0x509000
128
+
#define ADF_GEN6_RL_C2S_OFFSET 0x508818
129
+
#define ADF_GEN6_RL_TOKEN_PCIEIN_BUCKET_OFFSET 0x508800
130
+
#define ADF_GEN6_RL_TOKEN_PCIEOUT_BUCKET_OFFSET 0x508804
131
+
125
132
/* Physical function fuses */
126
133
#define ADF_6XXX_ACCELENGINES_MASK GENMASK(8, 0)
127
134
#define ADF_6XXX_ADMIN_AE_MASK GENMASK(8, 8)
···
139
132
#define ADF_6XXX_CY_OBJ "qat_6xxx_cy.bin"
140
133
#define ADF_6XXX_DC_OBJ "qat_6xxx_dc.bin"
141
134
#define ADF_6XXX_ADMIN_OBJ "qat_6xxx_admin.bin"
135
+
136
+
/* RL constants */
137
+
#define ADF_6XXX_RL_PCIE_SCALE_FACTOR_DIV 100
138
+
#define ADF_6XXX_RL_PCIE_SCALE_FACTOR_MUL 102
139
+
#define ADF_6XXX_RL_SCANS_PER_SEC 954
140
+
#define ADF_6XXX_RL_MAX_TP_ASYM 173750UL
141
+
#define ADF_6XXX_RL_MAX_TP_SYM 95000UL
142
+
#define ADF_6XXX_RL_MAX_TP_DC 40000UL
143
+
#define ADF_6XXX_RL_MAX_TP_DECOMP 40000UL
144
+
#define ADF_6XXX_RL_SLICE_REF 1000UL
145
+
146
+
/* Clock frequency */
147
+
#define ADF_6XXX_AE_FREQ (1000 * HZ_PER_MHZ)
142
148
143
149
enum icp_qat_gen6_slice_mask {
144
150
ICP_ACCEL_GEN6_MASK_UCS_SLICE = BIT(0),
+4
drivers/crypto/intel/qat/qat_common/Makefile
+4
drivers/crypto/intel/qat/qat_common/Makefile
···
4
4
intel_qat-y := adf_accel_engine.o \
5
5
adf_admin.o \
6
6
adf_aer.o \
7
+
adf_bank_state.o \
7
8
adf_cfg.o \
8
9
adf_cfg_services.o \
9
10
adf_clock.o \
···
49
48
adf_fw_counters.o \
50
49
adf_gen4_pm_debugfs.o \
51
50
adf_gen4_tl.o \
51
+
adf_gen6_pm_dbgfs.o \
52
+
adf_gen6_tl.o \
52
53
adf_heartbeat_dbgfs.o \
53
54
adf_heartbeat.o \
54
55
adf_pm_dbgfs.o \
56
+
adf_pm_dbgfs_utils.o \
55
57
adf_telemetry.o \
56
58
adf_tl_debugfs.o \
57
59
adf_transport_debug.o
+5
-35
drivers/crypto/intel/qat/qat_common/adf_accel_devices.h
+5
-35
drivers/crypto/intel/qat/qat_common/adf_accel_devices.h
···
157
157
u32 mailbox_offset;
158
158
};
159
159
160
-
struct ring_config {
161
-
u64 base;
162
-
u32 config;
163
-
u32 head;
164
-
u32 tail;
165
-
u32 reserved0;
166
-
};
167
-
168
-
struct bank_state {
169
-
u32 ringstat0;
170
-
u32 ringstat1;
171
-
u32 ringuostat;
172
-
u32 ringestat;
173
-
u32 ringnestat;
174
-
u32 ringnfstat;
175
-
u32 ringfstat;
176
-
u32 ringcstat0;
177
-
u32 ringcstat1;
178
-
u32 ringcstat2;
179
-
u32 ringcstat3;
180
-
u32 iaintflagen;
181
-
u32 iaintflagreg;
182
-
u32 iaintflagsrcsel0;
183
-
u32 iaintflagsrcsel1;
184
-
u32 iaintcolen;
185
-
u32 iaintcolctl;
186
-
u32 iaintflagandcolen;
187
-
u32 ringexpstat;
188
-
u32 ringexpintenable;
189
-
u32 ringsrvarben;
190
-
u32 reserved0;
191
-
struct ring_config rings[ADF_ETR_MAX_RINGS_PER_BANK];
192
-
};
160
+
struct adf_bank_state;
193
161
194
162
struct adf_hw_csr_ops {
195
163
u64 (*build_csr_ring_base_addr)(dma_addr_t addr, u32 size);
···
306
338
void (*set_ssm_wdtimer)(struct adf_accel_dev *accel_dev);
307
339
int (*ring_pair_reset)(struct adf_accel_dev *accel_dev, u32 bank_nr);
308
340
int (*bank_state_save)(struct adf_accel_dev *accel_dev, u32 bank_number,
309
-
struct bank_state *state);
341
+
struct adf_bank_state *state);
310
342
int (*bank_state_restore)(struct adf_accel_dev *accel_dev,
311
-
u32 bank_number, struct bank_state *state);
343
+
u32 bank_number, struct adf_bank_state *state);
312
344
void (*reset_device)(struct adf_accel_dev *accel_dev);
313
345
void (*set_msix_rttable)(struct adf_accel_dev *accel_dev);
314
346
const char *(*uof_get_name)(struct adf_accel_dev *accel_dev, u32 obj_num);
···
319
351
u32 (*get_ena_thd_mask)(struct adf_accel_dev *accel_dev, u32 obj_num);
320
352
int (*dev_config)(struct adf_accel_dev *accel_dev);
321
353
bool (*services_supported)(unsigned long mask);
354
+
u32 (*get_svc_slice_cnt)(struct adf_accel_dev *accel_dev,
355
+
enum adf_base_services svc);
322
356
struct adf_pfvf_ops pfvf_ops;
323
357
struct adf_hw_csr_ops csr_ops;
324
358
struct adf_dc_ops dc_ops;
+1
-1
drivers/crypto/intel/qat/qat_common/adf_aer.c
+1
-1
drivers/crypto/intel/qat/qat_common/adf_aer.c
···
229
229
};
230
230
EXPORT_SYMBOL_GPL(adf_err_handler);
231
231
232
-
int adf_dev_autoreset(struct adf_accel_dev *accel_dev)
232
+
static int adf_dev_autoreset(struct adf_accel_dev *accel_dev)
233
233
{
234
234
if (accel_dev->autoreset_on_error)
235
235
return adf_dev_aer_schedule_reset(accel_dev, ADF_DEV_RESET_ASYNC);
+238
drivers/crypto/intel/qat/qat_common/adf_bank_state.c
+238
drivers/crypto/intel/qat/qat_common/adf_bank_state.c
···
1
+
// SPDX-License-Identifier: GPL-2.0-only
2
+
/* Copyright(c) 2025 Intel Corporation */
3
+
4
+
#define pr_fmt(fmt) "QAT: " fmt
5
+
6
+
#include <linux/bits.h>
7
+
#include <linux/dev_printk.h>
8
+
#include <linux/printk.h>
9
+
#include "adf_accel_devices.h"
10
+
#include "adf_bank_state.h"
11
+
#include "adf_common_drv.h"
12
+
13
+
/* Ring interrupt masks */
14
+
#define ADF_RP_INT_SRC_SEL_F_RISE_MASK GENMASK(1, 0)
15
+
#define ADF_RP_INT_SRC_SEL_F_FALL_MASK GENMASK(2, 0)
16
+
#define ADF_RP_INT_SRC_SEL_RANGE_WIDTH 4
17
+
18
+
static inline int check_stat(u32 (*op)(void __iomem *, u32), u32 expect_val,
19
+
const char *name, void __iomem *base, u32 bank)
20
+
{
21
+
u32 actual_val = op(base, bank);
22
+
23
+
if (expect_val == actual_val)
24
+
return 0;
25
+
26
+
pr_err("Fail to restore %s register. Expected %#x, actual %#x\n",
27
+
name, expect_val, actual_val);
28
+
29
+
return -EINVAL;
30
+
}
31
+
32
+
static void bank_state_save(struct adf_hw_csr_ops *ops, void __iomem *base,
33
+
u32 bank, struct adf_bank_state *state, u32 num_rings)
34
+
{
35
+
u32 i;
36
+
37
+
state->ringstat0 = ops->read_csr_stat(base, bank);
38
+
state->ringuostat = ops->read_csr_uo_stat(base, bank);
39
+
state->ringestat = ops->read_csr_e_stat(base, bank);
40
+
state->ringnestat = ops->read_csr_ne_stat(base, bank);
41
+
state->ringnfstat = ops->read_csr_nf_stat(base, bank);
42
+
state->ringfstat = ops->read_csr_f_stat(base, bank);
43
+
state->ringcstat0 = ops->read_csr_c_stat(base, bank);
44
+
state->iaintflagen = ops->read_csr_int_en(base, bank);
45
+
state->iaintflagreg = ops->read_csr_int_flag(base, bank);
46
+
state->iaintflagsrcsel0 = ops->read_csr_int_srcsel(base, bank);
47
+
state->iaintcolen = ops->read_csr_int_col_en(base, bank);
48
+
state->iaintcolctl = ops->read_csr_int_col_ctl(base, bank);
49
+
state->iaintflagandcolen = ops->read_csr_int_flag_and_col(base, bank);
50
+
state->ringexpstat = ops->read_csr_exp_stat(base, bank);
51
+
state->ringexpintenable = ops->read_csr_exp_int_en(base, bank);
52
+
state->ringsrvarben = ops->read_csr_ring_srv_arb_en(base, bank);
53
+
54
+
for (i = 0; i < num_rings; i++) {
55
+
state->rings[i].head = ops->read_csr_ring_head(base, bank, i);
56
+
state->rings[i].tail = ops->read_csr_ring_tail(base, bank, i);
57
+
state->rings[i].config = ops->read_csr_ring_config(base, bank, i);
58
+
state->rings[i].base = ops->read_csr_ring_base(base, bank, i);
59
+
}
60
+
}
61
+
62
+
static int bank_state_restore(struct adf_hw_csr_ops *ops, void __iomem *base,
63
+
u32 bank, struct adf_bank_state *state, u32 num_rings,
64
+
int tx_rx_gap)
65
+
{
66
+
u32 val, tmp_val, i;
67
+
int ret;
68
+
69
+
for (i = 0; i < num_rings; i++)
70
+
ops->write_csr_ring_base(base, bank, i, state->rings[i].base);
71
+
72
+
for (i = 0; i < num_rings; i++)
73
+
ops->write_csr_ring_config(base, bank, i, state->rings[i].config);
74
+
75
+
for (i = 0; i < num_rings / 2; i++) {
76
+
int tx = i * (tx_rx_gap + 1);
77
+
int rx = tx + tx_rx_gap;
78
+
79
+
ops->write_csr_ring_head(base, bank, tx, state->rings[tx].head);
80
+
ops->write_csr_ring_tail(base, bank, tx, state->rings[tx].tail);
81
+
82
+
/*
83
+
* The TX ring head needs to be updated again to make sure that
84
+
* the HW will not consider the ring as full when it is empty
85
+
* and the correct state flags are set to match the recovered state.
86
+
*/
87
+
if (state->ringestat & BIT(tx)) {
88
+
val = ops->read_csr_int_srcsel(base, bank);
89
+
val |= ADF_RP_INT_SRC_SEL_F_RISE_MASK;
90
+
ops->write_csr_int_srcsel_w_val(base, bank, val);
91
+
ops->write_csr_ring_head(base, bank, tx, state->rings[tx].head);
92
+
}
93
+
94
+
ops->write_csr_ring_tail(base, bank, rx, state->rings[rx].tail);
95
+
val = ops->read_csr_int_srcsel(base, bank);
96
+
val |= ADF_RP_INT_SRC_SEL_F_RISE_MASK << ADF_RP_INT_SRC_SEL_RANGE_WIDTH;
97
+
ops->write_csr_int_srcsel_w_val(base, bank, val);
98
+
99
+
ops->write_csr_ring_head(base, bank, rx, state->rings[rx].head);
100
+
val = ops->read_csr_int_srcsel(base, bank);
101
+
val |= ADF_RP_INT_SRC_SEL_F_FALL_MASK << ADF_RP_INT_SRC_SEL_RANGE_WIDTH;
102
+
ops->write_csr_int_srcsel_w_val(base, bank, val);
103
+
104
+
/*
105
+
* The RX ring tail needs to be updated again to make sure that
106
+
* the HW will not consider the ring as empty when it is full
107
+
* and the correct state flags are set to match the recovered state.
108
+
*/
109
+
if (state->ringfstat & BIT(rx))
110
+
ops->write_csr_ring_tail(base, bank, rx, state->rings[rx].tail);
111
+
}
112
+
113
+
ops->write_csr_int_flag_and_col(base, bank, state->iaintflagandcolen);
114
+
ops->write_csr_int_en(base, bank, state->iaintflagen);
115
+
ops->write_csr_int_col_en(base, bank, state->iaintcolen);
116
+
ops->write_csr_int_srcsel_w_val(base, bank, state->iaintflagsrcsel0);
117
+
ops->write_csr_exp_int_en(base, bank, state->ringexpintenable);
118
+
ops->write_csr_int_col_ctl(base, bank, state->iaintcolctl);
119
+
120
+
/*
121
+
* Verify whether any exceptions were raised during the bank save process.
122
+
* If exceptions occurred, the status and exception registers cannot
123
+
* be directly restored. Consequently, further restoration is not
124
+
* feasible, and the current state of the ring should be maintained.
125
+
*/
126
+
val = state->ringexpstat;
127
+
if (val) {
128
+
pr_info("Bank %u state not fully restored due to exception in saved state (%#x)\n",
129
+
bank, val);
130
+
return 0;
131
+
}
132
+
133
+
/* Ensure that the restoration process completed without exceptions */
134
+
tmp_val = ops->read_csr_exp_stat(base, bank);
135
+
if (tmp_val) {
136
+
pr_err("Bank %u restored with exception: %#x\n", bank, tmp_val);
137
+
return -EFAULT;
138
+
}
139
+
140
+
ops->write_csr_ring_srv_arb_en(base, bank, state->ringsrvarben);
141
+
142
+
/* Check that all ring statuses match the saved state. */
143
+
ret = check_stat(ops->read_csr_stat, state->ringstat0, "ringstat",
144
+
base, bank);
145
+
if (ret)
146
+
return ret;
147
+
148
+
ret = check_stat(ops->read_csr_e_stat, state->ringestat, "ringestat",
149
+
base, bank);
150
+
if (ret)
151
+
return ret;
152
+
153
+
ret = check_stat(ops->read_csr_ne_stat, state->ringnestat, "ringnestat",
154
+
base, bank);
155
+
if (ret)
156
+
return ret;
157
+
158
+
ret = check_stat(ops->read_csr_nf_stat, state->ringnfstat, "ringnfstat",
159
+
base, bank);
160
+
if (ret)
161
+
return ret;
162
+
163
+
ret = check_stat(ops->read_csr_f_stat, state->ringfstat, "ringfstat",
164
+
base, bank);
165
+
if (ret)
166
+
return ret;
167
+
168
+
ret = check_stat(ops->read_csr_c_stat, state->ringcstat0, "ringcstat",
169
+
base, bank);
170
+
if (ret)
171
+
return ret;
172
+
173
+
return 0;
174
+
}
175
+
176
+
/**
177
+
* adf_bank_state_save() - save state of bank-related registers
178
+
* @accel_dev: Pointer to the device structure
179
+
* @bank_number: Bank number
180
+
* @state: Pointer to bank state structure
181
+
*
182
+
* This function saves the state of a bank by reading the bank CSRs and
183
+
* writing them in the @state structure.
184
+
*
185
+
* Returns 0 on success, error code otherwise
186
+
*/
187
+
int adf_bank_state_save(struct adf_accel_dev *accel_dev, u32 bank_number,
188
+
struct adf_bank_state *state)
189
+
{
190
+
struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev);
191
+
struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev);
192
+
void __iomem *csr_base = adf_get_etr_base(accel_dev);
193
+
194
+
if (bank_number >= hw_data->num_banks || !state)
195
+
return -EINVAL;
196
+
197
+
dev_dbg(&GET_DEV(accel_dev), "Saving state of bank %d\n", bank_number);
198
+
199
+
bank_state_save(csr_ops, csr_base, bank_number, state,
200
+
hw_data->num_rings_per_bank);
201
+
202
+
return 0;
203
+
}
204
+
EXPORT_SYMBOL_GPL(adf_bank_state_save);
205
+
206
+
/**
207
+
* adf_bank_state_restore() - restore state of bank-related registers
208
+
* @accel_dev: Pointer to the device structure
209
+
* @bank_number: Bank number
210
+
* @state: Pointer to bank state structure
211
+
*
212
+
* This function attempts to restore the state of a bank by writing the
213
+
* bank CSRs to the values in the state structure.
214
+
*
215
+
* Returns 0 on success, error code otherwise
216
+
*/
217
+
int adf_bank_state_restore(struct adf_accel_dev *accel_dev, u32 bank_number,
218
+
struct adf_bank_state *state)
219
+
{
220
+
struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev);
221
+
struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev);
222
+
void __iomem *csr_base = adf_get_etr_base(accel_dev);
223
+
int ret;
224
+
225
+
if (bank_number >= hw_data->num_banks || !state)
226
+
return -EINVAL;
227
+
228
+
dev_dbg(&GET_DEV(accel_dev), "Restoring state of bank %d\n", bank_number);
229
+
230
+
ret = bank_state_restore(csr_ops, csr_base, bank_number, state,
231
+
hw_data->num_rings_per_bank, hw_data->tx_rx_gap);
232
+
if (ret)
233
+
dev_err(&GET_DEV(accel_dev),
234
+
"Unable to restore state of bank %d\n", bank_number);
235
+
236
+
return ret;
237
+
}
238
+
EXPORT_SYMBOL_GPL(adf_bank_state_restore);
+49
drivers/crypto/intel/qat/qat_common/adf_bank_state.h
+49
drivers/crypto/intel/qat/qat_common/adf_bank_state.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0-only */
2
+
/* Copyright(c) 2025 Intel Corporation */
3
+
#ifndef ADF_BANK_STATE_H_
4
+
#define ADF_BANK_STATE_H_
5
+
6
+
#include <linux/types.h>
7
+
8
+
struct adf_accel_dev;
9
+
10
+
struct ring_config {
11
+
u64 base;
12
+
u32 config;
13
+
u32 head;
14
+
u32 tail;
15
+
u32 reserved0;
16
+
};
17
+
18
+
struct adf_bank_state {
19
+
u32 ringstat0;
20
+
u32 ringstat1;
21
+
u32 ringuostat;
22
+
u32 ringestat;
23
+
u32 ringnestat;
24
+
u32 ringnfstat;
25
+
u32 ringfstat;
26
+
u32 ringcstat0;
27
+
u32 ringcstat1;
28
+
u32 ringcstat2;
29
+
u32 ringcstat3;
30
+
u32 iaintflagen;
31
+
u32 iaintflagreg;
32
+
u32 iaintflagsrcsel0;
33
+
u32 iaintflagsrcsel1;
34
+
u32 iaintcolen;
35
+
u32 iaintcolctl;
36
+
u32 iaintflagandcolen;
37
+
u32 ringexpstat;
38
+
u32 ringexpintenable;
39
+
u32 ringsrvarben;
40
+
u32 reserved0;
41
+
struct ring_config rings[ADF_ETR_MAX_RINGS_PER_BANK];
42
+
};
43
+
44
+
int adf_bank_state_restore(struct adf_accel_dev *accel_dev, u32 bank_number,
45
+
struct adf_bank_state *state);
46
+
int adf_bank_state_save(struct adf_accel_dev *accel_dev, u32 bank_number,
47
+
struct adf_bank_state *state);
48
+
49
+
#endif
+1
drivers/crypto/intel/qat/qat_common/adf_cfg_common.h
+1
drivers/crypto/intel/qat/qat_common/adf_cfg_common.h
+41
-4
drivers/crypto/intel/qat/qat_common/adf_cfg_services.c
+41
-4
drivers/crypto/intel/qat/qat_common/adf_cfg_services.c
···
7
7
#include <linux/pci.h>
8
8
#include <linux/string.h>
9
9
#include "adf_cfg.h"
10
+
#include "adf_cfg_common.h"
10
11
#include "adf_cfg_services.h"
11
12
#include "adf_cfg_strings.h"
12
13
···
16
15
[SVC_SYM] = ADF_CFG_SYM,
17
16
[SVC_DC] = ADF_CFG_DC,
18
17
[SVC_DCC] = ADF_CFG_DCC,
18
+
[SVC_DECOMP] = ADF_CFG_DECOMP,
19
19
};
20
20
21
21
/*
22
22
* Ensure that the size of the array matches the number of services,
23
-
* SVC_BASE_COUNT, that is used to size the bitmap.
23
+
* SVC_COUNT, that is used to size the bitmap.
24
24
*/
25
-
static_assert(ARRAY_SIZE(adf_cfg_services) == SVC_BASE_COUNT);
25
+
static_assert(ARRAY_SIZE(adf_cfg_services) == SVC_COUNT);
26
26
27
27
/*
28
28
* Ensure that the maximum number of concurrent services that can be
···
36
34
* Ensure that the number of services fit a single unsigned long, as each
37
35
* service is represented by a bit in the mask.
38
36
*/
39
-
static_assert(BITS_PER_LONG >= SVC_BASE_COUNT);
37
+
static_assert(BITS_PER_LONG >= SVC_COUNT);
40
38
41
39
/*
42
40
* Ensure that size of the concatenation of all service strings is smaller
···
45
43
static_assert(sizeof(ADF_CFG_SYM ADF_SERVICES_DELIMITER
46
44
ADF_CFG_ASYM ADF_SERVICES_DELIMITER
47
45
ADF_CFG_DC ADF_SERVICES_DELIMITER
46
+
ADF_CFG_DECOMP ADF_SERVICES_DELIMITER
48
47
ADF_CFG_DCC) < ADF_CFG_MAX_VAL_LEN_IN_BYTES);
49
48
50
49
static int adf_service_string_to_mask(struct adf_accel_dev *accel_dev, const char *buf,
···
91
88
if (len < ADF_CFG_MAX_VAL_LEN_IN_BYTES)
92
89
return -ENOSPC;
93
90
94
-
for_each_set_bit(bit, &mask, SVC_BASE_COUNT) {
91
+
for_each_set_bit(bit, &mask, SVC_COUNT) {
95
92
if (offset)
96
93
offset += scnprintf(buf + offset, len - offset,
97
94
ADF_SERVICES_DELIMITER);
···
170
167
if (test_bit(SVC_DC, &mask))
171
168
return SVC_DC;
172
169
170
+
if (test_bit(SVC_DECOMP, &mask))
171
+
return SVC_DECOMP;
172
+
173
173
if (test_bit(SVC_DCC, &mask))
174
174
return SVC_DCC;
175
175
176
176
return -EINVAL;
177
177
}
178
178
EXPORT_SYMBOL_GPL(adf_get_service_enabled);
179
+
180
+
enum adf_cfg_service_type adf_srv_to_cfg_svc_type(enum adf_base_services svc)
181
+
{
182
+
switch (svc) {
183
+
case SVC_ASYM:
184
+
return ASYM;
185
+
case SVC_SYM:
186
+
return SYM;
187
+
case SVC_DC:
188
+
return COMP;
189
+
case SVC_DECOMP:
190
+
return DECOMP;
191
+
default:
192
+
return UNUSED;
193
+
}
194
+
}
195
+
196
+
bool adf_is_service_enabled(struct adf_accel_dev *accel_dev, enum adf_base_services svc)
197
+
{
198
+
enum adf_cfg_service_type arb_srv = adf_srv_to_cfg_svc_type(svc);
199
+
struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev);
200
+
u8 rps_per_bundle = hw_data->num_banks_per_vf;
201
+
int i;
202
+
203
+
for (i = 0; i < rps_per_bundle; i++) {
204
+
if (GET_SRV_TYPE(accel_dev, i) == arb_srv)
205
+
return true;
206
+
}
207
+
208
+
return false;
209
+
}
+10
-3
drivers/crypto/intel/qat/qat_common/adf_cfg_services.h
+10
-3
drivers/crypto/intel/qat/qat_common/adf_cfg_services.h
···
7
7
8
8
struct adf_accel_dev;
9
9
10
-
enum adf_services {
10
+
enum adf_base_services {
11
11
SVC_ASYM = 0,
12
12
SVC_SYM,
13
13
SVC_DC,
14
-
SVC_DCC,
14
+
SVC_DECOMP,
15
15
SVC_BASE_COUNT
16
16
};
17
17
18
+
enum adf_extended_services {
19
+
SVC_DCC = SVC_BASE_COUNT,
20
+
SVC_COUNT
21
+
};
22
+
18
23
enum adf_composed_services {
19
-
SVC_SYM_ASYM = SVC_BASE_COUNT,
24
+
SVC_SYM_ASYM = SVC_COUNT,
20
25
SVC_SYM_DC,
21
26
SVC_ASYM_DC,
22
27
};
···
38
33
size_t in_len, char *out, size_t out_len);
39
34
int adf_get_service_enabled(struct adf_accel_dev *accel_dev);
40
35
int adf_get_service_mask(struct adf_accel_dev *accel_dev, unsigned long *mask);
36
+
enum adf_cfg_service_type adf_srv_to_cfg_svc_type(enum adf_base_services svc);
37
+
bool adf_is_service_enabled(struct adf_accel_dev *accel_dev, enum adf_base_services svc);
41
38
42
39
#endif
+1
drivers/crypto/intel/qat/qat_common/adf_cfg_strings.h
+1
drivers/crypto/intel/qat/qat_common/adf_cfg_strings.h
+1
-1
drivers/crypto/intel/qat/qat_common/adf_common_drv.h
+1
-1
drivers/crypto/intel/qat/qat_common/adf_common_drv.h
···
86
86
extern const struct pci_error_handlers adf_err_handler;
87
87
void adf_reset_sbr(struct adf_accel_dev *accel_dev);
88
88
void adf_reset_flr(struct adf_accel_dev *accel_dev);
89
-
int adf_dev_autoreset(struct adf_accel_dev *accel_dev);
90
89
void adf_dev_restore(struct adf_accel_dev *accel_dev);
91
90
int adf_init_aer(void);
92
91
void adf_exit_aer(void);
···
188
189
bool adf_misc_wq_queue_work(struct work_struct *work);
189
190
bool adf_misc_wq_queue_delayed_work(struct delayed_work *work,
190
191
unsigned long delay);
192
+
void adf_misc_wq_flush(void);
191
193
#if defined(CONFIG_PCI_IOV)
192
194
int adf_sriov_configure(struct pci_dev *pdev, int numvfs);
193
195
void adf_disable_sriov(struct adf_accel_dev *accel_dev);
+47
-182
drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.c
+47
-182
drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.c
···
1
1
// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
2
2
/* Copyright(c) 2020 Intel Corporation */
3
+
4
+
#define pr_fmt(fmt) "QAT: " fmt
5
+
3
6
#include <linux/bitops.h>
4
7
#include <linux/iopoll.h>
5
8
#include <asm/div64.h>
···
262
259
{
263
260
unsigned long num_svc = hweight_long(mask);
264
261
265
-
if (mask >= BIT(SVC_BASE_COUNT))
262
+
if (mask >= BIT(SVC_COUNT))
263
+
return false;
264
+
265
+
if (test_bit(SVC_DECOMP, &mask))
266
266
return false;
267
267
268
268
switch (num_svc) {
···
491
485
return ret;
492
486
}
493
487
494
-
static void bank_state_save(struct adf_hw_csr_ops *ops, void __iomem *base,
495
-
u32 bank, struct bank_state *state, u32 num_rings)
496
-
{
497
-
u32 i;
498
-
499
-
state->ringstat0 = ops->read_csr_stat(base, bank);
500
-
state->ringuostat = ops->read_csr_uo_stat(base, bank);
501
-
state->ringestat = ops->read_csr_e_stat(base, bank);
502
-
state->ringnestat = ops->read_csr_ne_stat(base, bank);
503
-
state->ringnfstat = ops->read_csr_nf_stat(base, bank);
504
-
state->ringfstat = ops->read_csr_f_stat(base, bank);
505
-
state->ringcstat0 = ops->read_csr_c_stat(base, bank);
506
-
state->iaintflagen = ops->read_csr_int_en(base, bank);
507
-
state->iaintflagreg = ops->read_csr_int_flag(base, bank);
508
-
state->iaintflagsrcsel0 = ops->read_csr_int_srcsel(base, bank);
509
-
state->iaintcolen = ops->read_csr_int_col_en(base, bank);
510
-
state->iaintcolctl = ops->read_csr_int_col_ctl(base, bank);
511
-
state->iaintflagandcolen = ops->read_csr_int_flag_and_col(base, bank);
512
-
state->ringexpstat = ops->read_csr_exp_stat(base, bank);
513
-
state->ringexpintenable = ops->read_csr_exp_int_en(base, bank);
514
-
state->ringsrvarben = ops->read_csr_ring_srv_arb_en(base, bank);
515
-
516
-
for (i = 0; i < num_rings; i++) {
517
-
state->rings[i].head = ops->read_csr_ring_head(base, bank, i);
518
-
state->rings[i].tail = ops->read_csr_ring_tail(base, bank, i);
519
-
state->rings[i].config = ops->read_csr_ring_config(base, bank, i);
520
-
state->rings[i].base = ops->read_csr_ring_base(base, bank, i);
521
-
}
522
-
}
523
-
524
-
#define CHECK_STAT(op, expect_val, name, args...) \
525
-
({ \
526
-
u32 __expect_val = (expect_val); \
527
-
u32 actual_val = op(args); \
528
-
(__expect_val == actual_val) ? 0 : \
529
-
(pr_err("QAT: Fail to restore %s register. Expected 0x%x, actual 0x%x\n", \
530
-
name, __expect_val, actual_val), -EINVAL); \
531
-
})
532
-
533
-
static int bank_state_restore(struct adf_hw_csr_ops *ops, void __iomem *base,
534
-
u32 bank, struct bank_state *state, u32 num_rings,
535
-
int tx_rx_gap)
536
-
{
537
-
u32 val, tmp_val, i;
538
-
int ret;
539
-
540
-
for (i = 0; i < num_rings; i++)
541
-
ops->write_csr_ring_base(base, bank, i, state->rings[i].base);
542
-
543
-
for (i = 0; i < num_rings; i++)
544
-
ops->write_csr_ring_config(base, bank, i, state->rings[i].config);
545
-
546
-
for (i = 0; i < num_rings / 2; i++) {
547
-
int tx = i * (tx_rx_gap + 1);
548
-
int rx = tx + tx_rx_gap;
549
-
550
-
ops->write_csr_ring_head(base, bank, tx, state->rings[tx].head);
551
-
ops->write_csr_ring_tail(base, bank, tx, state->rings[tx].tail);
552
-
553
-
/*
554
-
* The TX ring head needs to be updated again to make sure that
555
-
* the HW will not consider the ring as full when it is empty
556
-
* and the correct state flags are set to match the recovered state.
557
-
*/
558
-
if (state->ringestat & BIT(tx)) {
559
-
val = ops->read_csr_int_srcsel(base, bank);
560
-
val |= ADF_RP_INT_SRC_SEL_F_RISE_MASK;
561
-
ops->write_csr_int_srcsel_w_val(base, bank, val);
562
-
ops->write_csr_ring_head(base, bank, tx, state->rings[tx].head);
563
-
}
564
-
565
-
ops->write_csr_ring_tail(base, bank, rx, state->rings[rx].tail);
566
-
val = ops->read_csr_int_srcsel(base, bank);
567
-
val |= ADF_RP_INT_SRC_SEL_F_RISE_MASK << ADF_RP_INT_SRC_SEL_RANGE_WIDTH;
568
-
ops->write_csr_int_srcsel_w_val(base, bank, val);
569
-
570
-
ops->write_csr_ring_head(base, bank, rx, state->rings[rx].head);
571
-
val = ops->read_csr_int_srcsel(base, bank);
572
-
val |= ADF_RP_INT_SRC_SEL_F_FALL_MASK << ADF_RP_INT_SRC_SEL_RANGE_WIDTH;
573
-
ops->write_csr_int_srcsel_w_val(base, bank, val);
574
-
575
-
/*
576
-
* The RX ring tail needs to be updated again to make sure that
577
-
* the HW will not consider the ring as empty when it is full
578
-
* and the correct state flags are set to match the recovered state.
579
-
*/
580
-
if (state->ringfstat & BIT(rx))
581
-
ops->write_csr_ring_tail(base, bank, rx, state->rings[rx].tail);
582
-
}
583
-
584
-
ops->write_csr_int_flag_and_col(base, bank, state->iaintflagandcolen);
585
-
ops->write_csr_int_en(base, bank, state->iaintflagen);
586
-
ops->write_csr_int_col_en(base, bank, state->iaintcolen);
587
-
ops->write_csr_int_srcsel_w_val(base, bank, state->iaintflagsrcsel0);
588
-
ops->write_csr_exp_int_en(base, bank, state->ringexpintenable);
589
-
ops->write_csr_int_col_ctl(base, bank, state->iaintcolctl);
590
-
ops->write_csr_ring_srv_arb_en(base, bank, state->ringsrvarben);
591
-
592
-
/* Check that all ring statuses match the saved state. */
593
-
ret = CHECK_STAT(ops->read_csr_stat, state->ringstat0, "ringstat",
594
-
base, bank);
595
-
if (ret)
596
-
return ret;
597
-
598
-
ret = CHECK_STAT(ops->read_csr_e_stat, state->ringestat, "ringestat",
599
-
base, bank);
600
-
if (ret)
601
-
return ret;
602
-
603
-
ret = CHECK_STAT(ops->read_csr_ne_stat, state->ringnestat, "ringnestat",
604
-
base, bank);
605
-
if (ret)
606
-
return ret;
607
-
608
-
ret = CHECK_STAT(ops->read_csr_nf_stat, state->ringnfstat, "ringnfstat",
609
-
base, bank);
610
-
if (ret)
611
-
return ret;
612
-
613
-
ret = CHECK_STAT(ops->read_csr_f_stat, state->ringfstat, "ringfstat",
614
-
base, bank);
615
-
if (ret)
616
-
return ret;
617
-
618
-
ret = CHECK_STAT(ops->read_csr_c_stat, state->ringcstat0, "ringcstat",
619
-
base, bank);
620
-
if (ret)
621
-
return ret;
622
-
623
-
tmp_val = ops->read_csr_exp_stat(base, bank);
624
-
val = state->ringexpstat;
625
-
if (tmp_val && !val) {
626
-
pr_err("QAT: Bank was restored with exception: 0x%x\n", val);
627
-
return -EINVAL;
628
-
}
629
-
630
-
return 0;
631
-
}
632
-
633
-
int adf_gen4_bank_state_save(struct adf_accel_dev *accel_dev, u32 bank_number,
634
-
struct bank_state *state)
635
-
{
636
-
struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev);
637
-
struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev);
638
-
void __iomem *csr_base = adf_get_etr_base(accel_dev);
639
-
640
-
if (bank_number >= hw_data->num_banks || !state)
641
-
return -EINVAL;
642
-
643
-
dev_dbg(&GET_DEV(accel_dev), "Saving state of bank %d\n", bank_number);
644
-
645
-
bank_state_save(csr_ops, csr_base, bank_number, state,
646
-
hw_data->num_rings_per_bank);
647
-
648
-
return 0;
649
-
}
650
-
EXPORT_SYMBOL_GPL(adf_gen4_bank_state_save);
651
-
652
-
int adf_gen4_bank_state_restore(struct adf_accel_dev *accel_dev, u32 bank_number,
653
-
struct bank_state *state)
654
-
{
655
-
struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev);
656
-
struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev);
657
-
void __iomem *csr_base = adf_get_etr_base(accel_dev);
658
-
int ret;
659
-
660
-
if (bank_number >= hw_data->num_banks || !state)
661
-
return -EINVAL;
662
-
663
-
dev_dbg(&GET_DEV(accel_dev), "Restoring state of bank %d\n", bank_number);
664
-
665
-
ret = bank_state_restore(csr_ops, csr_base, bank_number, state,
666
-
hw_data->num_rings_per_bank, hw_data->tx_rx_gap);
667
-
if (ret)
668
-
dev_err(&GET_DEV(accel_dev),
669
-
"Unable to restore state of bank %d\n", bank_number);
670
-
671
-
return ret;
672
-
}
673
-
EXPORT_SYMBOL_GPL(adf_gen4_bank_state_restore);
674
-
675
488
static int adf_gen4_build_comp_block(void *ctx, enum adf_dc_algo algo)
676
489
{
677
490
struct icp_qat_fw_comp_req *req_tmpl = ctx;
···
558
733
dc_ops->build_decomp_block = adf_gen4_build_decomp_block;
559
734
}
560
735
EXPORT_SYMBOL_GPL(adf_gen4_init_dc_ops);
736
+
737
+
void adf_gen4_init_num_svc_aes(struct adf_rl_hw_data *device_data)
738
+
{
739
+
struct adf_hw_device_data *hw_data;
740
+
unsigned int i;
741
+
u32 ae_cnt;
742
+
743
+
hw_data = container_of(device_data, struct adf_hw_device_data, rl_data);
744
+
ae_cnt = hweight32(hw_data->get_ae_mask(hw_data));
745
+
if (!ae_cnt)
746
+
return;
747
+
748
+
for (i = 0; i < SVC_BASE_COUNT; i++)
749
+
device_data->svc_ae_mask[i] = ae_cnt - 1;
750
+
751
+
/*
752
+
* The decompression service is not supported on QAT GEN4 devices.
753
+
* Therefore, set svc_ae_mask to 0.
754
+
*/
755
+
device_data->svc_ae_mask[SVC_DECOMP] = 0;
756
+
}
757
+
EXPORT_SYMBOL_GPL(adf_gen4_init_num_svc_aes);
758
+
759
+
u32 adf_gen4_get_svc_slice_cnt(struct adf_accel_dev *accel_dev,
760
+
enum adf_base_services svc)
761
+
{
762
+
struct adf_rl_hw_data *device_data = &accel_dev->hw_device->rl_data;
763
+
764
+
switch (svc) {
765
+
case SVC_SYM:
766
+
return device_data->slices.cph_cnt;
767
+
case SVC_ASYM:
768
+
return device_data->slices.pke_cnt;
769
+
case SVC_DC:
770
+
return device_data->slices.dcpr_cnt;
771
+
default:
772
+
return 0;
773
+
}
774
+
}
775
+
EXPORT_SYMBOL_GPL(adf_gen4_get_svc_slice_cnt);
+3
-7
drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.h
+3
-7
drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.h
···
84
84
#define ADF_WQM_CSR_RPRESETSTS(bank) (ADF_WQM_CSR_RPRESETCTL(bank) + 4)
85
85
86
86
/* Ring interrupt */
87
-
#define ADF_RP_INT_SRC_SEL_F_RISE_MASK GENMASK(1, 0)
88
-
#define ADF_RP_INT_SRC_SEL_F_FALL_MASK GENMASK(2, 0)
89
-
#define ADF_RP_INT_SRC_SEL_RANGE_WIDTH 4
90
87
#define ADF_COALESCED_POLL_TIMEOUT_US (1 * USEC_PER_SEC)
91
88
#define ADF_COALESCED_POLL_DELAY_US 1000
92
89
#define ADF_WQM_CSR_RPINTSOU(bank) (0x200000 + ((bank) << 12))
···
173
176
u32 bank_number, int timeout_us);
174
177
void adf_gen4_bank_drain_finish(struct adf_accel_dev *accel_dev,
175
178
u32 bank_number);
176
-
int adf_gen4_bank_state_save(struct adf_accel_dev *accel_dev, u32 bank_number,
177
-
struct bank_state *state);
178
-
int adf_gen4_bank_state_restore(struct adf_accel_dev *accel_dev,
179
-
u32 bank_number, struct bank_state *state);
180
179
bool adf_gen4_services_supported(unsigned long service_mask);
181
180
void adf_gen4_init_dc_ops(struct adf_dc_ops *dc_ops);
181
+
void adf_gen4_init_num_svc_aes(struct adf_rl_hw_data *device_data);
182
+
u32 adf_gen4_get_svc_slice_cnt(struct adf_accel_dev *accel_dev,
183
+
enum adf_base_services svc);
182
184
183
185
#endif
+19
-86
drivers/crypto/intel/qat/qat_common/adf_gen4_pm_debugfs.c
+19
-86
drivers/crypto/intel/qat/qat_common/adf_gen4_pm_debugfs.c
···
1
1
// SPDX-License-Identifier: GPL-2.0-only
2
2
/* Copyright(c) 2023 Intel Corporation */
3
3
#include <linux/dma-mapping.h>
4
-
#include <linux/kernel.h>
5
4
#include <linux/string_helpers.h>
6
-
#include <linux/stringify.h>
7
5
8
6
#include "adf_accel_devices.h"
9
7
#include "adf_admin.h"
10
8
#include "adf_common_drv.h"
11
9
#include "adf_gen4_pm.h"
10
+
#include "adf_pm_dbgfs_utils.h"
12
11
#include "icp_qat_fw_init_admin.h"
13
-
14
-
/*
15
-
* This is needed because a variable is used to index the mask at
16
-
* pm_scnprint_table(), making it not compile time constant, so the compile
17
-
* asserts from FIELD_GET() or u32_get_bits() won't be fulfilled.
18
-
*/
19
-
#define field_get(_mask, _reg) (((_reg) & (_mask)) >> (ffs(_mask) - 1))
20
-
21
-
#define PM_INFO_MEMBER_OFF(member) \
22
-
(offsetof(struct icp_qat_fw_init_admin_pm_info, member) / sizeof(u32))
23
-
24
-
#define PM_INFO_REGSET_ENTRY_MASK(_reg_, _field_, _mask_) \
25
-
{ \
26
-
.reg_offset = PM_INFO_MEMBER_OFF(_reg_), \
27
-
.key = __stringify(_field_), \
28
-
.field_mask = _mask_, \
29
-
}
30
-
31
-
#define PM_INFO_REGSET_ENTRY32(_reg_, _field_) \
32
-
PM_INFO_REGSET_ENTRY_MASK(_reg_, _field_, GENMASK(31, 0))
33
12
34
13
#define PM_INFO_REGSET_ENTRY(_reg_, _field_) \
35
14
PM_INFO_REGSET_ENTRY_MASK(_reg_, _field_, ADF_GEN4_PM_##_field_##_MASK)
36
-
37
-
#define PM_INFO_MAX_KEY_LEN 21
38
-
39
-
struct pm_status_row {
40
-
int reg_offset;
41
-
u32 field_mask;
42
-
const char *key;
43
-
};
44
15
45
16
static const struct pm_status_row pm_fuse_rows[] = {
46
17
PM_INFO_REGSET_ENTRY(fusectl0, ENABLE_PM),
···
80
109
PM_INFO_REGSET_ENTRY32(pm.pwrreq, CPM_PM_PWRREQ),
81
110
};
82
111
83
-
static int pm_scnprint_table(char *buff, const struct pm_status_row *table,
84
-
u32 *pm_info_regs, size_t buff_size, int table_len,
85
-
bool lowercase)
86
-
{
87
-
char key[PM_INFO_MAX_KEY_LEN];
88
-
int wr = 0;
89
-
int i;
90
-
91
-
for (i = 0; i < table_len; i++) {
92
-
if (lowercase)
93
-
string_lower(key, table[i].key);
94
-
else
95
-
string_upper(key, table[i].key);
96
-
97
-
wr += scnprintf(&buff[wr], buff_size - wr, "%s: %#x\n", key,
98
-
field_get(table[i].field_mask,
99
-
pm_info_regs[table[i].reg_offset]));
100
-
}
101
-
102
-
return wr;
103
-
}
104
-
105
-
static int pm_scnprint_table_upper_keys(char *buff, const struct pm_status_row *table,
106
-
u32 *pm_info_regs, size_t buff_size,
107
-
int table_len)
108
-
{
109
-
return pm_scnprint_table(buff, table, pm_info_regs, buff_size,
110
-
table_len, false);
111
-
}
112
-
113
-
static int pm_scnprint_table_lower_keys(char *buff, const struct pm_status_row *table,
114
-
u32 *pm_info_regs, size_t buff_size,
115
-
int table_len)
116
-
{
117
-
return pm_scnprint_table(buff, table, pm_info_regs, buff_size,
118
-
table_len, true);
119
-
}
120
-
121
112
static_assert(sizeof(struct icp_qat_fw_init_admin_pm_info) < PAGE_SIZE);
122
113
123
114
static ssize_t adf_gen4_print_pm_status(struct adf_accel_dev *accel_dev,
···
124
191
/* Fusectl related */
125
192
len += scnprintf(&pm_kv[len], PAGE_SIZE - len,
126
193
"----------- PM Fuse info ---------\n");
127
-
len += pm_scnprint_table_lower_keys(&pm_kv[len], pm_fuse_rows,
128
-
pm_info_regs, PAGE_SIZE - len,
129
-
ARRAY_SIZE(pm_fuse_rows));
194
+
len += adf_pm_scnprint_table_lower_keys(&pm_kv[len], pm_fuse_rows,
195
+
pm_info_regs, PAGE_SIZE - len,
196
+
ARRAY_SIZE(pm_fuse_rows));
130
197
len += scnprintf(&pm_kv[len], PAGE_SIZE - len, "max_pwrreq: %#x\n",
131
198
pm_info->max_pwrreq);
132
199
len += scnprintf(&pm_kv[len], PAGE_SIZE - len, "min_pwrreq: %#x\n",
···
137
204
"------------ PM Info ------------\n");
138
205
len += scnprintf(&pm_kv[len], PAGE_SIZE - len, "power_level: %s\n",
139
206
pm_info->pwr_state == PM_SET_MIN ? "min" : "max");
140
-
len += pm_scnprint_table_lower_keys(&pm_kv[len], pm_info_rows,
141
-
pm_info_regs, PAGE_SIZE - len,
142
-
ARRAY_SIZE(pm_info_rows));
207
+
len += adf_pm_scnprint_table_lower_keys(&pm_kv[len], pm_info_rows,
208
+
pm_info_regs, PAGE_SIZE - len,
209
+
ARRAY_SIZE(pm_info_rows));
143
210
len += scnprintf(&pm_kv[len], PAGE_SIZE - len, "pm_mode: STATIC\n");
144
211
145
212
/* SSM related */
146
213
len += scnprintf(&pm_kv[len], PAGE_SIZE - len,
147
214
"----------- SSM_PM Info ----------\n");
148
-
len += pm_scnprint_table_lower_keys(&pm_kv[len], pm_ssm_rows,
149
-
pm_info_regs, PAGE_SIZE - len,
150
-
ARRAY_SIZE(pm_ssm_rows));
215
+
len += adf_pm_scnprint_table_lower_keys(&pm_kv[len], pm_ssm_rows,
216
+
pm_info_regs, PAGE_SIZE - len,
217
+
ARRAY_SIZE(pm_ssm_rows));
151
218
152
219
/* Log related */
153
220
len += scnprintf(&pm_kv[len], PAGE_SIZE - len,
154
221
"------------- PM Log -------------\n");
155
-
len += pm_scnprint_table_lower_keys(&pm_kv[len], pm_log_rows,
156
-
pm_info_regs, PAGE_SIZE - len,
157
-
ARRAY_SIZE(pm_log_rows));
222
+
len += adf_pm_scnprint_table_lower_keys(&pm_kv[len], pm_log_rows,
223
+
pm_info_regs, PAGE_SIZE - len,
224
+
ARRAY_SIZE(pm_log_rows));
158
225
159
-
len += pm_scnprint_table_lower_keys(&pm_kv[len], pm_event_rows,
160
-
pm_info_regs, PAGE_SIZE - len,
161
-
ARRAY_SIZE(pm_event_rows));
226
+
len += adf_pm_scnprint_table_lower_keys(&pm_kv[len], pm_event_rows,
227
+
pm_info_regs, PAGE_SIZE - len,
228
+
ARRAY_SIZE(pm_event_rows));
162
229
163
230
len += scnprintf(&pm_kv[len], PAGE_SIZE - len, "idle_irq_count: %#x\n",
164
231
pm->idle_irq_counters);
···
174
241
/* CSRs content */
175
242
len += scnprintf(&pm_kv[len], PAGE_SIZE - len,
176
243
"----------- HW PM CSRs -----------\n");
177
-
len += pm_scnprint_table_upper_keys(&pm_kv[len], pm_csrs_rows,
178
-
pm_info_regs, PAGE_SIZE - len,
179
-
ARRAY_SIZE(pm_csrs_rows));
244
+
len += adf_pm_scnprint_table_upper_keys(&pm_kv[len], pm_csrs_rows,
245
+
pm_info_regs, PAGE_SIZE - len,
246
+
ARRAY_SIZE(pm_csrs_rows));
180
247
181
248
val = ADF_CSR_RD(pmisc, ADF_GEN4_PM_HOST_MSG);
182
249
len += scnprintf(&pm_kv[len], PAGE_SIZE - len,
+4
-3
drivers/crypto/intel/qat/qat_common/adf_gen4_vf_mig.c
+4
-3
drivers/crypto/intel/qat/qat_common/adf_gen4_vf_mig.c
···
9
9
#include <asm/errno.h>
10
10
11
11
#include "adf_accel_devices.h"
12
+
#include "adf_bank_state.h"
12
13
#include "adf_common_drv.h"
13
14
#include "adf_gen4_hw_data.h"
14
15
#include "adf_gen4_pfvf.h"
···
359
358
360
359
pf_bank_nr = vf_bank_info->bank_nr + vf_bank_info->vf_nr * hw_data->num_banks_per_vf;
361
360
ret = hw_data->bank_state_restore(accel_dev, pf_bank_nr,
362
-
(struct bank_state *)state);
361
+
(struct adf_bank_state *)state);
363
362
if (ret) {
364
363
dev_err(&GET_DEV(accel_dev),
365
364
"Failed to load regs for vf%d bank%d\n",
···
586
585
pf_bank_nr += vf_bank_info->vf_nr * hw_data->num_banks_per_vf;
587
586
588
587
ret = hw_data->bank_state_save(accel_dev, pf_bank_nr,
589
-
(struct bank_state *)state);
588
+
(struct adf_bank_state *)state);
590
589
if (ret) {
591
590
dev_err(&GET_DEV(accel_dev),
592
591
"Failed to save regs for vf%d bank%d\n",
···
594
593
return ret;
595
594
}
596
595
597
-
return sizeof(struct bank_state);
596
+
return sizeof(struct adf_bank_state);
598
597
}
599
598
600
599
static int adf_gen4_vfmig_save_etr_bank(struct adf_accel_dev *accel_dev,
+24
drivers/crypto/intel/qat/qat_common/adf_gen6_pm.h
+24
drivers/crypto/intel/qat/qat_common/adf_gen6_pm.h
···
24
24
25
25
/* cpm_pm_status bitfields */
26
26
#define ADF_GEN6_PM_INIT_STATE BIT(21)
27
+
#define ADF_GEN6_PM_CPM_PM_STATE_MASK GENMASK(22, 20)
28
+
29
+
/* fusectl0 bitfields */
30
+
#define ADF_GEN6_PM_ENABLE_PM_MASK BIT(21)
31
+
#define ADF_GEN6_PM_ENABLE_PM_IDLE_MASK BIT(22)
32
+
#define ADF_GEN6_PM_ENABLE_DEEP_PM_IDLE_MASK BIT(23)
33
+
34
+
/* cpm_pm_fw_init bitfields */
35
+
#define ADF_GEN6_PM_IDLE_FILTER_MASK GENMASK(5, 3)
36
+
#define ADF_GEN6_PM_IDLE_ENABLE_MASK BIT(2)
37
+
38
+
/* ssm_pm_enable bitfield */
39
+
#define ADF_GEN6_PM_SSM_PM_ENABLE_MASK BIT(0)
40
+
41
+
/* ssm_pm_domain_status bitfield */
42
+
#define ADF_GEN6_PM_DOMAIN_POWERED_UP_MASK BIT(0)
43
+
44
+
#ifdef CONFIG_DEBUG_FS
45
+
void adf_gen6_init_dev_pm_data(struct adf_accel_dev *accel_dev);
46
+
#else
47
+
static inline void adf_gen6_init_dev_pm_data(struct adf_accel_dev *accel_dev)
48
+
{
49
+
}
50
+
#endif /* CONFIG_DEBUG_FS */
27
51
28
52
#endif /* ADF_GEN6_PM_H */
+124
drivers/crypto/intel/qat/qat_common/adf_gen6_pm_dbgfs.c
+124
drivers/crypto/intel/qat/qat_common/adf_gen6_pm_dbgfs.c
···
1
+
// SPDX-License-Identifier: GPL-2.0-only
2
+
/* Copyright(c) 2025 Intel Corporation */
3
+
#include <linux/dma-mapping.h>
4
+
#include <linux/export.h>
5
+
#include <linux/string_helpers.h>
6
+
7
+
#include "adf_admin.h"
8
+
#include "adf_common_drv.h"
9
+
#include "adf_gen6_pm.h"
10
+
#include "adf_pm_dbgfs_utils.h"
11
+
#include "icp_qat_fw_init_admin.h"
12
+
13
+
#define PM_INFO_REGSET_ENTRY(_reg_, _field_) \
14
+
PM_INFO_REGSET_ENTRY_MASK(_reg_, _field_, ADF_GEN6_PM_##_field_##_MASK)
15
+
16
+
static struct pm_status_row pm_fuse_rows[] = {
17
+
PM_INFO_REGSET_ENTRY(fusectl0, ENABLE_PM),
18
+
PM_INFO_REGSET_ENTRY(fusectl0, ENABLE_PM_IDLE),
19
+
PM_INFO_REGSET_ENTRY(fusectl0, ENABLE_DEEP_PM_IDLE),
20
+
};
21
+
22
+
static struct pm_status_row pm_info_rows[] = {
23
+
PM_INFO_REGSET_ENTRY(pm.status, CPM_PM_STATE),
24
+
PM_INFO_REGSET_ENTRY(pm.fw_init, IDLE_ENABLE),
25
+
PM_INFO_REGSET_ENTRY(pm.fw_init, IDLE_FILTER),
26
+
};
27
+
28
+
static struct pm_status_row pm_ssm_rows[] = {
29
+
PM_INFO_REGSET_ENTRY(ssm.pm_enable, SSM_PM_ENABLE),
30
+
PM_INFO_REGSET_ENTRY(ssm.pm_domain_status, DOMAIN_POWERED_UP),
31
+
};
32
+
33
+
static struct pm_status_row pm_csrs_rows[] = {
34
+
PM_INFO_REGSET_ENTRY32(pm.fw_init, CPM_PM_FW_INIT),
35
+
PM_INFO_REGSET_ENTRY32(pm.status, CPM_PM_STATUS),
36
+
};
37
+
38
+
static_assert(sizeof(struct icp_qat_fw_init_admin_pm_info) < PAGE_SIZE);
39
+
40
+
static ssize_t adf_gen6_print_pm_status(struct adf_accel_dev *accel_dev,
41
+
char __user *buf, size_t count,
42
+
loff_t *pos)
43
+
{
44
+
void __iomem *pmisc = adf_get_pmisc_base(accel_dev);
45
+
struct icp_qat_fw_init_admin_pm_info *pm_info;
46
+
dma_addr_t p_state_addr;
47
+
u32 *pm_info_regs;
48
+
size_t len = 0;
49
+
char *pm_kv;
50
+
u32 val;
51
+
int ret;
52
+
53
+
pm_info = kzalloc(PAGE_SIZE, GFP_KERNEL);
54
+
if (!pm_info)
55
+
return -ENOMEM;
56
+
57
+
pm_kv = kzalloc(PAGE_SIZE, GFP_KERNEL);
58
+
if (!pm_kv) {
59
+
kfree(pm_info);
60
+
return -ENOMEM;
61
+
}
62
+
63
+
p_state_addr = dma_map_single(&GET_DEV(accel_dev), pm_info, PAGE_SIZE,
64
+
DMA_FROM_DEVICE);
65
+
ret = dma_mapping_error(&GET_DEV(accel_dev), p_state_addr);
66
+
if (ret)
67
+
goto out_free;
68
+
69
+
/* Query power management information from QAT FW */
70
+
ret = adf_get_pm_info(accel_dev, p_state_addr, PAGE_SIZE);
71
+
dma_unmap_single(&GET_DEV(accel_dev), p_state_addr, PAGE_SIZE,
72
+
DMA_FROM_DEVICE);
73
+
if (ret)
74
+
goto out_free;
75
+
76
+
pm_info_regs = (u32 *)pm_info;
77
+
78
+
/* Fuse control register */
79
+
len += scnprintf(&pm_kv[len], PAGE_SIZE - len,
80
+
"----------- PM Fuse info ---------\n");
81
+
len += adf_pm_scnprint_table_lower_keys(&pm_kv[len], pm_fuse_rows,
82
+
pm_info_regs, PAGE_SIZE - len,
83
+
ARRAY_SIZE(pm_fuse_rows));
84
+
85
+
/* Power management */
86
+
len += scnprintf(&pm_kv[len], PAGE_SIZE - len,
87
+
"----------- PM Info --------------\n");
88
+
89
+
len += adf_pm_scnprint_table_lower_keys(&pm_kv[len], pm_info_rows,
90
+
pm_info_regs, PAGE_SIZE - len,
91
+
ARRAY_SIZE(pm_info_rows));
92
+
len += scnprintf(&pm_kv[len], PAGE_SIZE - len, "pm_mode: ACTIVE\n");
93
+
94
+
/* Shared Slice Module */
95
+
len += scnprintf(&pm_kv[len], PAGE_SIZE - len,
96
+
"----------- SSM_PM Info ----------\n");
97
+
len += adf_pm_scnprint_table_lower_keys(&pm_kv[len], pm_ssm_rows,
98
+
pm_info_regs, PAGE_SIZE - len,
99
+
ARRAY_SIZE(pm_ssm_rows));
100
+
101
+
/* Control status register content */
102
+
len += scnprintf(&pm_kv[len], PAGE_SIZE - len,
103
+
"----------- HW PM CSRs -----------\n");
104
+
len += adf_pm_scnprint_table_upper_keys(&pm_kv[len], pm_csrs_rows,
105
+
pm_info_regs, PAGE_SIZE - len,
106
+
ARRAY_SIZE(pm_csrs_rows));
107
+
108
+
val = ADF_CSR_RD(pmisc, ADF_GEN6_PM_INTERRUPT);
109
+
len += scnprintf(&pm_kv[len], PAGE_SIZE - len, "CPM_PM_INTERRUPT: %#x\n", val);
110
+
ret = simple_read_from_buffer(buf, count, pos, pm_kv, len);
111
+
112
+
out_free:
113
+
kfree(pm_info);
114
+
kfree(pm_kv);
115
+
116
+
return ret;
117
+
}
118
+
119
+
void adf_gen6_init_dev_pm_data(struct adf_accel_dev *accel_dev)
120
+
{
121
+
accel_dev->power_management.print_pm_status = adf_gen6_print_pm_status;
122
+
accel_dev->power_management.present = true;
123
+
}
124
+
EXPORT_SYMBOL_GPL(adf_gen6_init_dev_pm_data);
+146
drivers/crypto/intel/qat/qat_common/adf_gen6_tl.c
+146
drivers/crypto/intel/qat/qat_common/adf_gen6_tl.c
···
1
+
// SPDX-License-Identifier: GPL-2.0-only
2
+
/* Copyright (c) 2025 Intel Corporation. */
3
+
#include <linux/export.h>
4
+
5
+
#include "adf_gen6_tl.h"
6
+
#include "adf_telemetry.h"
7
+
#include "adf_tl_debugfs.h"
8
+
#include "icp_qat_fw_init_admin.h"
9
+
10
+
#define ADF_GEN6_TL_DEV_REG_OFF(reg) ADF_TL_DEV_REG_OFF(reg, gen6)
11
+
12
+
#define ADF_GEN6_TL_RP_REG_OFF(reg) ADF_TL_RP_REG_OFF(reg, gen6)
13
+
14
+
#define ADF_GEN6_TL_SL_UTIL_COUNTER(_name) \
15
+
ADF_TL_COUNTER("util_" #_name, ADF_TL_SIMPLE_COUNT, \
16
+
ADF_TL_SLICE_REG_OFF(_name, reg_tm_slice_util, gen6))
17
+
18
+
#define ADF_GEN6_TL_SL_EXEC_COUNTER(_name) \
19
+
ADF_TL_COUNTER("exec_" #_name, ADF_TL_SIMPLE_COUNT, \
20
+
ADF_TL_SLICE_REG_OFF(_name, reg_tm_slice_exec_cnt, gen6))
21
+
22
+
#define SLICE_IDX(sl) offsetof(struct icp_qat_fw_init_admin_slice_cnt, sl##_cnt)
23
+
24
+
/* Device level counters. */
25
+
static const struct adf_tl_dbg_counter dev_counters[] = {
26
+
/* PCIe partial transactions. */
27
+
ADF_TL_COUNTER(PCI_TRANS_CNT_NAME, ADF_TL_SIMPLE_COUNT,
28
+
ADF_GEN6_TL_DEV_REG_OFF(reg_tl_prt_trans_cnt)),
29
+
/* Max read latency[ns]. */
30
+
ADF_TL_COUNTER(MAX_RD_LAT_NAME, ADF_TL_COUNTER_NS,
31
+
ADF_GEN6_TL_DEV_REG_OFF(reg_tl_rd_lat_max)),
32
+
/* Read latency average[ns]. */
33
+
ADF_TL_COUNTER_LATENCY(RD_LAT_ACC_NAME, ADF_TL_COUNTER_NS_AVG,
34
+
ADF_GEN6_TL_DEV_REG_OFF(reg_tl_rd_lat_acc),
35
+
ADF_GEN6_TL_DEV_REG_OFF(reg_tl_rd_cmpl_cnt)),
36
+
/* Max "get to put" latency[ns]. */
37
+
ADF_TL_COUNTER(MAX_LAT_NAME, ADF_TL_COUNTER_NS,
38
+
ADF_GEN6_TL_DEV_REG_OFF(reg_tl_gp_lat_max)),
39
+
/* "Get to put" latency average[ns]. */
40
+
ADF_TL_COUNTER_LATENCY(LAT_ACC_NAME, ADF_TL_COUNTER_NS_AVG,
41
+
ADF_GEN6_TL_DEV_REG_OFF(reg_tl_gp_lat_acc),
42
+
ADF_GEN6_TL_DEV_REG_OFF(reg_tl_ae_put_cnt)),
43
+
/* PCIe write bandwidth[Mbps]. */
44
+
ADF_TL_COUNTER(BW_IN_NAME, ADF_TL_COUNTER_MBPS,
45
+
ADF_GEN6_TL_DEV_REG_OFF(reg_tl_bw_in)),
46
+
/* PCIe read bandwidth[Mbps]. */
47
+
ADF_TL_COUNTER(BW_OUT_NAME, ADF_TL_COUNTER_MBPS,
48
+
ADF_GEN6_TL_DEV_REG_OFF(reg_tl_bw_out)),
49
+
/* Page request latency average[ns]. */
50
+
ADF_TL_COUNTER_LATENCY(PAGE_REQ_LAT_NAME, ADF_TL_COUNTER_NS_AVG,
51
+
ADF_GEN6_TL_DEV_REG_OFF(reg_tl_at_page_req_lat_acc),
52
+
ADF_GEN6_TL_DEV_REG_OFF(reg_tl_at_page_req_cnt)),
53
+
/* Page translation latency average[ns]. */
54
+
ADF_TL_COUNTER_LATENCY(AT_TRANS_LAT_NAME, ADF_TL_COUNTER_NS_AVG,
55
+
ADF_GEN6_TL_DEV_REG_OFF(reg_tl_at_trans_lat_acc),
56
+
ADF_GEN6_TL_DEV_REG_OFF(reg_tl_at_trans_lat_cnt)),
57
+
/* Maximum uTLB used. */
58
+
ADF_TL_COUNTER(AT_MAX_UTLB_USED_NAME, ADF_TL_SIMPLE_COUNT,
59
+
ADF_GEN6_TL_DEV_REG_OFF(reg_tl_at_max_utlb_used)),
60
+
};
61
+
62
+
/* Accelerator utilization counters */
63
+
static const struct adf_tl_dbg_counter sl_util_counters[ADF_TL_SL_CNT_COUNT] = {
64
+
/* Compression accelerator utilization. */
65
+
[SLICE_IDX(cpr)] = ADF_GEN6_TL_SL_UTIL_COUNTER(cnv),
66
+
/* Decompression accelerator utilization. */
67
+
[SLICE_IDX(dcpr)] = ADF_GEN6_TL_SL_UTIL_COUNTER(dcprz),
68
+
/* PKE accelerator utilization. */
69
+
[SLICE_IDX(pke)] = ADF_GEN6_TL_SL_UTIL_COUNTER(pke),
70
+
/* Wireless Authentication accelerator utilization. */
71
+
[SLICE_IDX(wat)] = ADF_GEN6_TL_SL_UTIL_COUNTER(wat),
72
+
/* Wireless Cipher accelerator utilization. */
73
+
[SLICE_IDX(wcp)] = ADF_GEN6_TL_SL_UTIL_COUNTER(wcp),
74
+
/* UCS accelerator utilization. */
75
+
[SLICE_IDX(ucs)] = ADF_GEN6_TL_SL_UTIL_COUNTER(ucs),
76
+
/* Authentication accelerator utilization. */
77
+
[SLICE_IDX(ath)] = ADF_GEN6_TL_SL_UTIL_COUNTER(ath),
78
+
};
79
+
80
+
/* Accelerator execution counters */
81
+
static const struct adf_tl_dbg_counter sl_exec_counters[ADF_TL_SL_CNT_COUNT] = {
82
+
/* Compression accelerator execution count. */
83
+
[SLICE_IDX(cpr)] = ADF_GEN6_TL_SL_EXEC_COUNTER(cnv),
84
+
/* Decompression accelerator execution count. */
85
+
[SLICE_IDX(dcpr)] = ADF_GEN6_TL_SL_EXEC_COUNTER(dcprz),
86
+
/* PKE execution count. */
87
+
[SLICE_IDX(pke)] = ADF_GEN6_TL_SL_EXEC_COUNTER(pke),
88
+
/* Wireless Authentication accelerator execution count. */
89
+
[SLICE_IDX(wat)] = ADF_GEN6_TL_SL_EXEC_COUNTER(wat),
90
+
/* Wireless Cipher accelerator execution count. */
91
+
[SLICE_IDX(wcp)] = ADF_GEN6_TL_SL_EXEC_COUNTER(wcp),
92
+
/* UCS accelerator execution count. */
93
+
[SLICE_IDX(ucs)] = ADF_GEN6_TL_SL_EXEC_COUNTER(ucs),
94
+
/* Authentication accelerator execution count. */
95
+
[SLICE_IDX(ath)] = ADF_GEN6_TL_SL_EXEC_COUNTER(ath),
96
+
};
97
+
98
+
/* Ring pair counters. */
99
+
static const struct adf_tl_dbg_counter rp_counters[] = {
100
+
/* PCIe partial transactions. */
101
+
ADF_TL_COUNTER(PCI_TRANS_CNT_NAME, ADF_TL_SIMPLE_COUNT,
102
+
ADF_GEN6_TL_RP_REG_OFF(reg_tl_prt_trans_cnt)),
103
+
/* "Get to put" latency average[ns]. */
104
+
ADF_TL_COUNTER_LATENCY(LAT_ACC_NAME, ADF_TL_COUNTER_NS_AVG,
105
+
ADF_GEN6_TL_RP_REG_OFF(reg_tl_gp_lat_acc),
106
+
ADF_GEN6_TL_RP_REG_OFF(reg_tl_ae_put_cnt)),
107
+
/* PCIe write bandwidth[Mbps]. */
108
+
ADF_TL_COUNTER(BW_IN_NAME, ADF_TL_COUNTER_MBPS,
109
+
ADF_GEN6_TL_RP_REG_OFF(reg_tl_bw_in)),
110
+
/* PCIe read bandwidth[Mbps]. */
111
+
ADF_TL_COUNTER(BW_OUT_NAME, ADF_TL_COUNTER_MBPS,
112
+
ADF_GEN6_TL_RP_REG_OFF(reg_tl_bw_out)),
113
+
/* Message descriptor DevTLB hit rate. */
114
+
ADF_TL_COUNTER(AT_GLOB_DTLB_HIT_NAME, ADF_TL_SIMPLE_COUNT,
115
+
ADF_GEN6_TL_RP_REG_OFF(reg_tl_at_glob_devtlb_hit)),
116
+
/* Message descriptor DevTLB miss rate. */
117
+
ADF_TL_COUNTER(AT_GLOB_DTLB_MISS_NAME, ADF_TL_SIMPLE_COUNT,
118
+
ADF_GEN6_TL_RP_REG_OFF(reg_tl_at_glob_devtlb_miss)),
119
+
/* Payload DevTLB hit rate. */
120
+
ADF_TL_COUNTER(AT_PAYLD_DTLB_HIT_NAME, ADF_TL_SIMPLE_COUNT,
121
+
ADF_GEN6_TL_RP_REG_OFF(reg_tl_at_payld_devtlb_hit)),
122
+
/* Payload DevTLB miss rate. */
123
+
ADF_TL_COUNTER(AT_PAYLD_DTLB_MISS_NAME, ADF_TL_SIMPLE_COUNT,
124
+
ADF_GEN6_TL_RP_REG_OFF(reg_tl_at_payld_devtlb_miss)),
125
+
};
126
+
127
+
void adf_gen6_init_tl_data(struct adf_tl_hw_data *tl_data)
128
+
{
129
+
tl_data->layout_sz = ADF_GEN6_TL_LAYOUT_SZ;
130
+
tl_data->slice_reg_sz = ADF_GEN6_TL_SLICE_REG_SZ;
131
+
tl_data->rp_reg_sz = ADF_GEN6_TL_RP_REG_SZ;
132
+
tl_data->num_hbuff = ADF_GEN6_TL_NUM_HIST_BUFFS;
133
+
tl_data->max_rp = ADF_GEN6_TL_MAX_RP_NUM;
134
+
tl_data->msg_cnt_off = ADF_GEN6_TL_MSG_CNT_OFF;
135
+
tl_data->cpp_ns_per_cycle = ADF_GEN6_CPP_NS_PER_CYCLE;
136
+
tl_data->bw_units_to_bytes = ADF_GEN6_TL_BW_HW_UNITS_TO_BYTES;
137
+
138
+
tl_data->dev_counters = dev_counters;
139
+
tl_data->num_dev_counters = ARRAY_SIZE(dev_counters);
140
+
tl_data->sl_util_counters = sl_util_counters;
141
+
tl_data->sl_exec_counters = sl_exec_counters;
142
+
tl_data->rp_counters = rp_counters;
143
+
tl_data->num_rp_counters = ARRAY_SIZE(rp_counters);
144
+
tl_data->max_sl_cnt = ADF_GEN6_TL_MAX_SLICES_PER_TYPE;
145
+
}
146
+
EXPORT_SYMBOL_GPL(adf_gen6_init_tl_data);
+198
drivers/crypto/intel/qat/qat_common/adf_gen6_tl.h
+198
drivers/crypto/intel/qat/qat_common/adf_gen6_tl.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0-only */
2
+
/* Copyright (c) 2025 Intel Corporation. */
3
+
#ifndef ADF_GEN6_TL_H
4
+
#define ADF_GEN6_TL_H
5
+
6
+
#include <linux/types.h>
7
+
8
+
struct adf_tl_hw_data;
9
+
10
+
/* Computation constants. */
11
+
#define ADF_GEN6_CPP_NS_PER_CYCLE 2
12
+
#define ADF_GEN6_TL_BW_HW_UNITS_TO_BYTES 64
13
+
14
+
/* Maximum aggregation time. Value is in milliseconds. */
15
+
#define ADF_GEN6_TL_MAX_AGGR_TIME_MS 4000
16
+
/* Number of buffers to store historic values. */
17
+
#define ADF_GEN6_TL_NUM_HIST_BUFFS \
18
+
(ADF_GEN6_TL_MAX_AGGR_TIME_MS / ADF_TL_DATA_WR_INTERVAL_MS)
19
+
20
+
/* Max number of HW resources of one type */
21
+
#define ADF_GEN6_TL_MAX_SLICES_PER_TYPE 32
22
+
#define MAX_ATH_SL_COUNT 7
23
+
#define MAX_CNV_SL_COUNT 2
24
+
#define MAX_DCPRZ_SL_COUNT 2
25
+
#define MAX_PKE_SL_COUNT 32
26
+
#define MAX_UCS_SL_COUNT 4
27
+
#define MAX_WAT_SL_COUNT 5
28
+
#define MAX_WCP_SL_COUNT 5
29
+
30
+
#define MAX_ATH_CMDQ_COUNT 14
31
+
#define MAX_CNV_CMDQ_COUNT 6
32
+
#define MAX_DCPRZ_CMDQ_COUNT 6
33
+
#define MAX_PKE_CMDQ_COUNT 32
34
+
#define MAX_UCS_CMDQ_COUNT 12
35
+
#define MAX_WAT_CMDQ_COUNT 35
36
+
#define MAX_WCP_CMDQ_COUNT 35
37
+
38
+
/* Max number of simultaneously monitored ring pairs. */
39
+
#define ADF_GEN6_TL_MAX_RP_NUM 4
40
+
41
+
/**
42
+
* struct adf_gen6_tl_slice_data_regs - HW slice data as populated by FW.
43
+
* @reg_tm_slice_exec_cnt: Slice execution count.
44
+
* @reg_tm_slice_util: Slice utilization.
45
+
*/
46
+
struct adf_gen6_tl_slice_data_regs {
47
+
__u32 reg_tm_slice_exec_cnt;
48
+
__u32 reg_tm_slice_util;
49
+
};
50
+
51
+
#define ADF_GEN6_TL_SLICE_REG_SZ sizeof(struct adf_gen6_tl_slice_data_regs)
52
+
53
+
/**
54
+
* struct adf_gen6_tl_cmdq_data_regs - HW CMDQ data as populated by FW.
55
+
* @reg_tm_cmdq_wait_cnt: CMDQ wait count.
56
+
* @reg_tm_cmdq_exec_cnt: CMDQ execution count.
57
+
* @reg_tm_cmdq_drain_cnt: CMDQ drain count.
58
+
*/
59
+
struct adf_gen6_tl_cmdq_data_regs {
60
+
__u32 reg_tm_cmdq_wait_cnt;
61
+
__u32 reg_tm_cmdq_exec_cnt;
62
+
__u32 reg_tm_cmdq_drain_cnt;
63
+
__u32 reserved;
64
+
};
65
+
66
+
#define ADF_GEN6_TL_CMDQ_REG_SZ sizeof(struct adf_gen6_tl_cmdq_data_regs)
67
+
68
+
/**
69
+
* struct adf_gen6_tl_device_data_regs - This structure stores device telemetry
70
+
* counter values as are being populated periodically by device.
71
+
* @reg_tl_rd_lat_acc: read latency accumulator
72
+
* @reg_tl_gp_lat_acc: "get to put" latency accumulator
73
+
* @reg_tl_at_page_req_lat_acc: AT/DevTLB page request latency accumulator
74
+
* @reg_tl_at_trans_lat_acc: DevTLB transaction latency accumulator
75
+
* @reg_tl_re_acc: accumulated ring empty time
76
+
* @reg_tl_prt_trans_cnt: PCIe partial transactions
77
+
* @reg_tl_rd_lat_max: maximum logged read latency
78
+
* @reg_tl_rd_cmpl_cnt: read requests completed count
79
+
* @reg_tl_gp_lat_max: maximum logged get to put latency
80
+
* @reg_tl_ae_put_cnt: Accelerator Engine put counts across all rings
81
+
* @reg_tl_bw_in: PCIe write bandwidth
82
+
* @reg_tl_bw_out: PCIe read bandwidth
83
+
* @reg_tl_at_page_req_cnt: DevTLB page requests count
84
+
* @reg_tl_at_trans_lat_cnt: DevTLB transaction latency samples count
85
+
* @reg_tl_at_max_utlb_used: maximum uTLB used
86
+
* @reg_tl_re_cnt: ring empty time samples count
87
+
* @reserved: reserved
88
+
* @ath_slices: array of Authentication slices utilization registers
89
+
* @cnv_slices: array of Compression slices utilization registers
90
+
* @dcprz_slices: array of Decompression slices utilization registers
91
+
* @pke_slices: array of PKE slices utilization registers
92
+
* @ucs_slices: array of UCS slices utilization registers
93
+
* @wat_slices: array of Wireless Authentication slices utilization registers
94
+
* @wcp_slices: array of Wireless Cipher slices utilization registers
95
+
* @ath_cmdq: array of Authentication cmdq telemetry registers
96
+
* @cnv_cmdq: array of Compression cmdq telemetry registers
97
+
* @dcprz_cmdq: array of Decomopression cmdq telemetry registers
98
+
* @pke_cmdq: array of PKE cmdq telemetry registers
99
+
* @ucs_cmdq: array of UCS cmdq telemetry registers
100
+
* @wat_cmdq: array of Wireless Authentication cmdq telemetry registers
101
+
* @wcp_cmdq: array of Wireless Cipher cmdq telemetry registers
102
+
*/
103
+
struct adf_gen6_tl_device_data_regs {
104
+
__u64 reg_tl_rd_lat_acc;
105
+
__u64 reg_tl_gp_lat_acc;
106
+
__u64 reg_tl_at_page_req_lat_acc;
107
+
__u64 reg_tl_at_trans_lat_acc;
108
+
__u64 reg_tl_re_acc;
109
+
__u32 reg_tl_prt_trans_cnt;
110
+
__u32 reg_tl_rd_lat_max;
111
+
__u32 reg_tl_rd_cmpl_cnt;
112
+
__u32 reg_tl_gp_lat_max;
113
+
__u32 reg_tl_ae_put_cnt;
114
+
__u32 reg_tl_bw_in;
115
+
__u32 reg_tl_bw_out;
116
+
__u32 reg_tl_at_page_req_cnt;
117
+
__u32 reg_tl_at_trans_lat_cnt;
118
+
__u32 reg_tl_at_max_utlb_used;
119
+
__u32 reg_tl_re_cnt;
120
+
__u32 reserved;
121
+
struct adf_gen6_tl_slice_data_regs ath_slices[MAX_ATH_SL_COUNT];
122
+
struct adf_gen6_tl_slice_data_regs cnv_slices[MAX_CNV_SL_COUNT];
123
+
struct adf_gen6_tl_slice_data_regs dcprz_slices[MAX_DCPRZ_SL_COUNT];
124
+
struct adf_gen6_tl_slice_data_regs pke_slices[MAX_PKE_SL_COUNT];
125
+
struct adf_gen6_tl_slice_data_regs ucs_slices[MAX_UCS_SL_COUNT];
126
+
struct adf_gen6_tl_slice_data_regs wat_slices[MAX_WAT_SL_COUNT];
127
+
struct adf_gen6_tl_slice_data_regs wcp_slices[MAX_WCP_SL_COUNT];
128
+
struct adf_gen6_tl_cmdq_data_regs ath_cmdq[MAX_ATH_CMDQ_COUNT];
129
+
struct adf_gen6_tl_cmdq_data_regs cnv_cmdq[MAX_CNV_CMDQ_COUNT];
130
+
struct adf_gen6_tl_cmdq_data_regs dcprz_cmdq[MAX_DCPRZ_CMDQ_COUNT];
131
+
struct adf_gen6_tl_cmdq_data_regs pke_cmdq[MAX_PKE_CMDQ_COUNT];
132
+
struct adf_gen6_tl_cmdq_data_regs ucs_cmdq[MAX_UCS_CMDQ_COUNT];
133
+
struct adf_gen6_tl_cmdq_data_regs wat_cmdq[MAX_WAT_CMDQ_COUNT];
134
+
struct adf_gen6_tl_cmdq_data_regs wcp_cmdq[MAX_WCP_CMDQ_COUNT];
135
+
};
136
+
137
+
/**
138
+
* struct adf_gen6_tl_ring_pair_data_regs - This structure stores ring pair
139
+
* telemetry counter values as they are being populated periodically by device.
140
+
* @reg_tl_gp_lat_acc: get-put latency accumulator
141
+
* @reg_tl_re_acc: accumulated ring empty time
142
+
* @reg_tl_pci_trans_cnt: PCIe partial transactions
143
+
* @reg_tl_ae_put_cnt: Accelerator Engine put counts across all rings
144
+
* @reg_tl_bw_in: PCIe write bandwidth
145
+
* @reg_tl_bw_out: PCIe read bandwidth
146
+
* @reg_tl_at_glob_devtlb_hit: Message descriptor DevTLB hit rate
147
+
* @reg_tl_at_glob_devtlb_miss: Message descriptor DevTLB miss rate
148
+
* @reg_tl_at_payld_devtlb_hit: Payload DevTLB hit rate
149
+
* @reg_tl_at_payld_devtlb_miss: Payload DevTLB miss rate
150
+
* @reg_tl_re_cnt: ring empty time samples count
151
+
* @reserved1: reserved
152
+
*/
153
+
struct adf_gen6_tl_ring_pair_data_regs {
154
+
__u64 reg_tl_gp_lat_acc;
155
+
__u64 reg_tl_re_acc;
156
+
__u32 reg_tl_prt_trans_cnt;
157
+
__u32 reg_tl_ae_put_cnt;
158
+
__u32 reg_tl_bw_in;
159
+
__u32 reg_tl_bw_out;
160
+
__u32 reg_tl_at_glob_devtlb_hit;
161
+
__u32 reg_tl_at_glob_devtlb_miss;
162
+
__u32 reg_tl_at_payld_devtlb_hit;
163
+
__u32 reg_tl_at_payld_devtlb_miss;
164
+
__u32 reg_tl_re_cnt;
165
+
__u32 reserved1;
166
+
};
167
+
168
+
#define ADF_GEN6_TL_RP_REG_SZ sizeof(struct adf_gen6_tl_ring_pair_data_regs)
169
+
170
+
/**
171
+
* struct adf_gen6_tl_layout - This structure represents the entire telemetry
172
+
* counters data: Device + 4 Ring Pairs as they are being populated periodically
173
+
* by device.
174
+
* @tl_device_data_regs: structure of device telemetry registers
175
+
* @tl_ring_pairs_data_regs: array of ring pairs telemetry registers
176
+
* @reg_tl_msg_cnt: telemetry message counter
177
+
* @reserved: reserved
178
+
*/
179
+
struct adf_gen6_tl_layout {
180
+
struct adf_gen6_tl_device_data_regs tl_device_data_regs;
181
+
struct adf_gen6_tl_ring_pair_data_regs
182
+
tl_ring_pairs_data_regs[ADF_GEN6_TL_MAX_RP_NUM];
183
+
__u32 reg_tl_msg_cnt;
184
+
__u32 reserved;
185
+
};
186
+
187
+
#define ADF_GEN6_TL_LAYOUT_SZ sizeof(struct adf_gen6_tl_layout)
188
+
#define ADF_GEN6_TL_MSG_CNT_OFF \
189
+
offsetof(struct adf_gen6_tl_layout, reg_tl_msg_cnt)
190
+
191
+
#ifdef CONFIG_DEBUG_FS
192
+
void adf_gen6_init_tl_data(struct adf_tl_hw_data *tl_data);
193
+
#else
194
+
static inline void adf_gen6_init_tl_data(struct adf_tl_hw_data *tl_data)
195
+
{
196
+
}
197
+
#endif /* CONFIG_DEBUG_FS */
198
+
#endif /* ADF_GEN6_TL_H */
+1
drivers/crypto/intel/qat/qat_common/adf_init.c
+1
drivers/crypto/intel/qat/qat_common/adf_init.c
+5
drivers/crypto/intel/qat/qat_common/adf_isr.c
+5
drivers/crypto/intel/qat/qat_common/adf_isr.c
+52
drivers/crypto/intel/qat/qat_common/adf_pm_dbgfs_utils.c
+52
drivers/crypto/intel/qat/qat_common/adf_pm_dbgfs_utils.c
···
1
+
// SPDX-License-Identifier: GPL-2.0-only
2
+
/* Copyright(c) 2025 Intel Corporation */
3
+
#include <linux/bitops.h>
4
+
#include <linux/sprintf.h>
5
+
#include <linux/string_helpers.h>
6
+
7
+
#include "adf_pm_dbgfs_utils.h"
8
+
9
+
/*
10
+
* This is needed because a variable is used to index the mask at
11
+
* pm_scnprint_table(), making it not compile time constant, so the compile
12
+
* asserts from FIELD_GET() or u32_get_bits() won't be fulfilled.
13
+
*/
14
+
#define field_get(_mask, _reg) (((_reg) & (_mask)) >> (ffs(_mask) - 1))
15
+
16
+
#define PM_INFO_MAX_KEY_LEN 21
17
+
18
+
static int pm_scnprint_table(char *buff, const struct pm_status_row *table,
19
+
u32 *pm_info_regs, size_t buff_size, int table_len,
20
+
bool lowercase)
21
+
{
22
+
char key[PM_INFO_MAX_KEY_LEN];
23
+
int wr = 0;
24
+
int i;
25
+
26
+
for (i = 0; i < table_len; i++) {
27
+
if (lowercase)
28
+
string_lower(key, table[i].key);
29
+
else
30
+
string_upper(key, table[i].key);
31
+
32
+
wr += scnprintf(&buff[wr], buff_size - wr, "%s: %#x\n", key,
33
+
field_get(table[i].field_mask,
34
+
pm_info_regs[table[i].reg_offset]));
35
+
}
36
+
37
+
return wr;
38
+
}
39
+
40
+
int adf_pm_scnprint_table_upper_keys(char *buff, const struct pm_status_row *table,
41
+
u32 *pm_info_regs, size_t buff_size, int table_len)
42
+
{
43
+
return pm_scnprint_table(buff, table, pm_info_regs, buff_size,
44
+
table_len, false);
45
+
}
46
+
47
+
int adf_pm_scnprint_table_lower_keys(char *buff, const struct pm_status_row *table,
48
+
u32 *pm_info_regs, size_t buff_size, int table_len)
49
+
{
50
+
return pm_scnprint_table(buff, table, pm_info_regs, buff_size,
51
+
table_len, true);
52
+
}
+36
drivers/crypto/intel/qat/qat_common/adf_pm_dbgfs_utils.h
+36
drivers/crypto/intel/qat/qat_common/adf_pm_dbgfs_utils.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0-only */
2
+
/* Copyright(c) 2025 Intel Corporation */
3
+
#ifndef ADF_PM_DBGFS_UTILS_H_
4
+
#define ADF_PM_DBGFS_UTILS_H_
5
+
6
+
#include <linux/stddef.h>
7
+
#include <linux/stringify.h>
8
+
#include <linux/types.h>
9
+
#include "icp_qat_fw_init_admin.h"
10
+
11
+
#define PM_INFO_MEMBER_OFF(member) \
12
+
(offsetof(struct icp_qat_fw_init_admin_pm_info, member) / sizeof(u32))
13
+
14
+
#define PM_INFO_REGSET_ENTRY_MASK(_reg_, _field_, _mask_) \
15
+
{ \
16
+
.reg_offset = PM_INFO_MEMBER_OFF(_reg_), \
17
+
.key = __stringify(_field_), \
18
+
.field_mask = _mask_, \
19
+
}
20
+
21
+
#define PM_INFO_REGSET_ENTRY32(_reg_, _field_) \
22
+
PM_INFO_REGSET_ENTRY_MASK(_reg_, _field_, GENMASK(31, 0))
23
+
24
+
struct pm_status_row {
25
+
int reg_offset;
26
+
u32 field_mask;
27
+
const char *key;
28
+
};
29
+
30
+
int adf_pm_scnprint_table_upper_keys(char *buff, const struct pm_status_row *table,
31
+
u32 *pm_info_regs, size_t buff_size, int table_len);
32
+
33
+
int adf_pm_scnprint_table_lower_keys(char *buff, const struct pm_status_row *table,
34
+
u32 *pm_info_regs, size_t buff_size, int table_len);
35
+
36
+
#endif /* ADF_PM_DBGFS_UTILS_H_ */
+27
-59
drivers/crypto/intel/qat/qat_common/adf_rl.c
+27
-59
drivers/crypto/intel/qat/qat_common/adf_rl.c
···
13
13
#include <linux/units.h>
14
14
15
15
#include "adf_accel_devices.h"
16
+
#include "adf_cfg_services.h"
16
17
#include "adf_common_drv.h"
17
18
#include "adf_rl_admin.h"
18
19
#include "adf_rl.h"
···
56
55
}
57
56
}
58
57
59
-
if (sla_in->srv >= ADF_SVC_NONE) {
58
+
if (sla_in->srv >= SVC_BASE_COUNT) {
60
59
dev_notice(&GET_DEV(accel_dev),
61
60
"Wrong service type\n");
62
61
return -EINVAL;
···
169
168
return NULL;
170
169
}
171
170
172
-
static enum adf_cfg_service_type srv_to_cfg_svc_type(enum adf_base_services rl_srv)
173
-
{
174
-
switch (rl_srv) {
175
-
case ADF_SVC_ASYM:
176
-
return ASYM;
177
-
case ADF_SVC_SYM:
178
-
return SYM;
179
-
case ADF_SVC_DC:
180
-
return COMP;
181
-
default:
182
-
return UNUSED;
183
-
}
184
-
}
185
-
186
171
/**
187
172
* adf_rl_get_sla_arr_of_type() - Returns a pointer to SLA type specific array
188
173
* @rl_data: pointer to ratelimiting data
···
196
209
}
197
210
}
198
211
199
-
static bool is_service_enabled(struct adf_accel_dev *accel_dev,
200
-
enum adf_base_services rl_srv)
201
-
{
202
-
enum adf_cfg_service_type arb_srv = srv_to_cfg_svc_type(rl_srv);
203
-
struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev);
204
-
u8 rps_per_bundle = hw_data->num_banks_per_vf;
205
-
int i;
206
-
207
-
for (i = 0; i < rps_per_bundle; i++) {
208
-
if (GET_SRV_TYPE(accel_dev, i) == arb_srv)
209
-
return true;
210
-
}
211
-
212
-
return false;
213
-
}
214
-
215
212
/**
216
213
* prepare_rp_ids() - Creates an array of ring pair IDs from bitmask
217
214
* @accel_dev: pointer to acceleration device structure
···
214
243
static int prepare_rp_ids(struct adf_accel_dev *accel_dev, struct rl_sla *sla,
215
244
const unsigned long rp_mask)
216
245
{
217
-
enum adf_cfg_service_type arb_srv = srv_to_cfg_svc_type(sla->srv);
246
+
enum adf_cfg_service_type arb_srv = adf_srv_to_cfg_svc_type(sla->srv);
218
247
u16 rps_per_bundle = GET_HW_DATA(accel_dev)->num_banks_per_vf;
219
248
bool *rp_in_use = accel_dev->rate_limiting->rp_in_use;
220
249
size_t rp_cnt_max = ARRAY_SIZE(sla->ring_pairs_ids);
···
529
558
if (!sla_val)
530
559
return 0;
531
560
561
+
/* Handle generation specific slice count adjustment */
532
562
avail_slice_cycles = hw_data->clock_frequency;
533
-
534
-
switch (svc_type) {
535
-
case ADF_SVC_ASYM:
536
-
avail_slice_cycles *= device_data->slices.pke_cnt;
537
-
break;
538
-
case ADF_SVC_SYM:
539
-
avail_slice_cycles *= device_data->slices.cph_cnt;
540
-
break;
541
-
case ADF_SVC_DC:
542
-
avail_slice_cycles *= device_data->slices.dcpr_cnt;
543
-
break;
544
-
default:
545
-
break;
546
-
}
563
+
avail_slice_cycles *= hw_data->get_svc_slice_cnt(accel_dev, svc_type);
547
564
548
565
do_div(avail_slice_cycles, device_data->scan_interval);
549
566
allocated_tokens = avail_slice_cycles * sla_val;
550
567
do_div(allocated_tokens, device_data->scale_ref);
551
568
552
569
return allocated_tokens;
570
+
}
571
+
572
+
static u32 adf_rl_get_num_svc_aes(struct adf_accel_dev *accel_dev,
573
+
enum adf_base_services svc)
574
+
{
575
+
struct adf_rl_hw_data *device_data = &accel_dev->hw_device->rl_data;
576
+
577
+
if (svc >= SVC_BASE_COUNT)
578
+
return 0;
579
+
580
+
return device_data->svc_ae_mask[svc];
553
581
}
554
582
555
583
u32 adf_rl_calculate_ae_cycles(struct adf_accel_dev *accel_dev, u32 sla_val,
···
562
592
return 0;
563
593
564
594
avail_ae_cycles = hw_data->clock_frequency;
565
-
avail_ae_cycles *= hw_data->get_num_aes(hw_data) - 1;
595
+
avail_ae_cycles *= adf_rl_get_num_svc_aes(accel_dev, svc_type);
566
596
do_div(avail_ae_cycles, device_data->scan_interval);
567
597
568
598
sla_val *= device_data->max_tp[svc_type];
···
587
617
sla_to_bytes *= device_data->max_tp[svc_type];
588
618
do_div(sla_to_bytes, device_data->scale_ref);
589
619
590
-
sla_to_bytes *= (svc_type == ADF_SVC_ASYM) ? RL_TOKEN_ASYM_SIZE :
591
-
BYTES_PER_MBIT;
592
-
if (svc_type == ADF_SVC_DC && is_bw_out)
620
+
sla_to_bytes *= (svc_type == SVC_ASYM) ? RL_TOKEN_ASYM_SIZE : BYTES_PER_MBIT;
621
+
if (svc_type == SVC_DC && is_bw_out)
593
622
sla_to_bytes *= device_data->slices.dcpr_cnt -
594
623
device_data->dcpr_correction;
595
624
···
629
660
}
630
661
*sla_out = sla;
631
662
632
-
if (!is_service_enabled(accel_dev, sla_in->srv)) {
663
+
if (!adf_is_service_enabled(accel_dev, sla_in->srv)) {
633
664
dev_notice(&GET_DEV(accel_dev),
634
665
"Provided service is not enabled\n");
635
666
ret = -EINVAL;
···
699
730
sla_in.type = RL_ROOT;
700
731
sla_in.parent_id = RL_PARENT_DEFAULT_ID;
701
732
702
-
for (i = 0; i < ADF_SVC_NONE; i++) {
703
-
if (!is_service_enabled(accel_dev, i))
733
+
for (i = 0; i < SVC_BASE_COUNT; i++) {
734
+
if (!adf_is_service_enabled(accel_dev, i))
704
735
continue;
705
736
706
737
sla_in.cir = device_data->scale_ref;
···
714
745
715
746
/* Init default cluster for each root */
716
747
sla_in.type = RL_CLUSTER;
717
-
for (i = 0; i < ADF_SVC_NONE; i++) {
748
+
for (i = 0; i < SVC_BASE_COUNT; i++) {
718
749
if (!rl_data->root[i])
719
750
continue;
720
-
721
751
sla_in.cir = rl_data->root[i]->cir;
722
752
sla_in.pir = sla_in.cir;
723
753
sla_in.srv = rl_data->root[i]->srv;
···
955
987
struct rl_sla *sla = NULL;
956
988
int i;
957
989
958
-
if (srv >= ADF_SVC_NONE)
990
+
if (srv >= SVC_BASE_COUNT)
959
991
return -EINVAL;
960
992
961
993
if (sla_id > RL_SLA_EMPTY_ID && !validate_sla_id(accel_dev, sla_id)) {
···
1054
1086
int ret = 0;
1055
1087
1056
1088
/* Validate device parameters */
1057
-
if (RL_VALIDATE_NON_ZERO(rl_hw_data->max_tp[ADF_SVC_ASYM]) ||
1058
-
RL_VALIDATE_NON_ZERO(rl_hw_data->max_tp[ADF_SVC_SYM]) ||
1059
-
RL_VALIDATE_NON_ZERO(rl_hw_data->max_tp[ADF_SVC_DC]) ||
1089
+
if (RL_VALIDATE_NON_ZERO(rl_hw_data->max_tp[SVC_ASYM]) ||
1090
+
RL_VALIDATE_NON_ZERO(rl_hw_data->max_tp[SVC_SYM]) ||
1091
+
RL_VALIDATE_NON_ZERO(rl_hw_data->max_tp[SVC_DC]) ||
1060
1092
RL_VALIDATE_NON_ZERO(rl_hw_data->scan_interval) ||
1061
1093
RL_VALIDATE_NON_ZERO(rl_hw_data->pcie_scale_div) ||
1062
1094
RL_VALIDATE_NON_ZERO(rl_hw_data->pcie_scale_mul) ||
+4
-7
drivers/crypto/intel/qat/qat_common/adf_rl.h
+4
-7
drivers/crypto/intel/qat/qat_common/adf_rl.h
···
7
7
#include <linux/mutex.h>
8
8
#include <linux/types.h>
9
9
10
+
#include "adf_cfg_services.h"
11
+
10
12
struct adf_accel_dev;
11
13
12
14
#define RL_ROOT_MAX 4
···
24
22
RL_ROOT,
25
23
RL_CLUSTER,
26
24
RL_LEAF,
27
-
};
28
-
29
-
enum adf_base_services {
30
-
ADF_SVC_ASYM = 0,
31
-
ADF_SVC_SYM,
32
-
ADF_SVC_DC,
33
-
ADF_SVC_NONE,
34
25
};
35
26
36
27
/**
···
68
73
u8 dcpr_cnt;
69
74
u8 pke_cnt;
70
75
u8 cph_cnt;
76
+
u8 cpr_cnt;
71
77
};
72
78
73
79
struct adf_rl_interface_data {
···
90
94
u32 pcie_scale_div;
91
95
u32 dcpr_correction;
92
96
u32 max_tp[RL_ROOT_MAX];
97
+
u32 svc_ae_mask[SVC_BASE_COUNT];
93
98
struct rl_slice_cnt slices;
94
99
};
95
100
+1
drivers/crypto/intel/qat/qat_common/adf_rl_admin.c
+1
drivers/crypto/intel/qat/qat_common/adf_rl_admin.c
-1
drivers/crypto/intel/qat/qat_common/adf_sriov.c
-1
drivers/crypto/intel/qat/qat_common/adf_sriov.c
+2
drivers/crypto/intel/qat/qat_common/adf_sysfs.c
+2
drivers/crypto/intel/qat/qat_common/adf_sysfs.c
+15
-6
drivers/crypto/intel/qat/qat_common/adf_sysfs_rl.c
+15
-6
drivers/crypto/intel/qat/qat_common/adf_sysfs_rl.c
···
32
32
};
33
33
34
34
static const char *const rl_services[] = {
35
-
[ADF_SVC_ASYM] = "asym",
36
-
[ADF_SVC_SYM] = "sym",
37
-
[ADF_SVC_DC] = "dc",
35
+
[SVC_ASYM] = "asym",
36
+
[SVC_SYM] = "sym",
37
+
[SVC_DC] = "dc",
38
+
[SVC_DECOMP] = "decomp",
38
39
};
39
40
40
41
static const char *const rl_operations[] = {
···
283
282
if (ret)
284
283
return ret;
285
284
286
-
if (get == ADF_SVC_NONE)
285
+
if (get == SVC_BASE_COUNT)
287
286
return -EINVAL;
288
287
289
288
return sysfs_emit(buf, "%s\n", rl_services[get]);
···
292
291
static ssize_t srv_store(struct device *dev, struct device_attribute *attr,
293
292
const char *buf, size_t count)
294
293
{
294
+
struct adf_accel_dev *accel_dev;
295
295
unsigned int val;
296
296
int ret;
297
+
298
+
accel_dev = adf_devmgr_pci_to_accel_dev(to_pci_dev(dev));
299
+
if (!accel_dev)
300
+
return -EINVAL;
297
301
298
302
ret = sysfs_match_string(rl_services, buf);
299
303
if (ret < 0)
300
304
return ret;
301
305
302
306
val = ret;
307
+
if (!adf_is_service_enabled(accel_dev, val))
308
+
return -EINVAL;
309
+
303
310
ret = set_param_u(dev, SRV, val);
304
311
if (ret)
305
312
return ret;
···
448
439
dev_err(&GET_DEV(accel_dev),
449
440
"Failed to create qat_rl attribute group\n");
450
441
451
-
data->cap_rem_srv = ADF_SVC_NONE;
452
-
data->input.srv = ADF_SVC_NONE;
442
+
data->cap_rem_srv = SVC_BASE_COUNT;
443
+
data->input.srv = SVC_BASE_COUNT;
453
444
data->sysfs_added = true;
454
445
455
446
return ret;
+3
drivers/crypto/intel/qat/qat_common/adf_tl_debugfs.c
+3
drivers/crypto/intel/qat/qat_common/adf_tl_debugfs.c
···
538
538
case ASYM:
539
539
seq_printf(s, "%*s\n", TL_VALUE_MIN_PADDING, ADF_CFG_ASYM);
540
540
break;
541
+
case DECOMP:
542
+
seq_printf(s, "%*s\n", TL_VALUE_MIN_PADDING, ADF_CFG_DECOMP);
543
+
break;
541
544
default:
542
545
seq_printf(s, "%*s\n", TL_VALUE_MIN_PADDING, TL_RP_SRV_UNKNOWN);
543
546
break;
+14
-7
drivers/crypto/intel/qat/qat_common/adf_transport_debug.c
+14
-7
drivers/crypto/intel/qat/qat_common/adf_transport_debug.c
···
10
10
static DEFINE_MUTEX(ring_read_lock);
11
11
static DEFINE_MUTEX(bank_read_lock);
12
12
13
+
#define ADF_RING_NUM_MSGS(ring) \
14
+
(ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size) / \
15
+
ADF_MSG_SIZE_TO_BYTES(ring->msg_size))
16
+
13
17
static void *adf_ring_start(struct seq_file *sfile, loff_t *pos)
14
18
{
15
19
struct adf_etr_ring_data *ring = sfile->private;
20
+
unsigned int num_msg = ADF_RING_NUM_MSGS(ring);
21
+
loff_t val = *pos;
16
22
17
23
mutex_lock(&ring_read_lock);
18
-
if (*pos == 0)
24
+
if (val == 0)
19
25
return SEQ_START_TOKEN;
20
26
21
-
if (*pos >= (ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size) /
22
-
ADF_MSG_SIZE_TO_BYTES(ring->msg_size)))
27
+
if (val >= num_msg)
23
28
return NULL;
24
29
25
30
return ring->base_addr +
···
34
29
static void *adf_ring_next(struct seq_file *sfile, void *v, loff_t *pos)
35
30
{
36
31
struct adf_etr_ring_data *ring = sfile->private;
32
+
unsigned int num_msg = ADF_RING_NUM_MSGS(ring);
33
+
loff_t val = *pos;
37
34
38
-
if (*pos >= (ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size) /
39
-
ADF_MSG_SIZE_TO_BYTES(ring->msg_size)))
35
+
(*pos)++;
36
+
37
+
if (val >= num_msg)
40
38
return NULL;
41
39
42
-
return ring->base_addr +
43
-
(ADF_MSG_SIZE_TO_BYTES(ring->msg_size) * (*pos)++);
40
+
return ring->base_addr + (ADF_MSG_SIZE_TO_BYTES(ring->msg_size) * val);
44
41
}
45
42
46
43
static int adf_ring_show(struct seq_file *sfile, void *v)
+6
-6
drivers/crypto/intel/qat/qat_common/qat_algs.c
+6
-6
drivers/crypto/intel/qat/qat_common/qat_algs.c
···
1277
1277
.base = {
1278
1278
.cra_name = "authenc(hmac(sha1),cbc(aes))",
1279
1279
.cra_driver_name = "qat_aes_cbc_hmac_sha1",
1280
-
.cra_priority = 4001,
1280
+
.cra_priority = 100,
1281
1281
.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
1282
1282
.cra_blocksize = AES_BLOCK_SIZE,
1283
1283
.cra_ctxsize = sizeof(struct qat_alg_aead_ctx),
···
1294
1294
.base = {
1295
1295
.cra_name = "authenc(hmac(sha256),cbc(aes))",
1296
1296
.cra_driver_name = "qat_aes_cbc_hmac_sha256",
1297
-
.cra_priority = 4001,
1297
+
.cra_priority = 100,
1298
1298
.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
1299
1299
.cra_blocksize = AES_BLOCK_SIZE,
1300
1300
.cra_ctxsize = sizeof(struct qat_alg_aead_ctx),
···
1311
1311
.base = {
1312
1312
.cra_name = "authenc(hmac(sha512),cbc(aes))",
1313
1313
.cra_driver_name = "qat_aes_cbc_hmac_sha512",
1314
-
.cra_priority = 4001,
1314
+
.cra_priority = 100,
1315
1315
.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
1316
1316
.cra_blocksize = AES_BLOCK_SIZE,
1317
1317
.cra_ctxsize = sizeof(struct qat_alg_aead_ctx),
···
1329
1329
static struct skcipher_alg qat_skciphers[] = { {
1330
1330
.base.cra_name = "cbc(aes)",
1331
1331
.base.cra_driver_name = "qat_aes_cbc",
1332
-
.base.cra_priority = 4001,
1332
+
.base.cra_priority = 100,
1333
1333
.base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
1334
1334
.base.cra_blocksize = AES_BLOCK_SIZE,
1335
1335
.base.cra_ctxsize = sizeof(struct qat_alg_skcipher_ctx),
···
1347
1347
}, {
1348
1348
.base.cra_name = "ctr(aes)",
1349
1349
.base.cra_driver_name = "qat_aes_ctr",
1350
-
.base.cra_priority = 4001,
1350
+
.base.cra_priority = 100,
1351
1351
.base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
1352
1352
.base.cra_blocksize = 1,
1353
1353
.base.cra_ctxsize = sizeof(struct qat_alg_skcipher_ctx),
···
1365
1365
}, {
1366
1366
.base.cra_name = "xts(aes)",
1367
1367
.base.cra_driver_name = "qat_aes_xts",
1368
-
.base.cra_priority = 4001,
1368
+
.base.cra_priority = 100,
1369
1369
.base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK |
1370
1370
CRYPTO_ALG_ALLOCATES_MEMORY,
1371
1371
.base.cra_blocksize = AES_BLOCK_SIZE,
+3
-3
drivers/crypto/intel/qat/qat_common/qat_bl.c
+3
-3
drivers/crypto/intel/qat/qat_common/qat_bl.c
···
38
38
for (i = 0; i < blout->num_mapped_bufs; i++) {
39
39
dma_unmap_single(dev, blout->buffers[i].addr,
40
40
blout->buffers[i].len,
41
-
DMA_FROM_DEVICE);
41
+
DMA_BIDIRECTIONAL);
42
42
}
43
43
dma_unmap_single(dev, blpout, sz_out, DMA_TO_DEVICE);
44
44
···
162
162
}
163
163
buffers[y].addr = dma_map_single(dev, sg_virt(sg) + left,
164
164
sg->length - left,
165
-
DMA_FROM_DEVICE);
165
+
DMA_BIDIRECTIONAL);
166
166
if (unlikely(dma_mapping_error(dev, buffers[y].addr)))
167
167
goto err_out;
168
168
buffers[y].len = sg->length;
···
204
204
if (!dma_mapping_error(dev, buflout->buffers[i].addr))
205
205
dma_unmap_single(dev, buflout->buffers[i].addr,
206
206
buflout->buffers[i].len,
207
-
DMA_FROM_DEVICE);
207
+
DMA_BIDIRECTIONAL);
208
208
}
209
209
210
210
if (!buf->sgl_dst_valid)
+4
-4
drivers/crypto/intel/qat/qat_common/qat_compression.c
+4
-4
drivers/crypto/intel/qat/qat_common/qat_compression.c
···
196
196
struct adf_dc_data *dc_data = NULL;
197
197
u8 *obuff = NULL;
198
198
199
-
dc_data = devm_kzalloc(dev, sizeof(*dc_data), GFP_KERNEL);
199
+
dc_data = kzalloc_node(sizeof(*dc_data), GFP_KERNEL, dev_to_node(dev));
200
200
if (!dc_data)
201
201
goto err;
202
202
···
204
204
if (!obuff)
205
205
goto err;
206
206
207
-
obuff_p = dma_map_single(dev, obuff, ovf_buff_sz, DMA_FROM_DEVICE);
207
+
obuff_p = dma_map_single(dev, obuff, ovf_buff_sz, DMA_BIDIRECTIONAL);
208
208
if (unlikely(dma_mapping_error(dev, obuff_p)))
209
209
goto err;
210
210
···
232
232
return;
233
233
234
234
dma_unmap_single(dev, dc_data->ovf_buff_p, dc_data->ovf_buff_sz,
235
-
DMA_FROM_DEVICE);
235
+
DMA_BIDIRECTIONAL);
236
236
kfree_sensitive(dc_data->ovf_buff);
237
-
devm_kfree(dev, dc_data);
237
+
kfree(dc_data);
238
238
accel_dev->dc_data = NULL;
239
239
}
240
240
+3
-1
drivers/crypto/marvell/cesa/cipher.c
+3
-1
drivers/crypto/marvell/cesa/cipher.c
···
75
75
static inline void mv_cesa_skcipher_cleanup(struct skcipher_request *req)
76
76
{
77
77
struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(req);
78
+
struct mv_cesa_engine *engine = creq->base.engine;
78
79
79
80
if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ)
80
81
mv_cesa_skcipher_dma_cleanup(req);
82
+
83
+
atomic_sub(req->cryptlen, &engine->load);
81
84
}
82
85
83
86
static void mv_cesa_skcipher_std_step(struct skcipher_request *req)
···
215
212
struct mv_cesa_engine *engine = creq->base.engine;
216
213
unsigned int ivsize;
217
214
218
-
atomic_sub(skreq->cryptlen, &engine->load);
219
215
ivsize = crypto_skcipher_ivsize(crypto_skcipher_reqtfm(skreq));
220
216
221
217
if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ) {
+4
-6
drivers/crypto/marvell/cesa/hash.c
+4
-6
drivers/crypto/marvell/cesa/hash.c
···
110
110
static inline void mv_cesa_ahash_cleanup(struct ahash_request *req)
111
111
{
112
112
struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);
113
+
struct mv_cesa_engine *engine = creq->base.engine;
113
114
114
115
if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ)
115
116
mv_cesa_ahash_dma_cleanup(req);
117
+
118
+
atomic_sub(req->nbytes, &engine->load);
116
119
}
117
120
118
121
static void mv_cesa_ahash_last_cleanup(struct ahash_request *req)
···
365
362
if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ &&
366
363
(creq->base.chain.last->flags & CESA_TDMA_TYPE_MSK) ==
367
364
CESA_TDMA_RESULT) {
368
-
__le32 *data = NULL;
365
+
const void *data;
369
366
370
367
/*
371
368
* Result is already in the correct endianness when the SA is
372
369
* used
373
370
*/
374
371
data = creq->base.chain.last->op->ctx.hash.hash;
375
-
for (i = 0; i < digsize / 4; i++)
376
-
creq->state[i] = le32_to_cpu(data[i]);
377
-
378
372
memcpy(ahashreq->result, data, digsize);
379
373
} else {
380
374
for (i = 0; i < digsize / 4; i++)
···
395
395
}
396
396
}
397
397
}
398
-
399
-
atomic_sub(ahashreq->nbytes, &engine->load);
400
398
}
401
399
402
400
static void mv_cesa_ahash_prepare(struct crypto_async_request *req,
+97
-31
drivers/crypto/marvell/octeontx2/otx2_cpt_reqmgr.h
+97
-31
drivers/crypto/marvell/octeontx2/otx2_cpt_reqmgr.h
···
34
34
#define SG_COMP_2 2
35
35
#define SG_COMP_1 1
36
36
37
+
#define OTX2_CPT_DPTR_RPTR_ALIGN 8
38
+
#define OTX2_CPT_RES_ADDR_ALIGN 32
39
+
37
40
union otx2_cpt_opcode {
38
41
u16 flags;
39
42
struct {
···
350
347
cn10k_sgv2_info_create(struct pci_dev *pdev, struct otx2_cpt_req_info *req,
351
348
gfp_t gfp)
352
349
{
353
-
u32 dlen = 0, g_len, sg_len, info_len;
354
-
int align = OTX2_CPT_DMA_MINALIGN;
350
+
u32 dlen = 0, g_len, s_len, sg_len, info_len;
355
351
struct otx2_cpt_inst_info *info;
356
-
u16 g_sz_bytes, s_sz_bytes;
357
352
u32 total_mem_len;
358
353
int i;
359
354
360
-
g_sz_bytes = ((req->in_cnt + 2) / 3) *
361
-
sizeof(struct cn10kb_cpt_sglist_component);
362
-
s_sz_bytes = ((req->out_cnt + 2) / 3) *
363
-
sizeof(struct cn10kb_cpt_sglist_component);
355
+
/* Allocate memory to meet below alignment requirement:
356
+
* ------------------------------------
357
+
* | struct otx2_cpt_inst_info |
358
+
* | (No alignment required) |
359
+
* | --------------------------------|
360
+
* | | padding for ARCH_DMA_MINALIGN |
361
+
* | | alignment |
362
+
* |------------------------------------|
363
+
* | SG List Gather/Input memory |
364
+
* | Length = multiple of 32Bytes |
365
+
* | Alignment = 8Byte |
366
+
* |---------------------------------- |
367
+
* | SG List Scatter/Output memory |
368
+
* | Length = multiple of 32Bytes |
369
+
* | Alignment = 8Byte |
370
+
* | -------------------------------|
371
+
* | | padding for 32B alignment |
372
+
* |------------------------------------|
373
+
* | Result response memory |
374
+
* | Alignment = 32Byte |
375
+
* ------------------------------------
376
+
*/
364
377
365
-
g_len = ALIGN(g_sz_bytes, align);
366
-
sg_len = ALIGN(g_len + s_sz_bytes, align);
367
-
info_len = ALIGN(sizeof(*info), align);
368
-
total_mem_len = sg_len + info_len + sizeof(union otx2_cpt_res_s);
378
+
info_len = sizeof(*info);
379
+
380
+
g_len = ((req->in_cnt + 2) / 3) *
381
+
sizeof(struct cn10kb_cpt_sglist_component);
382
+
s_len = ((req->out_cnt + 2) / 3) *
383
+
sizeof(struct cn10kb_cpt_sglist_component);
384
+
sg_len = g_len + s_len;
385
+
386
+
/* Allocate extra memory for SG and response address alignment */
387
+
total_mem_len = ALIGN(info_len, OTX2_CPT_DPTR_RPTR_ALIGN);
388
+
total_mem_len += (ARCH_DMA_MINALIGN - 1) &
389
+
~(OTX2_CPT_DPTR_RPTR_ALIGN - 1);
390
+
total_mem_len += ALIGN(sg_len, OTX2_CPT_RES_ADDR_ALIGN);
391
+
total_mem_len += sizeof(union otx2_cpt_res_s);
369
392
370
393
info = kzalloc(total_mem_len, gfp);
371
394
if (unlikely(!info))
···
401
372
dlen += req->in[i].size;
402
373
403
374
info->dlen = dlen;
404
-
info->in_buffer = (u8 *)info + info_len;
375
+
info->in_buffer = PTR_ALIGN((u8 *)info + info_len, ARCH_DMA_MINALIGN);
376
+
info->out_buffer = info->in_buffer + g_len;
405
377
info->gthr_sz = req->in_cnt;
406
378
info->sctr_sz = req->out_cnt;
407
379
···
414
384
}
415
385
416
386
if (sgv2io_components_setup(pdev, req->out, req->out_cnt,
417
-
&info->in_buffer[g_len])) {
387
+
info->out_buffer)) {
418
388
dev_err(&pdev->dev, "Failed to setup scatter list\n");
419
389
goto destroy_info;
420
390
}
···
431
401
* Get buffer for union otx2_cpt_res_s response
432
402
* structure and its physical address
433
403
*/
434
-
info->completion_addr = info->in_buffer + sg_len;
435
-
info->comp_baddr = info->dptr_baddr + sg_len;
404
+
info->completion_addr = PTR_ALIGN((info->in_buffer + sg_len),
405
+
OTX2_CPT_RES_ADDR_ALIGN);
406
+
info->comp_baddr = ALIGN((info->dptr_baddr + sg_len),
407
+
OTX2_CPT_RES_ADDR_ALIGN);
436
408
437
409
return info;
438
410
···
449
417
otx2_sg_info_create(struct pci_dev *pdev, struct otx2_cpt_req_info *req,
450
418
gfp_t gfp)
451
419
{
452
-
int align = OTX2_CPT_DMA_MINALIGN;
453
420
struct otx2_cpt_inst_info *info;
454
-
u32 dlen, align_dlen, info_len;
455
-
u16 g_sz_bytes, s_sz_bytes;
421
+
u32 dlen, info_len;
422
+
u16 g_len, s_len;
456
423
u32 total_mem_len;
457
424
458
425
if (unlikely(req->in_cnt > OTX2_CPT_MAX_SG_IN_CNT ||
···
460
429
return NULL;
461
430
}
462
431
463
-
g_sz_bytes = ((req->in_cnt + 3) / 4) *
464
-
sizeof(struct otx2_cpt_sglist_component);
465
-
s_sz_bytes = ((req->out_cnt + 3) / 4) *
466
-
sizeof(struct otx2_cpt_sglist_component);
432
+
/* Allocate memory to meet below alignment requirement:
433
+
* ------------------------------------
434
+
* | struct otx2_cpt_inst_info |
435
+
* | (No alignment required) |
436
+
* | --------------------------------|
437
+
* | | padding for ARCH_DMA_MINALIGN |
438
+
* | | alignment |
439
+
* |------------------------------------|
440
+
* | SG List Header of 8 Byte |
441
+
* |------------------------------------|
442
+
* | SG List Gather/Input memory |
443
+
* | Length = multiple of 32Bytes |
444
+
* | Alignment = 8Byte |
445
+
* |---------------------------------- |
446
+
* | SG List Scatter/Output memory |
447
+
* | Length = multiple of 32Bytes |
448
+
* | Alignment = 8Byte |
449
+
* | -------------------------------|
450
+
* | | padding for 32B alignment |
451
+
* |------------------------------------|
452
+
* | Result response memory |
453
+
* | Alignment = 32Byte |
454
+
* ------------------------------------
455
+
*/
467
456
468
-
dlen = g_sz_bytes + s_sz_bytes + SG_LIST_HDR_SIZE;
469
-
align_dlen = ALIGN(dlen, align);
470
-
info_len = ALIGN(sizeof(*info), align);
471
-
total_mem_len = align_dlen + info_len + sizeof(union otx2_cpt_res_s);
457
+
info_len = sizeof(*info);
458
+
459
+
g_len = ((req->in_cnt + 3) / 4) *
460
+
sizeof(struct otx2_cpt_sglist_component);
461
+
s_len = ((req->out_cnt + 3) / 4) *
462
+
sizeof(struct otx2_cpt_sglist_component);
463
+
464
+
dlen = g_len + s_len + SG_LIST_HDR_SIZE;
465
+
466
+
/* Allocate extra memory for SG and response address alignment */
467
+
total_mem_len = ALIGN(info_len, OTX2_CPT_DPTR_RPTR_ALIGN);
468
+
total_mem_len += (ARCH_DMA_MINALIGN - 1) &
469
+
~(OTX2_CPT_DPTR_RPTR_ALIGN - 1);
470
+
total_mem_len += ALIGN(dlen, OTX2_CPT_RES_ADDR_ALIGN);
471
+
total_mem_len += sizeof(union otx2_cpt_res_s);
472
472
473
473
info = kzalloc(total_mem_len, gfp);
474
474
if (unlikely(!info))
475
475
return NULL;
476
476
477
477
info->dlen = dlen;
478
-
info->in_buffer = (u8 *)info + info_len;
478
+
info->in_buffer = PTR_ALIGN((u8 *)info + info_len, ARCH_DMA_MINALIGN);
479
+
info->out_buffer = info->in_buffer + SG_LIST_HDR_SIZE + g_len;
479
480
480
481
((u16 *)info->in_buffer)[0] = req->out_cnt;
481
482
((u16 *)info->in_buffer)[1] = req->in_cnt;
···
523
460
}
524
461
525
462
if (setup_sgio_components(pdev, req->out, req->out_cnt,
526
-
&info->in_buffer[8 + g_sz_bytes])) {
463
+
info->out_buffer)) {
527
464
dev_err(&pdev->dev, "Failed to setup scatter list\n");
528
465
goto destroy_info;
529
466
}
···
539
476
* Get buffer for union otx2_cpt_res_s response
540
477
* structure and its physical address
541
478
*/
542
-
info->completion_addr = info->in_buffer + align_dlen;
543
-
info->comp_baddr = info->dptr_baddr + align_dlen;
479
+
info->completion_addr = PTR_ALIGN((info->in_buffer + dlen),
480
+
OTX2_CPT_RES_ADDR_ALIGN);
481
+
info->comp_baddr = ALIGN((info->dptr_baddr + dlen),
482
+
OTX2_CPT_RES_ADDR_ALIGN);
544
483
545
484
return info;
546
485
···
555
490
int otx2_cpt_do_request(struct pci_dev *pdev, struct otx2_cpt_req_info *req,
556
491
int cpu_num);
557
492
void otx2_cpt_post_process(struct otx2_cptlf_wqe *wqe);
558
-
int otx2_cpt_get_kcrypto_eng_grp_num(struct pci_dev *pdev);
493
+
int otx2_cpt_get_eng_grp_num(struct pci_dev *pdev,
494
+
enum otx2_cpt_eng_type);
559
495
560
496
#endif /* __OTX2_CPT_REQMGR_H */
+2
-1
drivers/crypto/marvell/octeontx2/otx2_cptlf.h
+2
-1
drivers/crypto/marvell/octeontx2/otx2_cptlf.h
···
124
124
struct cpt_hw_ops *ops;
125
125
u8 are_lfs_attached; /* Whether CPT LFs are attached */
126
126
u8 lfs_num; /* Number of CPT LFs */
127
-
u8 kcrypto_eng_grp_num; /* Kernel crypto engine group number */
127
+
u8 kcrypto_se_eng_grp_num; /* Crypto symmetric engine group number */
128
+
u8 kcrypto_ae_eng_grp_num; /* Crypto asymmetric engine group number */
128
129
u8 kvf_limits; /* Kernel crypto limits */
129
130
atomic_t state; /* LF's state. started/reset */
130
131
int blkaddr; /* CPT blkaddr: BLKADDR_CPT0/BLKADDR_CPT1 */
+35
-16
drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c
+35
-16
drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c
···
1493
1493
union otx2_cpt_opcode opcode;
1494
1494
union otx2_cpt_res_s *result;
1495
1495
union otx2_cpt_inst_s inst;
1496
+
dma_addr_t result_baddr;
1496
1497
dma_addr_t rptr_baddr;
1497
1498
struct pci_dev *pdev;
1498
-
u32 len, compl_rlen;
1499
+
int timeout = 10000;
1500
+
void *base, *rptr;
1499
1501
int ret, etype;
1500
-
void *rptr;
1502
+
u32 len;
1501
1503
1502
1504
/*
1503
1505
* We don't get capabilities if it was already done
···
1522
1520
if (ret)
1523
1521
goto delete_grps;
1524
1522
1525
-
compl_rlen = ALIGN(sizeof(union otx2_cpt_res_s), OTX2_CPT_DMA_MINALIGN);
1526
-
len = compl_rlen + LOADFVC_RLEN;
1523
+
/* Allocate extra memory for "rptr" and "result" pointer alignment */
1524
+
len = LOADFVC_RLEN + ARCH_DMA_MINALIGN +
1525
+
sizeof(union otx2_cpt_res_s) + OTX2_CPT_RES_ADDR_ALIGN;
1527
1526
1528
-
result = kzalloc(len, GFP_KERNEL);
1529
-
if (!result) {
1527
+
base = kzalloc(len, GFP_KERNEL);
1528
+
if (!base) {
1530
1529
ret = -ENOMEM;
1531
1530
goto lf_cleanup;
1532
1531
}
1533
-
rptr_baddr = dma_map_single(&pdev->dev, (void *)result, len,
1534
-
DMA_BIDIRECTIONAL);
1532
+
1533
+
rptr = PTR_ALIGN(base, ARCH_DMA_MINALIGN);
1534
+
rptr_baddr = dma_map_single(&pdev->dev, rptr, len, DMA_BIDIRECTIONAL);
1535
1535
if (dma_mapping_error(&pdev->dev, rptr_baddr)) {
1536
1536
dev_err(&pdev->dev, "DMA mapping failed\n");
1537
1537
ret = -EFAULT;
1538
-
goto free_result;
1538
+
goto free_rptr;
1539
1539
}
1540
-
rptr = (u8 *)result + compl_rlen;
1540
+
1541
+
result = (union otx2_cpt_res_s *)PTR_ALIGN(rptr + LOADFVC_RLEN,
1542
+
OTX2_CPT_RES_ADDR_ALIGN);
1543
+
result_baddr = ALIGN(rptr_baddr + LOADFVC_RLEN,
1544
+
OTX2_CPT_RES_ADDR_ALIGN);
1541
1545
1542
1546
/* Fill in the command */
1543
1547
opcode.s.major = LOADFVC_MAJOR_OP;
···
1555
1547
/* 64-bit swap for microcode data reads, not needed for addresses */
1556
1548
cpu_to_be64s(&iq_cmd.cmd.u);
1557
1549
iq_cmd.dptr = 0;
1558
-
iq_cmd.rptr = rptr_baddr + compl_rlen;
1550
+
iq_cmd.rptr = rptr_baddr;
1559
1551
iq_cmd.cptr.u = 0;
1560
1552
1561
1553
for (etype = 1; etype < OTX2_CPT_MAX_ENG_TYPES; etype++) {
1562
1554
result->s.compcode = OTX2_CPT_COMPLETION_CODE_INIT;
1563
1555
iq_cmd.cptr.s.grp = otx2_cpt_get_eng_grp(&cptpf->eng_grps,
1564
1556
etype);
1565
-
otx2_cpt_fill_inst(&inst, &iq_cmd, rptr_baddr);
1557
+
otx2_cpt_fill_inst(&inst, &iq_cmd, result_baddr);
1566
1558
lfs->ops->send_cmd(&inst, 1, &cptpf->lfs.lf[0]);
1559
+
timeout = 10000;
1567
1560
1568
1561
while (lfs->ops->cpt_get_compcode(result) ==
1569
-
OTX2_CPT_COMPLETION_CODE_INIT)
1562
+
OTX2_CPT_COMPLETION_CODE_INIT) {
1570
1563
cpu_relax();
1564
+
udelay(1);
1565
+
timeout--;
1566
+
if (!timeout) {
1567
+
ret = -ENODEV;
1568
+
cptpf->is_eng_caps_discovered = false;
1569
+
dev_warn(&pdev->dev, "Timeout on CPT load_fvc completion poll\n");
1570
+
goto error_no_response;
1571
+
}
1572
+
}
1571
1573
1572
1574
cptpf->eng_caps[etype].u = be64_to_cpup(rptr);
1573
1575
}
1574
-
dma_unmap_single(&pdev->dev, rptr_baddr, len, DMA_BIDIRECTIONAL);
1575
1576
cptpf->is_eng_caps_discovered = true;
1576
1577
1577
-
free_result:
1578
-
kfree(result);
1578
+
error_no_response:
1579
+
dma_unmap_single(&pdev->dev, rptr_baddr, len, DMA_BIDIRECTIONAL);
1580
+
free_rptr:
1581
+
kfree(base);
1579
1582
lf_cleanup:
1580
1583
otx2_cptlf_shutdown(lfs);
1581
1584
delete_grps:
+4
-2
drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c
+4
-2
drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c
···
384
384
req_info->req_type = OTX2_CPT_ENC_DEC_REQ;
385
385
req_info->is_enc = enc;
386
386
req_info->is_trunc_hmac = false;
387
-
req_info->ctrl.s.grp = otx2_cpt_get_kcrypto_eng_grp_num(pdev);
387
+
req_info->ctrl.s.grp = otx2_cpt_get_eng_grp_num(pdev,
388
+
OTX2_CPT_SE_TYPES);
388
389
389
390
req_info->req.cptr = ctx->er_ctx.hw_ctx;
390
391
req_info->req.cptr_dma = ctx->er_ctx.cptr_dma;
···
1289
1288
if (status)
1290
1289
return status;
1291
1290
1292
-
req_info->ctrl.s.grp = otx2_cpt_get_kcrypto_eng_grp_num(pdev);
1291
+
req_info->ctrl.s.grp = otx2_cpt_get_eng_grp_num(pdev,
1292
+
OTX2_CPT_SE_TYPES);
1293
1293
1294
1294
/*
1295
1295
* We perform an asynchronous send and once
+22
-6
drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c
+22
-6
drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c
···
265
265
u8 eng_grp_msk;
266
266
267
267
/* Get engine group number for symmetric crypto */
268
-
cptvf->lfs.kcrypto_eng_grp_num = OTX2_CPT_INVALID_CRYPTO_ENG_GRP;
268
+
cptvf->lfs.kcrypto_se_eng_grp_num = OTX2_CPT_INVALID_CRYPTO_ENG_GRP;
269
269
ret = otx2_cptvf_send_eng_grp_num_msg(cptvf, OTX2_CPT_SE_TYPES);
270
270
if (ret)
271
271
return ret;
272
272
273
-
if (cptvf->lfs.kcrypto_eng_grp_num == OTX2_CPT_INVALID_CRYPTO_ENG_GRP) {
274
-
dev_err(dev, "Engine group for kernel crypto not available\n");
275
-
ret = -ENOENT;
276
-
return ret;
273
+
if (cptvf->lfs.kcrypto_se_eng_grp_num ==
274
+
OTX2_CPT_INVALID_CRYPTO_ENG_GRP) {
275
+
dev_err(dev,
276
+
"Symmetric Engine group for crypto not available\n");
277
+
return -ENOENT;
277
278
}
278
-
eng_grp_msk = 1 << cptvf->lfs.kcrypto_eng_grp_num;
279
+
280
+
/* Get engine group number for asymmetric crypto */
281
+
cptvf->lfs.kcrypto_ae_eng_grp_num = OTX2_CPT_INVALID_CRYPTO_ENG_GRP;
282
+
ret = otx2_cptvf_send_eng_grp_num_msg(cptvf, OTX2_CPT_AE_TYPES);
283
+
if (ret)
284
+
return ret;
285
+
286
+
if (cptvf->lfs.kcrypto_ae_eng_grp_num ==
287
+
OTX2_CPT_INVALID_CRYPTO_ENG_GRP) {
288
+
dev_err(dev,
289
+
"Asymmetric Engine group for crypto not available\n");
290
+
return -ENOENT;
291
+
}
292
+
293
+
eng_grp_msk = BIT(cptvf->lfs.kcrypto_se_eng_grp_num) |
294
+
BIT(cptvf->lfs.kcrypto_ae_eng_grp_num);
279
295
280
296
ret = otx2_cptvf_send_kvf_limits_msg(cptvf);
281
297
if (ret)
+6
-1
drivers/crypto/marvell/octeontx2/otx2_cptvf_mbox.c
+6
-1
drivers/crypto/marvell/octeontx2/otx2_cptvf_mbox.c
···
75
75
struct otx2_cpt_caps_rsp *eng_caps;
76
76
struct cpt_rd_wr_reg_msg *rsp_reg;
77
77
struct msix_offset_rsp *rsp_msix;
78
+
u8 grp_num;
78
79
int i;
79
80
80
81
if (msg->id >= MBOX_MSG_MAX) {
···
123
122
break;
124
123
case MBOX_MSG_GET_ENG_GRP_NUM:
125
124
rsp_grp = (struct otx2_cpt_egrp_num_rsp *) msg;
126
-
cptvf->lfs.kcrypto_eng_grp_num = rsp_grp->eng_grp_num;
125
+
grp_num = rsp_grp->eng_grp_num;
126
+
if (rsp_grp->eng_type == OTX2_CPT_SE_TYPES)
127
+
cptvf->lfs.kcrypto_se_eng_grp_num = grp_num;
128
+
else if (rsp_grp->eng_type == OTX2_CPT_AE_TYPES)
129
+
cptvf->lfs.kcrypto_ae_eng_grp_num = grp_num;
127
130
break;
128
131
case MBOX_MSG_GET_KVF_LIMITS:
129
132
rsp_limits = (struct otx2_cpt_kvf_limits_rsp *) msg;
+12
-2
drivers/crypto/marvell/octeontx2/otx2_cptvf_reqmgr.c
+12
-2
drivers/crypto/marvell/octeontx2/otx2_cptvf_reqmgr.c
···
391
391
&wqe->lfs->lf[wqe->lf_num].pqueue);
392
392
}
393
393
394
-
int otx2_cpt_get_kcrypto_eng_grp_num(struct pci_dev *pdev)
394
+
int otx2_cpt_get_eng_grp_num(struct pci_dev *pdev,
395
+
enum otx2_cpt_eng_type eng_type)
395
396
{
396
397
struct otx2_cptvf_dev *cptvf = pci_get_drvdata(pdev);
397
398
398
-
return cptvf->lfs.kcrypto_eng_grp_num;
399
+
switch (eng_type) {
400
+
case OTX2_CPT_SE_TYPES:
401
+
return cptvf->lfs.kcrypto_se_eng_grp_num;
402
+
case OTX2_CPT_AE_TYPES:
403
+
return cptvf->lfs.kcrypto_ae_eng_grp_num;
404
+
default:
405
+
dev_err(&cptvf->pdev->dev, "Unsupported engine type");
406
+
break;
407
+
}
408
+
return -ENXIO;
399
409
}
-1
drivers/crypto/omap-aes-gcm.c
-1
drivers/crypto/omap-aes-gcm.c
-1
drivers/crypto/omap-aes.c
-1
drivers/crypto/omap-aes.c
-1
drivers/crypto/omap-des.c
-1
drivers/crypto/omap-des.c
-1
drivers/crypto/omap-sham.c
-1
drivers/crypto/omap-sham.c
-1
drivers/crypto/stm32/stm32-cryp.c
-1
drivers/crypto/stm32/stm32-cryp.c
-1
drivers/crypto/stm32/stm32-hash.c
-1
drivers/crypto/stm32/stm32-hash.c
-2
drivers/crypto/virtio/virtio_crypto_common.h
-2
drivers/crypto/virtio/virtio_crypto_common.h
···
113
113
int virtcrypto_devmgr_add_dev(struct virtio_crypto *vcrypto_dev);
114
114
struct list_head *virtcrypto_devmgr_get_head(void);
115
115
void virtcrypto_devmgr_rm_dev(struct virtio_crypto *vcrypto_dev);
116
-
struct virtio_crypto *virtcrypto_devmgr_get_first(void);
117
-
int virtcrypto_dev_in_use(struct virtio_crypto *vcrypto_dev);
118
116
int virtcrypto_dev_get(struct virtio_crypto *vcrypto_dev);
119
117
void virtcrypto_dev_put(struct virtio_crypto *vcrypto_dev);
120
118
int virtcrypto_dev_started(struct virtio_crypto *vcrypto_dev);
+1
-1
drivers/crypto/virtio/virtio_crypto_core.c
+1
-1
drivers/crypto/virtio/virtio_crypto_core.c
···
139
139
spin_lock_init(&vi->data_vq[i].lock);
140
140
vi->data_vq[i].vq = vqs[i];
141
141
/* Initialize crypto engine */
142
-
vi->data_vq[i].engine = crypto_engine_alloc_init_and_set(dev, true, NULL, true,
142
+
vi->data_vq[i].engine = crypto_engine_alloc_init_and_set(dev, true, true,
143
143
virtqueue_get_vring_size(vqs[i]));
144
144
if (!vi->data_vq[i].engine) {
145
145
ret = -ENOMEM;
-36
drivers/crypto/virtio/virtio_crypto_mgr.c
-36
drivers/crypto/virtio/virtio_crypto_mgr.c
···
82
82
}
83
83
84
84
/*
85
-
* virtcrypto_devmgr_get_first()
86
-
*
87
-
* Function returns the first virtio crypto device from the acceleration
88
-
* framework.
89
-
*
90
-
* To be used by virtio crypto device specific drivers.
91
-
*
92
-
* Return: pointer to vcrypto_dev or NULL if not found.
93
-
*/
94
-
struct virtio_crypto *virtcrypto_devmgr_get_first(void)
95
-
{
96
-
struct virtio_crypto *dev = NULL;
97
-
98
-
mutex_lock(&table_lock);
99
-
if (!list_empty(&virtio_crypto_table))
100
-
dev = list_first_entry(&virtio_crypto_table,
101
-
struct virtio_crypto,
102
-
list);
103
-
mutex_unlock(&table_lock);
104
-
return dev;
105
-
}
106
-
107
-
/*
108
-
* virtcrypto_dev_in_use() - Check whether vcrypto_dev is currently in use
109
-
* @vcrypto_dev: Pointer to virtio crypto device.
110
-
*
111
-
* To be used by virtio crypto device specific drivers.
112
-
*
113
-
* Return: 1 when device is in use, 0 otherwise.
114
-
*/
115
-
int virtcrypto_dev_in_use(struct virtio_crypto *vcrypto_dev)
116
-
{
117
-
return atomic_read(&vcrypto_dev->ref_count) != 0;
118
-
}
119
-
120
-
/*
121
85
* virtcrypto_dev_get() - Increment vcrypto_dev reference count
122
86
* @vcrypto_dev: Pointer to virtio crypto device.
123
87
*
-1
include/crypto/engine.h
-1
include/crypto/engine.h
···
76
76
struct crypto_engine *crypto_engine_alloc_init(struct device *dev, bool rt);
77
77
struct crypto_engine *crypto_engine_alloc_init_and_set(struct device *dev,
78
78
bool retry_support,
79
-
int (*cbk_do_batch)(struct crypto_engine *engine),
80
79
bool rt, int qlen);
81
80
void crypto_engine_exit(struct crypto_engine *engine);
82
81
+1
-4
include/crypto/internal/acompress.h
+1
-4
include/crypto/internal/acompress.h
···
63
63
struct crypto_acomp_streams {
64
64
/* These must come first because of struct scomp_alg. */
65
65
void *(*alloc_ctx)(void);
66
-
union {
67
-
void (*free_ctx)(void *);
68
-
void (*cfree_ctx)(const void *);
69
-
};
66
+
void (*free_ctx)(void *);
70
67
71
68
struct crypto_acomp_stream __percpu *streams;
72
69
struct work_struct stream_work;
-15
include/crypto/internal/engine.h
-15
include/crypto/internal/engine.h
···
21
21
/*
22
22
* struct crypto_engine - crypto hardware engine
23
23
* @name: the engine name
24
-
* @idling: the engine is entering idle state
25
24
* @busy: request pump is busy
26
25
* @running: the engine is on working
27
26
* @retry_support: indication that the hardware allows re-execution
···
30
31
* @list: link with the global crypto engine list
31
32
* @queue_lock: spinlock to synchronise access to request queue
32
33
* @queue: the crypto queue of the engine
33
-
* @prepare_crypt_hardware: a request will soon arrive from the queue
34
-
* so the subsystem requests the driver to prepare the hardware
35
-
* by issuing this call
36
-
* @unprepare_crypt_hardware: there are currently no more requests on the
37
-
* queue so the subsystem notifies the driver that it may relax the
38
-
* hardware by issuing this call
39
-
* @do_batch_requests: execute a batch of requests. Depends on multiple
40
-
* requests support.
41
34
* @kworker: kthread worker struct for request pump
42
35
* @pump_requests: work struct for scheduling work to the request pump
43
36
* @priv_data: the engine private data
···
37
46
*/
38
47
struct crypto_engine {
39
48
char name[ENGINE_NAME_LEN];
40
-
bool idling;
41
49
bool busy;
42
50
bool running;
43
51
···
47
57
spinlock_t queue_lock;
48
58
struct crypto_queue queue;
49
59
struct device *dev;
50
-
51
-
int (*prepare_crypt_hardware)(struct crypto_engine *engine);
52
-
int (*unprepare_crypt_hardware)(struct crypto_engine *engine);
53
-
int (*do_batch_requests)(struct crypto_engine *engine);
54
-
55
60
56
61
struct kthread_worker *kworker;
57
62
struct kthread_work pump_requests;
+36
include/crypto/internal/hash.h
+36
include/crypto/internal/hash.h
···
30
30
__##name##_req, (req))
31
31
32
32
struct ahash_request;
33
+
struct scatterlist;
34
+
35
+
struct crypto_hash_walk {
36
+
const char *data;
37
+
38
+
unsigned int offset;
39
+
unsigned int flags;
40
+
41
+
struct page *pg;
42
+
unsigned int entrylen;
43
+
44
+
unsigned int total;
45
+
struct scatterlist *sg;
46
+
};
33
47
34
48
struct ahash_instance {
35
49
void (*free)(struct ahash_instance *inst);
···
75
61
struct crypto_spawn base;
76
62
};
77
63
64
+
int crypto_hash_walk_done(struct crypto_hash_walk *walk, int err);
65
+
int crypto_hash_walk_first(struct ahash_request *req,
66
+
struct crypto_hash_walk *walk);
67
+
68
+
static inline int crypto_hash_walk_last(struct crypto_hash_walk *walk)
69
+
{
70
+
return !(walk->entrylen | walk->total);
71
+
}
72
+
78
73
int crypto_register_ahash(struct ahash_alg *alg);
79
74
void crypto_unregister_ahash(struct ahash_alg *alg);
80
75
int crypto_register_ahashes(struct ahash_alg *algs, int count);
···
112
89
{
113
90
return crypto_hash_alg_has_setkey(alg) &&
114
91
!(alg->base.cra_flags & CRYPTO_ALG_OPTIONAL_KEY);
92
+
}
93
+
94
+
static inline bool crypto_hash_no_export_core(struct crypto_ahash *tfm)
95
+
{
96
+
return crypto_hash_alg_common(tfm)->base.cra_flags &
97
+
CRYPTO_AHASH_ALG_NO_EXPORT_CORE;
115
98
}
116
99
117
100
int crypto_grab_ahash(struct crypto_ahash_spawn *spawn,
···
194
165
unsigned int reqsize)
195
166
{
196
167
tfm->reqsize = reqsize;
168
+
}
169
+
170
+
static inline bool crypto_ahash_tested(struct crypto_ahash *tfm)
171
+
{
172
+
struct crypto_tfm *tfm_base = crypto_ahash_tfm(tfm);
173
+
174
+
return tfm_base->__crt_alg->cra_flags & CRYPTO_ALG_TESTED;
197
175
}
198
176
199
177
static inline void crypto_ahash_set_reqsize_dma(struct crypto_ahash *ahash,
+3
include/linux/crypto.h
+3
include/linux/crypto.h
···
136
136
/* Set if the algorithm supports virtual addresses. */
137
137
#define CRYPTO_ALG_REQ_VIRT 0x00040000
138
138
139
+
/* Set if the algorithm cannot have a fallback (e.g., phmac). */
140
+
#define CRYPTO_ALG_NO_FALLBACK 0x00080000
141
+
139
142
/* The high bits 0xff000000 are reserved for type-specific flags. */
140
143
141
144
/*
+2
-2
include/linux/hisi_acc_qm.h
+2
-2
include/linux/hisi_acc_qm.h
···
556
556
struct hisi_acc_sgl_pool;
557
557
struct hisi_acc_hw_sgl *hisi_acc_sg_buf_map_to_hw_sgl(struct device *dev,
558
558
struct scatterlist *sgl, struct hisi_acc_sgl_pool *pool,
559
-
u32 index, dma_addr_t *hw_sgl_dma);
559
+
u32 index, dma_addr_t *hw_sgl_dma, enum dma_data_direction dir);
560
560
void hisi_acc_sg_buf_unmap(struct device *dev, struct scatterlist *sgl,
561
-
struct hisi_acc_hw_sgl *hw_sgl);
561
+
struct hisi_acc_hw_sgl *hw_sgl, enum dma_data_direction dir);
562
562
struct hisi_acc_sgl_pool *hisi_acc_create_sgl_pool(struct device *dev,
563
563
u32 count, u32 sge_nr);
564
564
void hisi_acc_free_sgl_pool(struct device *dev,
-4
include/linux/padata.h
-4
include/linux/padata.h
···
90
90
* @processed: Number of already processed objects.
91
91
* @cpu: Next CPU to be processed.
92
92
* @cpumask: The cpumasks in use for parallel and serial workers.
93
-
* @reorder_work: work struct for reordering.
94
-
* @lock: Reorder lock.
95
93
*/
96
94
struct parallel_data {
97
95
struct padata_shell *ps;
···
100
102
unsigned int processed;
101
103
int cpu;
102
104
struct padata_cpumask cpumask;
103
-
struct work_struct reorder_work;
104
-
spinlock_t ____cacheline_aligned lock;
105
105
};
106
106
107
107
/**
+40
-114
kernel/padata.c
+40
-114
kernel/padata.c
···
63
63
padata_put_pd_cnt(pd, 1);
64
64
}
65
65
66
-
static int padata_index_to_cpu(struct parallel_data *pd, int cpu_index)
67
-
{
68
-
int cpu, target_cpu;
69
-
70
-
target_cpu = cpumask_first(pd->cpumask.pcpu);
71
-
for (cpu = 0; cpu < cpu_index; cpu++)
72
-
target_cpu = cpumask_next(target_cpu, pd->cpumask.pcpu);
73
-
74
-
return target_cpu;
75
-
}
76
-
77
66
static int padata_cpu_hash(struct parallel_data *pd, unsigned int seq_nr)
78
67
{
79
68
/*
···
71
82
*/
72
83
int cpu_index = seq_nr % cpumask_weight(pd->cpumask.pcpu);
73
84
74
-
return padata_index_to_cpu(pd, cpu_index);
85
+
return cpumask_nth(cpu_index, pd->cpumask.pcpu);
75
86
}
76
87
77
88
static struct padata_work *padata_work_alloc(void)
···
181
192
struct padata_priv *padata, int *cb_cpu)
182
193
{
183
194
struct padata_instance *pinst = ps->pinst;
184
-
int i, cpu, cpu_index, err;
185
195
struct parallel_data *pd;
186
196
struct padata_work *pw;
197
+
int cpu_index, err;
187
198
188
199
rcu_read_lock_bh();
189
200
···
199
210
200
211
/* Select an alternate fallback CPU and notify the caller. */
201
212
cpu_index = *cb_cpu % cpumask_weight(pd->cpumask.cbcpu);
202
-
203
-
cpu = cpumask_first(pd->cpumask.cbcpu);
204
-
for (i = 0; i < cpu_index; i++)
205
-
cpu = cpumask_next(cpu, pd->cpumask.cbcpu);
206
-
207
-
*cb_cpu = cpu;
213
+
*cb_cpu = cpumask_nth(cpu_index, pd->cpumask.cbcpu);
208
214
}
209
215
210
216
err = -EBUSY;
···
245
261
* be parallel processed by another cpu and is not yet present in
246
262
* the cpu's reorder queue.
247
263
*/
248
-
static struct padata_priv *padata_find_next(struct parallel_data *pd,
249
-
bool remove_object)
264
+
static struct padata_priv *padata_find_next(struct parallel_data *pd, int cpu,
265
+
unsigned int processed)
250
266
{
251
267
struct padata_priv *padata;
252
268
struct padata_list *reorder;
253
-
int cpu = pd->cpu;
254
269
255
270
reorder = per_cpu_ptr(pd->reorder_list, cpu);
256
271
257
272
spin_lock(&reorder->lock);
258
-
if (list_empty(&reorder->list)) {
259
-
spin_unlock(&reorder->lock);
260
-
return NULL;
261
-
}
273
+
if (list_empty(&reorder->list))
274
+
goto notfound;
262
275
263
276
padata = list_entry(reorder->list.next, struct padata_priv, list);
264
277
···
263
282
* Checks the rare case where two or more parallel jobs have hashed to
264
283
* the same CPU and one of the later ones finishes first.
265
284
*/
266
-
if (padata->seq_nr != pd->processed) {
267
-
spin_unlock(&reorder->lock);
268
-
return NULL;
269
-
}
285
+
if (padata->seq_nr != processed)
286
+
goto notfound;
270
287
271
-
if (remove_object) {
272
-
list_del_init(&padata->list);
273
-
++pd->processed;
274
-
pd->cpu = cpumask_next_wrap(cpu, pd->cpumask.pcpu);
275
-
}
276
-
288
+
list_del_init(&padata->list);
277
289
spin_unlock(&reorder->lock);
278
290
return padata;
291
+
292
+
notfound:
293
+
pd->processed = processed;
294
+
pd->cpu = cpu;
295
+
spin_unlock(&reorder->lock);
296
+
return NULL;
279
297
}
280
298
281
-
static void padata_reorder(struct parallel_data *pd)
299
+
static void padata_reorder(struct padata_priv *padata)
282
300
{
301
+
struct parallel_data *pd = padata->pd;
283
302
struct padata_instance *pinst = pd->ps->pinst;
284
-
int cb_cpu;
285
-
struct padata_priv *padata;
286
-
struct padata_serial_queue *squeue;
287
-
struct padata_list *reorder;
303
+
unsigned int processed;
304
+
int cpu;
288
305
289
-
/*
290
-
* We need to ensure that only one cpu can work on dequeueing of
291
-
* the reorder queue the time. Calculating in which percpu reorder
292
-
* queue the next object will arrive takes some time. A spinlock
293
-
* would be highly contended. Also it is not clear in which order
294
-
* the objects arrive to the reorder queues. So a cpu could wait to
295
-
* get the lock just to notice that there is nothing to do at the
296
-
* moment. Therefore we use a trylock and let the holder of the lock
297
-
* care for all the objects enqueued during the holdtime of the lock.
298
-
*/
299
-
if (!spin_trylock_bh(&pd->lock))
300
-
return;
306
+
processed = pd->processed;
307
+
cpu = pd->cpu;
301
308
302
-
while (1) {
303
-
padata = padata_find_next(pd, true);
309
+
do {
310
+
struct padata_serial_queue *squeue;
311
+
int cb_cpu;
304
312
305
-
/*
306
-
* If the next object that needs serialization is parallel
307
-
* processed by another cpu and is still on it's way to the
308
-
* cpu's reorder queue, nothing to do for now.
309
-
*/
310
-
if (!padata)
311
-
break;
313
+
cpu = cpumask_next_wrap(cpu, pd->cpumask.pcpu);
314
+
processed++;
312
315
313
316
cb_cpu = padata->cb_cpu;
314
317
squeue = per_cpu_ptr(pd->squeue, cb_cpu);
315
318
316
319
spin_lock(&squeue->serial.lock);
317
320
list_add_tail(&padata->list, &squeue->serial.list);
318
-
spin_unlock(&squeue->serial.lock);
319
-
320
321
queue_work_on(cb_cpu, pinst->serial_wq, &squeue->work);
321
-
}
322
322
323
-
spin_unlock_bh(&pd->lock);
324
-
325
-
/*
326
-
* The next object that needs serialization might have arrived to
327
-
* the reorder queues in the meantime.
328
-
*
329
-
* Ensure reorder queue is read after pd->lock is dropped so we see
330
-
* new objects from another task in padata_do_serial. Pairs with
331
-
* smp_mb in padata_do_serial.
332
-
*/
333
-
smp_mb();
334
-
335
-
reorder = per_cpu_ptr(pd->reorder_list, pd->cpu);
336
-
if (!list_empty(&reorder->list) && padata_find_next(pd, false)) {
337
323
/*
338
-
* Other context(eg. the padata_serial_worker) can finish the request.
339
-
* To avoid UAF issue, add pd ref here, and put pd ref after reorder_work finish.
324
+
* If the next object that needs serialization is parallel
325
+
* processed by another cpu and is still on it's way to the
326
+
* cpu's reorder queue, end the loop.
340
327
*/
341
-
padata_get_pd(pd);
342
-
if (!queue_work(pinst->serial_wq, &pd->reorder_work))
343
-
padata_put_pd(pd);
344
-
}
345
-
}
346
-
347
-
static void invoke_padata_reorder(struct work_struct *work)
348
-
{
349
-
struct parallel_data *pd;
350
-
351
-
local_bh_disable();
352
-
pd = container_of(work, struct parallel_data, reorder_work);
353
-
padata_reorder(pd);
354
-
local_bh_enable();
355
-
/* Pairs with putting the reorder_work in the serial_wq */
356
-
padata_put_pd(pd);
328
+
padata = padata_find_next(pd, cpu, processed);
329
+
spin_unlock(&squeue->serial.lock);
330
+
} while (padata);
357
331
}
358
332
359
333
static void padata_serial_worker(struct work_struct *serial_work)
···
359
423
struct padata_list *reorder = per_cpu_ptr(pd->reorder_list, hashed_cpu);
360
424
struct padata_priv *cur;
361
425
struct list_head *pos;
426
+
bool gotit = true;
362
427
363
428
spin_lock(&reorder->lock);
364
429
/* Sort in ascending order of sequence number. */
···
369
432
if ((signed int)(cur->seq_nr - padata->seq_nr) < 0)
370
433
break;
371
434
}
372
-
list_add(&padata->list, pos);
435
+
if (padata->seq_nr != pd->processed) {
436
+
gotit = false;
437
+
list_add(&padata->list, pos);
438
+
}
373
439
spin_unlock(&reorder->lock);
374
440
375
-
/*
376
-
* Ensure the addition to the reorder list is ordered correctly
377
-
* with the trylock of pd->lock in padata_reorder. Pairs with smp_mb
378
-
* in padata_reorder.
379
-
*/
380
-
smp_mb();
381
-
382
-
padata_reorder(pd);
441
+
if (gotit)
442
+
padata_reorder(padata);
383
443
}
384
444
EXPORT_SYMBOL(padata_do_serial);
385
445
···
566
632
padata_init_squeues(pd);
567
633
pd->seq_nr = -1;
568
634
refcount_set(&pd->refcnt, 1);
569
-
spin_lock_init(&pd->lock);
570
635
pd->cpu = cpumask_first(pd->cpumask.pcpu);
571
-
INIT_WORK(&pd->reorder_work, invoke_padata_reorder);
572
636
573
637
return pd;
574
638
···
1075
1143
1076
1144
if (!ps)
1077
1145
return;
1078
-
1079
-
/*
1080
-
* Wait for all _do_serial calls to finish to avoid touching
1081
-
* freed pd's and ps's.
1082
-
*/
1083
-
synchronize_rcu();
1084
1146
1085
1147
mutex_lock(&ps->pinst->lock);
1086
1148
list_del(&ps->list);