Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
lib/crypto: powerpc/ghash: Migrate optimized code into library
Remove the "p8_ghash" crypto_shash algorithm. Move the corresponding
assembly code into lib/crypto/, and wire it up to the GHASH library.
This makes the GHASH library be optimized for POWER8. It also greatly
reduces the amount of powerpc-specific glue code that is needed, and it
fixes the issue where this optimized GHASH code was disabled by default.
Note that previously the C code defined the POWER8 GHASH key format as
"u128 htable[16]", despite the assembly code only using four entries.
Fix the C code to use the correct key format. To fulfill the library
API contract, also make the key preparation work in all contexts.
Note that the POWER8 assembly code takes the accumulator in GHASH
format, but it actually byte-reflects it to get it into POLYVAL format.
The library already works with POLYVAL natively. For now, just wire up
this existing code by converting it to/from GHASH format in C code.
This should be cleaned up to eliminate the unnecessary conversion later.
Acked-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20260319061723.1140720-12-ebiggers@kernel.org
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
+143
-186
+2
-2
MAINTAINERS
+2
-2
MAINTAINERS
···
12267
12267
F: arch/powerpc/crypto/aes_ctr.c
12268
12268
F: arch/powerpc/crypto/aes_xts.c
12269
12269
F: arch/powerpc/crypto/aesp8-ppc.*
12270
-
F: arch/powerpc/crypto/ghash.c
12271
-
F: arch/powerpc/crypto/ghashp8-ppc.pl
12272
12270
F: arch/powerpc/crypto/ppc-xlate.pl
12273
12271
F: arch/powerpc/crypto/vmx.c
12272
+
F: lib/crypto/powerpc/gf128hash.h
12273
+
F: lib/crypto/powerpc/ghashp8-ppc.pl
12274
12274
12275
12275
IBM ServeRAID RAID DRIVER
12276
12276
S: Orphan
+2
-3
arch/powerpc/crypto/Kconfig
+2
-3
arch/powerpc/crypto/Kconfig
···
54
54
select CRYPTO_AES
55
55
select CRYPTO_CBC
56
56
select CRYPTO_CTR
57
-
select CRYPTO_GHASH
58
57
select CRYPTO_XTS
59
58
default m
60
59
help
61
60
Support for VMX cryptographic acceleration instructions on Power8 CPU.
62
-
This module supports acceleration for AES and GHASH in hardware. If you
63
-
choose 'M' here, this module will be called vmx-crypto.
61
+
This module supports acceleration for AES in hardware. If you choose
62
+
'M' here, this module will be called vmx-crypto.
64
63
65
64
endmenu
+2
-6
arch/powerpc/crypto/Makefile
+2
-6
arch/powerpc/crypto/Makefile
···
11
11
12
12
aes-ppc-spe-y := aes-spe-glue.o
13
13
aes-gcm-p10-crypto-y := aes-gcm-p10-glue.o aes-gcm-p10.o ghashp10-ppc.o aesp10-ppc.o
14
-
vmx-crypto-objs := vmx.o ghashp8-ppc.o aes_cbc.o aes_ctr.o aes_xts.o ghash.o
14
+
vmx-crypto-objs := vmx.o aes_cbc.o aes_ctr.o aes_xts.o
15
15
16
16
ifeq ($(CONFIG_CPU_LITTLE_ENDIAN),y)
17
17
override flavour := linux-ppc64le
···
26
26
quiet_cmd_perl = PERL $@
27
27
cmd_perl = $(PERL) $< $(flavour) > $@
28
28
29
-
targets += aesp10-ppc.S ghashp10-ppc.S ghashp8-ppc.S
29
+
targets += aesp10-ppc.S ghashp10-ppc.S
30
30
31
31
$(obj)/aesp10-ppc.S $(obj)/ghashp10-ppc.S: $(obj)/%.S: $(src)/%.pl FORCE
32
32
$(call if_changed,perl)
33
33
34
-
$(obj)/ghashp8-ppc.S: $(obj)/%.S: $(src)/%.pl FORCE
35
-
$(call if_changed,perl)
36
-
37
34
OBJECT_FILES_NON_STANDARD_aesp10-ppc.o := y
38
35
OBJECT_FILES_NON_STANDARD_ghashp10-ppc.o := y
39
-
OBJECT_FILES_NON_STANDARD_ghashp8-ppc.o := y
-1
arch/powerpc/crypto/aesp8-ppc.h
-1
arch/powerpc/crypto/aesp8-ppc.h
-160
arch/powerpc/crypto/ghash.c
-160
arch/powerpc/crypto/ghash.c
···
1
-
// SPDX-License-Identifier: GPL-2.0
2
-
/*
3
-
* GHASH routines supporting VMX instructions on the Power 8
4
-
*
5
-
* Copyright (C) 2015, 2019 International Business Machines Inc.
6
-
*
7
-
* Author: Marcelo Henrique Cerri <mhcerri@br.ibm.com>
8
-
*
9
-
* Extended by Daniel Axtens <dja@axtens.net> to replace the fallback
10
-
* mechanism. The new approach is based on arm64 code, which is:
11
-
* Copyright (C) 2014 - 2018 Linaro Ltd. <ard.biesheuvel@linaro.org>
12
-
*/
13
-
14
-
#include "aesp8-ppc.h"
15
-
#include <asm/switch_to.h>
16
-
#include <crypto/aes.h>
17
-
#include <crypto/gf128mul.h>
18
-
#include <crypto/ghash.h>
19
-
#include <crypto/internal/hash.h>
20
-
#include <crypto/internal/simd.h>
21
-
#include <linux/err.h>
22
-
#include <linux/kernel.h>
23
-
#include <linux/module.h>
24
-
#include <linux/string.h>
25
-
#include <linux/uaccess.h>
26
-
27
-
void gcm_init_p8(u128 htable[16], const u64 Xi[2]);
28
-
void gcm_gmult_p8(u64 Xi[2], const u128 htable[16]);
29
-
void gcm_ghash_p8(u64 Xi[2], const u128 htable[16],
30
-
const u8 *in, size_t len);
31
-
32
-
struct p8_ghash_ctx {
33
-
/* key used by vector asm */
34
-
u128 htable[16];
35
-
/* key used by software fallback */
36
-
be128 key;
37
-
};
38
-
39
-
struct p8_ghash_desc_ctx {
40
-
u64 shash[2];
41
-
};
42
-
43
-
static int p8_ghash_init(struct shash_desc *desc)
44
-
{
45
-
struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
46
-
47
-
memset(dctx->shash, 0, GHASH_DIGEST_SIZE);
48
-
return 0;
49
-
}
50
-
51
-
static int p8_ghash_setkey(struct crypto_shash *tfm, const u8 *key,
52
-
unsigned int keylen)
53
-
{
54
-
struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(tfm));
55
-
56
-
if (keylen != GHASH_BLOCK_SIZE)
57
-
return -EINVAL;
58
-
59
-
preempt_disable();
60
-
pagefault_disable();
61
-
enable_kernel_vsx();
62
-
gcm_init_p8(ctx->htable, (const u64 *) key);
63
-
disable_kernel_vsx();
64
-
pagefault_enable();
65
-
preempt_enable();
66
-
67
-
memcpy(&ctx->key, key, GHASH_BLOCK_SIZE);
68
-
69
-
return 0;
70
-
}
71
-
72
-
static inline void __ghash_block(struct p8_ghash_ctx *ctx,
73
-
struct p8_ghash_desc_ctx *dctx,
74
-
const u8 *src)
75
-
{
76
-
if (crypto_simd_usable()) {
77
-
preempt_disable();
78
-
pagefault_disable();
79
-
enable_kernel_vsx();
80
-
gcm_ghash_p8(dctx->shash, ctx->htable, src, GHASH_BLOCK_SIZE);
81
-
disable_kernel_vsx();
82
-
pagefault_enable();
83
-
preempt_enable();
84
-
} else {
85
-
crypto_xor((u8 *)dctx->shash, src, GHASH_BLOCK_SIZE);
86
-
gf128mul_lle((be128 *)dctx->shash, &ctx->key);
87
-
}
88
-
}
89
-
90
-
static inline int __ghash_blocks(struct p8_ghash_ctx *ctx,
91
-
struct p8_ghash_desc_ctx *dctx,
92
-
const u8 *src, unsigned int srclen)
93
-
{
94
-
int remain = srclen - round_down(srclen, GHASH_BLOCK_SIZE);
95
-
96
-
srclen -= remain;
97
-
if (crypto_simd_usable()) {
98
-
preempt_disable();
99
-
pagefault_disable();
100
-
enable_kernel_vsx();
101
-
gcm_ghash_p8(dctx->shash, ctx->htable,
102
-
src, srclen);
103
-
disable_kernel_vsx();
104
-
pagefault_enable();
105
-
preempt_enable();
106
-
} else {
107
-
do {
108
-
crypto_xor((u8 *)dctx->shash, src, GHASH_BLOCK_SIZE);
109
-
gf128mul_lle((be128 *)dctx->shash, &ctx->key);
110
-
srclen -= GHASH_BLOCK_SIZE;
111
-
src += GHASH_BLOCK_SIZE;
112
-
} while (srclen);
113
-
}
114
-
115
-
return remain;
116
-
}
117
-
118
-
static int p8_ghash_update(struct shash_desc *desc,
119
-
const u8 *src, unsigned int srclen)
120
-
{
121
-
struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(desc->tfm));
122
-
struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
123
-
124
-
return __ghash_blocks(ctx, dctx, src, srclen);
125
-
}
126
-
127
-
static int p8_ghash_finup(struct shash_desc *desc, const u8 *src,
128
-
unsigned int len, u8 *out)
129
-
{
130
-
struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(desc->tfm));
131
-
struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
132
-
133
-
if (len) {
134
-
u8 buf[GHASH_BLOCK_SIZE] = {};
135
-
136
-
memcpy(buf, src, len);
137
-
__ghash_block(ctx, dctx, buf);
138
-
memzero_explicit(buf, sizeof(buf));
139
-
}
140
-
memcpy(out, dctx->shash, GHASH_DIGEST_SIZE);
141
-
return 0;
142
-
}
143
-
144
-
struct shash_alg p8_ghash_alg = {
145
-
.digestsize = GHASH_DIGEST_SIZE,
146
-
.init = p8_ghash_init,
147
-
.update = p8_ghash_update,
148
-
.finup = p8_ghash_finup,
149
-
.setkey = p8_ghash_setkey,
150
-
.descsize = sizeof(struct p8_ghash_desc_ctx),
151
-
.base = {
152
-
.cra_name = "ghash",
153
-
.cra_driver_name = "p8_ghash",
154
-
.cra_priority = 1000,
155
-
.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY,
156
-
.cra_blocksize = GHASH_BLOCK_SIZE,
157
-
.cra_ctxsize = sizeof(struct p8_ghash_ctx),
158
-
.cra_module = THIS_MODULE,
159
-
},
160
-
};
+1
arch/powerpc/crypto/ghashp8-ppc.pl
lib/crypto/powerpc/ghashp8-ppc.pl
+1
arch/powerpc/crypto/ghashp8-ppc.pl
lib/crypto/powerpc/ghashp8-ppc.pl
···
47
47
$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
48
48
( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
49
49
( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
50
+
( $xlate="${dir}../../../arch/powerpc/crypto/ppc-xlate.pl" and -f $xlate) or
50
51
die "can't locate ppc-xlate.pl";
51
52
52
53
open STDOUT,"| $^X $xlate $flavour $output" || die "can't call $xlate: $!";
+1
-9
arch/powerpc/crypto/vmx.c
+1
-9
arch/powerpc/crypto/vmx.c
···
14
14
#include <linux/cpufeature.h>
15
15
#include <linux/crypto.h>
16
16
#include <asm/cputable.h>
17
-
#include <crypto/internal/hash.h>
18
17
#include <crypto/internal/skcipher.h>
19
18
20
19
#include "aesp8-ppc.h"
···
22
23
{
23
24
int ret;
24
25
25
-
ret = crypto_register_shash(&p8_ghash_alg);
26
-
if (ret)
27
-
goto err;
28
-
29
26
ret = crypto_register_skcipher(&p8_aes_cbc_alg);
30
27
if (ret)
31
-
goto err_unregister_ghash;
28
+
goto err;
32
29
33
30
ret = crypto_register_skcipher(&p8_aes_ctr_alg);
34
31
if (ret)
···
40
45
crypto_unregister_skcipher(&p8_aes_ctr_alg);
41
46
err_unregister_aes_cbc:
42
47
crypto_unregister_skcipher(&p8_aes_cbc_alg);
43
-
err_unregister_ghash:
44
-
crypto_unregister_shash(&p8_ghash_alg);
45
48
err:
46
49
return ret;
47
50
}
···
49
56
crypto_unregister_skcipher(&p8_aes_xts_alg);
50
57
crypto_unregister_skcipher(&p8_aes_ctr_alg);
51
58
crypto_unregister_skcipher(&p8_aes_cbc_alg);
52
-
crypto_unregister_shash(&p8_ghash_alg);
53
59
}
54
60
55
61
module_cpu_feature_match(PPC_MODULE_FEATURE_VEC_CRYPTO, p8_init);
+4
include/crypto/gf128hash.h
+4
include/crypto/gf128hash.h
···
41
41
* Use ghash_preparekey() to initialize this.
42
42
*/
43
43
struct ghash_key {
44
+
#if defined(CONFIG_CRYPTO_LIB_GF128HASH_ARCH) && defined(CONFIG_PPC64)
45
+
/** @htable: GHASH key format used by the POWER8 assembly code */
46
+
u64 htable[4][2];
47
+
#endif
44
48
/** @h: The hash key H, in POLYVAL format */
45
49
struct polyval_elem h;
46
50
};
+1
lib/crypto/Kconfig
+1
lib/crypto/Kconfig
+20
-5
lib/crypto/Makefile
+20
-5
lib/crypto/Makefile
···
8
8
quiet_cmd_perlasm_with_args = PERLASM $@
9
9
cmd_perlasm_with_args = $(PERL) $(<) void $(@)
10
10
11
+
ppc64-perlasm-flavour-y := linux-ppc64
12
+
ppc64-perlasm-flavour-$(CONFIG_PPC64_ELF_ABI_V2) := linux-ppc64-elfv2
13
+
ppc64-perlasm-flavour-$(CONFIG_CPU_LITTLE_ENDIAN) := linux-ppc64le
14
+
11
15
obj-$(CONFIG_KUNIT) += tests/
12
16
13
17
obj-$(CONFIG_CRYPTO_HASH_INFO) += hash_info.o
···
40
36
powerpc/aes-tab-4k.o
41
37
else
42
38
libaes-y += powerpc/aesp8-ppc.o
43
-
aes-perlasm-flavour-y := linux-ppc64
44
-
aes-perlasm-flavour-$(CONFIG_PPC64_ELF_ABI_V2) := linux-ppc64-elfv2
45
-
aes-perlasm-flavour-$(CONFIG_CPU_LITTLE_ENDIAN) := linux-ppc64le
46
39
quiet_cmd_perlasm_aes = PERLASM $@
47
-
cmd_perlasm_aes = $(PERL) $< $(aes-perlasm-flavour-y) $@
40
+
cmd_perlasm_aes = $(PERL) $< $(ppc64-perlasm-flavour-y) $@
48
41
# Use if_changed instead of cmd, in case the flavour changed.
49
42
$(obj)/powerpc/aesp8-ppc.S: $(src)/powerpc/aesp8-ppc.pl FORCE
50
43
$(call if_changed,perlasm_aes)
···
162
161
libgf128hash-$(CONFIG_ARM) += arm/ghash-neon-core.o
163
162
libgf128hash-$(CONFIG_ARM64) += arm64/ghash-neon-core.o \
164
163
arm64/polyval-ce-core.o
165
-
libgf128hash-$(CONFIG_X86) += x86/polyval-pclmul-avx.o
164
+
165
+
ifeq ($(CONFIG_PPC),y)
166
+
libgf128hash-y += powerpc/ghashp8-ppc.o
167
+
quiet_cmd_perlasm_ghash = PERLASM $@
168
+
cmd_perlasm_ghash = $(PERL) $< $(ppc64-perlasm-flavour-y) $@
169
+
$(obj)/powerpc/ghashp8-ppc.S: $(src)/powerpc/ghashp8-ppc.pl FORCE
170
+
$(call if_changed,perlasm_ghash)
171
+
targets += powerpc/ghashp8-ppc.S
172
+
OBJECT_FILES_NON_STANDARD_powerpc/ghashp8-ppc.o := y
166
173
endif
174
+
175
+
libgf128hash-$(CONFIG_X86) += x86/polyval-pclmul-avx.o
176
+
endif # CONFIG_CRYPTO_LIB_GF128HASH_ARCH
177
+
178
+
# clean-files must be defined unconditionally
179
+
clean-files += powerpc/ghashp8-ppc.S
167
180
168
181
################################################################################
169
182
+1
lib/crypto/powerpc/.gitignore
+1
lib/crypto/powerpc/.gitignore
+109
lib/crypto/powerpc/gf128hash.h
+109
lib/crypto/powerpc/gf128hash.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0 */
2
+
/*
3
+
* GHASH routines supporting VMX instructions on the Power 8
4
+
*
5
+
* Copyright (C) 2015, 2019 International Business Machines Inc.
6
+
* Copyright (C) 2014 - 2018 Linaro Ltd.
7
+
* Copyright 2026 Google LLC
8
+
*/
9
+
10
+
#include <asm/simd.h>
11
+
#include <asm/switch_to.h>
12
+
#include <linux/cpufeature.h>
13
+
#include <linux/jump_label.h>
14
+
#include <linux/preempt.h>
15
+
#include <linux/uaccess.h>
16
+
17
+
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_vec_crypto);
18
+
19
+
void gcm_init_p8(u64 htable[4][2], const u8 h[16]);
20
+
void gcm_gmult_p8(u8 Xi[16], const u64 htable[4][2]);
21
+
void gcm_ghash_p8(u8 Xi[16], const u64 htable[4][2], const u8 *in, size_t len);
22
+
23
+
#define ghash_preparekey_arch ghash_preparekey_arch
24
+
static void ghash_preparekey_arch(struct ghash_key *key,
25
+
const u8 raw_key[GHASH_BLOCK_SIZE])
26
+
{
27
+
ghash_key_to_polyval(raw_key, &key->h);
28
+
29
+
if (static_branch_likely(&have_vec_crypto) && likely(may_use_simd())) {
30
+
preempt_disable();
31
+
pagefault_disable();
32
+
enable_kernel_vsx();
33
+
gcm_init_p8(key->htable, raw_key);
34
+
disable_kernel_vsx();
35
+
pagefault_enable();
36
+
preempt_enable();
37
+
} else {
38
+
/* This reproduces gcm_init_p8() on both LE and BE systems. */
39
+
key->htable[0][0] = 0;
40
+
key->htable[0][1] = 0xc200000000000000;
41
+
42
+
key->htable[1][0] = 0;
43
+
key->htable[1][1] = le64_to_cpu(key->h.lo);
44
+
45
+
key->htable[2][0] = le64_to_cpu(key->h.lo);
46
+
key->htable[2][1] = le64_to_cpu(key->h.hi);
47
+
48
+
key->htable[3][0] = le64_to_cpu(key->h.hi);
49
+
key->htable[3][1] = 0;
50
+
}
51
+
}
52
+
53
+
#define ghash_mul_arch ghash_mul_arch
54
+
static void ghash_mul_arch(struct polyval_elem *acc,
55
+
const struct ghash_key *key)
56
+
{
57
+
if (static_branch_likely(&have_vec_crypto) && likely(may_use_simd())) {
58
+
u8 ghash_acc[GHASH_BLOCK_SIZE];
59
+
60
+
polyval_acc_to_ghash(acc, ghash_acc);
61
+
62
+
preempt_disable();
63
+
pagefault_disable();
64
+
enable_kernel_vsx();
65
+
gcm_gmult_p8(ghash_acc, key->htable);
66
+
disable_kernel_vsx();
67
+
pagefault_enable();
68
+
preempt_enable();
69
+
70
+
ghash_acc_to_polyval(ghash_acc, acc);
71
+
memzero_explicit(ghash_acc, sizeof(ghash_acc));
72
+
} else {
73
+
polyval_mul_generic(acc, &key->h);
74
+
}
75
+
}
76
+
77
+
#define ghash_blocks_arch ghash_blocks_arch
78
+
static void ghash_blocks_arch(struct polyval_elem *acc,
79
+
const struct ghash_key *key,
80
+
const u8 *data, size_t nblocks)
81
+
{
82
+
if (static_branch_likely(&have_vec_crypto) && likely(may_use_simd())) {
83
+
u8 ghash_acc[GHASH_BLOCK_SIZE];
84
+
85
+
polyval_acc_to_ghash(acc, ghash_acc);
86
+
87
+
preempt_disable();
88
+
pagefault_disable();
89
+
enable_kernel_vsx();
90
+
gcm_ghash_p8(ghash_acc, key->htable, data,
91
+
nblocks * GHASH_BLOCK_SIZE);
92
+
disable_kernel_vsx();
93
+
pagefault_enable();
94
+
preempt_enable();
95
+
96
+
ghash_acc_to_polyval(ghash_acc, acc);
97
+
memzero_explicit(ghash_acc, sizeof(ghash_acc));
98
+
} else {
99
+
ghash_blocks_generic(acc, &key->h, data, nblocks);
100
+
}
101
+
}
102
+
103
+
#define gf128hash_mod_init_arch gf128hash_mod_init_arch
104
+
static void gf128hash_mod_init_arch(void)
105
+
{
106
+
if (cpu_has_feature(CPU_FTR_ARCH_207S) &&
107
+
(cur_cpu_spec->cpu_user_features2 & PPC_FEATURE2_VEC_CRYPTO))
108
+
static_branch_enable(&have_vec_crypto);
109
+
}