Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
lib/crypto: s390/ghash: Migrate optimized code into library
Remove the "ghash-s390" crypto_shash algorithm, and replace it with an
implementation of ghash_blocks_arch() for the GHASH library.
This makes the GHASH library be optimized with CPACF. It also greatly
reduces the amount of s390-specific glue code that is needed, and it
fixes the issue where this GHASH optimization was disabled by default.
Acked-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20260319061723.1140720-14-ebiggers@kernel.org
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
+57
-158
-1
arch/s390/configs/debug_defconfig
-1
arch/s390/configs/debug_defconfig
-1
arch/s390/configs/defconfig
-1
arch/s390/configs/defconfig
-10
arch/s390/crypto/Kconfig
-10
arch/s390/crypto/Kconfig
···
2
2
3
3
menu "Accelerated Cryptographic Algorithms for CPU (s390)"
4
4
5
-
config CRYPTO_GHASH_S390
6
-
tristate "Hash functions: GHASH"
7
-
select CRYPTO_HASH
8
-
help
9
-
GCM GHASH hash function (NIST SP800-38D)
10
-
11
-
Architecture: s390
12
-
13
-
It is available as of z196.
14
-
15
5
config CRYPTO_AES_S390
16
6
tristate "Ciphers: AES, modes: ECB, CBC, CTR, XTS, GCM"
17
7
select CRYPTO_SKCIPHER
-1
arch/s390/crypto/Makefile
-1
arch/s390/crypto/Makefile
···
7
7
obj-$(CONFIG_CRYPTO_AES_S390) += aes_s390.o
8
8
obj-$(CONFIG_CRYPTO_PAES_S390) += paes_s390.o
9
9
obj-$(CONFIG_S390_PRNG) += prng.o
10
-
obj-$(CONFIG_CRYPTO_GHASH_S390) += ghash_s390.o
11
10
obj-$(CONFIG_CRYPTO_HMAC_S390) += hmac_s390.o
12
11
obj-$(CONFIG_CRYPTO_PHMAC_S390) += phmac_s390.o
13
12
obj-y += arch_random.o
-144
arch/s390/crypto/ghash_s390.c
-144
arch/s390/crypto/ghash_s390.c
···
1
-
// SPDX-License-Identifier: GPL-2.0
2
-
/*
3
-
* Cryptographic API.
4
-
*
5
-
* s390 implementation of the GHASH algorithm for GCM (Galois/Counter Mode).
6
-
*
7
-
* Copyright IBM Corp. 2011
8
-
* Author(s): Gerald Schaefer <gerald.schaefer@de.ibm.com>
9
-
*/
10
-
11
-
#include <asm/cpacf.h>
12
-
#include <crypto/ghash.h>
13
-
#include <crypto/internal/hash.h>
14
-
#include <linux/cpufeature.h>
15
-
#include <linux/err.h>
16
-
#include <linux/kernel.h>
17
-
#include <linux/module.h>
18
-
#include <linux/string.h>
19
-
20
-
struct s390_ghash_ctx {
21
-
u8 key[GHASH_BLOCK_SIZE];
22
-
};
23
-
24
-
struct s390_ghash_desc_ctx {
25
-
u8 icv[GHASH_BLOCK_SIZE];
26
-
u8 key[GHASH_BLOCK_SIZE];
27
-
};
28
-
29
-
static int ghash_init(struct shash_desc *desc)
30
-
{
31
-
struct s390_ghash_ctx *ctx = crypto_shash_ctx(desc->tfm);
32
-
struct s390_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
33
-
34
-
memset(dctx, 0, sizeof(*dctx));
35
-
memcpy(dctx->key, ctx->key, GHASH_BLOCK_SIZE);
36
-
37
-
return 0;
38
-
}
39
-
40
-
static int ghash_setkey(struct crypto_shash *tfm,
41
-
const u8 *key, unsigned int keylen)
42
-
{
43
-
struct s390_ghash_ctx *ctx = crypto_shash_ctx(tfm);
44
-
45
-
if (keylen != GHASH_BLOCK_SIZE)
46
-
return -EINVAL;
47
-
48
-
memcpy(ctx->key, key, GHASH_BLOCK_SIZE);
49
-
50
-
return 0;
51
-
}
52
-
53
-
static int ghash_update(struct shash_desc *desc,
54
-
const u8 *src, unsigned int srclen)
55
-
{
56
-
struct s390_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
57
-
unsigned int n;
58
-
59
-
n = srclen & ~(GHASH_BLOCK_SIZE - 1);
60
-
cpacf_kimd(CPACF_KIMD_GHASH, dctx, src, n);
61
-
return srclen - n;
62
-
}
63
-
64
-
static void ghash_flush(struct s390_ghash_desc_ctx *dctx, const u8 *src,
65
-
unsigned int len)
66
-
{
67
-
if (len) {
68
-
u8 buf[GHASH_BLOCK_SIZE] = {};
69
-
70
-
memcpy(buf, src, len);
71
-
cpacf_kimd(CPACF_KIMD_GHASH, dctx, buf, GHASH_BLOCK_SIZE);
72
-
memzero_explicit(buf, sizeof(buf));
73
-
}
74
-
}
75
-
76
-
static int ghash_finup(struct shash_desc *desc, const u8 *src,
77
-
unsigned int len, u8 *dst)
78
-
{
79
-
struct s390_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
80
-
81
-
ghash_flush(dctx, src, len);
82
-
memcpy(dst, dctx->icv, GHASH_BLOCK_SIZE);
83
-
return 0;
84
-
}
85
-
86
-
static int ghash_export(struct shash_desc *desc, void *out)
87
-
{
88
-
struct s390_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
89
-
90
-
memcpy(out, dctx->icv, GHASH_DIGEST_SIZE);
91
-
return 0;
92
-
}
93
-
94
-
static int ghash_import(struct shash_desc *desc, const void *in)
95
-
{
96
-
struct s390_ghash_ctx *ctx = crypto_shash_ctx(desc->tfm);
97
-
struct s390_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
98
-
99
-
memcpy(dctx->icv, in, GHASH_DIGEST_SIZE);
100
-
memcpy(dctx->key, ctx->key, GHASH_BLOCK_SIZE);
101
-
return 0;
102
-
}
103
-
104
-
static struct shash_alg ghash_alg = {
105
-
.digestsize = GHASH_DIGEST_SIZE,
106
-
.init = ghash_init,
107
-
.update = ghash_update,
108
-
.finup = ghash_finup,
109
-
.setkey = ghash_setkey,
110
-
.export = ghash_export,
111
-
.import = ghash_import,
112
-
.statesize = sizeof(struct ghash_desc_ctx),
113
-
.descsize = sizeof(struct s390_ghash_desc_ctx),
114
-
.base = {
115
-
.cra_name = "ghash",
116
-
.cra_driver_name = "ghash-s390",
117
-
.cra_priority = 300,
118
-
.cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY,
119
-
.cra_blocksize = GHASH_BLOCK_SIZE,
120
-
.cra_ctxsize = sizeof(struct s390_ghash_ctx),
121
-
.cra_module = THIS_MODULE,
122
-
},
123
-
};
124
-
125
-
static int __init ghash_mod_init(void)
126
-
{
127
-
if (!cpacf_query_func(CPACF_KIMD, CPACF_KIMD_GHASH))
128
-
return -ENODEV;
129
-
130
-
return crypto_register_shash(&ghash_alg);
131
-
}
132
-
133
-
static void __exit ghash_mod_exit(void)
134
-
{
135
-
crypto_unregister_shash(&ghash_alg);
136
-
}
137
-
138
-
module_cpu_feature_match(S390_CPU_FEATURE_MSA, ghash_mod_init);
139
-
module_exit(ghash_mod_exit);
140
-
141
-
MODULE_ALIAS_CRYPTO("ghash");
142
-
143
-
MODULE_LICENSE("GPL");
144
-
MODULE_DESCRIPTION("GHASH hash function, s390 implementation");
+2
-1
include/crypto/gf128hash.h
+2
-1
include/crypto/gf128hash.h
···
44
44
#if defined(CONFIG_CRYPTO_LIB_GF128HASH_ARCH) && defined(CONFIG_PPC64)
45
45
/** @htable: GHASH key format used by the POWER8 assembly code */
46
46
u64 htable[4][2];
47
-
#elif defined(CONFIG_CRYPTO_LIB_GF128HASH_ARCH) && defined(CONFIG_RISCV)
47
+
#elif defined(CONFIG_CRYPTO_LIB_GF128HASH_ARCH) && \
48
+
(defined(CONFIG_RISCV) || defined(CONFIG_S390))
48
49
/** @h_raw: The hash key H, in GHASH format */
49
50
u8 h_raw[GHASH_BLOCK_SIZE];
50
51
#endif
+1
lib/crypto/Kconfig
+1
lib/crypto/Kconfig
+54
lib/crypto/s390/gf128hash.h
+54
lib/crypto/s390/gf128hash.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0-or-later */
2
+
/*
3
+
* GHASH optimized using the CP Assist for Cryptographic Functions (CPACF)
4
+
*
5
+
* Copyright 2026 Google LLC
6
+
*/
7
+
#include <asm/cpacf.h>
8
+
#include <linux/cpufeature.h>
9
+
10
+
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_cpacf_ghash);
11
+
12
+
#define ghash_preparekey_arch ghash_preparekey_arch
13
+
static void ghash_preparekey_arch(struct ghash_key *key,
14
+
const u8 raw_key[GHASH_BLOCK_SIZE])
15
+
{
16
+
/* Save key in POLYVAL format for fallback */
17
+
ghash_key_to_polyval(raw_key, &key->h);
18
+
19
+
/* Save key in GHASH format for CPACF_KIMD_GHASH */
20
+
memcpy(key->h_raw, raw_key, GHASH_BLOCK_SIZE);
21
+
}
22
+
23
+
#define ghash_blocks_arch ghash_blocks_arch
24
+
static void ghash_blocks_arch(struct polyval_elem *acc,
25
+
const struct ghash_key *key,
26
+
const u8 *data, size_t nblocks)
27
+
{
28
+
if (static_branch_likely(&have_cpacf_ghash)) {
29
+
/*
30
+
* CPACF_KIMD_GHASH requires the accumulator and key in a single
31
+
* buffer, each using the GHASH convention.
32
+
*/
33
+
u8 ctx[2][GHASH_BLOCK_SIZE] __aligned(8);
34
+
35
+
polyval_acc_to_ghash(acc, ctx[0]);
36
+
memcpy(ctx[1], key->h_raw, GHASH_BLOCK_SIZE);
37
+
38
+
cpacf_kimd(CPACF_KIMD_GHASH, ctx, data,
39
+
nblocks * GHASH_BLOCK_SIZE);
40
+
41
+
ghash_acc_to_polyval(ctx[0], acc);
42
+
memzero_explicit(ctx, sizeof(ctx));
43
+
} else {
44
+
ghash_blocks_generic(acc, &key->h, data, nblocks);
45
+
}
46
+
}
47
+
48
+
#define gf128hash_mod_init_arch gf128hash_mod_init_arch
49
+
static void gf128hash_mod_init_arch(void)
50
+
{
51
+
if (cpu_have_feature(S390_CPU_FEATURE_MSA) &&
52
+
cpacf_query_func(CPACF_KIMD, CPACF_KIMD_GHASH))
53
+
static_branch_enable(&have_cpacf_ghash);
54
+
}