Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
powerpc/crc32: expose CRC32 functions through lib
Move the powerpc CRC32C assembly code into the lib directory and wire it
up to the library interface. This allows it to be used without going
through the crypto API. It remains usable via the crypto API too via
the shash algorithms that use the library interface. Thus all the
arch-specific "shash" code becomes unnecessary and is removed.
Note: to see the diff from arch/powerpc/crypto/crc32c-vpmsum_glue.c to
arch/powerpc/lib/crc32-glue.c, view this commit with 'git show -M10'.
Reviewed-by: Ard Biesheuvel <ardb@kernel.org>
Acked-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://lore.kernel.org/r/20241202010844.144356-9-ebiggers@kernel.org
Signed-off-by: Eric Biggers <ebiggers@google.com>
+98
-192
+1
arch/powerpc/Kconfig
+1
arch/powerpc/Kconfig
···
127
127
select ARCH_ENABLE_MEMORY_HOTPLUG
128
128
select ARCH_ENABLE_MEMORY_HOTREMOVE
129
129
select ARCH_HAS_COPY_MC if PPC64
130
+
select ARCH_HAS_CRC32 if PPC64 && ALTIVEC
130
131
select ARCH_HAS_CURRENT_STACK_POINTER
131
132
select ARCH_HAS_DEBUG_VIRTUAL
132
133
select ARCH_HAS_DEBUG_VM_PGTABLE
-1
arch/powerpc/configs/powernv_defconfig
-1
arch/powerpc/configs/powernv_defconfig
-1
arch/powerpc/configs/ppc64_defconfig
-1
arch/powerpc/configs/ppc64_defconfig
+1
-14
arch/powerpc/crypto/Kconfig
+1
-14
arch/powerpc/crypto/Kconfig
···
13
13
Architecture: PowerPC64
14
14
- Little-endian
15
15
16
-
config CRYPTO_CRC32C_VPMSUM
17
-
tristate "CRC32c"
18
-
depends on PPC64 && ALTIVEC
19
-
select CRYPTO_HASH
20
-
select CRC32
21
-
help
22
-
CRC32c CRC algorithm with the iSCSI polynomial (RFC 3385 and RFC 3720)
23
-
24
-
Architecture: powerpc64 using
25
-
- AltiVec extensions
26
-
27
-
Enable on POWER8 and newer processors for improved performance.
28
-
29
16
config CRYPTO_CRCT10DIF_VPMSUM
30
17
tristate "CRC32T10DIF"
31
18
depends on PPC64 && ALTIVEC && CRC_T10DIF
···
27
40
28
41
config CRYPTO_VPMSUM_TESTER
29
42
tristate "CRC32c and CRC32T10DIF hardware acceleration tester"
30
-
depends on CRYPTO_CRCT10DIF_VPMSUM && CRYPTO_CRC32C_VPMSUM
43
+
depends on CRYPTO_CRCT10DIF_VPMSUM && CRYPTO_CRC32C && CRC32_ARCH
31
44
help
32
45
Stress test for CRC32c and CRCT10DIF algorithms implemented with
33
46
powerpc64 AltiVec extensions (POWER8 vpmsum instructions).
-2
arch/powerpc/crypto/Makefile
-2
arch/powerpc/crypto/Makefile
···
10
10
obj-$(CONFIG_CRYPTO_SHA1_PPC) += sha1-powerpc.o
11
11
obj-$(CONFIG_CRYPTO_SHA1_PPC_SPE) += sha1-ppc-spe.o
12
12
obj-$(CONFIG_CRYPTO_SHA256_PPC_SPE) += sha256-ppc-spe.o
13
-
obj-$(CONFIG_CRYPTO_CRC32C_VPMSUM) += crc32c-vpmsum.o
14
13
obj-$(CONFIG_CRYPTO_CRCT10DIF_VPMSUM) += crct10dif-vpmsum.o
15
14
obj-$(CONFIG_CRYPTO_VPMSUM_TESTER) += crc-vpmsum_test.o
16
15
obj-$(CONFIG_CRYPTO_AES_GCM_P10) += aes-gcm-p10-crypto.o
···
23
24
sha1-powerpc-y := sha1-powerpc-asm.o sha1.o
24
25
sha1-ppc-spe-y := sha1-spe-asm.o sha1-spe-glue.o
25
26
sha256-ppc-spe-y := sha256-spe-asm.o sha256-spe-glue.o
26
-
crc32c-vpmsum-y := crc32c-vpmsum_asm.o crc32c-vpmsum_glue.o
27
27
crct10dif-vpmsum-y := crct10dif-vpmsum_asm.o crct10dif-vpmsum_glue.o
28
28
aes-gcm-p10-crypto-y := aes-gcm-p10-glue.o aes-gcm-p10.o ghashp10-ppc.o aesp10-ppc.o
29
29
chacha-p10-crypto-y := chacha-p10-glue.o chacha-p10le-8x.o
arch/powerpc/crypto/crc32-vpmsum_core.S
arch/powerpc/lib/crc32-vpmsum_core.S
arch/powerpc/crypto/crc32-vpmsum_core.S
arch/powerpc/lib/crc32-vpmsum_core.S
arch/powerpc/crypto/crc32c-vpmsum_asm.S
arch/powerpc/lib/crc32c-vpmsum_asm.S
arch/powerpc/crypto/crc32c-vpmsum_asm.S
arch/powerpc/lib/crc32c-vpmsum_asm.S
-173
arch/powerpc/crypto/crc32c-vpmsum_glue.c
-173
arch/powerpc/crypto/crc32c-vpmsum_glue.c
···
1
-
// SPDX-License-Identifier: GPL-2.0-only
2
-
#include <linux/crc32.h>
3
-
#include <crypto/internal/hash.h>
4
-
#include <crypto/internal/simd.h>
5
-
#include <linux/init.h>
6
-
#include <linux/module.h>
7
-
#include <linux/string.h>
8
-
#include <linux/kernel.h>
9
-
#include <linux/cpufeature.h>
10
-
#include <asm/simd.h>
11
-
#include <asm/switch_to.h>
12
-
13
-
#define CHKSUM_BLOCK_SIZE 1
14
-
#define CHKSUM_DIGEST_SIZE 4
15
-
16
-
#define VMX_ALIGN 16
17
-
#define VMX_ALIGN_MASK (VMX_ALIGN-1)
18
-
19
-
#define VECTOR_BREAKPOINT 512
20
-
21
-
u32 __crc32c_vpmsum(u32 crc, unsigned char const *p, size_t len);
22
-
23
-
static u32 crc32c_vpmsum(u32 crc, unsigned char const *p, size_t len)
24
-
{
25
-
unsigned int prealign;
26
-
unsigned int tail;
27
-
28
-
if (len < (VECTOR_BREAKPOINT + VMX_ALIGN) || !crypto_simd_usable())
29
-
return __crc32c_le(crc, p, len);
30
-
31
-
if ((unsigned long)p & VMX_ALIGN_MASK) {
32
-
prealign = VMX_ALIGN - ((unsigned long)p & VMX_ALIGN_MASK);
33
-
crc = __crc32c_le(crc, p, prealign);
34
-
len -= prealign;
35
-
p += prealign;
36
-
}
37
-
38
-
if (len & ~VMX_ALIGN_MASK) {
39
-
preempt_disable();
40
-
pagefault_disable();
41
-
enable_kernel_altivec();
42
-
crc = __crc32c_vpmsum(crc, p, len & ~VMX_ALIGN_MASK);
43
-
disable_kernel_altivec();
44
-
pagefault_enable();
45
-
preempt_enable();
46
-
}
47
-
48
-
tail = len & VMX_ALIGN_MASK;
49
-
if (tail) {
50
-
p += len & ~VMX_ALIGN_MASK;
51
-
crc = __crc32c_le(crc, p, tail);
52
-
}
53
-
54
-
return crc;
55
-
}
56
-
57
-
static int crc32c_vpmsum_cra_init(struct crypto_tfm *tfm)
58
-
{
59
-
u32 *key = crypto_tfm_ctx(tfm);
60
-
61
-
*key = ~0;
62
-
63
-
return 0;
64
-
}
65
-
66
-
/*
67
-
* Setting the seed allows arbitrary accumulators and flexible XOR policy
68
-
* If your algorithm starts with ~0, then XOR with ~0 before you set
69
-
* the seed.
70
-
*/
71
-
static int crc32c_vpmsum_setkey(struct crypto_shash *hash, const u8 *key,
72
-
unsigned int keylen)
73
-
{
74
-
u32 *mctx = crypto_shash_ctx(hash);
75
-
76
-
if (keylen != sizeof(u32))
77
-
return -EINVAL;
78
-
*mctx = le32_to_cpup((__le32 *)key);
79
-
return 0;
80
-
}
81
-
82
-
static int crc32c_vpmsum_init(struct shash_desc *desc)
83
-
{
84
-
u32 *mctx = crypto_shash_ctx(desc->tfm);
85
-
u32 *crcp = shash_desc_ctx(desc);
86
-
87
-
*crcp = *mctx;
88
-
89
-
return 0;
90
-
}
91
-
92
-
static int crc32c_vpmsum_update(struct shash_desc *desc, const u8 *data,
93
-
unsigned int len)
94
-
{
95
-
u32 *crcp = shash_desc_ctx(desc);
96
-
97
-
*crcp = crc32c_vpmsum(*crcp, data, len);
98
-
99
-
return 0;
100
-
}
101
-
102
-
static int __crc32c_vpmsum_finup(u32 *crcp, const u8 *data, unsigned int len,
103
-
u8 *out)
104
-
{
105
-
*(__le32 *)out = ~cpu_to_le32(crc32c_vpmsum(*crcp, data, len));
106
-
107
-
return 0;
108
-
}
109
-
110
-
static int crc32c_vpmsum_finup(struct shash_desc *desc, const u8 *data,
111
-
unsigned int len, u8 *out)
112
-
{
113
-
return __crc32c_vpmsum_finup(shash_desc_ctx(desc), data, len, out);
114
-
}
115
-
116
-
static int crc32c_vpmsum_final(struct shash_desc *desc, u8 *out)
117
-
{
118
-
u32 *crcp = shash_desc_ctx(desc);
119
-
120
-
*(__le32 *)out = ~cpu_to_le32p(crcp);
121
-
122
-
return 0;
123
-
}
124
-
125
-
static int crc32c_vpmsum_digest(struct shash_desc *desc, const u8 *data,
126
-
unsigned int len, u8 *out)
127
-
{
128
-
return __crc32c_vpmsum_finup(crypto_shash_ctx(desc->tfm), data, len,
129
-
out);
130
-
}
131
-
132
-
static struct shash_alg alg = {
133
-
.setkey = crc32c_vpmsum_setkey,
134
-
.init = crc32c_vpmsum_init,
135
-
.update = crc32c_vpmsum_update,
136
-
.final = crc32c_vpmsum_final,
137
-
.finup = crc32c_vpmsum_finup,
138
-
.digest = crc32c_vpmsum_digest,
139
-
.descsize = sizeof(u32),
140
-
.digestsize = CHKSUM_DIGEST_SIZE,
141
-
.base = {
142
-
.cra_name = "crc32c",
143
-
.cra_driver_name = "crc32c-vpmsum",
144
-
.cra_priority = 200,
145
-
.cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
146
-
.cra_blocksize = CHKSUM_BLOCK_SIZE,
147
-
.cra_ctxsize = sizeof(u32),
148
-
.cra_module = THIS_MODULE,
149
-
.cra_init = crc32c_vpmsum_cra_init,
150
-
}
151
-
};
152
-
153
-
static int __init crc32c_vpmsum_mod_init(void)
154
-
{
155
-
if (!cpu_has_feature(CPU_FTR_ARCH_207S))
156
-
return -ENODEV;
157
-
158
-
return crypto_register_shash(&alg);
159
-
}
160
-
161
-
static void __exit crc32c_vpmsum_mod_fini(void)
162
-
{
163
-
crypto_unregister_shash(&alg);
164
-
}
165
-
166
-
module_cpu_feature_match(PPC_MODULE_FEATURE_VEC_CRYPTO, crc32c_vpmsum_mod_init);
167
-
module_exit(crc32c_vpmsum_mod_fini);
168
-
169
-
MODULE_AUTHOR("Anton Blanchard <anton@samba.org>");
170
-
MODULE_DESCRIPTION("CRC32C using vector polynomial multiply-sum instructions");
171
-
MODULE_LICENSE("GPL");
172
-
MODULE_ALIAS_CRYPTO("crc32c");
173
-
MODULE_ALIAS_CRYPTO("crc32c-vpmsum");
+1
-1
arch/powerpc/crypto/crct10dif-vpmsum_asm.S
+1
-1
arch/powerpc/crypto/crct10dif-vpmsum_asm.S
+3
arch/powerpc/lib/Makefile
+3
arch/powerpc/lib/Makefile
+92
arch/powerpc/lib/crc32-glue.c
+92
arch/powerpc/lib/crc32-glue.c
···
1
+
// SPDX-License-Identifier: GPL-2.0-only
2
+
#include <linux/crc32.h>
3
+
#include <crypto/internal/simd.h>
4
+
#include <linux/init.h>
5
+
#include <linux/module.h>
6
+
#include <linux/kernel.h>
7
+
#include <linux/cpufeature.h>
8
+
#include <asm/simd.h>
9
+
#include <asm/switch_to.h>
10
+
11
+
#define VMX_ALIGN 16
12
+
#define VMX_ALIGN_MASK (VMX_ALIGN-1)
13
+
14
+
#define VECTOR_BREAKPOINT 512
15
+
16
+
static DEFINE_STATIC_KEY_FALSE(have_vec_crypto);
17
+
18
+
u32 __crc32c_vpmsum(u32 crc, const u8 *p, size_t len);
19
+
20
+
u32 crc32_le_arch(u32 crc, const u8 *p, size_t len)
21
+
{
22
+
return crc32_le_base(crc, p, len);
23
+
}
24
+
EXPORT_SYMBOL(crc32_le_arch);
25
+
26
+
u32 crc32c_le_arch(u32 crc, const u8 *p, size_t len)
27
+
{
28
+
unsigned int prealign;
29
+
unsigned int tail;
30
+
31
+
if (len < (VECTOR_BREAKPOINT + VMX_ALIGN) ||
32
+
!static_branch_likely(&have_vec_crypto) || !crypto_simd_usable())
33
+
return crc32c_le_base(crc, p, len);
34
+
35
+
if ((unsigned long)p & VMX_ALIGN_MASK) {
36
+
prealign = VMX_ALIGN - ((unsigned long)p & VMX_ALIGN_MASK);
37
+
crc = crc32c_le_base(crc, p, prealign);
38
+
len -= prealign;
39
+
p += prealign;
40
+
}
41
+
42
+
if (len & ~VMX_ALIGN_MASK) {
43
+
preempt_disable();
44
+
pagefault_disable();
45
+
enable_kernel_altivec();
46
+
crc = __crc32c_vpmsum(crc, p, len & ~VMX_ALIGN_MASK);
47
+
disable_kernel_altivec();
48
+
pagefault_enable();
49
+
preempt_enable();
50
+
}
51
+
52
+
tail = len & VMX_ALIGN_MASK;
53
+
if (tail) {
54
+
p += len & ~VMX_ALIGN_MASK;
55
+
crc = crc32c_le_base(crc, p, tail);
56
+
}
57
+
58
+
return crc;
59
+
}
60
+
EXPORT_SYMBOL(crc32c_le_arch);
61
+
62
+
u32 crc32_be_arch(u32 crc, const u8 *p, size_t len)
63
+
{
64
+
return crc32_be_base(crc, p, len);
65
+
}
66
+
EXPORT_SYMBOL(crc32_be_arch);
67
+
68
+
static int __init crc32_powerpc_init(void)
69
+
{
70
+
if (cpu_has_feature(CPU_FTR_ARCH_207S) &&
71
+
(cur_cpu_spec->cpu_user_features2 & PPC_FEATURE2_VEC_CRYPTO))
72
+
static_branch_enable(&have_vec_crypto);
73
+
return 0;
74
+
}
75
+
arch_initcall(crc32_powerpc_init);
76
+
77
+
static void __exit crc32_powerpc_exit(void)
78
+
{
79
+
}
80
+
module_exit(crc32_powerpc_exit);
81
+
82
+
u32 crc32_optimizations(void)
83
+
{
84
+
if (static_key_enabled(&have_vec_crypto))
85
+
return CRC32C_OPTIMIZATION;
86
+
return 0;
87
+
}
88
+
EXPORT_SYMBOL(crc32_optimizations);
89
+
90
+
MODULE_AUTHOR("Anton Blanchard <anton@samba.org>");
91
+
MODULE_DESCRIPTION("CRC32C using vector polynomial multiply-sum instructions");
92
+
MODULE_LICENSE("GPL");