tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

arm/crc32: expose CRC32 functions through lib

Move the arm CRC32 assembly code into the lib directory and wire it up
to the library interface. This allows it to be used without going
through the crypto API. It remains usable via the crypto API too via
the shash algorithms that use the library interface. Thus all the
arch-specific "shash" code becomes unnecessary and is removed.

Note: to see the diff from arch/arm/crypto/crc32-ce-glue.c to
arch/arm/lib/crc32-glue.c, view this commit with 'git show -M10'.

Reviewed-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20241202010844.144356-6-ebiggers@kernel.org
Signed-off-by: Eric Biggers <ebiggers@google.com>

Eric Biggers 1e1b6dbc 780acb25

+129 -268
+1
arch/arm/Kconfig
··· 7 7 select ARCH_HAS_BINFMT_FLAT 8 8 select ARCH_HAS_CPU_CACHE_ALIASING 9 9 select ARCH_HAS_CPU_FINALIZE_INIT if MMU 10 + select ARCH_HAS_CRC32 if KERNEL_MODE_NEON 10 11 select ARCH_HAS_CURRENT_STACK_POINTER 11 12 select ARCH_HAS_DEBUG_VIRTUAL if MMU 12 13 select ARCH_HAS_DMA_ALLOC if MMU
-1
arch/arm/configs/milbeaut_m10v_defconfig
··· 107 107 CONFIG_CRYPTO_AES_ARM_BS=m 108 108 CONFIG_CRYPTO_AES_ARM_CE=m 109 109 CONFIG_CRYPTO_CHACHA20_NEON=m 110 - CONFIG_CRYPTO_CRC32_ARM_CE=m 111 110 # CONFIG_CRYPTO_HW is not set 112 111 CONFIG_CRC_CCITT=m 113 112 CONFIG_CRC_ITU_T=m
-1
arch/arm/configs/multi_v7_defconfig
··· 1306 1306 CONFIG_CRYPTO_AES_ARM_BS=m 1307 1307 CONFIG_CRYPTO_AES_ARM_CE=m 1308 1308 CONFIG_CRYPTO_CHACHA20_NEON=m 1309 - CONFIG_CRYPTO_CRC32_ARM_CE=m 1310 1309 CONFIG_CRYPTO_DEV_SUN4I_SS=m 1311 1310 CONFIG_CRYPTO_DEV_FSL_CAAM=m 1312 1311 CONFIG_CRYPTO_DEV_EXYNOS_RNG=m
-14
arch/arm/crypto/Kconfig
··· 222 222 Architecture: arm using: 223 223 - NEON (Advanced SIMD) extensions 224 224 225 - config CRYPTO_CRC32_ARM_CE 226 - tristate "CRC32C and CRC32" 227 - depends on KERNEL_MODE_NEON 228 - depends on CRC32 229 - select CRYPTO_HASH 230 - help 231 - CRC32c CRC algorithm with the iSCSI polynomial (RFC 3385 and RFC 3720) 232 - and CRC32 CRC algorithm (IEEE 802.3) 233 - 234 - Architecture: arm using: 235 - - CRC and/or PMULL instructions 236 - 237 - Drivers: crc32-arm-ce and crc32c-arm-ce 238 - 239 225 config CRYPTO_CRCT10DIF_ARM_CE 240 226 tristate "CRCT10DIF" 241 227 depends on KERNEL_MODE_NEON
-2
arch/arm/crypto/Makefile
··· 21 21 obj-$(CONFIG_CRYPTO_SHA2_ARM_CE) += sha2-arm-ce.o 22 22 obj-$(CONFIG_CRYPTO_GHASH_ARM_CE) += ghash-arm-ce.o 23 23 obj-$(CONFIG_CRYPTO_CRCT10DIF_ARM_CE) += crct10dif-arm-ce.o 24 - obj-$(CONFIG_CRYPTO_CRC32_ARM_CE) += crc32-arm-ce.o 25 24 26 25 aes-arm-y := aes-cipher-core.o aes-cipher-glue.o 27 26 aes-arm-bs-y := aes-neonbs-core.o aes-neonbs-glue.o ··· 37 38 aes-arm-ce-y := aes-ce-core.o aes-ce-glue.o 38 39 ghash-arm-ce-y := ghash-ce-core.o ghash-ce-glue.o 39 40 crct10dif-arm-ce-y := crct10dif-ce-core.o crct10dif-ce-glue.o 40 - crc32-arm-ce-y:= crc32-ce-core.o crc32-ce-glue.o 41 41 chacha-neon-y := chacha-scalar-core.o chacha-glue.o 42 42 chacha-neon-$(CONFIG_KERNEL_MODE_NEON) += chacha-neon-core.o 43 43 poly1305-arm-y := poly1305-core.o poly1305-glue.o
+2 -3
arch/arm/crypto/crc32-ce-core.S arch/arm/lib/crc32-core.S
··· 48 48 */ 49 49 50 50 #include <linux/linkage.h> 51 - #include <linux/cfi_types.h> 52 51 #include <asm/assembler.h> 53 52 54 53 .text ··· 296 297 .endm 297 298 298 299 .align 5 299 - SYM_TYPED_FUNC_START(crc32_armv8_le) 300 + SYM_FUNC_START(crc32_armv8_le) 300 301 __crc32 301 302 SYM_FUNC_END(crc32_armv8_le) 302 303 303 304 .align 5 304 - SYM_TYPED_FUNC_START(crc32c_armv8_le) 305 + SYM_FUNC_START(crc32c_armv8_le) 305 306 __crc32 c 306 307 SYM_FUNC_END(crc32c_armv8_le)
-247
arch/arm/crypto/crc32-ce-glue.c
··· 1 - // SPDX-License-Identifier: GPL-2.0-only 2 - /* 3 - * Accelerated CRC32(C) using ARM CRC, NEON and Crypto Extensions instructions 4 - * 5 - * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org> 6 - */ 7 - 8 - #include <linux/cpufeature.h> 9 - #include <linux/crc32.h> 10 - #include <linux/init.h> 11 - #include <linux/kernel.h> 12 - #include <linux/module.h> 13 - #include <linux/string.h> 14 - 15 - #include <crypto/internal/hash.h> 16 - #include <crypto/internal/simd.h> 17 - 18 - #include <asm/hwcap.h> 19 - #include <asm/neon.h> 20 - #include <asm/simd.h> 21 - #include <linux/unaligned.h> 22 - 23 - #define PMULL_MIN_LEN 64L /* minimum size of buffer 24 - * for crc32_pmull_le_16 */ 25 - #define SCALE_F 16L /* size of NEON register */ 26 - 27 - asmlinkage u32 crc32_pmull_le(const u8 buf[], u32 len, u32 init_crc); 28 - asmlinkage u32 crc32_armv8_le(u32 init_crc, const u8 buf[], u32 len); 29 - 30 - asmlinkage u32 crc32c_pmull_le(const u8 buf[], u32 len, u32 init_crc); 31 - asmlinkage u32 crc32c_armv8_le(u32 init_crc, const u8 buf[], u32 len); 32 - 33 - static u32 (*fallback_crc32)(u32 init_crc, const u8 buf[], u32 len); 34 - static u32 (*fallback_crc32c)(u32 init_crc, const u8 buf[], u32 len); 35 - 36 - static int crc32_cra_init(struct crypto_tfm *tfm) 37 - { 38 - u32 *key = crypto_tfm_ctx(tfm); 39 - 40 - *key = 0; 41 - return 0; 42 - } 43 - 44 - static int crc32c_cra_init(struct crypto_tfm *tfm) 45 - { 46 - u32 *key = crypto_tfm_ctx(tfm); 47 - 48 - *key = ~0; 49 - return 0; 50 - } 51 - 52 - static int crc32_setkey(struct crypto_shash *hash, const u8 *key, 53 - unsigned int keylen) 54 - { 55 - u32 *mctx = crypto_shash_ctx(hash); 56 - 57 - if (keylen != sizeof(u32)) 58 - return -EINVAL; 59 - *mctx = le32_to_cpup((__le32 *)key); 60 - return 0; 61 - } 62 - 63 - static int crc32_init(struct shash_desc *desc) 64 - { 65 - u32 *mctx = crypto_shash_ctx(desc->tfm); 66 - u32 *crc = shash_desc_ctx(desc); 67 - 68 - *crc = *mctx; 69 - return 0; 70 - } 71 - 72 - static int crc32_update(struct shash_desc *desc, const u8 *data, 73 - unsigned int length) 74 - { 75 - u32 *crc = shash_desc_ctx(desc); 76 - 77 - *crc = crc32_armv8_le(*crc, data, length); 78 - return 0; 79 - } 80 - 81 - static int crc32c_update(struct shash_desc *desc, const u8 *data, 82 - unsigned int length) 83 - { 84 - u32 *crc = shash_desc_ctx(desc); 85 - 86 - *crc = crc32c_armv8_le(*crc, data, length); 87 - return 0; 88 - } 89 - 90 - static int crc32_final(struct shash_desc *desc, u8 *out) 91 - { 92 - u32 *crc = shash_desc_ctx(desc); 93 - 94 - put_unaligned_le32(*crc, out); 95 - return 0; 96 - } 97 - 98 - static int crc32c_final(struct shash_desc *desc, u8 *out) 99 - { 100 - u32 *crc = shash_desc_ctx(desc); 101 - 102 - put_unaligned_le32(~*crc, out); 103 - return 0; 104 - } 105 - 106 - static int crc32_pmull_update(struct shash_desc *desc, const u8 *data, 107 - unsigned int length) 108 - { 109 - u32 *crc = shash_desc_ctx(desc); 110 - unsigned int l; 111 - 112 - if (crypto_simd_usable()) { 113 - if ((u32)data % SCALE_F) { 114 - l = min_t(u32, length, SCALE_F - ((u32)data % SCALE_F)); 115 - 116 - *crc = fallback_crc32(*crc, data, l); 117 - 118 - data += l; 119 - length -= l; 120 - } 121 - 122 - if (length >= PMULL_MIN_LEN) { 123 - l = round_down(length, SCALE_F); 124 - 125 - kernel_neon_begin(); 126 - *crc = crc32_pmull_le(data, l, *crc); 127 - kernel_neon_end(); 128 - 129 - data += l; 130 - length -= l; 131 - } 132 - } 133 - 134 - if (length > 0) 135 - *crc = fallback_crc32(*crc, data, length); 136 - 137 - return 0; 138 - } 139 - 140 - static int crc32c_pmull_update(struct shash_desc *desc, const u8 *data, 141 - unsigned int length) 142 - { 143 - u32 *crc = shash_desc_ctx(desc); 144 - unsigned int l; 145 - 146 - if (crypto_simd_usable()) { 147 - if ((u32)data % SCALE_F) { 148 - l = min_t(u32, length, SCALE_F - ((u32)data % SCALE_F)); 149 - 150 - *crc = fallback_crc32c(*crc, data, l); 151 - 152 - data += l; 153 - length -= l; 154 - } 155 - 156 - if (length >= PMULL_MIN_LEN) { 157 - l = round_down(length, SCALE_F); 158 - 159 - kernel_neon_begin(); 160 - *crc = crc32c_pmull_le(data, l, *crc); 161 - kernel_neon_end(); 162 - 163 - data += l; 164 - length -= l; 165 - } 166 - } 167 - 168 - if (length > 0) 169 - *crc = fallback_crc32c(*crc, data, length); 170 - 171 - return 0; 172 - } 173 - 174 - static struct shash_alg crc32_pmull_algs[] = { { 175 - .setkey = crc32_setkey, 176 - .init = crc32_init, 177 - .update = crc32_update, 178 - .final = crc32_final, 179 - .descsize = sizeof(u32), 180 - .digestsize = sizeof(u32), 181 - 182 - .base.cra_ctxsize = sizeof(u32), 183 - .base.cra_init = crc32_cra_init, 184 - .base.cra_name = "crc32", 185 - .base.cra_driver_name = "crc32-arm-ce", 186 - .base.cra_priority = 200, 187 - .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, 188 - .base.cra_blocksize = 1, 189 - .base.cra_module = THIS_MODULE, 190 - }, { 191 - .setkey = crc32_setkey, 192 - .init = crc32_init, 193 - .update = crc32c_update, 194 - .final = crc32c_final, 195 - .descsize = sizeof(u32), 196 - .digestsize = sizeof(u32), 197 - 198 - .base.cra_ctxsize = sizeof(u32), 199 - .base.cra_init = crc32c_cra_init, 200 - .base.cra_name = "crc32c", 201 - .base.cra_driver_name = "crc32c-arm-ce", 202 - .base.cra_priority = 200, 203 - .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, 204 - .base.cra_blocksize = 1, 205 - .base.cra_module = THIS_MODULE, 206 - } }; 207 - 208 - static int __init crc32_pmull_mod_init(void) 209 - { 210 - if (elf_hwcap2 & HWCAP2_PMULL) { 211 - crc32_pmull_algs[0].update = crc32_pmull_update; 212 - crc32_pmull_algs[1].update = crc32c_pmull_update; 213 - 214 - if (elf_hwcap2 & HWCAP2_CRC32) { 215 - fallback_crc32 = crc32_armv8_le; 216 - fallback_crc32c = crc32c_armv8_le; 217 - } else { 218 - fallback_crc32 = crc32_le; 219 - fallback_crc32c = __crc32c_le; 220 - } 221 - } else if (!(elf_hwcap2 & HWCAP2_CRC32)) { 222 - return -ENODEV; 223 - } 224 - 225 - return crypto_register_shashes(crc32_pmull_algs, 226 - ARRAY_SIZE(crc32_pmull_algs)); 227 - } 228 - 229 - static void __exit crc32_pmull_mod_exit(void) 230 - { 231 - crypto_unregister_shashes(crc32_pmull_algs, 232 - ARRAY_SIZE(crc32_pmull_algs)); 233 - } 234 - 235 - static const struct cpu_feature __maybe_unused crc32_cpu_feature[] = { 236 - { cpu_feature(CRC32) }, { cpu_feature(PMULL) }, { } 237 - }; 238 - MODULE_DEVICE_TABLE(cpu, crc32_cpu_feature); 239 - 240 - module_init(crc32_pmull_mod_init); 241 - module_exit(crc32_pmull_mod_exit); 242 - 243 - MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 244 - MODULE_DESCRIPTION("Accelerated CRC32(C) using ARM CRC, NEON and Crypto Extensions"); 245 - MODULE_LICENSE("GPL v2"); 246 - MODULE_ALIAS_CRYPTO("crc32"); 247 - MODULE_ALIAS_CRYPTO("crc32c");
+3
arch/arm/lib/Makefile
··· 45 45 endif 46 46 47 47 obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o 48 + 49 + obj-$(CONFIG_CRC32_ARCH) += crc32-arm.o 50 + crc32-arm-y := crc32-glue.o crc32-core.o
+123
arch/arm/lib/crc32-glue.c
··· 1 + // SPDX-License-Identifier: GPL-2.0-only 2 + /* 3 + * Accelerated CRC32(C) using ARM CRC, NEON and Crypto Extensions instructions 4 + * 5 + * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org> 6 + */ 7 + 8 + #include <linux/cpufeature.h> 9 + #include <linux/crc32.h> 10 + #include <linux/init.h> 11 + #include <linux/kernel.h> 12 + #include <linux/module.h> 13 + #include <linux/string.h> 14 + 15 + #include <crypto/internal/simd.h> 16 + 17 + #include <asm/hwcap.h> 18 + #include <asm/neon.h> 19 + #include <asm/simd.h> 20 + 21 + static DEFINE_STATIC_KEY_FALSE(have_crc32); 22 + static DEFINE_STATIC_KEY_FALSE(have_pmull); 23 + 24 + #define PMULL_MIN_LEN 64 /* min size of buffer for pmull functions */ 25 + 26 + asmlinkage u32 crc32_pmull_le(const u8 buf[], u32 len, u32 init_crc); 27 + asmlinkage u32 crc32_armv8_le(u32 init_crc, const u8 buf[], u32 len); 28 + 29 + asmlinkage u32 crc32c_pmull_le(const u8 buf[], u32 len, u32 init_crc); 30 + asmlinkage u32 crc32c_armv8_le(u32 init_crc, const u8 buf[], u32 len); 31 + 32 + static u32 crc32_le_scalar(u32 crc, const u8 *p, size_t len) 33 + { 34 + if (static_branch_likely(&have_crc32)) 35 + return crc32_armv8_le(crc, p, len); 36 + return crc32_le_base(crc, p, len); 37 + } 38 + 39 + u32 crc32_le_arch(u32 crc, const u8 *p, size_t len) 40 + { 41 + if (len >= PMULL_MIN_LEN + 15 && 42 + static_branch_likely(&have_pmull) && crypto_simd_usable()) { 43 + size_t n = -(uintptr_t)p & 15; 44 + 45 + /* align p to 16-byte boundary */ 46 + if (n) { 47 + crc = crc32_le_scalar(crc, p, n); 48 + p += n; 49 + len -= n; 50 + } 51 + n = round_down(len, 16); 52 + kernel_neon_begin(); 53 + crc = crc32_pmull_le(p, n, crc); 54 + kernel_neon_end(); 55 + p += n; 56 + len -= n; 57 + } 58 + return crc32_le_scalar(crc, p, len); 59 + } 60 + EXPORT_SYMBOL(crc32_le_arch); 61 + 62 + static u32 crc32c_le_scalar(u32 crc, const u8 *p, size_t len) 63 + { 64 + if (static_branch_likely(&have_crc32)) 65 + return crc32c_armv8_le(crc, p, len); 66 + return crc32c_le_base(crc, p, len); 67 + } 68 + 69 + u32 crc32c_le_arch(u32 crc, const u8 *p, size_t len) 70 + { 71 + if (len >= PMULL_MIN_LEN + 15 && 72 + static_branch_likely(&have_pmull) && crypto_simd_usable()) { 73 + size_t n = -(uintptr_t)p & 15; 74 + 75 + /* align p to 16-byte boundary */ 76 + if (n) { 77 + crc = crc32c_le_scalar(crc, p, n); 78 + p += n; 79 + len -= n; 80 + } 81 + n = round_down(len, 16); 82 + kernel_neon_begin(); 83 + crc = crc32c_pmull_le(p, n, crc); 84 + kernel_neon_end(); 85 + p += n; 86 + len -= n; 87 + } 88 + return crc32c_le_scalar(crc, p, len); 89 + } 90 + EXPORT_SYMBOL(crc32c_le_arch); 91 + 92 + u32 crc32_be_arch(u32 crc, const u8 *p, size_t len) 93 + { 94 + return crc32_be_base(crc, p, len); 95 + } 96 + EXPORT_SYMBOL(crc32_be_arch); 97 + 98 + static int __init crc32_arm_init(void) 99 + { 100 + if (elf_hwcap2 & HWCAP2_CRC32) 101 + static_branch_enable(&have_crc32); 102 + if (elf_hwcap2 & HWCAP2_PMULL) 103 + static_branch_enable(&have_pmull); 104 + return 0; 105 + } 106 + arch_initcall(crc32_arm_init); 107 + 108 + static void __exit crc32_arm_exit(void) 109 + { 110 + } 111 + module_exit(crc32_arm_exit); 112 + 113 + u32 crc32_optimizations(void) 114 + { 115 + if (elf_hwcap2 & (HWCAP2_CRC32 | HWCAP2_PMULL)) 116 + return CRC32_LE_OPTIMIZATION | CRC32C_OPTIMIZATION; 117 + return 0; 118 + } 119 + EXPORT_SYMBOL(crc32_optimizations); 120 + 121 + MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 122 + MODULE_DESCRIPTION("Accelerated CRC32(C) using ARM CRC, NEON and Crypto Extensions"); 123 + MODULE_LICENSE("GPL v2");