+10 -8

Documentation/core-api/kernel-api.rst

reviewed

··· 136 136 CRC Functions 137 137 ------------- 138 138 139 139 - .. kernel-doc:: lib/crc4.c 139 139 + .. kernel-doc:: lib/crc/crc4.c 140 140 :export: 141 141 142 142 - .. kernel-doc:: lib/crc7.c 142 142 + .. kernel-doc:: lib/crc/crc7.c 143 143 :export: 144 144 145 145 - .. kernel-doc:: lib/crc8.c 145 145 + .. kernel-doc:: lib/crc/crc8.c 146 146 :export: 147 147 148 148 - .. kernel-doc:: lib/crc16.c 148 148 + .. kernel-doc:: lib/crc/crc16.c 149 149 :export: 150 150 151 151 - .. kernel-doc:: lib/crc32.c 152 152 - 153 153 - .. kernel-doc:: lib/crc-ccitt.c 151 151 + .. kernel-doc:: lib/crc/crc-ccitt.c 154 152 :export: 155 153 156 156 - .. kernel-doc:: lib/crc-itu-t.c 154 154 + .. kernel-doc:: lib/crc/crc-itu-t.c 157 155 :export: 156 156 + 157 157 + .. kernel-doc:: include/linux/crc32.h 158 158 + 159 159 + .. kernel-doc:: include/linux/crc64.h 158 160 159 161 Base 2 log and power Functions 160 162 ------------------------------

+1 -3

MAINTAINERS

reviewed

··· 6362 6362 S: Maintained 6363 6363 T: git https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git crc-next 6364 6364 F: Documentation/staging/crc* 6365 6365 - F: arch/*/lib/crc* 6366 6365 F: include/linux/crc* 6367 6367 - F: lib/crc* 6368 6368 - F: lib/tests/crc_kunit.c 6366 6366 + F: lib/crc/ 6369 6367 F: scripts/gen-crc-consts.py 6370 6368 6371 6369 CREATIVE SB0540

-2

arch/arm/Kconfig

reviewed

··· 8 8 select ARCH_HAS_CACHE_LINE_SIZE if OF 9 9 select ARCH_HAS_CPU_CACHE_ALIASING 10 10 select ARCH_HAS_CPU_FINALIZE_INIT if MMU 11 11 - select ARCH_HAS_CRC32 if KERNEL_MODE_NEON 12 12 - select ARCH_HAS_CRC_T10DIF if KERNEL_MODE_NEON 13 11 select ARCH_HAS_CURRENT_STACK_POINTER 14 12 select ARCH_HAS_DEBUG_VIRTUAL if MMU 15 13 select ARCH_HAS_DMA_ALLOC if MMU

-1

arch/arm/configs/multi_v7_defconfig

reviewed

··· 1298 1298 CONFIG_CRYPTO_DEV_QCE=m 1299 1299 CONFIG_CRYPTO_DEV_QCOM_RNG=m 1300 1300 CONFIG_CRYPTO_DEV_ROCKCHIP=m 1301 1301 - CONFIG_CRYPTO_DEV_STM32_CRC=m 1302 1301 CONFIG_CRYPTO_DEV_STM32_HASH=m 1303 1302 CONFIG_CRYPTO_DEV_STM32_CRYP=m 1304 1303 CONFIG_CMA_SIZE_MBYTES=64

-6

arch/arm/lib/Makefile

reviewed

··· 47 47 endif 48 48 49 49 obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o 50 50 - 51 51 - obj-$(CONFIG_CRC32_ARCH) += crc32-arm.o 52 52 - crc32-arm-y := crc32.o crc32-core.o 53 53 - 54 54 - obj-$(CONFIG_CRC_T10DIF_ARCH) += crc-t10dif-arm.o 55 55 - crc-t10dif-arm-y := crc-t10dif.o crc-t10dif-core.o

arch/arm/lib/crc-t10dif-core.S lib/crc/arm/crc-t10dif-core.S

reviewed

-72

arch/arm/lib/crc-t10dif.c

reviewed

··· 1 1 - // SPDX-License-Identifier: GPL-2.0-only 2 2 - /* 3 3 - * Accelerated CRC-T10DIF using ARM NEON and Crypto Extensions instructions 4 4 - * 5 5 - * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org> 6 6 - */ 7 7 - 8 8 - #include <linux/crc-t10dif.h> 9 9 - #include <linux/init.h> 10 10 - #include <linux/kernel.h> 11 11 - #include <linux/module.h> 12 12 - #include <linux/string.h> 13 13 - 14 14 - #include <crypto/internal/simd.h> 15 15 - 16 16 - #include <asm/neon.h> 17 17 - #include <asm/simd.h> 18 18 - 19 19 - static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); 20 20 - static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull); 21 21 - 22 22 - #define CRC_T10DIF_PMULL_CHUNK_SIZE 16U 23 23 - 24 24 - asmlinkage u16 crc_t10dif_pmull64(u16 init_crc, const u8 *buf, size_t len); 25 25 - asmlinkage void crc_t10dif_pmull8(u16 init_crc, const u8 *buf, size_t len, 26 26 - u8 out[16]); 27 27 - 28 28 - u16 crc_t10dif_arch(u16 crc, const u8 *data, size_t length) 29 29 - { 30 30 - if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE) { 31 31 - if (static_branch_likely(&have_pmull)) { 32 32 - if (crypto_simd_usable()) { 33 33 - kernel_neon_begin(); 34 34 - crc = crc_t10dif_pmull64(crc, data, length); 35 35 - kernel_neon_end(); 36 36 - return crc; 37 37 - } 38 38 - } else if (length > CRC_T10DIF_PMULL_CHUNK_SIZE && 39 39 - static_branch_likely(&have_neon) && 40 40 - crypto_simd_usable()) { 41 41 - u8 buf[16] __aligned(16); 42 42 - 43 43 - kernel_neon_begin(); 44 44 - crc_t10dif_pmull8(crc, data, length, buf); 45 45 - kernel_neon_end(); 46 46 - 47 47 - return crc_t10dif_generic(0, buf, sizeof(buf)); 48 48 - } 49 49 - } 50 50 - return crc_t10dif_generic(crc, data, length); 51 51 - } 52 52 - EXPORT_SYMBOL(crc_t10dif_arch); 53 53 - 54 54 - static int __init crc_t10dif_arm_init(void) 55 55 - { 56 56 - if (elf_hwcap & HWCAP_NEON) { 57 57 - static_branch_enable(&have_neon); 58 58 - if (elf_hwcap2 & HWCAP2_PMULL) 59 59 - static_branch_enable(&have_pmull); 60 60 - } 61 61 - return 0; 62 62 - } 63 63 - subsys_initcall(crc_t10dif_arm_init); 64 64 - 65 65 - static void __exit crc_t10dif_arm_exit(void) 66 66 - { 67 67 - } 68 68 - module_exit(crc_t10dif_arm_exit); 69 69 - 70 70 - MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 71 71 - MODULE_DESCRIPTION("Accelerated CRC-T10DIF using ARM NEON and Crypto Extensions"); 72 72 - MODULE_LICENSE("GPL v2");

arch/arm/lib/crc32-core.S lib/crc/arm/crc32-core.S

reviewed

-123

arch/arm/lib/crc32.c

reviewed

··· 1 1 - // SPDX-License-Identifier: GPL-2.0-only 2 2 - /* 3 3 - * Accelerated CRC32(C) using ARM CRC, NEON and Crypto Extensions instructions 4 4 - * 5 5 - * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org> 6 6 - */ 7 7 - 8 8 - #include <linux/cpufeature.h> 9 9 - #include <linux/crc32.h> 10 10 - #include <linux/init.h> 11 11 - #include <linux/kernel.h> 12 12 - #include <linux/module.h> 13 13 - #include <linux/string.h> 14 14 - 15 15 - #include <crypto/internal/simd.h> 16 16 - 17 17 - #include <asm/hwcap.h> 18 18 - #include <asm/neon.h> 19 19 - #include <asm/simd.h> 20 20 - 21 21 - static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_crc32); 22 22 - static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull); 23 23 - 24 24 - #define PMULL_MIN_LEN 64 /* min size of buffer for pmull functions */ 25 25 - 26 26 - asmlinkage u32 crc32_pmull_le(const u8 buf[], u32 len, u32 init_crc); 27 27 - asmlinkage u32 crc32_armv8_le(u32 init_crc, const u8 buf[], u32 len); 28 28 - 29 29 - asmlinkage u32 crc32c_pmull_le(const u8 buf[], u32 len, u32 init_crc); 30 30 - asmlinkage u32 crc32c_armv8_le(u32 init_crc, const u8 buf[], u32 len); 31 31 - 32 32 - static u32 crc32_le_scalar(u32 crc, const u8 *p, size_t len) 33 33 - { 34 34 - if (static_branch_likely(&have_crc32)) 35 35 - return crc32_armv8_le(crc, p, len); 36 36 - return crc32_le_base(crc, p, len); 37 37 - } 38 38 - 39 39 - u32 crc32_le_arch(u32 crc, const u8 *p, size_t len) 40 40 - { 41 41 - if (len >= PMULL_MIN_LEN + 15 && 42 42 - static_branch_likely(&have_pmull) && crypto_simd_usable()) { 43 43 - size_t n = -(uintptr_t)p & 15; 44 44 - 45 45 - /* align p to 16-byte boundary */ 46 46 - if (n) { 47 47 - crc = crc32_le_scalar(crc, p, n); 48 48 - p += n; 49 49 - len -= n; 50 50 - } 51 51 - n = round_down(len, 16); 52 52 - kernel_neon_begin(); 53 53 - crc = crc32_pmull_le(p, n, crc); 54 54 - kernel_neon_end(); 55 55 - p += n; 56 56 - len -= n; 57 57 - } 58 58 - return crc32_le_scalar(crc, p, len); 59 59 - } 60 60 - EXPORT_SYMBOL(crc32_le_arch); 61 61 - 62 62 - static u32 crc32c_scalar(u32 crc, const u8 *p, size_t len) 63 63 - { 64 64 - if (static_branch_likely(&have_crc32)) 65 65 - return crc32c_armv8_le(crc, p, len); 66 66 - return crc32c_base(crc, p, len); 67 67 - } 68 68 - 69 69 - u32 crc32c_arch(u32 crc, const u8 *p, size_t len) 70 70 - { 71 71 - if (len >= PMULL_MIN_LEN + 15 && 72 72 - static_branch_likely(&have_pmull) && crypto_simd_usable()) { 73 73 - size_t n = -(uintptr_t)p & 15; 74 74 - 75 75 - /* align p to 16-byte boundary */ 76 76 - if (n) { 77 77 - crc = crc32c_scalar(crc, p, n); 78 78 - p += n; 79 79 - len -= n; 80 80 - } 81 81 - n = round_down(len, 16); 82 82 - kernel_neon_begin(); 83 83 - crc = crc32c_pmull_le(p, n, crc); 84 84 - kernel_neon_end(); 85 85 - p += n; 86 86 - len -= n; 87 87 - } 88 88 - return crc32c_scalar(crc, p, len); 89 89 - } 90 90 - EXPORT_SYMBOL(crc32c_arch); 91 91 - 92 92 - u32 crc32_be_arch(u32 crc, const u8 *p, size_t len) 93 93 - { 94 94 - return crc32_be_base(crc, p, len); 95 95 - } 96 96 - EXPORT_SYMBOL(crc32_be_arch); 97 97 - 98 98 - static int __init crc32_arm_init(void) 99 99 - { 100 100 - if (elf_hwcap2 & HWCAP2_CRC32) 101 101 - static_branch_enable(&have_crc32); 102 102 - if (elf_hwcap2 & HWCAP2_PMULL) 103 103 - static_branch_enable(&have_pmull); 104 104 - return 0; 105 105 - } 106 106 - subsys_initcall(crc32_arm_init); 107 107 - 108 108 - static void __exit crc32_arm_exit(void) 109 109 - { 110 110 - } 111 111 - module_exit(crc32_arm_exit); 112 112 - 113 113 - u32 crc32_optimizations(void) 114 114 - { 115 115 - if (elf_hwcap2 & (HWCAP2_CRC32 | HWCAP2_PMULL)) 116 116 - return CRC32_LE_OPTIMIZATION | CRC32C_OPTIMIZATION; 117 117 - return 0; 118 118 - } 119 119 - EXPORT_SYMBOL(crc32_optimizations); 120 120 - 121 121 - MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 122 122 - MODULE_DESCRIPTION("Accelerated CRC32(C) using ARM CRC, NEON and Crypto Extensions"); 123 123 - MODULE_LICENSE("GPL v2");

-2

arch/arm64/Kconfig

reviewed

··· 21 21 select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE 22 22 select ARCH_HAS_CACHE_LINE_SIZE 23 23 select ARCH_HAS_CC_PLATFORM 24 24 - select ARCH_HAS_CRC32 25 25 - select ARCH_HAS_CRC_T10DIF if KERNEL_MODE_NEON 26 24 select ARCH_HAS_CURRENT_STACK_POINTER 27 25 select ARCH_HAS_DEBUG_VIRTUAL 28 26 select ARCH_HAS_DEBUG_VM_PGTABLE

-6

arch/arm64/lib/Makefile

reviewed

··· 16 16 17 17 lib-$(CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE) += uaccess_flushcache.o 18 18 19 19 - obj-$(CONFIG_CRC32_ARCH) += crc32-arm64.o 20 20 - crc32-arm64-y := crc32.o crc32-core.o 21 21 - 22 22 - obj-$(CONFIG_CRC_T10DIF_ARCH) += crc-t10dif-arm64.o 23 23 - crc-t10dif-arm64-y := crc-t10dif.o crc-t10dif-core.o 24 24 - 25 19 obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o 26 20 27 21 obj-$(CONFIG_ARM64_MTE) += mte.o

arch/arm64/lib/crc-t10dif-core.S lib/crc/arm64/crc-t10dif-core.S

reviewed

-73

arch/arm64/lib/crc-t10dif.c

reviewed

··· 1 1 - // SPDX-License-Identifier: GPL-2.0-only 2 2 - /* 3 3 - * Accelerated CRC-T10DIF using arm64 NEON and Crypto Extensions instructions 4 4 - * 5 5 - * Copyright (C) 2016 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org> 6 6 - */ 7 7 - 8 8 - #include <linux/cpufeature.h> 9 9 - #include <linux/crc-t10dif.h> 10 10 - #include <linux/init.h> 11 11 - #include <linux/kernel.h> 12 12 - #include <linux/module.h> 13 13 - #include <linux/string.h> 14 14 - 15 15 - #include <crypto/internal/simd.h> 16 16 - 17 17 - #include <asm/neon.h> 18 18 - #include <asm/simd.h> 19 19 - 20 20 - static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_asimd); 21 21 - static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull); 22 22 - 23 23 - #define CRC_T10DIF_PMULL_CHUNK_SIZE 16U 24 24 - 25 25 - asmlinkage void crc_t10dif_pmull_p8(u16 init_crc, const u8 *buf, size_t len, 26 26 - u8 out[16]); 27 27 - asmlinkage u16 crc_t10dif_pmull_p64(u16 init_crc, const u8 *buf, size_t len); 28 28 - 29 29 - u16 crc_t10dif_arch(u16 crc, const u8 *data, size_t length) 30 30 - { 31 31 - if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE) { 32 32 - if (static_branch_likely(&have_pmull)) { 33 33 - if (crypto_simd_usable()) { 34 34 - kernel_neon_begin(); 35 35 - crc = crc_t10dif_pmull_p64(crc, data, length); 36 36 - kernel_neon_end(); 37 37 - return crc; 38 38 - } 39 39 - } else if (length > CRC_T10DIF_PMULL_CHUNK_SIZE && 40 40 - static_branch_likely(&have_asimd) && 41 41 - crypto_simd_usable()) { 42 42 - u8 buf[16]; 43 43 - 44 44 - kernel_neon_begin(); 45 45 - crc_t10dif_pmull_p8(crc, data, length, buf); 46 46 - kernel_neon_end(); 47 47 - 48 48 - return crc_t10dif_generic(0, buf, sizeof(buf)); 49 49 - } 50 50 - } 51 51 - return crc_t10dif_generic(crc, data, length); 52 52 - } 53 53 - EXPORT_SYMBOL(crc_t10dif_arch); 54 54 - 55 55 - static int __init crc_t10dif_arm64_init(void) 56 56 - { 57 57 - if (cpu_have_named_feature(ASIMD)) { 58 58 - static_branch_enable(&have_asimd); 59 59 - if (cpu_have_named_feature(PMULL)) 60 60 - static_branch_enable(&have_pmull); 61 61 - } 62 62 - return 0; 63 63 - } 64 64 - subsys_initcall(crc_t10dif_arm64_init); 65 65 - 66 66 - static void __exit crc_t10dif_arm64_exit(void) 67 67 - { 68 68 - } 69 69 - module_exit(crc_t10dif_arm64_exit); 70 70 - 71 71 - MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 72 72 - MODULE_DESCRIPTION("CRC-T10DIF using arm64 NEON and Crypto Extensions"); 73 73 - MODULE_LICENSE("GPL v2");

arch/arm64/lib/crc32-core.S lib/crc/arm64/crc32-core.S

reviewed

-99

arch/arm64/lib/crc32.c

reviewed

··· 1 1 - // SPDX-License-Identifier: GPL-2.0-only 2 2 - 3 3 - #include <linux/crc32.h> 4 4 - #include <linux/linkage.h> 5 5 - #include <linux/module.h> 6 6 - 7 7 - #include <asm/alternative.h> 8 8 - #include <asm/cpufeature.h> 9 9 - #include <asm/neon.h> 10 10 - #include <asm/simd.h> 11 11 - 12 12 - #include <crypto/internal/simd.h> 13 13 - 14 14 - // The minimum input length to consider the 4-way interleaved code path 15 15 - static const size_t min_len = 1024; 16 16 - 17 17 - asmlinkage u32 crc32_le_arm64(u32 crc, unsigned char const *p, size_t len); 18 18 - asmlinkage u32 crc32c_le_arm64(u32 crc, unsigned char const *p, size_t len); 19 19 - asmlinkage u32 crc32_be_arm64(u32 crc, unsigned char const *p, size_t len); 20 20 - 21 21 - asmlinkage u32 crc32_le_arm64_4way(u32 crc, unsigned char const *p, size_t len); 22 22 - asmlinkage u32 crc32c_le_arm64_4way(u32 crc, unsigned char const *p, size_t len); 23 23 - asmlinkage u32 crc32_be_arm64_4way(u32 crc, unsigned char const *p, size_t len); 24 24 - 25 25 - u32 crc32_le_arch(u32 crc, const u8 *p, size_t len) 26 26 - { 27 27 - if (!alternative_has_cap_likely(ARM64_HAS_CRC32)) 28 28 - return crc32_le_base(crc, p, len); 29 29 - 30 30 - if (len >= min_len && cpu_have_named_feature(PMULL) && crypto_simd_usable()) { 31 31 - kernel_neon_begin(); 32 32 - crc = crc32_le_arm64_4way(crc, p, len); 33 33 - kernel_neon_end(); 34 34 - 35 35 - p += round_down(len, 64); 36 36 - len %= 64; 37 37 - 38 38 - if (!len) 39 39 - return crc; 40 40 - } 41 41 - 42 42 - return crc32_le_arm64(crc, p, len); 43 43 - } 44 44 - EXPORT_SYMBOL(crc32_le_arch); 45 45 - 46 46 - u32 crc32c_arch(u32 crc, const u8 *p, size_t len) 47 47 - { 48 48 - if (!alternative_has_cap_likely(ARM64_HAS_CRC32)) 49 49 - return crc32c_base(crc, p, len); 50 50 - 51 51 - if (len >= min_len && cpu_have_named_feature(PMULL) && crypto_simd_usable()) { 52 52 - kernel_neon_begin(); 53 53 - crc = crc32c_le_arm64_4way(crc, p, len); 54 54 - kernel_neon_end(); 55 55 - 56 56 - p += round_down(len, 64); 57 57 - len %= 64; 58 58 - 59 59 - if (!len) 60 60 - return crc; 61 61 - } 62 62 - 63 63 - return crc32c_le_arm64(crc, p, len); 64 64 - } 65 65 - EXPORT_SYMBOL(crc32c_arch); 66 66 - 67 67 - u32 crc32_be_arch(u32 crc, const u8 *p, size_t len) 68 68 - { 69 69 - if (!alternative_has_cap_likely(ARM64_HAS_CRC32)) 70 70 - return crc32_be_base(crc, p, len); 71 71 - 72 72 - if (len >= min_len && cpu_have_named_feature(PMULL) && crypto_simd_usable()) { 73 73 - kernel_neon_begin(); 74 74 - crc = crc32_be_arm64_4way(crc, p, len); 75 75 - kernel_neon_end(); 76 76 - 77 77 - p += round_down(len, 64); 78 78 - len %= 64; 79 79 - 80 80 - if (!len) 81 81 - return crc; 82 82 - } 83 83 - 84 84 - return crc32_be_arm64(crc, p, len); 85 85 - } 86 86 - EXPORT_SYMBOL(crc32_be_arch); 87 87 - 88 88 - u32 crc32_optimizations(void) 89 89 - { 90 90 - if (alternative_has_cap_likely(ARM64_HAS_CRC32)) 91 91 - return CRC32_LE_OPTIMIZATION | 92 92 - CRC32_BE_OPTIMIZATION | 93 93 - CRC32C_OPTIMIZATION; 94 94 - return 0; 95 95 - } 96 96 - EXPORT_SYMBOL(crc32_optimizations); 97 97 - 98 98 - MODULE_LICENSE("GPL"); 99 99 - MODULE_DESCRIPTION("arm64-optimized CRC32 functions");

-1

arch/loongarch/Kconfig

reviewed

··· 15 15 select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE 16 16 select ARCH_HAS_ACPI_TABLE_UPGRADE if ACPI 17 17 select ARCH_HAS_CPU_FINALIZE_INIT 18 18 - select ARCH_HAS_CRC32 19 18 select ARCH_HAS_CURRENT_STACK_POINTER 20 19 select ARCH_HAS_DEBUG_VM_PGTABLE 21 20 select ARCH_HAS_FAST_MULTIPLIER

-2

arch/loongarch/lib/Makefile

reviewed

··· 11 11 obj-$(CONFIG_CPU_HAS_LSX) += xor_simd.o xor_simd_glue.o 12 12 13 13 obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o 14 14 - 15 15 - obj-$(CONFIG_CRC32_ARCH) += crc32-loongarch.o

-136

arch/loongarch/lib/crc32-loongarch.c

reviewed

··· 1 1 - // SPDX-License-Identifier: GPL-2.0 2 2 - /* 3 3 - * CRC32 and CRC32C using LoongArch crc* instructions 4 4 - * 5 5 - * Module based on mips/crypto/crc32-mips.c 6 6 - * 7 7 - * Copyright (C) 2014 Linaro Ltd <yazen.ghannam@linaro.org> 8 8 - * Copyright (C) 2018 MIPS Tech, LLC 9 9 - * Copyright (C) 2020-2023 Loongson Technology Corporation Limited 10 10 - */ 11 11 - 12 12 - #include <asm/cpu-features.h> 13 13 - #include <linux/crc32.h> 14 14 - #include <linux/export.h> 15 15 - #include <linux/module.h> 16 16 - #include <linux/unaligned.h> 17 17 - 18 18 - #define _CRC32(crc, value, size, type) \ 19 19 - do { \ 20 20 - __asm__ __volatile__( \ 21 21 - #type ".w." #size ".w" " %0, %1, %0\n\t"\ 22 22 - : "+r" (crc) \ 23 23 - : "r" (value) \ 24 24 - : "memory"); \ 25 25 - } while (0) 26 26 - 27 27 - #define CRC32(crc, value, size) _CRC32(crc, value, size, crc) 28 28 - #define CRC32C(crc, value, size) _CRC32(crc, value, size, crcc) 29 29 - 30 30 - static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_crc32); 31 31 - 32 32 - u32 crc32_le_arch(u32 crc, const u8 *p, size_t len) 33 33 - { 34 34 - if (!static_branch_likely(&have_crc32)) 35 35 - return crc32_le_base(crc, p, len); 36 36 - 37 37 - while (len >= sizeof(u64)) { 38 38 - u64 value = get_unaligned_le64(p); 39 39 - 40 40 - CRC32(crc, value, d); 41 41 - p += sizeof(u64); 42 42 - len -= sizeof(u64); 43 43 - } 44 44 - 45 45 - if (len & sizeof(u32)) { 46 46 - u32 value = get_unaligned_le32(p); 47 47 - 48 48 - CRC32(crc, value, w); 49 49 - p += sizeof(u32); 50 50 - } 51 51 - 52 52 - if (len & sizeof(u16)) { 53 53 - u16 value = get_unaligned_le16(p); 54 54 - 55 55 - CRC32(crc, value, h); 56 56 - p += sizeof(u16); 57 57 - } 58 58 - 59 59 - if (len & sizeof(u8)) { 60 60 - u8 value = *p++; 61 61 - 62 62 - CRC32(crc, value, b); 63 63 - } 64 64 - 65 65 - return crc; 66 66 - } 67 67 - EXPORT_SYMBOL(crc32_le_arch); 68 68 - 69 69 - u32 crc32c_arch(u32 crc, const u8 *p, size_t len) 70 70 - { 71 71 - if (!static_branch_likely(&have_crc32)) 72 72 - return crc32c_base(crc, p, len); 73 73 - 74 74 - while (len >= sizeof(u64)) { 75 75 - u64 value = get_unaligned_le64(p); 76 76 - 77 77 - CRC32C(crc, value, d); 78 78 - p += sizeof(u64); 79 79 - len -= sizeof(u64); 80 80 - } 81 81 - 82 82 - if (len & sizeof(u32)) { 83 83 - u32 value = get_unaligned_le32(p); 84 84 - 85 85 - CRC32C(crc, value, w); 86 86 - p += sizeof(u32); 87 87 - } 88 88 - 89 89 - if (len & sizeof(u16)) { 90 90 - u16 value = get_unaligned_le16(p); 91 91 - 92 92 - CRC32C(crc, value, h); 93 93 - p += sizeof(u16); 94 94 - } 95 95 - 96 96 - if (len & sizeof(u8)) { 97 97 - u8 value = *p++; 98 98 - 99 99 - CRC32C(crc, value, b); 100 100 - } 101 101 - 102 102 - return crc; 103 103 - } 104 104 - EXPORT_SYMBOL(crc32c_arch); 105 105 - 106 106 - u32 crc32_be_arch(u32 crc, const u8 *p, size_t len) 107 107 - { 108 108 - return crc32_be_base(crc, p, len); 109 109 - } 110 110 - EXPORT_SYMBOL(crc32_be_arch); 111 111 - 112 112 - static int __init crc32_loongarch_init(void) 113 113 - { 114 114 - if (cpu_has_crc32) 115 115 - static_branch_enable(&have_crc32); 116 116 - return 0; 117 117 - } 118 118 - subsys_initcall(crc32_loongarch_init); 119 119 - 120 120 - static void __exit crc32_loongarch_exit(void) 121 121 - { 122 122 - } 123 123 - module_exit(crc32_loongarch_exit); 124 124 - 125 125 - u32 crc32_optimizations(void) 126 126 - { 127 127 - if (static_key_enabled(&have_crc32)) 128 128 - return CRC32_LE_OPTIMIZATION | CRC32C_OPTIMIZATION; 129 129 - return 0; 130 130 - } 131 131 - EXPORT_SYMBOL(crc32_optimizations); 132 132 - 133 133 - MODULE_AUTHOR("Min Zhou <zhoumin@loongson.cn>"); 134 134 - MODULE_AUTHOR("Huacai Chen <chenhuacai@loongson.cn>"); 135 135 - MODULE_DESCRIPTION("CRC32 and CRC32C using LoongArch crc* instructions"); 136 136 - MODULE_LICENSE("GPL v2");

-1

arch/mips/Kconfig

reviewed

··· 2024 2024 config CPU_MIPSR6 2025 2025 bool 2026 2026 default y if CPU_MIPS32_R6 || CPU_MIPS64_R6 2027 2027 - select ARCH_HAS_CRC32 2028 2027 select CPU_HAS_RIXI 2029 2028 select CPU_HAS_DIEI if !CPU_DIEI_BROKEN 2030 2029 select HAVE_ARCH_BITREVERSE

-2

arch/mips/lib/Makefile

reviewed

··· 16 16 obj-$(CONFIG_CPU_GENERIC_DUMP_TLB) += dump_tlb.o 17 17 obj-$(CONFIG_CPU_R3000) += r3k_dump_tlb.o 18 18 19 19 - obj-$(CONFIG_CRC32_ARCH) += crc32-mips.o 20 20 - 21 19 # libgcc-style stuff needed in the kernel 22 20 obj-y += bswapsi.o bswapdi.o multi3.o

-183

arch/mips/lib/crc32-mips.c

reviewed

··· 1 1 - // SPDX-License-Identifier: GPL-2.0 2 2 - /* 3 3 - * crc32-mips.c - CRC32 and CRC32C using optional MIPSr6 instructions 4 4 - * 5 5 - * Module based on arm64/crypto/crc32-arm.c 6 6 - * 7 7 - * Copyright (C) 2014 Linaro Ltd <yazen.ghannam@linaro.org> 8 8 - * Copyright (C) 2018 MIPS Tech, LLC 9 9 - */ 10 10 - 11 11 - #include <linux/cpufeature.h> 12 12 - #include <linux/crc32.h> 13 13 - #include <linux/init.h> 14 14 - #include <linux/kernel.h> 15 15 - #include <linux/module.h> 16 16 - #include <asm/mipsregs.h> 17 17 - #include <linux/unaligned.h> 18 18 - 19 19 - #ifndef TOOLCHAIN_SUPPORTS_CRC 20 20 - #define _ASM_SET_CRC(OP, SZ, TYPE) \ 21 21 - _ASM_MACRO_3R(OP, rt, rs, rt2, \ 22 22 - ".ifnc \\rt, \\rt2\n\t" \ 23 23 - ".error \"invalid operands \\\"" #OP " \\rt,\\rs,\\rt2\\\"\"\n\t" \ 24 24 - ".endif\n\t" \ 25 25 - _ASM_INSN_IF_MIPS(0x7c00000f | (__rt << 16) | (__rs << 21) | \ 26 26 - ((SZ) << 6) | ((TYPE) << 8)) \ 27 27 - _ASM_INSN32_IF_MM(0x00000030 | (__rs << 16) | (__rt << 21) | \ 28 28 - ((SZ) << 14) | ((TYPE) << 3))) 29 29 - #define _ASM_UNSET_CRC(op, SZ, TYPE) ".purgem " #op "\n\t" 30 30 - #else /* !TOOLCHAIN_SUPPORTS_CRC */ 31 31 - #define _ASM_SET_CRC(op, SZ, TYPE) ".set\tcrc\n\t" 32 32 - #define _ASM_UNSET_CRC(op, SZ, TYPE) 33 33 - #endif 34 34 - 35 35 - #define __CRC32(crc, value, op, SZ, TYPE) \ 36 36 - do { \ 37 37 - __asm__ __volatile__( \ 38 38 - ".set push\n\t" \ 39 39 - _ASM_SET_CRC(op, SZ, TYPE) \ 40 40 - #op " %0, %1, %0\n\t" \ 41 41 - _ASM_UNSET_CRC(op, SZ, TYPE) \ 42 42 - ".set pop" \ 43 43 - : "+r" (crc) \ 44 44 - : "r" (value)); \ 45 45 - } while (0) 46 46 - 47 47 - #define _CRC32_crc32b(crc, value) __CRC32(crc, value, crc32b, 0, 0) 48 48 - #define _CRC32_crc32h(crc, value) __CRC32(crc, value, crc32h, 1, 0) 49 49 - #define _CRC32_crc32w(crc, value) __CRC32(crc, value, crc32w, 2, 0) 50 50 - #define _CRC32_crc32d(crc, value) __CRC32(crc, value, crc32d, 3, 0) 51 51 - #define _CRC32_crc32cb(crc, value) __CRC32(crc, value, crc32cb, 0, 1) 52 52 - #define _CRC32_crc32ch(crc, value) __CRC32(crc, value, crc32ch, 1, 1) 53 53 - #define _CRC32_crc32cw(crc, value) __CRC32(crc, value, crc32cw, 2, 1) 54 54 - #define _CRC32_crc32cd(crc, value) __CRC32(crc, value, crc32cd, 3, 1) 55 55 - 56 56 - #define _CRC32(crc, value, size, op) \ 57 57 - _CRC32_##op##size(crc, value) 58 58 - 59 59 - #define CRC32(crc, value, size) \ 60 60 - _CRC32(crc, value, size, crc32) 61 61 - 62 62 - #define CRC32C(crc, value, size) \ 63 63 - _CRC32(crc, value, size, crc32c) 64 64 - 65 65 - static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_crc32); 66 66 - 67 67 - u32 crc32_le_arch(u32 crc, const u8 *p, size_t len) 68 68 - { 69 69 - if (!static_branch_likely(&have_crc32)) 70 70 - return crc32_le_base(crc, p, len); 71 71 - 72 72 - if (IS_ENABLED(CONFIG_64BIT)) { 73 73 - for (; len >= sizeof(u64); p += sizeof(u64), len -= sizeof(u64)) { 74 74 - u64 value = get_unaligned_le64(p); 75 75 - 76 76 - CRC32(crc, value, d); 77 77 - } 78 78 - 79 79 - if (len & sizeof(u32)) { 80 80 - u32 value = get_unaligned_le32(p); 81 81 - 82 82 - CRC32(crc, value, w); 83 83 - p += sizeof(u32); 84 84 - } 85 85 - } else { 86 86 - for (; len >= sizeof(u32); len -= sizeof(u32)) { 87 87 - u32 value = get_unaligned_le32(p); 88 88 - 89 89 - CRC32(crc, value, w); 90 90 - p += sizeof(u32); 91 91 - } 92 92 - } 93 93 - 94 94 - if (len & sizeof(u16)) { 95 95 - u16 value = get_unaligned_le16(p); 96 96 - 97 97 - CRC32(crc, value, h); 98 98 - p += sizeof(u16); 99 99 - } 100 100 - 101 101 - if (len & sizeof(u8)) { 102 102 - u8 value = *p++; 103 103 - 104 104 - CRC32(crc, value, b); 105 105 - } 106 106 - 107 107 - return crc; 108 108 - } 109 109 - EXPORT_SYMBOL(crc32_le_arch); 110 110 - 111 111 - u32 crc32c_arch(u32 crc, const u8 *p, size_t len) 112 112 - { 113 113 - if (!static_branch_likely(&have_crc32)) 114 114 - return crc32c_base(crc, p, len); 115 115 - 116 116 - if (IS_ENABLED(CONFIG_64BIT)) { 117 117 - for (; len >= sizeof(u64); p += sizeof(u64), len -= sizeof(u64)) { 118 118 - u64 value = get_unaligned_le64(p); 119 119 - 120 120 - CRC32C(crc, value, d); 121 121 - } 122 122 - 123 123 - if (len & sizeof(u32)) { 124 124 - u32 value = get_unaligned_le32(p); 125 125 - 126 126 - CRC32C(crc, value, w); 127 127 - p += sizeof(u32); 128 128 - } 129 129 - } else { 130 130 - for (; len >= sizeof(u32); len -= sizeof(u32)) { 131 131 - u32 value = get_unaligned_le32(p); 132 132 - 133 133 - CRC32C(crc, value, w); 134 134 - p += sizeof(u32); 135 135 - } 136 136 - } 137 137 - 138 138 - if (len & sizeof(u16)) { 139 139 - u16 value = get_unaligned_le16(p); 140 140 - 141 141 - CRC32C(crc, value, h); 142 142 - p += sizeof(u16); 143 143 - } 144 144 - 145 145 - if (len & sizeof(u8)) { 146 146 - u8 value = *p++; 147 147 - 148 148 - CRC32C(crc, value, b); 149 149 - } 150 150 - return crc; 151 151 - } 152 152 - EXPORT_SYMBOL(crc32c_arch); 153 153 - 154 154 - u32 crc32_be_arch(u32 crc, const u8 *p, size_t len) 155 155 - { 156 156 - return crc32_be_base(crc, p, len); 157 157 - } 158 158 - EXPORT_SYMBOL(crc32_be_arch); 159 159 - 160 160 - static int __init crc32_mips_init(void) 161 161 - { 162 162 - if (cpu_have_feature(cpu_feature(MIPS_CRC32))) 163 163 - static_branch_enable(&have_crc32); 164 164 - return 0; 165 165 - } 166 166 - subsys_initcall(crc32_mips_init); 167 167 - 168 168 - static void __exit crc32_mips_exit(void) 169 169 - { 170 170 - } 171 171 - module_exit(crc32_mips_exit); 172 172 - 173 173 - u32 crc32_optimizations(void) 174 174 - { 175 175 - if (static_key_enabled(&have_crc32)) 176 176 - return CRC32_LE_OPTIMIZATION | CRC32C_OPTIMIZATION; 177 177 - return 0; 178 178 - } 179 179 - EXPORT_SYMBOL(crc32_optimizations); 180 180 - 181 181 - MODULE_AUTHOR("Marcin Nowakowski <marcin.nowakowski@mips.com"); 182 182 - MODULE_DESCRIPTION("CRC32 and CRC32C using optional MIPS instructions"); 183 183 - MODULE_LICENSE("GPL v2");

-2

arch/powerpc/Kconfig

reviewed

··· 127 127 select ARCH_ENABLE_MEMORY_HOTPLUG 128 128 select ARCH_ENABLE_MEMORY_HOTREMOVE 129 129 select ARCH_HAS_COPY_MC if PPC64 130 130 - select ARCH_HAS_CRC32 if PPC64 && ALTIVEC 131 131 - select ARCH_HAS_CRC_T10DIF if PPC64 && ALTIVEC 132 130 select ARCH_HAS_CURRENT_STACK_POINTER 133 131 select ARCH_HAS_DEBUG_VIRTUAL 134 132 select ARCH_HAS_DEBUG_VM_PGTABLE

-6

arch/powerpc/lib/Makefile

reviewed

··· 80 80 # Enable <altivec.h> 81 81 CFLAGS_xor_vmx.o += -isystem $(shell $(CC) -print-file-name=include) 82 82 83 83 - obj-$(CONFIG_CRC32_ARCH) += crc32-powerpc.o 84 84 - crc32-powerpc-y := crc32.o crc32c-vpmsum_asm.o 85 85 - 86 86 - obj-$(CONFIG_CRC_T10DIF_ARCH) += crc-t10dif-powerpc.o 87 87 - crc-t10dif-powerpc-y := crc-t10dif.o crct10dif-vpmsum_asm.o 88 88 - 89 83 obj-$(CONFIG_PPC64) += $(obj64-y)

-83

arch/powerpc/lib/crc-t10dif.c

reviewed

··· 1 1 - // SPDX-License-Identifier: GPL-2.0-or-later 2 2 - /* 3 3 - * Calculate a CRC T10-DIF with vpmsum acceleration 4 4 - * 5 5 - * Copyright 2017, Daniel Axtens, IBM Corporation. 6 6 - * [based on crc32c-vpmsum_glue.c] 7 7 - */ 8 8 - 9 9 - #include <asm/switch_to.h> 10 10 - #include <crypto/internal/simd.h> 11 11 - #include <linux/cpufeature.h> 12 12 - #include <linux/crc-t10dif.h> 13 13 - #include <linux/jump_label.h> 14 14 - #include <linux/kernel.h> 15 15 - #include <linux/module.h> 16 16 - #include <linux/preempt.h> 17 17 - #include <linux/uaccess.h> 18 18 - 19 19 - #define VMX_ALIGN 16 20 20 - #define VMX_ALIGN_MASK (VMX_ALIGN-1) 21 21 - 22 22 - #define VECTOR_BREAKPOINT 64 23 23 - 24 24 - static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_vec_crypto); 25 25 - 26 26 - u32 __crct10dif_vpmsum(u32 crc, unsigned char const *p, size_t len); 27 27 - 28 28 - u16 crc_t10dif_arch(u16 crci, const u8 *p, size_t len) 29 29 - { 30 30 - unsigned int prealign; 31 31 - unsigned int tail; 32 32 - u32 crc = crci; 33 33 - 34 34 - if (len < (VECTOR_BREAKPOINT + VMX_ALIGN) || 35 35 - !static_branch_likely(&have_vec_crypto) || !crypto_simd_usable()) 36 36 - return crc_t10dif_generic(crc, p, len); 37 37 - 38 38 - if ((unsigned long)p & VMX_ALIGN_MASK) { 39 39 - prealign = VMX_ALIGN - ((unsigned long)p & VMX_ALIGN_MASK); 40 40 - crc = crc_t10dif_generic(crc, p, prealign); 41 41 - len -= prealign; 42 42 - p += prealign; 43 43 - } 44 44 - 45 45 - if (len & ~VMX_ALIGN_MASK) { 46 46 - crc <<= 16; 47 47 - preempt_disable(); 48 48 - pagefault_disable(); 49 49 - enable_kernel_altivec(); 50 50 - crc = __crct10dif_vpmsum(crc, p, len & ~VMX_ALIGN_MASK); 51 51 - disable_kernel_altivec(); 52 52 - pagefault_enable(); 53 53 - preempt_enable(); 54 54 - crc >>= 16; 55 55 - } 56 56 - 57 57 - tail = len & VMX_ALIGN_MASK; 58 58 - if (tail) { 59 59 - p += len & ~VMX_ALIGN_MASK; 60 60 - crc = crc_t10dif_generic(crc, p, tail); 61 61 - } 62 62 - 63 63 - return crc & 0xffff; 64 64 - } 65 65 - EXPORT_SYMBOL(crc_t10dif_arch); 66 66 - 67 67 - static int __init crc_t10dif_powerpc_init(void) 68 68 - { 69 69 - if (cpu_has_feature(CPU_FTR_ARCH_207S) && 70 70 - (cur_cpu_spec->cpu_user_features2 & PPC_FEATURE2_VEC_CRYPTO)) 71 71 - static_branch_enable(&have_vec_crypto); 72 72 - return 0; 73 73 - } 74 74 - subsys_initcall(crc_t10dif_powerpc_init); 75 75 - 76 76 - static void __exit crc_t10dif_powerpc_exit(void) 77 77 - { 78 78 - } 79 79 - module_exit(crc_t10dif_powerpc_exit); 80 80 - 81 81 - MODULE_AUTHOR("Daniel Axtens <dja@axtens.net>"); 82 82 - MODULE_DESCRIPTION("CRCT10DIF using vector polynomial multiply-sum instructions"); 83 83 - MODULE_LICENSE("GPL");

arch/powerpc/lib/crc-vpmsum-template.S lib/crc/powerpc/crc-vpmsum-template.S

reviewed

-93

arch/powerpc/lib/crc32.c

reviewed

··· 1 1 - // SPDX-License-Identifier: GPL-2.0-only 2 2 - #include <asm/switch_to.h> 3 3 - #include <crypto/internal/simd.h> 4 4 - #include <linux/cpufeature.h> 5 5 - #include <linux/crc32.h> 6 6 - #include <linux/jump_label.h> 7 7 - #include <linux/kernel.h> 8 8 - #include <linux/module.h> 9 9 - #include <linux/preempt.h> 10 10 - #include <linux/uaccess.h> 11 11 - 12 12 - #define VMX_ALIGN 16 13 13 - #define VMX_ALIGN_MASK (VMX_ALIGN-1) 14 14 - 15 15 - #define VECTOR_BREAKPOINT 512 16 16 - 17 17 - static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_vec_crypto); 18 18 - 19 19 - u32 __crc32c_vpmsum(u32 crc, const u8 *p, size_t len); 20 20 - 21 21 - u32 crc32_le_arch(u32 crc, const u8 *p, size_t len) 22 22 - { 23 23 - return crc32_le_base(crc, p, len); 24 24 - } 25 25 - EXPORT_SYMBOL(crc32_le_arch); 26 26 - 27 27 - u32 crc32c_arch(u32 crc, const u8 *p, size_t len) 28 28 - { 29 29 - unsigned int prealign; 30 30 - unsigned int tail; 31 31 - 32 32 - if (len < (VECTOR_BREAKPOINT + VMX_ALIGN) || 33 33 - !static_branch_likely(&have_vec_crypto) || !crypto_simd_usable()) 34 34 - return crc32c_base(crc, p, len); 35 35 - 36 36 - if ((unsigned long)p & VMX_ALIGN_MASK) { 37 37 - prealign = VMX_ALIGN - ((unsigned long)p & VMX_ALIGN_MASK); 38 38 - crc = crc32c_base(crc, p, prealign); 39 39 - len -= prealign; 40 40 - p += prealign; 41 41 - } 42 42 - 43 43 - if (len & ~VMX_ALIGN_MASK) { 44 44 - preempt_disable(); 45 45 - pagefault_disable(); 46 46 - enable_kernel_altivec(); 47 47 - crc = __crc32c_vpmsum(crc, p, len & ~VMX_ALIGN_MASK); 48 48 - disable_kernel_altivec(); 49 49 - pagefault_enable(); 50 50 - preempt_enable(); 51 51 - } 52 52 - 53 53 - tail = len & VMX_ALIGN_MASK; 54 54 - if (tail) { 55 55 - p += len & ~VMX_ALIGN_MASK; 56 56 - crc = crc32c_base(crc, p, tail); 57 57 - } 58 58 - 59 59 - return crc; 60 60 - } 61 61 - EXPORT_SYMBOL(crc32c_arch); 62 62 - 63 63 - u32 crc32_be_arch(u32 crc, const u8 *p, size_t len) 64 64 - { 65 65 - return crc32_be_base(crc, p, len); 66 66 - } 67 67 - EXPORT_SYMBOL(crc32_be_arch); 68 68 - 69 69 - static int __init crc32_powerpc_init(void) 70 70 - { 71 71 - if (cpu_has_feature(CPU_FTR_ARCH_207S) && 72 72 - (cur_cpu_spec->cpu_user_features2 & PPC_FEATURE2_VEC_CRYPTO)) 73 73 - static_branch_enable(&have_vec_crypto); 74 74 - return 0; 75 75 - } 76 76 - subsys_initcall(crc32_powerpc_init); 77 77 - 78 78 - static void __exit crc32_powerpc_exit(void) 79 79 - { 80 80 - } 81 81 - module_exit(crc32_powerpc_exit); 82 82 - 83 83 - u32 crc32_optimizations(void) 84 84 - { 85 85 - if (static_key_enabled(&have_vec_crypto)) 86 86 - return CRC32C_OPTIMIZATION; 87 87 - return 0; 88 88 - } 89 89 - EXPORT_SYMBOL(crc32_optimizations); 90 90 - 91 91 - MODULE_AUTHOR("Anton Blanchard <anton@samba.org>"); 92 92 - MODULE_DESCRIPTION("CRC32C using vector polynomial multiply-sum instructions"); 93 93 - MODULE_LICENSE("GPL");

arch/powerpc/lib/crc32c-vpmsum_asm.S lib/crc/powerpc/crc32c-vpmsum_asm.S

reviewed

arch/powerpc/lib/crct10dif-vpmsum_asm.S lib/crc/powerpc/crct10dif-vpmsum_asm.S

reviewed

-3

arch/riscv/Kconfig

reviewed

··· 24 24 select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2 25 25 select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE 26 26 select ARCH_HAS_BINFMT_FLAT 27 27 - select ARCH_HAS_CRC32 if RISCV_ISA_ZBC 28 28 - select ARCH_HAS_CRC64 if 64BIT && RISCV_ISA_ZBC 29 29 - select ARCH_HAS_CRC_T10DIF if RISCV_ISA_ZBC 30 27 select ARCH_HAS_CURRENT_STACK_POINTER 31 28 select ARCH_HAS_DEBUG_VIRTUAL if MMU 32 29 select ARCH_HAS_DEBUG_VM_PGTABLE

-6

arch/riscv/lib/Makefile

reviewed

··· 16 16 lib-$(CONFIG_MMU) += uaccess.o 17 17 lib-$(CONFIG_64BIT) += tishift.o 18 18 lib-$(CONFIG_RISCV_ISA_ZICBOZ) += clear_page.o 19 19 - obj-$(CONFIG_CRC32_ARCH) += crc32-riscv.o 20 20 - crc32-riscv-y := crc32.o crc32_msb.o crc32_lsb.o 21 21 - obj-$(CONFIG_CRC64_ARCH) += crc64-riscv.o 22 22 - crc64-riscv-y := crc64.o crc64_msb.o crc64_lsb.o 23 23 - obj-$(CONFIG_CRC_T10DIF_ARCH) += crc-t10dif-riscv.o 24 24 - crc-t10dif-riscv-y := crc-t10dif.o crc16_msb.o 25 19 obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o 26 20 lib-$(CONFIG_RISCV_ISA_V) += xor.o 27 21 lib-$(CONFIG_RISCV_ISA_V) += riscv_v_helpers.o

arch/riscv/lib/crc-clmul-consts.h lib/crc/riscv/crc-clmul-consts.h

reviewed

arch/riscv/lib/crc-clmul-template.h lib/crc/riscv/crc-clmul-template.h

reviewed

arch/riscv/lib/crc-clmul.h lib/crc/riscv/crc-clmul.h

reviewed

-24

arch/riscv/lib/crc-t10dif.c

reviewed

··· 1 1 - // SPDX-License-Identifier: GPL-2.0-or-later 2 2 - /* 3 3 - * RISC-V optimized CRC-T10DIF function 4 4 - * 5 5 - * Copyright 2025 Google LLC 6 6 - */ 7 7 - 8 8 - #include <asm/hwcap.h> 9 9 - #include <asm/alternative-macros.h> 10 10 - #include <linux/crc-t10dif.h> 11 11 - #include <linux/module.h> 12 12 - 13 13 - #include "crc-clmul.h" 14 14 - 15 15 - u16 crc_t10dif_arch(u16 crc, const u8 *p, size_t len) 16 16 - { 17 17 - if (riscv_has_extension_likely(RISCV_ISA_EXT_ZBC)) 18 18 - return crc16_msb_clmul(crc, p, len, &crc16_msb_0x8bb7_consts); 19 19 - return crc_t10dif_generic(crc, p, len); 20 20 - } 21 21 - EXPORT_SYMBOL(crc_t10dif_arch); 22 22 - 23 23 - MODULE_DESCRIPTION("RISC-V optimized CRC-T10DIF function"); 24 24 - MODULE_LICENSE("GPL");

arch/riscv/lib/crc16_msb.c lib/crc/riscv/crc16_msb.c

reviewed

-53

arch/riscv/lib/crc32.c

reviewed

··· 1 1 - // SPDX-License-Identifier: GPL-2.0-or-later 2 2 - /* 3 3 - * RISC-V optimized CRC32 functions 4 4 - * 5 5 - * Copyright 2025 Google LLC 6 6 - */ 7 7 - 8 8 - #include <asm/hwcap.h> 9 9 - #include <asm/alternative-macros.h> 10 10 - #include <linux/crc32.h> 11 11 - #include <linux/module.h> 12 12 - 13 13 - #include "crc-clmul.h" 14 14 - 15 15 - u32 crc32_le_arch(u32 crc, const u8 *p, size_t len) 16 16 - { 17 17 - if (riscv_has_extension_likely(RISCV_ISA_EXT_ZBC)) 18 18 - return crc32_lsb_clmul(crc, p, len, 19 19 - &crc32_lsb_0xedb88320_consts); 20 20 - return crc32_le_base(crc, p, len); 21 21 - } 22 22 - EXPORT_SYMBOL(crc32_le_arch); 23 23 - 24 24 - u32 crc32_be_arch(u32 crc, const u8 *p, size_t len) 25 25 - { 26 26 - if (riscv_has_extension_likely(RISCV_ISA_EXT_ZBC)) 27 27 - return crc32_msb_clmul(crc, p, len, 28 28 - &crc32_msb_0x04c11db7_consts); 29 29 - return crc32_be_base(crc, p, len); 30 30 - } 31 31 - EXPORT_SYMBOL(crc32_be_arch); 32 32 - 33 33 - u32 crc32c_arch(u32 crc, const u8 *p, size_t len) 34 34 - { 35 35 - if (riscv_has_extension_likely(RISCV_ISA_EXT_ZBC)) 36 36 - return crc32_lsb_clmul(crc, p, len, 37 37 - &crc32_lsb_0x82f63b78_consts); 38 38 - return crc32c_base(crc, p, len); 39 39 - } 40 40 - EXPORT_SYMBOL(crc32c_arch); 41 41 - 42 42 - u32 crc32_optimizations(void) 43 43 - { 44 44 - if (riscv_has_extension_likely(RISCV_ISA_EXT_ZBC)) 45 45 - return CRC32_LE_OPTIMIZATION | 46 46 - CRC32_BE_OPTIMIZATION | 47 47 - CRC32C_OPTIMIZATION; 48 48 - return 0; 49 49 - } 50 50 - EXPORT_SYMBOL(crc32_optimizations); 51 51 - 52 52 - MODULE_DESCRIPTION("RISC-V optimized CRC32 functions"); 53 53 - MODULE_LICENSE("GPL");

arch/riscv/lib/crc32_lsb.c lib/crc/riscv/crc32_lsb.c

reviewed

arch/riscv/lib/crc32_msb.c lib/crc/riscv/crc32_msb.c

reviewed

-34

arch/riscv/lib/crc64.c

reviewed

··· 1 1 - // SPDX-License-Identifier: GPL-2.0-or-later 2 2 - /* 3 3 - * RISC-V optimized CRC64 functions 4 4 - * 5 5 - * Copyright 2025 Google LLC 6 6 - */ 7 7 - 8 8 - #include <asm/hwcap.h> 9 9 - #include <asm/alternative-macros.h> 10 10 - #include <linux/crc64.h> 11 11 - #include <linux/module.h> 12 12 - 13 13 - #include "crc-clmul.h" 14 14 - 15 15 - u64 crc64_be_arch(u64 crc, const u8 *p, size_t len) 16 16 - { 17 17 - if (riscv_has_extension_likely(RISCV_ISA_EXT_ZBC)) 18 18 - return crc64_msb_clmul(crc, p, len, 19 19 - &crc64_msb_0x42f0e1eba9ea3693_consts); 20 20 - return crc64_be_generic(crc, p, len); 21 21 - } 22 22 - EXPORT_SYMBOL(crc64_be_arch); 23 23 - 24 24 - u64 crc64_nvme_arch(u64 crc, const u8 *p, size_t len) 25 25 - { 26 26 - if (riscv_has_extension_likely(RISCV_ISA_EXT_ZBC)) 27 27 - return crc64_lsb_clmul(crc, p, len, 28 28 - &crc64_lsb_0x9a6c9329ac4bc9b5_consts); 29 29 - return crc64_nvme_generic(crc, p, len); 30 30 - } 31 31 - EXPORT_SYMBOL(crc64_nvme_arch); 32 32 - 33 33 - MODULE_DESCRIPTION("RISC-V optimized CRC64 functions"); 34 34 - MODULE_LICENSE("GPL");

arch/riscv/lib/crc64_lsb.c lib/crc/riscv/crc64_lsb.c

reviewed

arch/riscv/lib/crc64_msb.c lib/crc/riscv/crc64_msb.c

reviewed

-1

arch/s390/Kconfig

reviewed

··· 75 75 select ARCH_ENABLE_MEMORY_HOTREMOVE 76 76 select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2 77 77 select ARCH_HAS_CPU_FINALIZE_INIT 78 78 - select ARCH_HAS_CRC32 79 78 select ARCH_HAS_CURRENT_STACK_POINTER 80 79 select ARCH_HAS_DEBUG_VIRTUAL 81 80 select ARCH_HAS_DEBUG_VM_PGTABLE

-3

arch/s390/lib/Makefile

reviewed

··· 25 25 lib-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o 26 26 27 27 obj-$(CONFIG_EXPOLINE_EXTERN) += expoline.o 28 28 - 29 29 - obj-$(CONFIG_CRC32_ARCH) += crc32-s390.o 30 30 - crc32-s390-y := crc32.o crc32le-vx.o crc32be-vx.o

arch/s390/lib/crc32-vx.h lib/crc/s390/crc32-vx.h

reviewed

-77

arch/s390/lib/crc32.c

reviewed

··· 1 1 - // SPDX-License-Identifier: GPL-2.0 2 2 - /* 3 3 - * CRC-32 implemented with the z/Architecture Vector Extension Facility. 4 4 - * 5 5 - * Copyright IBM Corp. 2015 6 6 - * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com> 7 7 - */ 8 8 - #define KMSG_COMPONENT "crc32-vx" 9 9 - #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt 10 10 - 11 11 - #include <linux/module.h> 12 12 - #include <linux/cpufeature.h> 13 13 - #include <linux/crc32.h> 14 14 - #include <asm/fpu.h> 15 15 - #include "crc32-vx.h" 16 16 - 17 17 - #define VX_MIN_LEN 64 18 18 - #define VX_ALIGNMENT 16L 19 19 - #define VX_ALIGN_MASK (VX_ALIGNMENT - 1) 20 20 - 21 21 - /* 22 22 - * DEFINE_CRC32_VX() - Define a CRC-32 function using the vector extension 23 23 - * 24 24 - * Creates a function to perform a particular CRC-32 computation. Depending 25 25 - * on the message buffer, the hardware-accelerated or software implementation 26 26 - * is used. Note that the message buffer is aligned to improve fetch 27 27 - * operations of VECTOR LOAD MULTIPLE instructions. 28 28 - */ 29 29 - #define DEFINE_CRC32_VX(___fname, ___crc32_vx, ___crc32_sw) \ 30 30 - u32 ___fname(u32 crc, const u8 *data, size_t datalen) \ 31 31 - { \ 32 32 - unsigned long prealign, aligned, remaining; \ 33 33 - DECLARE_KERNEL_FPU_ONSTACK16(vxstate); \ 34 34 - \ 35 35 - if (datalen < VX_MIN_LEN + VX_ALIGN_MASK || !cpu_has_vx()) \ 36 36 - return ___crc32_sw(crc, data, datalen); \ 37 37 - \ 38 38 - if ((unsigned long)data & VX_ALIGN_MASK) { \ 39 39 - prealign = VX_ALIGNMENT - \ 40 40 - ((unsigned long)data & VX_ALIGN_MASK); \ 41 41 - datalen -= prealign; \ 42 42 - crc = ___crc32_sw(crc, data, prealign); \ 43 43 - data = (void *)((unsigned long)data + prealign); \ 44 44 - } \ 45 45 - \ 46 46 - aligned = datalen & ~VX_ALIGN_MASK; \ 47 47 - remaining = datalen & VX_ALIGN_MASK; \ 48 48 - \ 49 49 - kernel_fpu_begin(&vxstate, KERNEL_VXR_LOW); \ 50 50 - crc = ___crc32_vx(crc, data, aligned); \ 51 51 - kernel_fpu_end(&vxstate, KERNEL_VXR_LOW); \ 52 52 - \ 53 53 - if (remaining) \ 54 54 - crc = ___crc32_sw(crc, data + aligned, remaining); \ 55 55 - \ 56 56 - return crc; \ 57 57 - } \ 58 58 - EXPORT_SYMBOL(___fname); 59 59 - 60 60 - DEFINE_CRC32_VX(crc32_le_arch, crc32_le_vgfm_16, crc32_le_base) 61 61 - DEFINE_CRC32_VX(crc32_be_arch, crc32_be_vgfm_16, crc32_be_base) 62 62 - DEFINE_CRC32_VX(crc32c_arch, crc32c_le_vgfm_16, crc32c_base) 63 63 - 64 64 - u32 crc32_optimizations(void) 65 65 - { 66 66 - if (cpu_has_vx()) { 67 67 - return CRC32_LE_OPTIMIZATION | 68 68 - CRC32_BE_OPTIMIZATION | 69 69 - CRC32C_OPTIMIZATION; 70 70 - } 71 71 - return 0; 72 72 - } 73 73 - EXPORT_SYMBOL(crc32_optimizations); 74 74 - 75 75 - MODULE_AUTHOR("Hendrik Brueckner <brueckner@linux.vnet.ibm.com>"); 76 76 - MODULE_DESCRIPTION("CRC-32 algorithms using z/Architecture Vector Extension Facility"); 77 77 - MODULE_LICENSE("GPL");

arch/s390/lib/crc32be-vx.c lib/crc/s390/crc32be-vx.c

reviewed

arch/s390/lib/crc32le-vx.c lib/crc/s390/crc32le-vx.c

reviewed

-1

arch/sparc/Kconfig

reviewed

··· 110 110 select HAVE_SETUP_PER_CPU_AREA 111 111 select NEED_PER_CPU_EMBED_FIRST_CHUNK 112 112 select NEED_PER_CPU_PAGE_FIRST_CHUNK 113 113 - select ARCH_HAS_CRC32 114 113 115 114 config ARCH_PROC_KCORE_TEXT 116 115 def_bool y

-2

arch/sparc/lib/Makefile

reviewed

··· 54 54 obj-$(CONFIG_SPARC64) += iomap.o 55 55 obj-$(CONFIG_SPARC32) += atomic32.o 56 56 obj-$(CONFIG_SPARC64) += PeeCeeI.o 57 57 - obj-$(CONFIG_CRC32_ARCH) += crc32-sparc.o 58 58 - crc32-sparc-y := crc32.o crc32c_asm.o

-93

arch/sparc/lib/crc32.c

reviewed

··· 1 1 - // SPDX-License-Identifier: GPL-2.0-only 2 2 - /* CRC32c (Castagnoli), sparc64 crc32c opcode accelerated 3 3 - * 4 4 - * This is based largely upon arch/x86/crypto/crc32c-intel.c 5 5 - * 6 6 - * Copyright (C) 2008 Intel Corporation 7 7 - * Authors: Austin Zhang <austin_zhang@linux.intel.com> 8 8 - * Kent Liu <kent.liu@intel.com> 9 9 - */ 10 10 - 11 11 - #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 12 12 - 13 13 - #include <linux/init.h> 14 14 - #include <linux/module.h> 15 15 - #include <linux/kernel.h> 16 16 - #include <linux/crc32.h> 17 17 - #include <asm/pstate.h> 18 18 - #include <asm/elf.h> 19 19 - 20 20 - static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_crc32c_opcode); 21 21 - 22 22 - u32 crc32_le_arch(u32 crc, const u8 *data, size_t len) 23 23 - { 24 24 - return crc32_le_base(crc, data, len); 25 25 - } 26 26 - EXPORT_SYMBOL(crc32_le_arch); 27 27 - 28 28 - void crc32c_sparc64(u32 *crcp, const u64 *data, size_t len); 29 29 - 30 30 - u32 crc32c_arch(u32 crc, const u8 *data, size_t len) 31 31 - { 32 32 - size_t n = -(uintptr_t)data & 7; 33 33 - 34 34 - if (!static_branch_likely(&have_crc32c_opcode)) 35 35 - return crc32c_base(crc, data, len); 36 36 - 37 37 - if (n) { 38 38 - /* Data isn't 8-byte aligned. Align it. */ 39 39 - n = min(n, len); 40 40 - crc = crc32c_base(crc, data, n); 41 41 - data += n; 42 42 - len -= n; 43 43 - } 44 44 - n = len & ~7U; 45 45 - if (n) { 46 46 - crc32c_sparc64(&crc, (const u64 *)data, n); 47 47 - data += n; 48 48 - len -= n; 49 49 - } 50 50 - if (len) 51 51 - crc = crc32c_base(crc, data, len); 52 52 - return crc; 53 53 - } 54 54 - EXPORT_SYMBOL(crc32c_arch); 55 55 - 56 56 - u32 crc32_be_arch(u32 crc, const u8 *data, size_t len) 57 57 - { 58 58 - return crc32_be_base(crc, data, len); 59 59 - } 60 60 - EXPORT_SYMBOL(crc32_be_arch); 61 61 - 62 62 - static int __init crc32_sparc_init(void) 63 63 - { 64 64 - unsigned long cfr; 65 65 - 66 66 - if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO)) 67 67 - return 0; 68 68 - 69 69 - __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr)); 70 70 - if (!(cfr & CFR_CRC32C)) 71 71 - return 0; 72 72 - 73 73 - static_branch_enable(&have_crc32c_opcode); 74 74 - pr_info("Using sparc64 crc32c opcode optimized CRC32C implementation\n"); 75 75 - return 0; 76 76 - } 77 77 - subsys_initcall(crc32_sparc_init); 78 78 - 79 79 - static void __exit crc32_sparc_exit(void) 80 80 - { 81 81 - } 82 82 - module_exit(crc32_sparc_exit); 83 83 - 84 84 - u32 crc32_optimizations(void) 85 85 - { 86 86 - if (static_key_enabled(&have_crc32c_opcode)) 87 87 - return CRC32C_OPTIMIZATION; 88 88 - return 0; 89 89 - } 90 90 - EXPORT_SYMBOL(crc32_optimizations); 91 91 - 92 92 - MODULE_LICENSE("GPL"); 93 93 - MODULE_DESCRIPTION("CRC32c (Castagnoli), sparc64 crc32c opcode accelerated");

arch/sparc/lib/crc32c_asm.S lib/crc/sparc/crc32c_asm.S

reviewed

-3

arch/x86/Kconfig

reviewed

··· 79 79 select ARCH_HAS_CPU_CACHE_INVALIDATE_MEMREGION 80 80 select ARCH_HAS_CPU_FINALIZE_INIT 81 81 select ARCH_HAS_CPU_PASID if IOMMU_SVA 82 82 - select ARCH_HAS_CRC32 83 83 - select ARCH_HAS_CRC64 if X86_64 84 84 - select ARCH_HAS_CRC_T10DIF 85 82 select ARCH_HAS_CURRENT_STACK_POINTER 86 83 select ARCH_HAS_DEBUG_VIRTUAL 87 84 select ARCH_HAS_DEBUG_VM_PGTABLE if !X86_PAE

-10

arch/x86/lib/Makefile

reviewed

··· 40 40 lib-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o 41 41 lib-$(CONFIG_MITIGATION_RETPOLINE) += retpoline.o 42 42 43 43 - obj-$(CONFIG_CRC32_ARCH) += crc32-x86.o 44 44 - crc32-x86-y := crc32.o crc32-pclmul.o 45 45 - crc32-x86-$(CONFIG_64BIT) += crc32c-3way.o 46 46 - 47 47 - obj-$(CONFIG_CRC64_ARCH) += crc64-x86.o 48 48 - crc64-x86-y := crc64.o crc64-pclmul.o 49 49 - 50 50 - obj-$(CONFIG_CRC_T10DIF_ARCH) += crc-t10dif-x86.o 51 51 - crc-t10dif-x86-y := crc-t10dif.o crc16-msb-pclmul.o 52 52 - 53 43 obj-y += msr.o msr-reg.o msr-reg-export.o hweight.o 54 44 obj-y += iomem.o 55 45

-195

arch/x86/lib/crc-pclmul-consts.h

reviewed

··· 1 1 - /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 2 - /* 3 3 - * CRC constants generated by: 4 4 - * 5 5 - * ./scripts/gen-crc-consts.py x86_pclmul crc16_msb_0x8bb7,crc32_lsb_0xedb88320,crc64_msb_0x42f0e1eba9ea3693,crc64_lsb_0x9a6c9329ac4bc9b5 6 6 - * 7 7 - * Do not edit manually. 8 8 - */ 9 9 - 10 10 - /* 11 11 - * CRC folding constants generated for most-significant-bit-first CRC-16 using 12 12 - * G(x) = x^16 + x^15 + x^11 + x^9 + x^8 + x^7 + x^5 + x^4 + x^2 + x^1 + x^0 13 13 - */ 14 14 - static const struct { 15 15 - u8 bswap_mask[16]; 16 16 - u64 fold_across_2048_bits_consts[2]; 17 17 - u64 fold_across_1024_bits_consts[2]; 18 18 - u64 fold_across_512_bits_consts[2]; 19 19 - u64 fold_across_256_bits_consts[2]; 20 20 - u64 fold_across_128_bits_consts[2]; 21 21 - u8 shuf_table[48]; 22 22 - u64 barrett_reduction_consts[2]; 23 23 - } crc16_msb_0x8bb7_consts ____cacheline_aligned __maybe_unused = { 24 24 - .bswap_mask = {15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0}, 25 25 - .fold_across_2048_bits_consts = { 26 26 - 0xdccf000000000000, /* LO64_TERMS: (x^2000 mod G) * x^48 */ 27 27 - 0x4b0b000000000000, /* HI64_TERMS: (x^2064 mod G) * x^48 */ 28 28 - }, 29 29 - .fold_across_1024_bits_consts = { 30 30 - 0x9d9d000000000000, /* LO64_TERMS: (x^976 mod G) * x^48 */ 31 31 - 0x7cf5000000000000, /* HI64_TERMS: (x^1040 mod G) * x^48 */ 32 32 - }, 33 33 - .fold_across_512_bits_consts = { 34 34 - 0x044c000000000000, /* LO64_TERMS: (x^464 mod G) * x^48 */ 35 35 - 0xe658000000000000, /* HI64_TERMS: (x^528 mod G) * x^48 */ 36 36 - }, 37 37 - .fold_across_256_bits_consts = { 38 38 - 0x6ee3000000000000, /* LO64_TERMS: (x^208 mod G) * x^48 */ 39 39 - 0xe7b5000000000000, /* HI64_TERMS: (x^272 mod G) * x^48 */ 40 40 - }, 41 41 - .fold_across_128_bits_consts = { 42 42 - 0x2d56000000000000, /* LO64_TERMS: (x^80 mod G) * x^48 */ 43 43 - 0x06df000000000000, /* HI64_TERMS: (x^144 mod G) * x^48 */ 44 44 - }, 45 45 - .shuf_table = { 46 46 - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 47 47 - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 48 48 - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 49 49 - }, 50 50 - .barrett_reduction_consts = { 51 51 - 0x8bb7000000000000, /* LO64_TERMS: (G - x^16) * x^48 */ 52 52 - 0xf65a57f81d33a48a, /* HI64_TERMS: (floor(x^79 / G) * x) - x^64 */ 53 53 - }, 54 54 - }; 55 55 - 56 56 - /* 57 57 - * CRC folding constants generated for least-significant-bit-first CRC-32 using 58 58 - * G(x) = x^32 + x^26 + x^23 + x^22 + x^16 + x^12 + x^11 + x^10 + x^8 + x^7 + 59 59 - * x^5 + x^4 + x^2 + x^1 + x^0 60 60 - */ 61 61 - static const struct { 62 62 - u64 fold_across_2048_bits_consts[2]; 63 63 - u64 fold_across_1024_bits_consts[2]; 64 64 - u64 fold_across_512_bits_consts[2]; 65 65 - u64 fold_across_256_bits_consts[2]; 66 66 - u64 fold_across_128_bits_consts[2]; 67 67 - u8 shuf_table[48]; 68 68 - u64 barrett_reduction_consts[2]; 69 69 - } crc32_lsb_0xedb88320_consts ____cacheline_aligned __maybe_unused = { 70 70 - .fold_across_2048_bits_consts = { 71 71 - 0x00000000ce3371cb, /* HI64_TERMS: (x^2079 mod G) * x^32 */ 72 72 - 0x00000000e95c1271, /* LO64_TERMS: (x^2015 mod G) * x^32 */ 73 73 - }, 74 74 - .fold_across_1024_bits_consts = { 75 75 - 0x0000000033fff533, /* HI64_TERMS: (x^1055 mod G) * x^32 */ 76 76 - 0x00000000910eeec1, /* LO64_TERMS: (x^991 mod G) * x^32 */ 77 77 - }, 78 78 - .fold_across_512_bits_consts = { 79 79 - 0x000000008f352d95, /* HI64_TERMS: (x^543 mod G) * x^32 */ 80 80 - 0x000000001d9513d7, /* LO64_TERMS: (x^479 mod G) * x^32 */ 81 81 - }, 82 82 - .fold_across_256_bits_consts = { 83 83 - 0x00000000f1da05aa, /* HI64_TERMS: (x^287 mod G) * x^32 */ 84 84 - 0x0000000081256527, /* LO64_TERMS: (x^223 mod G) * x^32 */ 85 85 - }, 86 86 - .fold_across_128_bits_consts = { 87 87 - 0x00000000ae689191, /* HI64_TERMS: (x^159 mod G) * x^32 */ 88 88 - 0x00000000ccaa009e, /* LO64_TERMS: (x^95 mod G) * x^32 */ 89 89 - }, 90 90 - .shuf_table = { 91 91 - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 92 92 - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 93 93 - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 94 94 - }, 95 95 - .barrett_reduction_consts = { 96 96 - 0xb4e5b025f7011641, /* HI64_TERMS: floor(x^95 / G) */ 97 97 - 0x00000001db710640, /* LO64_TERMS: (G - x^32) * x^31 */ 98 98 - }, 99 99 - }; 100 100 - 101 101 - /* 102 102 - * CRC folding constants generated for most-significant-bit-first CRC-64 using 103 103 - * G(x) = x^64 + x^62 + x^57 + x^55 + x^54 + x^53 + x^52 + x^47 + x^46 + x^45 + 104 104 - * x^40 + x^39 + x^38 + x^37 + x^35 + x^33 + x^32 + x^31 + x^29 + x^27 + 105 105 - * x^24 + x^23 + x^22 + x^21 + x^19 + x^17 + x^13 + x^12 + x^10 + x^9 + 106 106 - * x^7 + x^4 + x^1 + x^0 107 107 - */ 108 108 - static const struct { 109 109 - u8 bswap_mask[16]; 110 110 - u64 fold_across_2048_bits_consts[2]; 111 111 - u64 fold_across_1024_bits_consts[2]; 112 112 - u64 fold_across_512_bits_consts[2]; 113 113 - u64 fold_across_256_bits_consts[2]; 114 114 - u64 fold_across_128_bits_consts[2]; 115 115 - u8 shuf_table[48]; 116 116 - u64 barrett_reduction_consts[2]; 117 117 - } crc64_msb_0x42f0e1eba9ea3693_consts ____cacheline_aligned __maybe_unused = { 118 118 - .bswap_mask = {15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0}, 119 119 - .fold_across_2048_bits_consts = { 120 120 - 0x7f52691a60ddc70d, /* LO64_TERMS: (x^2048 mod G) * x^0 */ 121 121 - 0x7036b0389f6a0c82, /* HI64_TERMS: (x^2112 mod G) * x^0 */ 122 122 - }, 123 123 - .fold_across_1024_bits_consts = { 124 124 - 0x05cf79dea9ac37d6, /* LO64_TERMS: (x^1024 mod G) * x^0 */ 125 125 - 0x001067e571d7d5c2, /* HI64_TERMS: (x^1088 mod G) * x^0 */ 126 126 - }, 127 127 - .fold_across_512_bits_consts = { 128 128 - 0x5f6843ca540df020, /* LO64_TERMS: (x^512 mod G) * x^0 */ 129 129 - 0xddf4b6981205b83f, /* HI64_TERMS: (x^576 mod G) * x^0 */ 130 130 - }, 131 131 - .fold_across_256_bits_consts = { 132 132 - 0x571bee0a227ef92b, /* LO64_TERMS: (x^256 mod G) * x^0 */ 133 133 - 0x44bef2a201b5200c, /* HI64_TERMS: (x^320 mod G) * x^0 */ 134 134 - }, 135 135 - .fold_across_128_bits_consts = { 136 136 - 0x05f5c3c7eb52fab6, /* LO64_TERMS: (x^128 mod G) * x^0 */ 137 137 - 0x4eb938a7d257740e, /* HI64_TERMS: (x^192 mod G) * x^0 */ 138 138 - }, 139 139 - .shuf_table = { 140 140 - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 141 141 - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 142 142 - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 143 143 - }, 144 144 - .barrett_reduction_consts = { 145 145 - 0x42f0e1eba9ea3693, /* LO64_TERMS: (G - x^64) * x^0 */ 146 146 - 0x578d29d06cc4f872, /* HI64_TERMS: (floor(x^127 / G) * x) - x^64 */ 147 147 - }, 148 148 - }; 149 149 - 150 150 - /* 151 151 - * CRC folding constants generated for least-significant-bit-first CRC-64 using 152 152 - * G(x) = x^64 + x^63 + x^61 + x^59 + x^58 + x^56 + x^55 + x^52 + x^49 + x^48 + 153 153 - * x^47 + x^46 + x^44 + x^41 + x^37 + x^36 + x^34 + x^32 + x^31 + x^28 + 154 154 - * x^26 + x^23 + x^22 + x^19 + x^16 + x^13 + x^12 + x^10 + x^9 + x^6 + 155 155 - * x^4 + x^3 + x^0 156 156 - */ 157 157 - static const struct { 158 158 - u64 fold_across_2048_bits_consts[2]; 159 159 - u64 fold_across_1024_bits_consts[2]; 160 160 - u64 fold_across_512_bits_consts[2]; 161 161 - u64 fold_across_256_bits_consts[2]; 162 162 - u64 fold_across_128_bits_consts[2]; 163 163 - u8 shuf_table[48]; 164 164 - u64 barrett_reduction_consts[2]; 165 165 - } crc64_lsb_0x9a6c9329ac4bc9b5_consts ____cacheline_aligned __maybe_unused = { 166 166 - .fold_across_2048_bits_consts = { 167 167 - 0x37ccd3e14069cabc, /* HI64_TERMS: (x^2111 mod G) * x^0 */ 168 168 - 0xa043808c0f782663, /* LO64_TERMS: (x^2047 mod G) * x^0 */ 169 169 - }, 170 170 - .fold_across_1024_bits_consts = { 171 171 - 0xa1ca681e733f9c40, /* HI64_TERMS: (x^1087 mod G) * x^0 */ 172 172 - 0x5f852fb61e8d92dc, /* LO64_TERMS: (x^1023 mod G) * x^0 */ 173 173 - }, 174 174 - .fold_across_512_bits_consts = { 175 175 - 0x0c32cdb31e18a84a, /* HI64_TERMS: (x^575 mod G) * x^0 */ 176 176 - 0x62242240ace5045a, /* LO64_TERMS: (x^511 mod G) * x^0 */ 177 177 - }, 178 178 - .fold_across_256_bits_consts = { 179 179 - 0xb0bc2e589204f500, /* HI64_TERMS: (x^319 mod G) * x^0 */ 180 180 - 0xe1e0bb9d45d7a44c, /* LO64_TERMS: (x^255 mod G) * x^0 */ 181 181 - }, 182 182 - .fold_across_128_bits_consts = { 183 183 - 0xeadc41fd2ba3d420, /* HI64_TERMS: (x^191 mod G) * x^0 */ 184 184 - 0x21e9761e252621ac, /* LO64_TERMS: (x^127 mod G) * x^0 */ 185 185 - }, 186 186 - .shuf_table = { 187 187 - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 188 188 - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 189 189 - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 190 190 - }, 191 191 - .barrett_reduction_consts = { 192 192 - 0x27ecfa329aef9f77, /* HI64_TERMS: floor(x^127 / G) */ 193 193 - 0x34d926535897936a, /* LO64_TERMS: (G - x^64 - x^0) / x */ 194 194 - }, 195 195 - };

-582

arch/x86/lib/crc-pclmul-template.S

reviewed

··· 1 1 - /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 2 - // 3 3 - // Template to generate [V]PCLMULQDQ-based CRC functions for x86 4 4 - // 5 5 - // Copyright 2025 Google LLC 6 6 - // 7 7 - // Author: Eric Biggers <ebiggers@google.com> 8 8 - 9 9 - #include <linux/linkage.h> 10 10 - #include <linux/objtool.h> 11 11 - 12 12 - // Offsets within the generated constants table 13 13 - .set OFFSETOF_BSWAP_MASK, -5*16 // msb-first CRCs only 14 14 - .set OFFSETOF_FOLD_ACROSS_2048_BITS_CONSTS, -4*16 // must precede next 15 15 - .set OFFSETOF_FOLD_ACROSS_1024_BITS_CONSTS, -3*16 // must precede next 16 16 - .set OFFSETOF_FOLD_ACROSS_512_BITS_CONSTS, -2*16 // must precede next 17 17 - .set OFFSETOF_FOLD_ACROSS_256_BITS_CONSTS, -1*16 // must precede next 18 18 - .set OFFSETOF_FOLD_ACROSS_128_BITS_CONSTS, 0*16 // must be 0 19 19 - .set OFFSETOF_SHUF_TABLE, 1*16 20 20 - .set OFFSETOF_BARRETT_REDUCTION_CONSTS, 4*16 21 21 - 22 22 - // Emit a VEX (or EVEX) coded instruction if allowed, or emulate it using the 23 23 - // corresponding non-VEX instruction plus any needed moves. The supported 24 24 - // instruction formats are: 25 25 - // 26 26 - // - Two-arg [src, dst], where the non-VEX format is the same. 27 27 - // - Three-arg [src1, src2, dst] where the non-VEX format is 28 28 - // [src1, src2_and_dst]. If src2 != dst, then src1 must != dst too. 29 29 - // 30 30 - // \insn gives the instruction without a "v" prefix and including any immediate 31 31 - // argument if needed to make the instruction follow one of the above formats. 32 32 - // If \unaligned_mem_tmp is given, then the emitted non-VEX code moves \arg1 to 33 33 - // it first; this is needed when \arg1 is an unaligned mem operand. 34 34 - .macro _cond_vex insn:req, arg1:req, arg2:req, arg3, unaligned_mem_tmp 35 35 - .if AVX_LEVEL == 0 36 36 - // VEX not allowed. Emulate it. 37 37 - .ifnb \arg3 // Three-arg [src1, src2, dst] 38 38 - .ifc "\arg2", "\arg3" // src2 == dst? 39 39 - .ifnb \unaligned_mem_tmp 40 40 - movdqu \arg1, \unaligned_mem_tmp 41 41 - \insn \unaligned_mem_tmp, \arg3 42 42 - .else 43 43 - \insn \arg1, \arg3 44 44 - .endif 45 45 - .else // src2 != dst 46 46 - .ifc "\arg1", "\arg3" 47 47 - .error "Can't have src1 == dst when src2 != dst" 48 48 - .endif 49 49 - .ifnb \unaligned_mem_tmp 50 50 - movdqu \arg1, \unaligned_mem_tmp 51 51 - movdqa \arg2, \arg3 52 52 - \insn \unaligned_mem_tmp, \arg3 53 53 - .else 54 54 - movdqa \arg2, \arg3 55 55 - \insn \arg1, \arg3 56 56 - .endif 57 57 - .endif 58 58 - .else // Two-arg [src, dst] 59 59 - .ifnb \unaligned_mem_tmp 60 60 - movdqu \arg1, \unaligned_mem_tmp 61 61 - \insn \unaligned_mem_tmp, \arg2 62 62 - .else 63 63 - \insn \arg1, \arg2 64 64 - .endif 65 65 - .endif 66 66 - .else 67 67 - // VEX is allowed. Emit the desired instruction directly. 68 68 - .ifnb \arg3 69 69 - v\insn \arg1, \arg2, \arg3 70 70 - .else 71 71 - v\insn \arg1, \arg2 72 72 - .endif 73 73 - .endif 74 74 - .endm 75 75 - 76 76 - // Broadcast an aligned 128-bit mem operand to all 128-bit lanes of a vector 77 77 - // register of length VL. 78 78 - .macro _vbroadcast src, dst 79 79 - .if VL == 16 80 80 - _cond_vex movdqa, \src, \dst 81 81 - .elseif VL == 32 82 82 - vbroadcasti128 \src, \dst 83 83 - .else 84 84 - vbroadcasti32x4 \src, \dst 85 85 - .endif 86 86 - .endm 87 87 - 88 88 - // Load \vl bytes from the unaligned mem operand \src into \dst, and if the CRC 89 89 - // is msb-first use \bswap_mask to reflect the bytes within each 128-bit lane. 90 90 - .macro _load_data vl, src, bswap_mask, dst 91 91 - .if \vl < 64 92 92 - _cond_vex movdqu, "\src", \dst 93 93 - .else 94 94 - vmovdqu8 \src, \dst 95 95 - .endif 96 96 - .if !LSB_CRC 97 97 - _cond_vex pshufb, \bswap_mask, \dst, \dst 98 98 - .endif 99 99 - .endm 100 100 - 101 101 - .macro _prepare_v0 vl, v0, v1, bswap_mask 102 102 - .if LSB_CRC 103 103 - .if \vl < 64 104 104 - _cond_vex pxor, (BUF), \v0, \v0, unaligned_mem_tmp=\v1 105 105 - .else 106 106 - vpxorq (BUF), \v0, \v0 107 107 - .endif 108 108 - .else 109 109 - _load_data \vl, (BUF), \bswap_mask, \v1 110 110 - .if \vl < 64 111 111 - _cond_vex pxor, \v1, \v0, \v0 112 112 - .else 113 113 - vpxorq \v1, \v0, \v0 114 114 - .endif 115 115 - .endif 116 116 - .endm 117 117 - 118 118 - // The x^0..x^63 terms, i.e. poly128 mod x^64, i.e. the physically low qword for 119 119 - // msb-first order or the physically high qword for lsb-first order 120 120 - #define LO64_TERMS 0 121 121 - 122 122 - // The x^64..x^127 terms, i.e. floor(poly128 / x^64), i.e. the physically high 123 123 - // qword for msb-first order or the physically low qword for lsb-first order 124 124 - #define HI64_TERMS 1 125 125 - 126 126 - // Multiply the given \src1_terms of each 128-bit lane of \src1 by the given 127 127 - // \src2_terms of each 128-bit lane of \src2, and write the result(s) to \dst. 128 128 - .macro _pclmulqdq src1, src1_terms, src2, src2_terms, dst 129 129 - _cond_vex "pclmulqdq $((\src1_terms ^ LSB_CRC) << 4) ^ (\src2_terms ^ LSB_CRC),", \ 130 130 - \src1, \src2, \dst 131 131 - .endm 132 132 - 133 133 - // Fold \acc into \data and store the result back into \acc. \data can be an 134 134 - // unaligned mem operand if using VEX is allowed and the CRC is lsb-first so no 135 135 - // byte-reflection is needed; otherwise it must be a vector register. \consts 136 136 - // is a vector register containing the needed fold constants, and \tmp is a 137 137 - // temporary vector register. All arguments must be the same length. 138 138 - .macro _fold_vec acc, data, consts, tmp 139 139 - _pclmulqdq \consts, HI64_TERMS, \acc, HI64_TERMS, \tmp 140 140 - _pclmulqdq \consts, LO64_TERMS, \acc, LO64_TERMS, \acc 141 141 - .if AVX_LEVEL <= 2 142 142 - _cond_vex pxor, \data, \tmp, \tmp 143 143 - _cond_vex pxor, \tmp, \acc, \acc 144 144 - .else 145 145 - vpternlogq $0x96, \data, \tmp, \acc 146 146 - .endif 147 147 - .endm 148 148 - 149 149 - // Fold \acc into \data and store the result back into \acc. \data is an 150 150 - // unaligned mem operand, \consts is a vector register containing the needed 151 151 - // fold constants, \bswap_mask is a vector register containing the 152 152 - // byte-reflection table if the CRC is msb-first, and \tmp1 and \tmp2 are 153 153 - // temporary vector registers. All arguments must have length \vl. 154 154 - .macro _fold_vec_mem vl, acc, data, consts, bswap_mask, tmp1, tmp2 155 155 - .if AVX_LEVEL == 0 || !LSB_CRC 156 156 - _load_data \vl, \data, \bswap_mask, \tmp1 157 157 - _fold_vec \acc, \tmp1, \consts, \tmp2 158 158 - .else 159 159 - _fold_vec \acc, \data, \consts, \tmp1 160 160 - .endif 161 161 - .endm 162 162 - 163 163 - // Load the constants for folding across 2**i vectors of length VL at a time 164 164 - // into all 128-bit lanes of the vector register CONSTS. 165 165 - .macro _load_vec_folding_consts i 166 166 - _vbroadcast OFFSETOF_FOLD_ACROSS_128_BITS_CONSTS+(4-LOG2_VL-\i)*16(CONSTS_PTR), \ 167 167 - CONSTS 168 168 - .endm 169 169 - 170 170 - // Given vector registers \v0 and \v1 of length \vl, fold \v0 into \v1 and store 171 171 - // the result back into \v0. If the remaining length mod \vl is nonzero, also 172 172 - // fold \vl data bytes from BUF. For both operations the fold distance is \vl. 173 173 - // \consts must be a register of length \vl containing the fold constants. 174 174 - .macro _fold_vec_final vl, v0, v1, consts, bswap_mask, tmp1, tmp2 175 175 - _fold_vec \v0, \v1, \consts, \tmp1 176 176 - test $\vl, LEN8 177 177 - jz .Lfold_vec_final_done\@ 178 178 - _fold_vec_mem \vl, \v0, (BUF), \consts, \bswap_mask, \tmp1, \tmp2 179 179 - add $\vl, BUF 180 180 - .Lfold_vec_final_done\@: 181 181 - .endm 182 182 - 183 183 - // This macro generates the body of a CRC function with the following prototype: 184 184 - // 185 185 - // crc_t crc_func(crc_t crc, const u8 *buf, size_t len, const void *consts); 186 186 - // 187 187 - // |crc| is the initial CRC, and crc_t is a data type wide enough to hold it. 188 188 - // |buf| is the data to checksum. |len| is the data length in bytes, which must 189 189 - // be at least 16. |consts| is a pointer to the fold_across_128_bits_consts 190 190 - // field of the constants struct that was generated for the chosen CRC variant. 191 191 - // 192 192 - // Moving onto the macro parameters, \n is the number of bits in the CRC, e.g. 193 193 - // 32 for a CRC-32. Currently the supported values are 8, 16, 32, and 64. If 194 194 - // the file is compiled in i386 mode, then the maximum supported value is 32. 195 195 - // 196 196 - // \lsb_crc is 1 if the CRC processes the least significant bit of each byte 197 197 - // first, i.e. maps bit0 to x^7, bit1 to x^6, ..., bit7 to x^0. \lsb_crc is 0 198 198 - // if the CRC processes the most significant bit of each byte first, i.e. maps 199 199 - // bit0 to x^0, bit1 to x^1, bit7 to x^7. 200 200 - // 201 201 - // \vl is the maximum length of vector register to use in bytes: 16, 32, or 64. 202 202 - // 203 203 - // \avx_level is the level of AVX support to use: 0 for SSE only, 2 for AVX2, or 204 204 - // 512 for AVX512. 205 205 - // 206 206 - // If \vl == 16 && \avx_level == 0, the generated code requires: 207 207 - // PCLMULQDQ && SSE4.1. (Note: all known CPUs with PCLMULQDQ also have SSE4.1.) 208 208 - // 209 209 - // If \vl == 32 && \avx_level == 2, the generated code requires: 210 210 - // VPCLMULQDQ && AVX2. 211 211 - // 212 212 - // If \vl == 64 && \avx_level == 512, the generated code requires: 213 213 - // VPCLMULQDQ && AVX512BW && AVX512VL. 214 214 - // 215 215 - // Other \vl and \avx_level combinations are either not supported or not useful. 216 216 - .macro _crc_pclmul n, lsb_crc, vl, avx_level 217 217 - .set LSB_CRC, \lsb_crc 218 218 - .set VL, \vl 219 219 - .set AVX_LEVEL, \avx_level 220 220 - 221 221 - // Define aliases for the xmm, ymm, or zmm registers according to VL. 222 222 - .irp i, 0,1,2,3,4,5,6,7 223 223 - .if VL == 16 224 224 - .set V\i, %xmm\i 225 225 - .set LOG2_VL, 4 226 226 - .elseif VL == 32 227 227 - .set V\i, %ymm\i 228 228 - .set LOG2_VL, 5 229 229 - .elseif VL == 64 230 230 - .set V\i, %zmm\i 231 231 - .set LOG2_VL, 6 232 232 - .else 233 233 - .error "Unsupported vector length" 234 234 - .endif 235 235 - .endr 236 236 - // Define aliases for the function parameters. 237 237 - // Note: when crc_t is shorter than u32, zero-extension to 32 bits is 238 238 - // guaranteed by the ABI. Zero-extension to 64 bits is *not* guaranteed 239 239 - // when crc_t is shorter than u64. 240 240 - #ifdef __x86_64__ 241 241 - .if \n <= 32 242 242 - .set CRC, %edi 243 243 - .else 244 244 - .set CRC, %rdi 245 245 - .endif 246 246 - .set BUF, %rsi 247 247 - .set LEN, %rdx 248 248 - .set LEN32, %edx 249 249 - .set LEN8, %dl 250 250 - .set CONSTS_PTR, %rcx 251 251 - #else 252 252 - // 32-bit support, assuming -mregparm=3 and not including support for 253 253 - // CRC-64 (which would use both eax and edx to pass the crc parameter). 254 254 - .set CRC, %eax 255 255 - .set BUF, %edx 256 256 - .set LEN, %ecx 257 257 - .set LEN32, %ecx 258 258 - .set LEN8, %cl 259 259 - .set CONSTS_PTR, %ebx // Passed on stack 260 260 - #endif 261 261 - 262 262 - // Define aliases for some local variables. V0-V5 are used without 263 263 - // aliases (for accumulators, data, temporary values, etc). Staying 264 264 - // within the first 8 vector registers keeps the code 32-bit SSE 265 265 - // compatible and reduces the size of 64-bit SSE code slightly. 266 266 - .set BSWAP_MASK, V6 267 267 - .set BSWAP_MASK_YMM, %ymm6 268 268 - .set BSWAP_MASK_XMM, %xmm6 269 269 - .set CONSTS, V7 270 270 - .set CONSTS_YMM, %ymm7 271 271 - .set CONSTS_XMM, %xmm7 272 272 - 273 273 - // Use ANNOTATE_NOENDBR to suppress an objtool warning, since the 274 274 - // functions generated by this macro are called only by static_call. 275 275 - ANNOTATE_NOENDBR 276 276 - 277 277 - #ifdef __i386__ 278 278 - push CONSTS_PTR 279 279 - mov 8(%esp), CONSTS_PTR 280 280 - #endif 281 281 - 282 282 - // Create a 128-bit vector that contains the initial CRC in the end 283 283 - // representing the high-order polynomial coefficients, and the rest 0. 284 284 - // If the CRC is msb-first, also load the byte-reflection table. 285 285 - .if \n <= 32 286 286 - _cond_vex movd, CRC, %xmm0 287 287 - .else 288 288 - _cond_vex movq, CRC, %xmm0 289 289 - .endif 290 290 - .if !LSB_CRC 291 291 - _cond_vex pslldq, $(128-\n)/8, %xmm0, %xmm0 292 292 - _vbroadcast OFFSETOF_BSWAP_MASK(CONSTS_PTR), BSWAP_MASK 293 293 - .endif 294 294 - 295 295 - // Load the first vector of data and XOR the initial CRC into the 296 296 - // appropriate end of the first 128-bit lane of data. If LEN < VL, then 297 297 - // use a short vector and jump ahead to the final reduction. (LEN >= 16 298 298 - // is guaranteed here but not necessarily LEN >= VL.) 299 299 - .if VL >= 32 300 300 - cmp $VL, LEN 301 301 - jae .Lat_least_1vec\@ 302 302 - .if VL == 64 303 303 - cmp $32, LEN32 304 304 - jb .Lless_than_32bytes\@ 305 305 - _prepare_v0 32, %ymm0, %ymm1, BSWAP_MASK_YMM 306 306 - add $32, BUF 307 307 - jmp .Lreduce_256bits_to_128bits\@ 308 308 - .Lless_than_32bytes\@: 309 309 - .endif 310 310 - _prepare_v0 16, %xmm0, %xmm1, BSWAP_MASK_XMM 311 311 - add $16, BUF 312 312 - vmovdqa OFFSETOF_FOLD_ACROSS_128_BITS_CONSTS(CONSTS_PTR), CONSTS_XMM 313 313 - jmp .Lcheck_for_partial_block\@ 314 314 - .Lat_least_1vec\@: 315 315 - .endif 316 316 - _prepare_v0 VL, V0, V1, BSWAP_MASK 317 317 - 318 318 - // Handle VL <= LEN < 4*VL. 319 319 - cmp $4*VL-1, LEN 320 320 - ja .Lat_least_4vecs\@ 321 321 - add $VL, BUF 322 322 - // If VL <= LEN < 2*VL, then jump ahead to the reduction from 1 vector. 323 323 - // If VL==16 then load fold_across_128_bits_consts first, as the final 324 324 - // reduction depends on it and it won't be loaded anywhere else. 325 325 - cmp $2*VL-1, LEN32 326 326 - .if VL == 16 327 327 - _cond_vex movdqa, OFFSETOF_FOLD_ACROSS_128_BITS_CONSTS(CONSTS_PTR), CONSTS_XMM 328 328 - .endif 329 329 - jbe .Lreduce_1vec_to_128bits\@ 330 330 - // Otherwise 2*VL <= LEN < 4*VL. Load one more vector and jump ahead to 331 331 - // the reduction from 2 vectors. 332 332 - _load_data VL, (BUF), BSWAP_MASK, V1 333 333 - add $VL, BUF 334 334 - jmp .Lreduce_2vecs_to_1\@ 335 335 - 336 336 - .Lat_least_4vecs\@: 337 337 - // Load 3 more vectors of data. 338 338 - _load_data VL, 1*VL(BUF), BSWAP_MASK, V1 339 339 - _load_data VL, 2*VL(BUF), BSWAP_MASK, V2 340 340 - _load_data VL, 3*VL(BUF), BSWAP_MASK, V3 341 341 - sub $-4*VL, BUF // Shorter than 'add 4*VL' when VL=32 342 342 - add $-4*VL, LEN // Shorter than 'sub 4*VL' when VL=32 343 343 - 344 344 - // Main loop: while LEN >= 4*VL, fold the 4 vectors V0-V3 into the next 345 345 - // 4 vectors of data and write the result back to V0-V3. 346 346 - cmp $4*VL-1, LEN // Shorter than 'cmp 4*VL' when VL=32 347 347 - jbe .Lreduce_4vecs_to_2\@ 348 348 - _load_vec_folding_consts 2 349 349 - .Lfold_4vecs_loop\@: 350 350 - _fold_vec_mem VL, V0, 0*VL(BUF), CONSTS, BSWAP_MASK, V4, V5 351 351 - _fold_vec_mem VL, V1, 1*VL(BUF), CONSTS, BSWAP_MASK, V4, V5 352 352 - _fold_vec_mem VL, V2, 2*VL(BUF), CONSTS, BSWAP_MASK, V4, V5 353 353 - _fold_vec_mem VL, V3, 3*VL(BUF), CONSTS, BSWAP_MASK, V4, V5 354 354 - sub $-4*VL, BUF 355 355 - add $-4*VL, LEN 356 356 - cmp $4*VL-1, LEN 357 357 - ja .Lfold_4vecs_loop\@ 358 358 - 359 359 - // Fold V0,V1 into V2,V3 and write the result back to V0,V1. Then fold 360 360 - // two more vectors of data from BUF, if at least that much remains. 361 361 - .Lreduce_4vecs_to_2\@: 362 362 - _load_vec_folding_consts 1 363 363 - _fold_vec V0, V2, CONSTS, V4 364 364 - _fold_vec V1, V3, CONSTS, V4 365 365 - test $2*VL, LEN8 366 366 - jz .Lreduce_2vecs_to_1\@ 367 367 - _fold_vec_mem VL, V0, 0*VL(BUF), CONSTS, BSWAP_MASK, V4, V5 368 368 - _fold_vec_mem VL, V1, 1*VL(BUF), CONSTS, BSWAP_MASK, V4, V5 369 369 - sub $-2*VL, BUF 370 370 - 371 371 - // Fold V0 into V1 and write the result back to V0. Then fold one more 372 372 - // vector of data from BUF, if at least that much remains. 373 373 - .Lreduce_2vecs_to_1\@: 374 374 - _load_vec_folding_consts 0 375 375 - _fold_vec_final VL, V0, V1, CONSTS, BSWAP_MASK, V4, V5 376 376 - 377 377 - .Lreduce_1vec_to_128bits\@: 378 378 - .if VL == 64 379 379 - // Reduce 512-bit %zmm0 to 256-bit %ymm0. Then fold 256 more bits of 380 380 - // data from BUF, if at least that much remains. 381 381 - vbroadcasti128 OFFSETOF_FOLD_ACROSS_256_BITS_CONSTS(CONSTS_PTR), CONSTS_YMM 382 382 - vextracti64x4 $1, %zmm0, %ymm1 383 383 - _fold_vec_final 32, %ymm0, %ymm1, CONSTS_YMM, BSWAP_MASK_YMM, %ymm4, %ymm5 384 384 - .Lreduce_256bits_to_128bits\@: 385 385 - .endif 386 386 - .if VL >= 32 387 387 - // Reduce 256-bit %ymm0 to 128-bit %xmm0. Then fold 128 more bits of 388 388 - // data from BUF, if at least that much remains. 389 389 - vmovdqa OFFSETOF_FOLD_ACROSS_128_BITS_CONSTS(CONSTS_PTR), CONSTS_XMM 390 390 - vextracti128 $1, %ymm0, %xmm1 391 391 - _fold_vec_final 16, %xmm0, %xmm1, CONSTS_XMM, BSWAP_MASK_XMM, %xmm4, %xmm5 392 392 - .Lcheck_for_partial_block\@: 393 393 - .endif 394 394 - and $15, LEN32 395 395 - jz .Lreduce_128bits_to_crc\@ 396 396 - 397 397 - // 1 <= LEN <= 15 data bytes remain in BUF. The polynomial is now 398 398 - // A*(x^(8*LEN)) + B, where A is the 128-bit polynomial stored in %xmm0 399 399 - // and B is the polynomial of the remaining LEN data bytes. To reduce 400 400 - // this to 128 bits without needing fold constants for each possible 401 401 - // LEN, rearrange this expression into C1*(x^128) + C2, where 402 402 - // C1 = floor(A / x^(128 - 8*LEN)) and C2 = A*x^(8*LEN) + B mod x^128. 403 403 - // Then fold C1 into C2, which is just another fold across 128 bits. 404 404 - 405 405 - .if !LSB_CRC || AVX_LEVEL == 0 406 406 - // Load the last 16 data bytes. Note that originally LEN was >= 16. 407 407 - _load_data 16, "-16(BUF,LEN)", BSWAP_MASK_XMM, %xmm2 408 408 - .endif // Else will use vpblendvb mem operand later. 409 409 - .if !LSB_CRC 410 410 - neg LEN // Needed for indexing shuf_table 411 411 - .endif 412 412 - 413 413 - // tmp = A*x^(8*LEN) mod x^128 414 414 - // lsb: pshufb by [LEN, LEN+1, ..., 15, -1, -1, ..., -1] 415 415 - // i.e. right-shift by LEN bytes. 416 416 - // msb: pshufb by [-1, -1, ..., -1, 0, 1, ..., 15-LEN] 417 417 - // i.e. left-shift by LEN bytes. 418 418 - _cond_vex movdqu, "OFFSETOF_SHUF_TABLE+16(CONSTS_PTR,LEN)", %xmm3 419 419 - _cond_vex pshufb, %xmm3, %xmm0, %xmm1 420 420 - 421 421 - // C1 = floor(A / x^(128 - 8*LEN)) 422 422 - // lsb: pshufb by [-1, -1, ..., -1, 0, 1, ..., LEN-1] 423 423 - // i.e. left-shift by 16-LEN bytes. 424 424 - // msb: pshufb by [16-LEN, 16-LEN+1, ..., 15, -1, -1, ..., -1] 425 425 - // i.e. right-shift by 16-LEN bytes. 426 426 - _cond_vex pshufb, "OFFSETOF_SHUF_TABLE+32*!LSB_CRC(CONSTS_PTR,LEN)", \ 427 427 - %xmm0, %xmm0, unaligned_mem_tmp=%xmm4 428 428 - 429 429 - // C2 = tmp + B. This is just a blend of tmp with the last 16 data 430 430 - // bytes (reflected if msb-first). The blend mask is the shuffle table 431 431 - // that was used to create tmp. 0 selects tmp, and 1 last16databytes. 432 432 - .if AVX_LEVEL == 0 433 433 - movdqa %xmm0, %xmm4 434 434 - movdqa %xmm3, %xmm0 435 435 - pblendvb %xmm2, %xmm1 // uses %xmm0 as implicit operand 436 436 - movdqa %xmm4, %xmm0 437 437 - .elseif LSB_CRC 438 438 - vpblendvb %xmm3, -16(BUF,LEN), %xmm1, %xmm1 439 439 - .else 440 440 - vpblendvb %xmm3, %xmm2, %xmm1, %xmm1 441 441 - .endif 442 442 - 443 443 - // Fold C1 into C2 and store the 128-bit result in %xmm0. 444 444 - _fold_vec %xmm0, %xmm1, CONSTS_XMM, %xmm4 445 445 - 446 446 - .Lreduce_128bits_to_crc\@: 447 447 - // Compute the CRC as %xmm0 * x^n mod G. Here %xmm0 means the 128-bit 448 448 - // polynomial stored in %xmm0 (using either lsb-first or msb-first bit 449 449 - // order according to LSB_CRC), and G is the CRC's generator polynomial. 450 450 - 451 451 - // First, multiply %xmm0 by x^n and reduce the result to 64+n bits: 452 452 - // 453 453 - // t0 := (x^(64+n) mod G) * floor(%xmm0 / x^64) + 454 454 - // x^n * (%xmm0 mod x^64) 455 455 - // 456 456 - // Store t0 * x^(64-n) in %xmm0. I.e., actually do: 457 457 - // 458 458 - // %xmm0 := ((x^(64+n) mod G) * x^(64-n)) * floor(%xmm0 / x^64) + 459 459 - // x^64 * (%xmm0 mod x^64) 460 460 - // 461 461 - // The extra unreduced factor of x^(64-n) makes floor(t0 / x^n) aligned 462 462 - // to the HI64_TERMS of %xmm0 so that the next pclmulqdq can easily 463 463 - // select it. The 64-bit constant (x^(64+n) mod G) * x^(64-n) in the 464 464 - // msb-first case, or (x^(63+n) mod G) * x^(64-n) in the lsb-first case 465 465 - // (considering the extra factor of x that gets implicitly introduced by 466 466 - // each pclmulqdq when using lsb-first order), is identical to the 467 467 - // constant that was used earlier for folding the LO64_TERMS across 128 468 468 - // bits. Thus it's already available in LO64_TERMS of CONSTS_XMM. 469 469 - _pclmulqdq CONSTS_XMM, LO64_TERMS, %xmm0, HI64_TERMS, %xmm1 470 470 - .if LSB_CRC 471 471 - _cond_vex psrldq, $8, %xmm0, %xmm0 // x^64 * (%xmm0 mod x^64) 472 472 - .else 473 473 - _cond_vex pslldq, $8, %xmm0, %xmm0 // x^64 * (%xmm0 mod x^64) 474 474 - .endif 475 475 - _cond_vex pxor, %xmm1, %xmm0, %xmm0 476 476 - // The HI64_TERMS of %xmm0 now contain floor(t0 / x^n). 477 477 - // The LO64_TERMS of %xmm0 now contain (t0 mod x^n) * x^(64-n). 478 478 - 479 479 - // First step of Barrett reduction: Compute floor(t0 / G). This is the 480 480 - // polynomial by which G needs to be multiplied to cancel out the x^n 481 481 - // and higher terms of t0, i.e. to reduce t0 mod G. First do: 482 482 - // 483 483 - // t1 := floor(x^(63+n) / G) * x * floor(t0 / x^n) 484 484 - // 485 485 - // Then the desired value floor(t0 / G) is floor(t1 / x^64). The 63 in 486 486 - // x^(63+n) is the maximum degree of floor(t0 / x^n) and thus the lowest 487 487 - // value that makes enough precision be carried through the calculation. 488 488 - // 489 489 - // The '* x' makes it so the result is floor(t1 / x^64) rather than 490 490 - // floor(t1 / x^63), making it qword-aligned in HI64_TERMS so that it 491 491 - // can be extracted much more easily in the next step. In the lsb-first 492 492 - // case the '* x' happens implicitly. In the msb-first case it must be 493 493 - // done explicitly; floor(x^(63+n) / G) * x is a 65-bit constant, so the 494 494 - // constant passed to pclmulqdq is (floor(x^(63+n) / G) * x) - x^64, and 495 495 - // the multiplication by the x^64 term is handled using a pxor. The 496 496 - // pxor causes the low 64 terms of t1 to be wrong, but they are unused. 497 497 - _cond_vex movdqa, OFFSETOF_BARRETT_REDUCTION_CONSTS(CONSTS_PTR), CONSTS_XMM 498 498 - _pclmulqdq CONSTS_XMM, HI64_TERMS, %xmm0, HI64_TERMS, %xmm1 499 499 - .if !LSB_CRC 500 500 - _cond_vex pxor, %xmm0, %xmm1, %xmm1 // += x^64 * floor(t0 / x^n) 501 501 - .endif 502 502 - // The HI64_TERMS of %xmm1 now contain floor(t1 / x^64) = floor(t0 / G). 503 503 - 504 504 - // Second step of Barrett reduction: Cancel out the x^n and higher terms 505 505 - // of t0 by subtracting the needed multiple of G. This gives the CRC: 506 506 - // 507 507 - // crc := t0 - (G * floor(t0 / G)) 508 508 - // 509 509 - // But %xmm0 contains t0 * x^(64-n), so it's more convenient to do: 510 510 - // 511 511 - // crc := ((t0 * x^(64-n)) - ((G * x^(64-n)) * floor(t0 / G))) / x^(64-n) 512 512 - // 513 513 - // Furthermore, since the resulting CRC is n-bit, if mod x^n is 514 514 - // explicitly applied to it then the x^n term of G makes no difference 515 515 - // in the result and can be omitted. This helps keep the constant 516 516 - // multiplier in 64 bits in most cases. This gives the following: 517 517 - // 518 518 - // %xmm0 := %xmm0 - (((G - x^n) * x^(64-n)) * floor(t0 / G)) 519 519 - // crc := (%xmm0 / x^(64-n)) mod x^n 520 520 - // 521 521 - // In the lsb-first case, each pclmulqdq implicitly introduces 522 522 - // an extra factor of x, so in that case the constant that needs to be 523 523 - // passed to pclmulqdq is actually '(G - x^n) * x^(63-n)' when n <= 63. 524 524 - // For lsb-first CRCs where n=64, the extra factor of x cannot be as 525 525 - // easily avoided. In that case, instead pass '(G - x^n - x^0) / x' to 526 526 - // pclmulqdq and handle the x^0 term (i.e. 1) separately. (All CRC 527 527 - // polynomials have nonzero x^n and x^0 terms.) It works out as: the 528 528 - // CRC has be XORed with the physically low qword of %xmm1, representing 529 529 - // floor(t0 / G). The most efficient way to do that is to move it to 530 530 - // the physically high qword and use a ternlog to combine the two XORs. 531 531 - .if LSB_CRC && \n == 64 532 532 - _cond_vex punpcklqdq, %xmm1, %xmm2, %xmm2 533 533 - _pclmulqdq CONSTS_XMM, LO64_TERMS, %xmm1, HI64_TERMS, %xmm1 534 534 - .if AVX_LEVEL <= 2 535 535 - _cond_vex pxor, %xmm2, %xmm0, %xmm0 536 536 - _cond_vex pxor, %xmm1, %xmm0, %xmm0 537 537 - .else 538 538 - vpternlogq $0x96, %xmm2, %xmm1, %xmm0 539 539 - .endif 540 540 - _cond_vex "pextrq $1,", %xmm0, %rax // (%xmm0 / x^0) mod x^64 541 541 - .else 542 542 - _pclmulqdq CONSTS_XMM, LO64_TERMS, %xmm1, HI64_TERMS, %xmm1 543 543 - _cond_vex pxor, %xmm1, %xmm0, %xmm0 544 544 - .if \n == 8 545 545 - _cond_vex "pextrb $7 + LSB_CRC,", %xmm0, %eax // (%xmm0 / x^56) mod x^8 546 546 - .elseif \n == 16 547 547 - _cond_vex "pextrw $3 + LSB_CRC,", %xmm0, %eax // (%xmm0 / x^48) mod x^16 548 548 - .elseif \n == 32 549 549 - _cond_vex "pextrd $1 + LSB_CRC,", %xmm0, %eax // (%xmm0 / x^32) mod x^32 550 550 - .else // \n == 64 && !LSB_CRC 551 551 - _cond_vex movq, %xmm0, %rax // (%xmm0 / x^0) mod x^64 552 552 - .endif 553 553 - .endif 554 554 - 555 555 - .if VL > 16 556 556 - vzeroupper // Needed when ymm or zmm registers may have been used. 557 557 - .endif 558 558 - #ifdef __i386__ 559 559 - pop CONSTS_PTR 560 560 - #endif 561 561 - RET 562 562 - .endm 563 563 - 564 564 - #ifdef CONFIG_AS_VPCLMULQDQ 565 565 - #define DEFINE_CRC_PCLMUL_FUNCS(prefix, bits, lsb) \ 566 566 - SYM_FUNC_START(prefix##_pclmul_sse); \ 567 567 - _crc_pclmul n=bits, lsb_crc=lsb, vl=16, avx_level=0; \ 568 568 - SYM_FUNC_END(prefix##_pclmul_sse); \ 569 569 - \ 570 570 - SYM_FUNC_START(prefix##_vpclmul_avx2); \ 571 571 - _crc_pclmul n=bits, lsb_crc=lsb, vl=32, avx_level=2; \ 572 572 - SYM_FUNC_END(prefix##_vpclmul_avx2); \ 573 573 - \ 574 574 - SYM_FUNC_START(prefix##_vpclmul_avx512); \ 575 575 - _crc_pclmul n=bits, lsb_crc=lsb, vl=64, avx_level=512; \ 576 576 - SYM_FUNC_END(prefix##_vpclmul_avx512); 577 577 - #else 578 578 - #define DEFINE_CRC_PCLMUL_FUNCS(prefix, bits, lsb) \ 579 579 - SYM_FUNC_START(prefix##_pclmul_sse); \ 580 580 - _crc_pclmul n=bits, lsb_crc=lsb, vl=16, avx_level=0; \ 581 581 - SYM_FUNC_END(prefix##_pclmul_sse); 582 582 - #endif // !CONFIG_AS_VPCLMULQDQ

-76

arch/x86/lib/crc-pclmul-template.h

reviewed

··· 1 1 - /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 2 - /* 3 3 - * Macros for accessing the [V]PCLMULQDQ-based CRC functions that are 4 4 - * instantiated by crc-pclmul-template.S 5 5 - * 6 6 - * Copyright 2025 Google LLC 7 7 - * 8 8 - * Author: Eric Biggers <ebiggers@google.com> 9 9 - */ 10 10 - #ifndef _CRC_PCLMUL_TEMPLATE_H 11 11 - #define _CRC_PCLMUL_TEMPLATE_H 12 12 - 13 13 - #include <asm/cpufeatures.h> 14 14 - #include <asm/simd.h> 15 15 - #include <crypto/internal/simd.h> 16 16 - #include <linux/static_call.h> 17 17 - #include "crc-pclmul-consts.h" 18 18 - 19 19 - #define DECLARE_CRC_PCLMUL_FUNCS(prefix, crc_t) \ 20 20 - crc_t prefix##_pclmul_sse(crc_t crc, const u8 *p, size_t len, \ 21 21 - const void *consts_ptr); \ 22 22 - crc_t prefix##_vpclmul_avx2(crc_t crc, const u8 *p, size_t len, \ 23 23 - const void *consts_ptr); \ 24 24 - crc_t prefix##_vpclmul_avx512(crc_t crc, const u8 *p, size_t len, \ 25 25 - const void *consts_ptr); \ 26 26 - DEFINE_STATIC_CALL(prefix##_pclmul, prefix##_pclmul_sse) 27 27 - 28 28 - #define INIT_CRC_PCLMUL(prefix) \ 29 29 - do { \ 30 30 - if (IS_ENABLED(CONFIG_AS_VPCLMULQDQ) && \ 31 31 - boot_cpu_has(X86_FEATURE_VPCLMULQDQ) && \ 32 32 - boot_cpu_has(X86_FEATURE_AVX2) && \ 33 33 - cpu_has_xfeatures(XFEATURE_MASK_YMM, NULL)) { \ 34 34 - if (boot_cpu_has(X86_FEATURE_AVX512BW) && \ 35 35 - boot_cpu_has(X86_FEATURE_AVX512VL) && \ 36 36 - !boot_cpu_has(X86_FEATURE_PREFER_YMM) && \ 37 37 - cpu_has_xfeatures(XFEATURE_MASK_AVX512, NULL)) { \ 38 38 - static_call_update(prefix##_pclmul, \ 39 39 - prefix##_vpclmul_avx512); \ 40 40 - } else { \ 41 41 - static_call_update(prefix##_pclmul, \ 42 42 - prefix##_vpclmul_avx2); \ 43 43 - } \ 44 44 - } \ 45 45 - } while (0) 46 46 - 47 47 - /* 48 48 - * Call a [V]PCLMULQDQ optimized CRC function if the data length is at least 16 49 49 - * bytes, the CPU has PCLMULQDQ support, and the current context may use SIMD. 50 50 - * 51 51 - * 16 bytes is the minimum length supported by the [V]PCLMULQDQ functions. 52 52 - * There is overhead associated with kernel_fpu_begin() and kernel_fpu_end(), 53 53 - * varying by CPU and factors such as which parts of the "FPU" state userspace 54 54 - * has touched, which could result in a larger cutoff being better. Indeed, a 55 55 - * larger cutoff is usually better for a *single* message. However, the 56 56 - * overhead of the FPU section gets amortized if multiple FPU sections get 57 57 - * executed before returning to userspace, since the XSAVE and XRSTOR occur only 58 58 - * once. Considering that and the fact that the [V]PCLMULQDQ code is lighter on 59 59 - * the dcache than the table-based code is, a 16-byte cutoff seems to work well. 60 60 - */ 61 61 - #define CRC_PCLMUL(crc, p, len, prefix, consts, have_pclmulqdq) \ 62 62 - do { \ 63 63 - if ((len) >= 16 && static_branch_likely(&(have_pclmulqdq)) && \ 64 64 - crypto_simd_usable()) { \ 65 65 - const void *consts_ptr; \ 66 66 - \ 67 67 - consts_ptr = (consts).fold_across_128_bits_consts; \ 68 68 - kernel_fpu_begin(); \ 69 69 - crc = static_call(prefix##_pclmul)((crc), (p), (len), \ 70 70 - consts_ptr); \ 71 71 - kernel_fpu_end(); \ 72 72 - return crc; \ 73 73 - } \ 74 74 - } while (0) 75 75 - 76 76 - #endif /* _CRC_PCLMUL_TEMPLATE_H */

-40

arch/x86/lib/crc-t10dif.c

reviewed

··· 1 1 - // SPDX-License-Identifier: GPL-2.0-or-later 2 2 - /* 3 3 - * CRC-T10DIF using [V]PCLMULQDQ instructions 4 4 - * 5 5 - * Copyright 2024 Google LLC 6 6 - */ 7 7 - 8 8 - #include <linux/crc-t10dif.h> 9 9 - #include <linux/module.h> 10 10 - #include "crc-pclmul-template.h" 11 11 - 12 12 - static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pclmulqdq); 13 13 - 14 14 - DECLARE_CRC_PCLMUL_FUNCS(crc16_msb, u16); 15 15 - 16 16 - u16 crc_t10dif_arch(u16 crc, const u8 *p, size_t len) 17 17 - { 18 18 - CRC_PCLMUL(crc, p, len, crc16_msb, crc16_msb_0x8bb7_consts, 19 19 - have_pclmulqdq); 20 20 - return crc_t10dif_generic(crc, p, len); 21 21 - } 22 22 - EXPORT_SYMBOL(crc_t10dif_arch); 23 23 - 24 24 - static int __init crc_t10dif_x86_init(void) 25 25 - { 26 26 - if (boot_cpu_has(X86_FEATURE_PCLMULQDQ)) { 27 27 - static_branch_enable(&have_pclmulqdq); 28 28 - INIT_CRC_PCLMUL(crc16_msb); 29 29 - } 30 30 - return 0; 31 31 - } 32 32 - subsys_initcall(crc_t10dif_x86_init); 33 33 - 34 34 - static void __exit crc_t10dif_x86_exit(void) 35 35 - { 36 36 - } 37 37 - module_exit(crc_t10dif_x86_exit); 38 38 - 39 39 - MODULE_DESCRIPTION("CRC-T10DIF using [V]PCLMULQDQ instructions"); 40 40 - MODULE_LICENSE("GPL");

arch/x86/lib/crc16-msb-pclmul.S lib/crc/x86/crc16-msb-pclmul.S

reviewed

arch/x86/lib/crc32-pclmul.S lib/crc/x86/crc32-pclmul.S

reviewed

-111

arch/x86/lib/crc32.c

reviewed

··· 1 1 - // SPDX-License-Identifier: GPL-2.0-only 2 2 - /* 3 3 - * x86-optimized CRC32 functions 4 4 - * 5 5 - * Copyright (C) 2008 Intel Corporation 6 6 - * Copyright 2012 Xyratex Technology Limited 7 7 - * Copyright 2024 Google LLC 8 8 - */ 9 9 - 10 10 - #include <linux/crc32.h> 11 11 - #include <linux/module.h> 12 12 - #include "crc-pclmul-template.h" 13 13 - 14 14 - static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_crc32); 15 15 - static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pclmulqdq); 16 16 - 17 17 - DECLARE_CRC_PCLMUL_FUNCS(crc32_lsb, u32); 18 18 - 19 19 - u32 crc32_le_arch(u32 crc, const u8 *p, size_t len) 20 20 - { 21 21 - CRC_PCLMUL(crc, p, len, crc32_lsb, crc32_lsb_0xedb88320_consts, 22 22 - have_pclmulqdq); 23 23 - return crc32_le_base(crc, p, len); 24 24 - } 25 25 - EXPORT_SYMBOL(crc32_le_arch); 26 26 - 27 27 - #ifdef CONFIG_X86_64 28 28 - #define CRC32_INST "crc32q %1, %q0" 29 29 - #else 30 30 - #define CRC32_INST "crc32l %1, %0" 31 31 - #endif 32 32 - 33 33 - /* 34 34 - * Use carryless multiply version of crc32c when buffer size is >= 512 to 35 35 - * account for FPU state save/restore overhead. 36 36 - */ 37 37 - #define CRC32C_PCLMUL_BREAKEVEN 512 38 38 - 39 39 - asmlinkage u32 crc32c_x86_3way(u32 crc, const u8 *buffer, size_t len); 40 40 - 41 41 - u32 crc32c_arch(u32 crc, const u8 *p, size_t len) 42 42 - { 43 43 - size_t num_longs; 44 44 - 45 45 - if (!static_branch_likely(&have_crc32)) 46 46 - return crc32c_base(crc, p, len); 47 47 - 48 48 - if (IS_ENABLED(CONFIG_X86_64) && len >= CRC32C_PCLMUL_BREAKEVEN && 49 49 - static_branch_likely(&have_pclmulqdq) && crypto_simd_usable()) { 50 50 - kernel_fpu_begin(); 51 51 - crc = crc32c_x86_3way(crc, p, len); 52 52 - kernel_fpu_end(); 53 53 - return crc; 54 54 - } 55 55 - 56 56 - for (num_longs = len / sizeof(unsigned long); 57 57 - num_longs != 0; num_longs--, p += sizeof(unsigned long)) 58 58 - asm(CRC32_INST : "+r" (crc) : ASM_INPUT_RM (*(unsigned long *)p)); 59 59 - 60 60 - if (sizeof(unsigned long) > 4 && (len & 4)) { 61 61 - asm("crc32l %1, %0" : "+r" (crc) : ASM_INPUT_RM (*(u32 *)p)); 62 62 - p += 4; 63 63 - } 64 64 - if (len & 2) { 65 65 - asm("crc32w %1, %0" : "+r" (crc) : ASM_INPUT_RM (*(u16 *)p)); 66 66 - p += 2; 67 67 - } 68 68 - if (len & 1) 69 69 - asm("crc32b %1, %0" : "+r" (crc) : ASM_INPUT_RM (*p)); 70 70 - 71 71 - return crc; 72 72 - } 73 73 - EXPORT_SYMBOL(crc32c_arch); 74 74 - 75 75 - u32 crc32_be_arch(u32 crc, const u8 *p, size_t len) 76 76 - { 77 77 - return crc32_be_base(crc, p, len); 78 78 - } 79 79 - EXPORT_SYMBOL(crc32_be_arch); 80 80 - 81 81 - static int __init crc32_x86_init(void) 82 82 - { 83 83 - if (boot_cpu_has(X86_FEATURE_XMM4_2)) 84 84 - static_branch_enable(&have_crc32); 85 85 - if (boot_cpu_has(X86_FEATURE_PCLMULQDQ)) { 86 86 - static_branch_enable(&have_pclmulqdq); 87 87 - INIT_CRC_PCLMUL(crc32_lsb); 88 88 - } 89 89 - return 0; 90 90 - } 91 91 - subsys_initcall(crc32_x86_init); 92 92 - 93 93 - static void __exit crc32_x86_exit(void) 94 94 - { 95 95 - } 96 96 - module_exit(crc32_x86_exit); 97 97 - 98 98 - u32 crc32_optimizations(void) 99 99 - { 100 100 - u32 optimizations = 0; 101 101 - 102 102 - if (static_key_enabled(&have_crc32)) 103 103 - optimizations |= CRC32C_OPTIMIZATION; 104 104 - if (static_key_enabled(&have_pclmulqdq)) 105 105 - optimizations |= CRC32_LE_OPTIMIZATION; 106 106 - return optimizations; 107 107 - } 108 108 - EXPORT_SYMBOL(crc32_optimizations); 109 109 - 110 110 - MODULE_DESCRIPTION("x86-optimized CRC32 functions"); 111 111 - MODULE_LICENSE("GPL");

arch/x86/lib/crc32c-3way.S lib/crc/x86/crc32c-3way.S

reviewed

arch/x86/lib/crc64-pclmul.S lib/crc/x86/crc64-pclmul.S

reviewed

-50

arch/x86/lib/crc64.c

reviewed

··· 1 1 - // SPDX-License-Identifier: GPL-2.0-or-later 2 2 - /* 3 3 - * CRC64 using [V]PCLMULQDQ instructions 4 4 - * 5 5 - * Copyright 2025 Google LLC 6 6 - */ 7 7 - 8 8 - #include <linux/crc64.h> 9 9 - #include <linux/module.h> 10 10 - #include "crc-pclmul-template.h" 11 11 - 12 12 - static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pclmulqdq); 13 13 - 14 14 - DECLARE_CRC_PCLMUL_FUNCS(crc64_msb, u64); 15 15 - DECLARE_CRC_PCLMUL_FUNCS(crc64_lsb, u64); 16 16 - 17 17 - u64 crc64_be_arch(u64 crc, const u8 *p, size_t len) 18 18 - { 19 19 - CRC_PCLMUL(crc, p, len, crc64_msb, crc64_msb_0x42f0e1eba9ea3693_consts, 20 20 - have_pclmulqdq); 21 21 - return crc64_be_generic(crc, p, len); 22 22 - } 23 23 - EXPORT_SYMBOL_GPL(crc64_be_arch); 24 24 - 25 25 - u64 crc64_nvme_arch(u64 crc, const u8 *p, size_t len) 26 26 - { 27 27 - CRC_PCLMUL(crc, p, len, crc64_lsb, crc64_lsb_0x9a6c9329ac4bc9b5_consts, 28 28 - have_pclmulqdq); 29 29 - return crc64_nvme_generic(crc, p, len); 30 30 - } 31 31 - EXPORT_SYMBOL_GPL(crc64_nvme_arch); 32 32 - 33 33 - static int __init crc64_x86_init(void) 34 34 - { 35 35 - if (boot_cpu_has(X86_FEATURE_PCLMULQDQ)) { 36 36 - static_branch_enable(&have_pclmulqdq); 37 37 - INIT_CRC_PCLMUL(crc64_msb); 38 38 - INIT_CRC_PCLMUL(crc64_lsb); 39 39 - } 40 40 - return 0; 41 41 - } 42 42 - subsys_initcall(crc64_x86_init); 43 43 - 44 44 - static void __exit crc64_x86_exit(void) 45 45 - { 46 46 - } 47 47 - module_exit(crc64_x86_exit); 48 48 - 49 49 - MODULE_DESCRIPTION("CRC64 using [V]PCLMULQDQ instructions"); 50 50 - MODULE_LICENSE("GPL");

-2

crypto/Makefile

reviewed

··· 154 154 obj-$(CONFIG_CRYPTO_MICHAEL_MIC) += michael_mic.o 155 155 obj-$(CONFIG_CRYPTO_CRC32C) += crc32c-cryptoapi.o 156 156 crc32c-cryptoapi-y := crc32c.o 157 157 - CFLAGS_crc32c.o += -DARCH=$(ARCH) 158 157 obj-$(CONFIG_CRYPTO_CRC32) += crc32-cryptoapi.o 159 158 crc32-cryptoapi-y := crc32.o 160 160 - CFLAGS_crc32.o += -DARCH=$(ARCH) 161 159 obj-$(CONFIG_CRYPTO_AUTHENC) += authenc.o authencesn.o 162 160 obj-$(CONFIG_CRYPTO_KRB5ENC) += krb5enc.o 163 161 obj-$(CONFIG_CRYPTO_LZO) += lzo.o lzo-rle.o

+6 -59

crypto/crc32.c

reviewed

··· 59 59 { 60 60 u32 *crcp = shash_desc_ctx(desc); 61 61 62 62 - *crcp = crc32_le_base(*crcp, data, len); 63 63 - return 0; 64 64 - } 65 65 - 66 66 - static int crc32_update_arch(struct shash_desc *desc, const u8 *data, 67 67 - unsigned int len) 68 68 - { 69 69 - u32 *crcp = shash_desc_ctx(desc); 70 70 - 71 62 *crcp = crc32_le(*crcp, data, len); 72 63 return 0; 73 64 } 74 65 75 66 /* No final XOR 0xFFFFFFFF, like crc32_le */ 76 76 - static int __crc32_finup(u32 *crcp, const u8 *data, unsigned int len, 77 77 - u8 *out) 78 78 - { 79 79 - put_unaligned_le32(crc32_le_base(*crcp, data, len), out); 80 80 - return 0; 81 81 - } 82 82 - 83 83 - static int __crc32_finup_arch(u32 *crcp, const u8 *data, unsigned int len, 84 84 - u8 *out) 67 67 + static int __crc32_finup(u32 *crcp, const u8 *data, unsigned int len, u8 *out) 85 68 { 86 69 put_unaligned_le32(crc32_le(*crcp, data, len), out); 87 70 return 0; ··· 74 91 unsigned int len, u8 *out) 75 92 { 76 93 return __crc32_finup(shash_desc_ctx(desc), data, len, out); 77 77 - } 78 78 - 79 79 - static int crc32_finup_arch(struct shash_desc *desc, const u8 *data, 80 80 - unsigned int len, u8 *out) 81 81 - { 82 82 - return __crc32_finup_arch(shash_desc_ctx(desc), data, len, out); 83 94 } 84 95 85 96 static int crc32_final(struct shash_desc *desc, u8 *out) ··· 90 113 return __crc32_finup(crypto_shash_ctx(desc->tfm), data, len, out); 91 114 } 92 115 93 93 - static int crc32_digest_arch(struct shash_desc *desc, const u8 *data, 94 94 - unsigned int len, u8 *out) 95 95 - { 96 96 - return __crc32_finup_arch(crypto_shash_ctx(desc->tfm), data, len, out); 97 97 - } 98 98 - 99 99 - static struct shash_alg algs[] = {{ 116 116 + static struct shash_alg alg = { 100 117 .setkey = crc32_setkey, 101 118 .init = crc32_init, 102 119 .update = crc32_update, ··· 101 130 .digestsize = CHKSUM_DIGEST_SIZE, 102 131 103 132 .base.cra_name = "crc32", 104 104 - .base.cra_driver_name = "crc32-generic", 133 133 + .base.cra_driver_name = "crc32-lib", 105 134 .base.cra_priority = 100, 106 135 .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, 107 136 .base.cra_blocksize = CHKSUM_BLOCK_SIZE, 108 137 .base.cra_ctxsize = sizeof(u32), 109 138 .base.cra_module = THIS_MODULE, 110 139 .base.cra_init = crc32_cra_init, 111 111 - }, { 112 112 - .setkey = crc32_setkey, 113 113 - .init = crc32_init, 114 114 - .update = crc32_update_arch, 115 115 - .final = crc32_final, 116 116 - .finup = crc32_finup_arch, 117 117 - .digest = crc32_digest_arch, 118 118 - .descsize = sizeof(u32), 119 119 - .digestsize = CHKSUM_DIGEST_SIZE, 120 120 - 121 121 - .base.cra_name = "crc32", 122 122 - .base.cra_driver_name = "crc32-" __stringify(ARCH), 123 123 - .base.cra_priority = 150, 124 124 - .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, 125 125 - .base.cra_blocksize = CHKSUM_BLOCK_SIZE, 126 126 - .base.cra_ctxsize = sizeof(u32), 127 127 - .base.cra_module = THIS_MODULE, 128 128 - .base.cra_init = crc32_cra_init, 129 129 - }}; 130 130 - 131 131 - static int num_algs; 140 140 + }; 132 141 133 142 static int __init crc32_mod_init(void) 134 143 { 135 135 - /* register the arch flavor only if it differs from the generic one */ 136 136 - num_algs = 1 + ((crc32_optimizations() & CRC32_LE_OPTIMIZATION) != 0); 137 137 - 138 138 - return crypto_register_shashes(algs, num_algs); 144 144 + return crypto_register_shash(&alg); 139 145 } 140 146 141 147 static void __exit crc32_mod_fini(void) 142 148 { 143 143 - crypto_unregister_shashes(algs, num_algs); 149 149 + crypto_unregister_shash(&alg); 144 150 } 145 151 146 152 module_init(crc32_mod_init); ··· 127 179 MODULE_DESCRIPTION("CRC32 calculations wrapper for lib/crc32"); 128 180 MODULE_LICENSE("GPL"); 129 181 MODULE_ALIAS_CRYPTO("crc32"); 130 130 - MODULE_ALIAS_CRYPTO("crc32-generic");

+5 -61

crypto/crc32c.c

reviewed

··· 85 85 { 86 86 struct chksum_desc_ctx *ctx = shash_desc_ctx(desc); 87 87 88 88 - ctx->crc = crc32c_base(ctx->crc, data, length); 89 89 - return 0; 90 90 - } 91 91 - 92 92 - static int chksum_update_arch(struct shash_desc *desc, const u8 *data, 93 93 - unsigned int length) 94 94 - { 95 95 - struct chksum_desc_ctx *ctx = shash_desc_ctx(desc); 96 96 - 97 88 ctx->crc = crc32c(ctx->crc, data, length); 98 89 return 0; 99 90 } ··· 99 108 100 109 static int __chksum_finup(u32 *crcp, const u8 *data, unsigned int len, u8 *out) 101 110 { 102 102 - put_unaligned_le32(~crc32c_base(*crcp, data, len), out); 103 103 - return 0; 104 104 - } 105 105 - 106 106 - static int __chksum_finup_arch(u32 *crcp, const u8 *data, unsigned int len, 107 107 - u8 *out) 108 108 - { 109 111 put_unaligned_le32(~crc32c(*crcp, data, len), out); 110 112 return 0; 111 113 } ··· 111 127 return __chksum_finup(&ctx->crc, data, len, out); 112 128 } 113 129 114 114 - static int chksum_finup_arch(struct shash_desc *desc, const u8 *data, 115 115 - unsigned int len, u8 *out) 116 116 - { 117 117 - struct chksum_desc_ctx *ctx = shash_desc_ctx(desc); 118 118 - 119 119 - return __chksum_finup_arch(&ctx->crc, data, len, out); 120 120 - } 121 121 - 122 130 static int chksum_digest(struct shash_desc *desc, const u8 *data, 123 131 unsigned int length, u8 *out) 124 132 { 125 133 struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm); 126 134 127 135 return __chksum_finup(&mctx->key, data, length, out); 128 128 - } 129 129 - 130 130 - static int chksum_digest_arch(struct shash_desc *desc, const u8 *data, 131 131 - unsigned int length, u8 *out) 132 132 - { 133 133 - struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm); 134 134 - 135 135 - return __chksum_finup_arch(&mctx->key, data, length, out); 136 136 } 137 137 138 138 static int crc32c_cra_init(struct crypto_tfm *tfm) ··· 127 159 return 0; 128 160 } 129 161 130 130 - static struct shash_alg algs[] = {{ 162 162 + static struct shash_alg alg = { 131 163 .digestsize = CHKSUM_DIGEST_SIZE, 132 164 .setkey = chksum_setkey, 133 165 .init = chksum_init, ··· 138 170 .descsize = sizeof(struct chksum_desc_ctx), 139 171 140 172 .base.cra_name = "crc32c", 141 141 - .base.cra_driver_name = "crc32c-generic", 173 173 + .base.cra_driver_name = "crc32c-lib", 142 174 .base.cra_priority = 100, 143 175 .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, 144 176 .base.cra_blocksize = CHKSUM_BLOCK_SIZE, 145 177 .base.cra_ctxsize = sizeof(struct chksum_ctx), 146 178 .base.cra_module = THIS_MODULE, 147 179 .base.cra_init = crc32c_cra_init, 148 148 - }, { 149 149 - .digestsize = CHKSUM_DIGEST_SIZE, 150 150 - .setkey = chksum_setkey, 151 151 - .init = chksum_init, 152 152 - .update = chksum_update_arch, 153 153 - .final = chksum_final, 154 154 - .finup = chksum_finup_arch, 155 155 - .digest = chksum_digest_arch, 156 156 - .descsize = sizeof(struct chksum_desc_ctx), 157 157 - 158 158 - .base.cra_name = "crc32c", 159 159 - .base.cra_driver_name = "crc32c-" __stringify(ARCH), 160 160 - .base.cra_priority = 150, 161 161 - .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, 162 162 - .base.cra_blocksize = CHKSUM_BLOCK_SIZE, 163 163 - .base.cra_ctxsize = sizeof(struct chksum_ctx), 164 164 - .base.cra_module = THIS_MODULE, 165 165 - .base.cra_init = crc32c_cra_init, 166 166 - }}; 167 167 - 168 168 - static int num_algs; 180 180 + }; 169 181 170 182 static int __init crc32c_mod_init(void) 171 183 { 172 172 - /* register the arch flavor only if it differs from the generic one */ 173 173 - num_algs = 1 + ((crc32_optimizations() & CRC32C_OPTIMIZATION) != 0); 174 174 - 175 175 - return crypto_register_shashes(algs, num_algs); 184 184 + return crypto_register_shash(&alg); 176 185 } 177 186 178 187 static void __exit crc32c_mod_fini(void) 179 188 { 180 180 - crypto_unregister_shashes(algs, num_algs); 189 189 + crypto_unregister_shash(&alg); 181 190 } 182 191 183 192 module_init(crc32c_mod_init); ··· 164 219 MODULE_DESCRIPTION("CRC32c (Castagnoli) calculations wrapper for lib/crc32c"); 165 220 MODULE_LICENSE("GPL"); 166 221 MODULE_ALIAS_CRYPTO("crc32c"); 167 167 - MODULE_ALIAS_CRYPTO("crc32c-generic");

+3 -54

crypto/testmgr.c

reviewed

··· 3550 3550 return err; 3551 3551 } 3552 3552 3553 3553 - static int alg_test_crc32c(const struct alg_test_desc *desc, 3554 3554 - const char *driver, u32 type, u32 mask) 3555 3555 - { 3556 3556 - struct crypto_shash *tfm; 3557 3557 - __le32 val; 3558 3558 - int err; 3559 3559 - 3560 3560 - err = alg_test_hash(desc, driver, type, mask); 3561 3561 - if (err) 3562 3562 - return err; 3563 3563 - 3564 3564 - tfm = crypto_alloc_shash(driver, type, mask); 3565 3565 - if (IS_ERR(tfm)) { 3566 3566 - if (PTR_ERR(tfm) == -ENOENT) { 3567 3567 - /* 3568 3568 - * This crc32c implementation is only available through 3569 3569 - * ahash API, not the shash API, so the remaining part 3570 3570 - * of the test is not applicable to it. 3571 3571 - */ 3572 3572 - return 0; 3573 3573 - } 3574 3574 - printk(KERN_ERR "alg: crc32c: Failed to load transform for %s: " 3575 3575 - "%ld\n", driver, PTR_ERR(tfm)); 3576 3576 - return PTR_ERR(tfm); 3577 3577 - } 3578 3578 - driver = crypto_shash_driver_name(tfm); 3579 3579 - 3580 3580 - do { 3581 3581 - SHASH_DESC_ON_STACK(shash, tfm); 3582 3582 - u32 *ctx = (u32 *)shash_desc_ctx(shash); 3583 3583 - 3584 3584 - shash->tfm = tfm; 3585 3585 - 3586 3586 - *ctx = 420553207; 3587 3587 - err = crypto_shash_final(shash, (u8 *)&val); 3588 3588 - if (err) { 3589 3589 - printk(KERN_ERR "alg: crc32c: Operation failed for " 3590 3590 - "%s: %d\n", driver, err); 3591 3591 - break; 3592 3592 - } 3593 3593 - 3594 3594 - if (val != cpu_to_le32(~420553207)) { 3595 3595 - pr_err("alg: crc32c: Test failed for %s: %u\n", 3596 3596 - driver, le32_to_cpu(val)); 3597 3597 - err = -EINVAL; 3598 3598 - } 3599 3599 - } while (0); 3600 3600 - 3601 3601 - crypto_free_shash(tfm); 3602 3602 - 3603 3603 - return err; 3604 3604 - } 3605 3605 - 3606 3553 static int alg_test_cprng(const struct alg_test_desc *desc, const char *driver, 3607 3554 u32 type, u32 mask) 3608 3555 { ··· 4502 4555 } 4503 4556 }, { 4504 4557 .alg = "crc32", 4558 4558 + .generic_driver = "crc32-lib", 4505 4559 .test = alg_test_hash, 4506 4560 .fips_allowed = 1, 4507 4561 .suite = { ··· 4510 4562 } 4511 4563 }, { 4512 4564 .alg = "crc32c", 4513 4513 - .test = alg_test_crc32c, 4565 4565 + .generic_driver = "crc32c-lib", 4566 4566 + .test = alg_test_hash, 4514 4567 .fips_allowed = 1, 4515 4568 .suite = { 4516 4569 .hash = __VECS(crc32c_tv_template)

-1

drivers/crypto/inside-secure/safexcel.c

reviewed

··· 1218 1218 &safexcel_alg_xts_aes, 1219 1219 &safexcel_alg_gcm, 1220 1220 &safexcel_alg_ccm, 1221 1221 - &safexcel_alg_crc32, 1222 1221 &safexcel_alg_cbcmac, 1223 1222 &safexcel_alg_xcbcmac, 1224 1223 &safexcel_alg_cmac,

-1

drivers/crypto/inside-secure/safexcel.h

reviewed

··· 959 959 extern struct safexcel_alg_template safexcel_alg_xts_aes; 960 960 extern struct safexcel_alg_template safexcel_alg_gcm; 961 961 extern struct safexcel_alg_template safexcel_alg_ccm; 962 962 - extern struct safexcel_alg_template safexcel_alg_crc32; 963 962 extern struct safexcel_alg_template safexcel_alg_cbcmac; 964 963 extern struct safexcel_alg_template safexcel_alg_xcbcmac; 965 964 extern struct safexcel_alg_template safexcel_alg_cmac;

+2 -90

drivers/crypto/inside-secure/safexcel_hash.c

reviewed

··· 289 289 return 1; 290 290 } 291 291 292 292 - if (unlikely(sreq->digest == CONTEXT_CONTROL_DIGEST_XCM && 293 293 - ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_CRC32)) { 294 294 - /* Undo final XOR with 0xffffffff ...*/ 295 295 - *(__le32 *)areq->result = ~sreq->state[0]; 296 296 - } else { 297 297 - memcpy(areq->result, sreq->state, 298 298 - crypto_ahash_digestsize(ahash)); 299 299 - } 292 292 + memcpy(areq->result, sreq->state, 293 293 + crypto_ahash_digestsize(ahash)); 300 294 } 301 295 302 296 cache_len = safexcel_queued_len(sreq); ··· 1868 1874 .cra_blocksize = MD5_HMAC_BLOCK_SIZE, 1869 1875 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1870 1876 .cra_init = safexcel_ahash_cra_init, 1871 1871 - .cra_exit = safexcel_ahash_cra_exit, 1872 1872 - .cra_module = THIS_MODULE, 1873 1873 - }, 1874 1874 - }, 1875 1875 - }, 1876 1876 - }; 1877 1877 - 1878 1878 - static int safexcel_crc32_cra_init(struct crypto_tfm *tfm) 1879 1879 - { 1880 1880 - struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm); 1881 1881 - int ret = safexcel_ahash_cra_init(tfm); 1882 1882 - 1883 1883 - /* Default 'key' is all zeroes */ 1884 1884 - memset(&ctx->base.ipad, 0, sizeof(u32)); 1885 1885 - return ret; 1886 1886 - } 1887 1887 - 1888 1888 - static int safexcel_crc32_init(struct ahash_request *areq) 1889 1889 - { 1890 1890 - struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); 1891 1891 - struct safexcel_ahash_req *req = ahash_request_ctx_dma(areq); 1892 1892 - 1893 1893 - memset(req, 0, sizeof(*req)); 1894 1894 - 1895 1895 - /* Start from loaded key */ 1896 1896 - req->state[0] = cpu_to_le32(~ctx->base.ipad.word[0]); 1897 1897 - /* Set processed to non-zero to enable invalidation detection */ 1898 1898 - req->len = sizeof(u32); 1899 1899 - req->processed = sizeof(u32); 1900 1900 - 1901 1901 - ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_CRC32; 1902 1902 - req->digest = CONTEXT_CONTROL_DIGEST_XCM; 1903 1903 - req->state_sz = sizeof(u32); 1904 1904 - req->digest_sz = sizeof(u32); 1905 1905 - req->block_sz = sizeof(u32); 1906 1906 - 1907 1907 - return 0; 1908 1908 - } 1909 1909 - 1910 1910 - static int safexcel_crc32_setkey(struct crypto_ahash *tfm, const u8 *key, 1911 1911 - unsigned int keylen) 1912 1912 - { 1913 1913 - struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); 1914 1914 - 1915 1915 - if (keylen != sizeof(u32)) 1916 1916 - return -EINVAL; 1917 1917 - 1918 1918 - memcpy(&ctx->base.ipad, key, sizeof(u32)); 1919 1919 - return 0; 1920 1920 - } 1921 1921 - 1922 1922 - static int safexcel_crc32_digest(struct ahash_request *areq) 1923 1923 - { 1924 1924 - return safexcel_crc32_init(areq) ?: safexcel_ahash_finup(areq); 1925 1925 - } 1926 1926 - 1927 1927 - struct safexcel_alg_template safexcel_alg_crc32 = { 1928 1928 - .type = SAFEXCEL_ALG_TYPE_AHASH, 1929 1929 - .algo_mask = 0, 1930 1930 - .alg.ahash = { 1931 1931 - .init = safexcel_crc32_init, 1932 1932 - .update = safexcel_ahash_update, 1933 1933 - .final = safexcel_ahash_final, 1934 1934 - .finup = safexcel_ahash_finup, 1935 1935 - .digest = safexcel_crc32_digest, 1936 1936 - .setkey = safexcel_crc32_setkey, 1937 1937 - .export = safexcel_ahash_export, 1938 1938 - .import = safexcel_ahash_import, 1939 1939 - .halg = { 1940 1940 - .digestsize = sizeof(u32), 1941 1941 - .statesize = sizeof(struct safexcel_ahash_export_state), 1942 1942 - .base = { 1943 1943 - .cra_name = "crc32", 1944 1944 - .cra_driver_name = "safexcel-crc32", 1945 1945 - .cra_priority = SAFEXCEL_CRA_PRIORITY, 1946 1946 - .cra_flags = CRYPTO_ALG_OPTIONAL_KEY | 1947 1947 - CRYPTO_ALG_ASYNC | 1948 1948 - CRYPTO_ALG_ALLOCATES_MEMORY | 1949 1949 - CRYPTO_ALG_KERN_DRIVER_ONLY, 1950 1950 - .cra_blocksize = 1, 1951 1951 - .cra_ctxsize = sizeof(struct safexcel_ahash_ctx), 1952 1952 - .cra_init = safexcel_crc32_cra_init, 1953 1877 .cra_exit = safexcel_ahash_cra_exit, 1954 1878 .cra_module = THIS_MODULE, 1955 1879 },

-9

drivers/crypto/stm32/Kconfig

reviewed

··· 1 1 # SPDX-License-Identifier: GPL-2.0-only 2 2 - config CRYPTO_DEV_STM32_CRC 3 3 - tristate "Support for STM32 crc accelerators" 4 4 - depends on ARCH_STM32 5 5 - select CRYPTO_HASH 6 6 - select CRC32 7 7 - help 8 8 - This enables support for the CRC32 hw accelerator which can be found 9 9 - on STMicroelectronics STM32 SOC. 10 10 - 11 2 config CRYPTO_DEV_STM32_HASH 12 3 tristate "Support for STM32 hash accelerators" 13 4 depends on ARCH_STM32 || ARCH_U8500

-1

drivers/crypto/stm32/Makefile

reviewed

··· 1 1 # SPDX-License-Identifier: GPL-2.0-only 2 2 - obj-$(CONFIG_CRYPTO_DEV_STM32_CRC) += stm32-crc32.o 3 2 obj-$(CONFIG_CRYPTO_DEV_STM32_HASH) += stm32-hash.o 4 3 obj-$(CONFIG_CRYPTO_DEV_STM32_CRYP) += stm32-cryp.o

-480

drivers/crypto/stm32/stm32-crc32.c

reviewed

··· 1 1 - // SPDX-License-Identifier: GPL-2.0-only 2 2 - /* 3 3 - * Copyright (C) STMicroelectronics SA 2017 4 4 - * Author: Fabien Dessenne <fabien.dessenne@st.com> 5 5 - */ 6 6 - 7 7 - #include <linux/bitrev.h> 8 8 - #include <linux/clk.h> 9 9 - #include <linux/crc32.h> 10 10 - #include <linux/crc32poly.h> 11 11 - #include <linux/io.h> 12 12 - #include <linux/kernel.h> 13 13 - #include <linux/module.h> 14 14 - #include <linux/mod_devicetable.h> 15 15 - #include <linux/platform_device.h> 16 16 - #include <linux/pm_runtime.h> 17 17 - 18 18 - #include <crypto/internal/hash.h> 19 19 - 20 20 - #include <linux/unaligned.h> 21 21 - 22 22 - #define DRIVER_NAME "stm32-crc32" 23 23 - #define CHKSUM_DIGEST_SIZE 4 24 24 - #define CHKSUM_BLOCK_SIZE 1 25 25 - 26 26 - /* Registers */ 27 27 - #define CRC_DR 0x00000000 28 28 - #define CRC_CR 0x00000008 29 29 - #define CRC_INIT 0x00000010 30 30 - #define CRC_POL 0x00000014 31 31 - 32 32 - /* Registers values */ 33 33 - #define CRC_CR_RESET BIT(0) 34 34 - #define CRC_CR_REV_IN_WORD (BIT(6) | BIT(5)) 35 35 - #define CRC_CR_REV_IN_BYTE BIT(5) 36 36 - #define CRC_CR_REV_OUT BIT(7) 37 37 - #define CRC32C_INIT_DEFAULT 0xFFFFFFFF 38 38 - 39 39 - #define CRC_AUTOSUSPEND_DELAY 50 40 40 - 41 41 - static unsigned int burst_size; 42 42 - module_param(burst_size, uint, 0644); 43 43 - MODULE_PARM_DESC(burst_size, "Select burst byte size (0 unlimited)"); 44 44 - 45 45 - struct stm32_crc { 46 46 - struct list_head list; 47 47 - struct device *dev; 48 48 - void __iomem *regs; 49 49 - struct clk *clk; 50 50 - spinlock_t lock; 51 51 - }; 52 52 - 53 53 - struct stm32_crc_list { 54 54 - struct list_head dev_list; 55 55 - spinlock_t lock; /* protect dev_list */ 56 56 - }; 57 57 - 58 58 - static struct stm32_crc_list crc_list = { 59 59 - .dev_list = LIST_HEAD_INIT(crc_list.dev_list), 60 60 - .lock = __SPIN_LOCK_UNLOCKED(crc_list.lock), 61 61 - }; 62 62 - 63 63 - struct stm32_crc_ctx { 64 64 - u32 key; 65 65 - u32 poly; 66 66 - }; 67 67 - 68 68 - struct stm32_crc_desc_ctx { 69 69 - u32 partial; /* crc32c: partial in first 4 bytes of that struct */ 70 70 - }; 71 71 - 72 72 - static int stm32_crc32_cra_init(struct crypto_tfm *tfm) 73 73 - { 74 74 - struct stm32_crc_ctx *mctx = crypto_tfm_ctx(tfm); 75 75 - 76 76 - mctx->key = 0; 77 77 - mctx->poly = CRC32_POLY_LE; 78 78 - return 0; 79 79 - } 80 80 - 81 81 - static int stm32_crc32c_cra_init(struct crypto_tfm *tfm) 82 82 - { 83 83 - struct stm32_crc_ctx *mctx = crypto_tfm_ctx(tfm); 84 84 - 85 85 - mctx->key = CRC32C_INIT_DEFAULT; 86 86 - mctx->poly = CRC32C_POLY_LE; 87 87 - return 0; 88 88 - } 89 89 - 90 90 - static int stm32_crc_setkey(struct crypto_shash *tfm, const u8 *key, 91 91 - unsigned int keylen) 92 92 - { 93 93 - struct stm32_crc_ctx *mctx = crypto_shash_ctx(tfm); 94 94 - 95 95 - if (keylen != sizeof(u32)) 96 96 - return -EINVAL; 97 97 - 98 98 - mctx->key = get_unaligned_le32(key); 99 99 - return 0; 100 100 - } 101 101 - 102 102 - static struct stm32_crc *stm32_crc_get_next_crc(void) 103 103 - { 104 104 - struct stm32_crc *crc; 105 105 - 106 106 - spin_lock_bh(&crc_list.lock); 107 107 - crc = list_first_entry_or_null(&crc_list.dev_list, struct stm32_crc, list); 108 108 - if (crc) 109 109 - list_move_tail(&crc->list, &crc_list.dev_list); 110 110 - spin_unlock_bh(&crc_list.lock); 111 111 - 112 112 - return crc; 113 113 - } 114 114 - 115 115 - static int stm32_crc_init(struct shash_desc *desc) 116 116 - { 117 117 - struct stm32_crc_desc_ctx *ctx = shash_desc_ctx(desc); 118 118 - struct stm32_crc_ctx *mctx = crypto_shash_ctx(desc->tfm); 119 119 - struct stm32_crc *crc; 120 120 - unsigned long flags; 121 121 - 122 122 - crc = stm32_crc_get_next_crc(); 123 123 - if (!crc) 124 124 - return -ENODEV; 125 125 - 126 126 - pm_runtime_get_sync(crc->dev); 127 127 - 128 128 - spin_lock_irqsave(&crc->lock, flags); 129 129 - 130 130 - /* Reset, set key, poly and configure in bit reverse mode */ 131 131 - writel_relaxed(bitrev32(mctx->key), crc->regs + CRC_INIT); 132 132 - writel_relaxed(bitrev32(mctx->poly), crc->regs + CRC_POL); 133 133 - writel_relaxed(CRC_CR_RESET | CRC_CR_REV_IN_WORD | CRC_CR_REV_OUT, 134 134 - crc->regs + CRC_CR); 135 135 - 136 136 - /* Store partial result */ 137 137 - ctx->partial = readl_relaxed(crc->regs + CRC_DR); 138 138 - 139 139 - spin_unlock_irqrestore(&crc->lock, flags); 140 140 - 141 141 - pm_runtime_mark_last_busy(crc->dev); 142 142 - pm_runtime_put_autosuspend(crc->dev); 143 143 - 144 144 - return 0; 145 145 - } 146 146 - 147 147 - static int burst_update(struct shash_desc *desc, const u8 *d8, 148 148 - size_t length) 149 149 - { 150 150 - struct stm32_crc_desc_ctx *ctx = shash_desc_ctx(desc); 151 151 - struct stm32_crc_ctx *mctx = crypto_shash_ctx(desc->tfm); 152 152 - struct stm32_crc *crc; 153 153 - 154 154 - crc = stm32_crc_get_next_crc(); 155 155 - if (!crc) 156 156 - return -ENODEV; 157 157 - 158 158 - pm_runtime_get_sync(crc->dev); 159 159 - 160 160 - if (!spin_trylock(&crc->lock)) { 161 161 - /* Hardware is busy, calculate crc32 by software */ 162 162 - if (mctx->poly == CRC32_POLY_LE) 163 163 - ctx->partial = crc32_le(ctx->partial, d8, length); 164 164 - else 165 165 - ctx->partial = crc32c(ctx->partial, d8, length); 166 166 - 167 167 - goto pm_out; 168 168 - } 169 169 - 170 170 - /* 171 171 - * Restore previously calculated CRC for this context as init value 172 172 - * Restore polynomial configuration 173 173 - * Configure in register for word input data, 174 174 - * Configure out register in reversed bit mode data. 175 175 - */ 176 176 - writel_relaxed(bitrev32(ctx->partial), crc->regs + CRC_INIT); 177 177 - writel_relaxed(bitrev32(mctx->poly), crc->regs + CRC_POL); 178 178 - writel_relaxed(CRC_CR_RESET | CRC_CR_REV_IN_WORD | CRC_CR_REV_OUT, 179 179 - crc->regs + CRC_CR); 180 180 - 181 181 - if (d8 != PTR_ALIGN(d8, sizeof(u32))) { 182 182 - /* Configure for byte data */ 183 183 - writel_relaxed(CRC_CR_REV_IN_BYTE | CRC_CR_REV_OUT, 184 184 - crc->regs + CRC_CR); 185 185 - while (d8 != PTR_ALIGN(d8, sizeof(u32)) && length) { 186 186 - writeb_relaxed(*d8++, crc->regs + CRC_DR); 187 187 - length--; 188 188 - } 189 189 - /* Configure for word data */ 190 190 - writel_relaxed(CRC_CR_REV_IN_WORD | CRC_CR_REV_OUT, 191 191 - crc->regs + CRC_CR); 192 192 - } 193 193 - 194 194 - for (; length >= sizeof(u32); d8 += sizeof(u32), length -= sizeof(u32)) 195 195 - writel_relaxed(*((u32 *)d8), crc->regs + CRC_DR); 196 196 - 197 197 - if (length) { 198 198 - /* Configure for byte data */ 199 199 - writel_relaxed(CRC_CR_REV_IN_BYTE | CRC_CR_REV_OUT, 200 200 - crc->regs + CRC_CR); 201 201 - while (length--) 202 202 - writeb_relaxed(*d8++, crc->regs + CRC_DR); 203 203 - } 204 204 - 205 205 - /* Store partial result */ 206 206 - ctx->partial = readl_relaxed(crc->regs + CRC_DR); 207 207 - 208 208 - spin_unlock(&crc->lock); 209 209 - 210 210 - pm_out: 211 211 - pm_runtime_mark_last_busy(crc->dev); 212 212 - pm_runtime_put_autosuspend(crc->dev); 213 213 - 214 214 - return 0; 215 215 - } 216 216 - 217 217 - static int stm32_crc_update(struct shash_desc *desc, const u8 *d8, 218 218 - unsigned int length) 219 219 - { 220 220 - const unsigned int burst_sz = burst_size; 221 221 - unsigned int rem_sz; 222 222 - const u8 *cur; 223 223 - size_t size; 224 224 - int ret; 225 225 - 226 226 - if (!burst_sz) 227 227 - return burst_update(desc, d8, length); 228 228 - 229 229 - /* Digest first bytes not 32bit aligned at first pass in the loop */ 230 230 - size = min_t(size_t, length, burst_sz + (size_t)d8 - 231 231 - ALIGN_DOWN((size_t)d8, sizeof(u32))); 232 232 - for (rem_sz = length, cur = d8; rem_sz; 233 233 - rem_sz -= size, cur += size, size = min(rem_sz, burst_sz)) { 234 234 - ret = burst_update(desc, cur, size); 235 235 - if (ret) 236 236 - return ret; 237 237 - } 238 238 - 239 239 - return 0; 240 240 - } 241 241 - 242 242 - static int stm32_crc_final(struct shash_desc *desc, u8 *out) 243 243 - { 244 244 - struct stm32_crc_desc_ctx *ctx = shash_desc_ctx(desc); 245 245 - struct stm32_crc_ctx *mctx = crypto_shash_ctx(desc->tfm); 246 246 - 247 247 - /* Send computed CRC */ 248 248 - put_unaligned_le32(mctx->poly == CRC32C_POLY_LE ? 249 249 - ~ctx->partial : ctx->partial, out); 250 250 - 251 251 - return 0; 252 252 - } 253 253 - 254 254 - static int stm32_crc_finup(struct shash_desc *desc, const u8 *data, 255 255 - unsigned int length, u8 *out) 256 256 - { 257 257 - return stm32_crc_update(desc, data, length) ?: 258 258 - stm32_crc_final(desc, out); 259 259 - } 260 260 - 261 261 - static int stm32_crc_digest(struct shash_desc *desc, const u8 *data, 262 262 - unsigned int length, u8 *out) 263 263 - { 264 264 - return stm32_crc_init(desc) ?: stm32_crc_finup(desc, data, length, out); 265 265 - } 266 266 - 267 267 - static unsigned int refcnt; 268 268 - static DEFINE_MUTEX(refcnt_lock); 269 269 - static struct shash_alg algs[] = { 270 270 - /* CRC-32 */ 271 271 - { 272 272 - .setkey = stm32_crc_setkey, 273 273 - .init = stm32_crc_init, 274 274 - .update = stm32_crc_update, 275 275 - .final = stm32_crc_final, 276 276 - .finup = stm32_crc_finup, 277 277 - .digest = stm32_crc_digest, 278 278 - .descsize = sizeof(struct stm32_crc_desc_ctx), 279 279 - .digestsize = CHKSUM_DIGEST_SIZE, 280 280 - .base = { 281 281 - .cra_name = "crc32", 282 282 - .cra_driver_name = "stm32-crc32-crc32", 283 283 - .cra_priority = 200, 284 284 - .cra_flags = CRYPTO_ALG_OPTIONAL_KEY, 285 285 - .cra_blocksize = CHKSUM_BLOCK_SIZE, 286 286 - .cra_ctxsize = sizeof(struct stm32_crc_ctx), 287 287 - .cra_module = THIS_MODULE, 288 288 - .cra_init = stm32_crc32_cra_init, 289 289 - } 290 290 - }, 291 291 - /* CRC-32Castagnoli */ 292 292 - { 293 293 - .setkey = stm32_crc_setkey, 294 294 - .init = stm32_crc_init, 295 295 - .update = stm32_crc_update, 296 296 - .final = stm32_crc_final, 297 297 - .finup = stm32_crc_finup, 298 298 - .digest = stm32_crc_digest, 299 299 - .descsize = sizeof(struct stm32_crc_desc_ctx), 300 300 - .digestsize = CHKSUM_DIGEST_SIZE, 301 301 - .base = { 302 302 - .cra_name = "crc32c", 303 303 - .cra_driver_name = "stm32-crc32-crc32c", 304 304 - .cra_priority = 200, 305 305 - .cra_flags = CRYPTO_ALG_OPTIONAL_KEY, 306 306 - .cra_blocksize = CHKSUM_BLOCK_SIZE, 307 307 - .cra_ctxsize = sizeof(struct stm32_crc_ctx), 308 308 - .cra_module = THIS_MODULE, 309 309 - .cra_init = stm32_crc32c_cra_init, 310 310 - } 311 311 - } 312 312 - }; 313 313 - 314 314 - static int stm32_crc_probe(struct platform_device *pdev) 315 315 - { 316 316 - struct device *dev = &pdev->dev; 317 317 - struct stm32_crc *crc; 318 318 - int ret; 319 319 - 320 320 - crc = devm_kzalloc(dev, sizeof(*crc), GFP_KERNEL); 321 321 - if (!crc) 322 322 - return -ENOMEM; 323 323 - 324 324 - crc->dev = dev; 325 325 - 326 326 - crc->regs = devm_platform_ioremap_resource(pdev, 0); 327 327 - if (IS_ERR(crc->regs)) { 328 328 - dev_err(dev, "Cannot map CRC IO\n"); 329 329 - return PTR_ERR(crc->regs); 330 330 - } 331 331 - 332 332 - crc->clk = devm_clk_get(dev, NULL); 333 333 - if (IS_ERR(crc->clk)) { 334 334 - dev_err(dev, "Could not get clock\n"); 335 335 - return PTR_ERR(crc->clk); 336 336 - } 337 337 - 338 338 - ret = clk_prepare_enable(crc->clk); 339 339 - if (ret) { 340 340 - dev_err(crc->dev, "Failed to enable clock\n"); 341 341 - return ret; 342 342 - } 343 343 - 344 344 - pm_runtime_set_autosuspend_delay(dev, CRC_AUTOSUSPEND_DELAY); 345 345 - pm_runtime_use_autosuspend(dev); 346 346 - 347 347 - pm_runtime_get_noresume(dev); 348 348 - pm_runtime_set_active(dev); 349 349 - pm_runtime_irq_safe(dev); 350 350 - pm_runtime_enable(dev); 351 351 - 352 352 - spin_lock_init(&crc->lock); 353 353 - 354 354 - platform_set_drvdata(pdev, crc); 355 355 - 356 356 - spin_lock(&crc_list.lock); 357 357 - list_add(&crc->list, &crc_list.dev_list); 358 358 - spin_unlock(&crc_list.lock); 359 359 - 360 360 - mutex_lock(&refcnt_lock); 361 361 - if (!refcnt) { 362 362 - ret = crypto_register_shashes(algs, ARRAY_SIZE(algs)); 363 363 - if (ret) { 364 364 - mutex_unlock(&refcnt_lock); 365 365 - dev_err(dev, "Failed to register\n"); 366 366 - clk_disable_unprepare(crc->clk); 367 367 - return ret; 368 368 - } 369 369 - } 370 370 - refcnt++; 371 371 - mutex_unlock(&refcnt_lock); 372 372 - 373 373 - dev_info(dev, "Initialized\n"); 374 374 - 375 375 - pm_runtime_put_sync(dev); 376 376 - 377 377 - return 0; 378 378 - } 379 379 - 380 380 - static void stm32_crc_remove(struct platform_device *pdev) 381 381 - { 382 382 - struct stm32_crc *crc = platform_get_drvdata(pdev); 383 383 - int ret = pm_runtime_get_sync(crc->dev); 384 384 - 385 385 - spin_lock(&crc_list.lock); 386 386 - list_del(&crc->list); 387 387 - spin_unlock(&crc_list.lock); 388 388 - 389 389 - mutex_lock(&refcnt_lock); 390 390 - if (!--refcnt) 391 391 - crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); 392 392 - mutex_unlock(&refcnt_lock); 393 393 - 394 394 - pm_runtime_disable(crc->dev); 395 395 - pm_runtime_put_noidle(crc->dev); 396 396 - 397 397 - if (ret >= 0) 398 398 - clk_disable(crc->clk); 399 399 - clk_unprepare(crc->clk); 400 400 - } 401 401 - 402 402 - static int __maybe_unused stm32_crc_suspend(struct device *dev) 403 403 - { 404 404 - struct stm32_crc *crc = dev_get_drvdata(dev); 405 405 - int ret; 406 406 - 407 407 - ret = pm_runtime_force_suspend(dev); 408 408 - if (ret) 409 409 - return ret; 410 410 - 411 411 - clk_unprepare(crc->clk); 412 412 - 413 413 - return 0; 414 414 - } 415 415 - 416 416 - static int __maybe_unused stm32_crc_resume(struct device *dev) 417 417 - { 418 418 - struct stm32_crc *crc = dev_get_drvdata(dev); 419 419 - int ret; 420 420 - 421 421 - ret = clk_prepare(crc->clk); 422 422 - if (ret) { 423 423 - dev_err(crc->dev, "Failed to prepare clock\n"); 424 424 - return ret; 425 425 - } 426 426 - 427 427 - return pm_runtime_force_resume(dev); 428 428 - } 429 429 - 430 430 - static int __maybe_unused stm32_crc_runtime_suspend(struct device *dev) 431 431 - { 432 432 - struct stm32_crc *crc = dev_get_drvdata(dev); 433 433 - 434 434 - clk_disable(crc->clk); 435 435 - 436 436 - return 0; 437 437 - } 438 438 - 439 439 - static int __maybe_unused stm32_crc_runtime_resume(struct device *dev) 440 440 - { 441 441 - struct stm32_crc *crc = dev_get_drvdata(dev); 442 442 - int ret; 443 443 - 444 444 - ret = clk_enable(crc->clk); 445 445 - if (ret) { 446 446 - dev_err(crc->dev, "Failed to enable clock\n"); 447 447 - return ret; 448 448 - } 449 449 - 450 450 - return 0; 451 451 - } 452 452 - 453 453 - static const struct dev_pm_ops stm32_crc_pm_ops = { 454 454 - SET_SYSTEM_SLEEP_PM_OPS(stm32_crc_suspend, 455 455 - stm32_crc_resume) 456 456 - SET_RUNTIME_PM_OPS(stm32_crc_runtime_suspend, 457 457 - stm32_crc_runtime_resume, NULL) 458 458 - }; 459 459 - 460 460 - static const struct of_device_id stm32_dt_ids[] = { 461 461 - { .compatible = "st,stm32f7-crc", }, 462 462 - {}, 463 463 - }; 464 464 - MODULE_DEVICE_TABLE(of, stm32_dt_ids); 465 465 - 466 466 - static struct platform_driver stm32_crc_driver = { 467 467 - .probe = stm32_crc_probe, 468 468 - .remove = stm32_crc_remove, 469 469 - .driver = { 470 470 - .name = DRIVER_NAME, 471 471 - .pm = &stm32_crc_pm_ops, 472 472 - .of_match_table = stm32_dt_ids, 473 473 - }, 474 474 - }; 475 475 - 476 476 - module_platform_driver(stm32_crc_driver); 477 477 - 478 478 - MODULE_AUTHOR("Fabien Dessenne <fabien.dessenne@st.com>"); 479 479 - MODULE_DESCRIPTION("STMicrolectronics STM32 CRC32 hardware driver"); 480 480 - MODULE_LICENSE("GPL");

+1 -1

drivers/nvmem/layouts/u-boot-env.c

reviewed

··· 148 148 crc32_data_len = dev_size - crc32_data_offset; 149 149 data_len = dev_size - data_offset; 150 150 151 151 - calc = crc32(~0, buf + crc32_data_offset, crc32_data_len) ^ ~0L; 151 151 + calc = crc32_le(~0, buf + crc32_data_offset, crc32_data_len) ^ ~0L; 152 152 if (calc != crc32) { 153 153 dev_err(dev, "Invalid calculated CRC32: 0x%08x (expected: 0x%08x)\n", calc, crc32); 154 154 err = -EINVAL;

+2 -6

fs/btrfs/disk-io.c

reviewed

··· 2029 2029 2030 2030 fs_info->csum_shash = csum_shash; 2031 2031 2032 2032 - /* 2033 2033 - * Check if the checksum implementation is a fast accelerated one. 2034 2034 - * As-is this is a bit of a hack and should be replaced once the csum 2035 2035 - * implementations provide that information themselves. 2036 2036 - */ 2032 2032 + /* Check if the checksum implementation is a fast accelerated one. */ 2037 2033 switch (csum_type) { 2038 2034 case BTRFS_CSUM_TYPE_CRC32: 2039 2039 - if (!strstr(crypto_shash_driver_name(csum_shash), "generic")) 2035 2035 + if (crc32_optimizations() & CRC32C_OPTIMIZATION) 2040 2036 set_bit(BTRFS_FS_CSUM_IMPL_FAST, &fs_info->flags); 2041 2037 break; 2042 2038 case BTRFS_CSUM_TYPE_XXHASH:

+1 -9

include/linux/crc-t10dif.h

reviewed

··· 4 4 5 5 #include <linux/types.h> 6 6 7 7 - u16 crc_t10dif_arch(u16 crc, const u8 *p, size_t len); 8 8 - u16 crc_t10dif_generic(u16 crc, const u8 *p, size_t len); 9 9 - 10 10 - static inline u16 crc_t10dif_update(u16 crc, const u8 *p, size_t len) 11 11 - { 12 12 - if (IS_ENABLED(CONFIG_CRC_T10DIF_ARCH)) 13 13 - return crc_t10dif_arch(crc, p, len); 14 14 - return crc_t10dif_generic(crc, p, len); 15 15 - } 7 7 + u16 crc_t10dif_update(u16 crc, const u8 *p, size_t len); 16 8 17 9 static inline u16 crc_t10dif(const u8 *p, size_t len) 18 10 {

+70 -49

include/linux/crc32.h

reviewed

··· 5 5 #include <linux/types.h> 6 6 #include <linux/bitrev.h> 7 7 8 8 - u32 crc32_le_arch(u32 crc, const u8 *p, size_t len); 9 9 - u32 crc32_le_base(u32 crc, const u8 *p, size_t len); 10 10 - u32 crc32_be_arch(u32 crc, const u8 *p, size_t len); 11 11 - u32 crc32_be_base(u32 crc, const u8 *p, size_t len); 12 12 - u32 crc32c_arch(u32 crc, const u8 *p, size_t len); 13 13 - u32 crc32c_base(u32 crc, const u8 *p, size_t len); 8 8 + /** 9 9 + * crc32_le() - Compute least-significant-bit-first IEEE CRC-32 10 10 + * @crc: Initial CRC value. ~0 (recommended) or 0 for a new CRC computation, or 11 11 + * the previous CRC value if computing incrementally. 12 12 + * @p: Pointer to the data buffer 13 13 + * @len: Length of data in bytes 14 14 + * 15 15 + * This implements the CRC variant that is often known as the IEEE CRC-32, or 16 16 + * simply CRC-32, and is widely used in Ethernet and other applications: 17 17 + * 18 18 + * - Polynomial: x^32 + x^26 + x^23 + x^22 + x^16 + x^12 + x^11 + x^10 + x^8 + 19 19 + * x^7 + x^5 + x^4 + x^2 + x^1 + x^0 20 20 + * - Bit order: Least-significant-bit-first 21 21 + * - Polynomial in integer form: 0xedb88320 22 22 + * 23 23 + * This does *not* invert the CRC at the beginning or end. The caller is 24 24 + * expected to do that if it needs to. Inverting at both ends is recommended. 25 25 + * 26 26 + * For new applications, prefer to use CRC-32C instead. See crc32c(). 27 27 + * 28 28 + * Context: Any context 29 29 + * Return: The new CRC value 30 30 + */ 31 31 + u32 crc32_le(u32 crc, const void *p, size_t len); 14 32 15 15 - static inline u32 crc32_le(u32 crc, const void *p, size_t len) 33 33 + /* This is just an alias for crc32_le(). */ 34 34 + static inline u32 crc32(u32 crc, const void *p, size_t len) 16 35 { 17 17 - if (IS_ENABLED(CONFIG_CRC32_ARCH)) 18 18 - return crc32_le_arch(crc, p, len); 19 19 - return crc32_le_base(crc, p, len); 36 36 + return crc32_le(crc, p, len); 20 37 } 21 38 22 22 - static inline u32 crc32_be(u32 crc, const void *p, size_t len) 23 23 - { 24 24 - if (IS_ENABLED(CONFIG_CRC32_ARCH)) 25 25 - return crc32_be_arch(crc, p, len); 26 26 - return crc32_be_base(crc, p, len); 27 27 - } 39 39 + /** 40 40 + * crc32_be() - Compute most-significant-bit-first IEEE CRC-32 41 41 + * @crc: Initial CRC value. ~0 (recommended) or 0 for a new CRC computation, or 42 42 + * the previous CRC value if computing incrementally. 43 43 + * @p: Pointer to the data buffer 44 44 + * @len: Length of data in bytes 45 45 + * 46 46 + * crc32_be() is the same as crc32_le() except that crc32_be() computes the 47 47 + * *most-significant-bit-first* variant of the CRC. I.e., within each byte, the 48 48 + * most significant bit is processed first (treated as highest order polynomial 49 49 + * coefficient). The same bit order is also used for the CRC value itself: 50 50 + * 51 51 + * - Polynomial: x^32 + x^26 + x^23 + x^22 + x^16 + x^12 + x^11 + x^10 + x^8 + 52 52 + * x^7 + x^5 + x^4 + x^2 + x^1 + x^0 53 53 + * - Bit order: Most-significant-bit-first 54 54 + * - Polynomial in integer form: 0x04c11db7 55 55 + * 56 56 + * Context: Any context 57 57 + * Return: The new CRC value 58 58 + */ 59 59 + u32 crc32_be(u32 crc, const void *p, size_t len); 28 60 29 29 - static inline u32 crc32c(u32 crc, const void *p, size_t len) 30 30 - { 31 31 - if (IS_ENABLED(CONFIG_CRC32_ARCH)) 32 32 - return crc32c_arch(crc, p, len); 33 33 - return crc32c_base(crc, p, len); 34 34 - } 61 61 + /** 62 62 + * crc32c() - Compute CRC-32C 63 63 + * @crc: Initial CRC value. ~0 (recommended) or 0 for a new CRC computation, or 64 64 + * the previous CRC value if computing incrementally. 65 65 + * @p: Pointer to the data buffer 66 66 + * @len: Length of data in bytes 67 67 + * 68 68 + * This implements CRC-32C, i.e. the Castagnoli CRC. This is the recommended 69 69 + * CRC variant to use in new applications that want a 32-bit CRC. 70 70 + * 71 71 + * - Polynomial: x^32 + x^28 + x^27 + x^26 + x^25 + x^23 + x^22 + x^20 + x^19 + 72 72 + * x^18 + x^14 + x^13 + x^11 + x^10 + x^9 + x^8 + x^6 + x^0 73 73 + * - Bit order: Least-significant-bit-first 74 74 + * - Polynomial in integer form: 0x82f63b78 75 75 + * 76 76 + * This does *not* invert the CRC at the beginning or end. The caller is 77 77 + * expected to do that if it needs to. Inverting at both ends is recommended. 78 78 + * 79 79 + * Context: Any context 80 80 + * Return: The new CRC value 81 81 + */ 82 82 + u32 crc32c(u32 crc, const void *p, size_t len); 35 83 36 84 /* 37 85 * crc32_optimizations() returns flags that indicate which CRC32 library ··· 95 47 #else 96 48 static inline u32 crc32_optimizations(void) { return 0; } 97 49 #endif 98 98 - 99 99 - /** 100 100 - * crc32_le_combine - Combine two crc32 check values into one. For two 101 101 - * sequences of bytes, seq1 and seq2 with lengths len1 102 102 - * and len2, crc32_le() check values were calculated 103 103 - * for each, crc1 and crc2. 104 104 - * 105 105 - * @crc1: crc32 of the first block 106 106 - * @crc2: crc32 of the second block 107 107 - * @len2: length of the second block 108 108 - * 109 109 - * Return: The crc32_le() check value of seq1 and seq2 concatenated, 110 110 - * requiring only crc1, crc2, and len2. Note: If seq_full denotes 111 111 - * the concatenated memory area of seq1 with seq2, and crc_full 112 112 - * the crc32_le() value of seq_full, then crc_full == 113 113 - * crc32_le_combine(crc1, crc2, len2) when crc_full was seeded 114 114 - * with the same initializer as crc1, and crc2 seed was 0. See 115 115 - * also crc32_combine_test(). 116 116 - */ 117 117 - u32 crc32_le_shift(u32 crc, size_t len); 118 118 - 119 119 - static inline u32 crc32_le_combine(u32 crc1, u32 crc2, size_t len2) 120 120 - { 121 121 - return crc32_le_shift(crc1, len2) ^ crc2; 122 122 - } 123 123 - 124 124 - #define crc32(seed, data, length) crc32_le(seed, (unsigned char const *)(data), length) 125 50 126 51 /* 127 52 * Helpers for hash table generation of ethernet nics:

+5 -11

include/linux/crc32poly.h

reviewed

··· 2 2 #ifndef _LINUX_CRC32_POLY_H 3 3 #define _LINUX_CRC32_POLY_H 4 4 5 5 - /* 6 6 - * There are multiple 16-bit CRC polynomials in common use, but this is 7 7 - * *the* standard CRC-32 polynomial, first popularized by Ethernet. 8 8 - * x^32+x^26+x^23+x^22+x^16+x^12+x^11+x^10+x^8+x^7+x^5+x^4+x^2+x^1+x^0 9 9 - */ 5 5 + /* The polynomial used by crc32_le(), in integer form. See crc32_le(). */ 10 6 #define CRC32_POLY_LE 0xedb88320 7 7 + 8 8 + /* The polynomial used by crc32_be(), in integer form. See crc32_be(). */ 11 9 #define CRC32_POLY_BE 0x04c11db7 12 10 13 13 - /* 14 14 - * This is the CRC32c polynomial, as outlined by Castagnoli. 15 15 - * x^32+x^28+x^27+x^26+x^25+x^23+x^22+x^20+x^19+x^18+x^14+x^13+x^11+x^10+x^9+ 16 16 - * x^8+x^6+x^0 17 17 - */ 18 18 - #define CRC32C_POLY_LE 0x82F63B78 11 11 + /* The polynomial used by crc32c(), in integer form. See crc32c(). */ 12 12 + #define CRC32C_POLY_LE 0x82f63b78 19 13 20 14 #endif /* _LINUX_CRC32_POLY_H */

+2 -20

include/linux/crc64.h

reviewed

··· 1 1 /* SPDX-License-Identifier: GPL-2.0 */ 2 2 - /* 3 3 - * See lib/crc64.c for the related specification and polynomial arithmetic. 4 4 - */ 5 2 #ifndef _LINUX_CRC64_H 6 3 #define _LINUX_CRC64_H 7 4 8 5 #include <linux/types.h> 9 9 - 10 10 - u64 crc64_be_arch(u64 crc, const u8 *p, size_t len); 11 11 - u64 crc64_be_generic(u64 crc, const u8 *p, size_t len); 12 12 - u64 crc64_nvme_arch(u64 crc, const u8 *p, size_t len); 13 13 - u64 crc64_nvme_generic(u64 crc, const u8 *p, size_t len); 14 6 15 7 /** 16 8 * crc64_be - Calculate bitwise big-endian ECMA-182 CRC64 ··· 11 19 * @p: pointer to buffer over which CRC64 is run 12 20 * @len: length of buffer @p 13 21 */ 14 14 - static inline u64 crc64_be(u64 crc, const void *p, size_t len) 15 15 - { 16 16 - if (IS_ENABLED(CONFIG_CRC64_ARCH)) 17 17 - return crc64_be_arch(crc, p, len); 18 18 - return crc64_be_generic(crc, p, len); 19 19 - } 22 22 + u64 crc64_be(u64 crc, const void *p, size_t len); 20 23 21 24 /** 22 25 * crc64_nvme - Calculate CRC64-NVME ··· 23 36 * This computes the CRC64 defined in the NVME NVM Command Set Specification, 24 37 * *including the bitwise inversion at the beginning and end*. 25 38 */ 26 26 - static inline u64 crc64_nvme(u64 crc, const void *p, size_t len) 27 27 - { 28 28 - if (IS_ENABLED(CONFIG_CRC64_ARCH)) 29 29 - return ~crc64_nvme_arch(~crc, p, len); 30 30 - return ~crc64_nvme_generic(~crc, p, len); 31 31 - } 39 39 + u64 crc64_nvme(u64 crc, const void *p, size_t len); 32 40 33 41 #endif /* _LINUX_CRC64_H */

+1 -87

lib/Kconfig

reviewed

··· 136 136 Create tracepoints for MMIO read/write operations. These trace events 137 137 can be used for logging all MMIO read/write operations. 138 138 139 139 + source "lib/crc/Kconfig" 139 140 source "lib/crypto/Kconfig" 140 140 - 141 141 - config CRC_CCITT 142 142 - tristate 143 143 - help 144 144 - The CRC-CCITT library functions. Select this if your module uses any 145 145 - of the functions from <linux/crc-ccitt.h>. 146 146 - 147 147 - config CRC16 148 148 - tristate 149 149 - help 150 150 - The CRC16 library functions. Select this if your module uses any of 151 151 - the functions from <linux/crc16.h>. 152 152 - 153 153 - config CRC_T10DIF 154 154 - tristate 155 155 - help 156 156 - The CRC-T10DIF library functions. Select this if your module uses 157 157 - any of the functions from <linux/crc-t10dif.h>. 158 158 - 159 159 - config ARCH_HAS_CRC_T10DIF 160 160 - bool 161 161 - 162 162 - config CRC_T10DIF_ARCH 163 163 - tristate 164 164 - default CRC_T10DIF if ARCH_HAS_CRC_T10DIF && CRC_OPTIMIZATIONS 165 165 - 166 166 - config CRC_ITU_T 167 167 - tristate 168 168 - help 169 169 - The CRC-ITU-T library functions. Select this if your module uses 170 170 - any of the functions from <linux/crc-itu-t.h>. 171 171 - 172 172 - config CRC32 173 173 - tristate 174 174 - select BITREVERSE 175 175 - help 176 176 - The CRC32 library functions. Select this if your module uses any of 177 177 - the functions from <linux/crc32.h> or <linux/crc32c.h>. 178 178 - 179 179 - config ARCH_HAS_CRC32 180 180 - bool 181 181 - 182 182 - config CRC32_ARCH 183 183 - tristate 184 184 - default CRC32 if ARCH_HAS_CRC32 && CRC_OPTIMIZATIONS 185 185 - 186 186 - config CRC64 187 187 - tristate 188 188 - help 189 189 - The CRC64 library functions. Select this if your module uses any of 190 190 - the functions from <linux/crc64.h>. 191 191 - 192 192 - config ARCH_HAS_CRC64 193 193 - bool 194 194 - 195 195 - config CRC64_ARCH 196 196 - tristate 197 197 - default CRC64 if ARCH_HAS_CRC64 && CRC_OPTIMIZATIONS 198 198 - 199 199 - config CRC4 200 200 - tristate 201 201 - help 202 202 - The CRC4 library functions. Select this if your module uses any of 203 203 - the functions from <linux/crc4.h>. 204 204 - 205 205 - config CRC7 206 206 - tristate 207 207 - help 208 208 - The CRC7 library functions. Select this if your module uses any of 209 209 - the functions from <linux/crc7.h>. 210 210 - 211 211 - config CRC8 212 212 - tristate 213 213 - help 214 214 - The CRC8 library functions. Select this if your module uses any of 215 215 - the functions from <linux/crc8.h>. 216 216 - 217 217 - config CRC_OPTIMIZATIONS 218 218 - bool "Enable optimized CRC implementations" if EXPERT 219 219 - default y 220 220 - help 221 221 - Disabling this option reduces code size slightly by disabling the 222 222 - architecture-optimized implementations of any CRC variants that are 223 223 - enabled. CRC checksumming performance may get much slower. 224 224 - 225 225 - Keep this enabled unless you're really trying to minimize the size of 226 226 - the kernel. 227 141 228 142 config XXHASH 229 143 tristate

-21

lib/Kconfig.debug

reviewed

··· 2910 2910 2911 2911 If unsure, say N. 2912 2912 2913 2913 - config CRC_KUNIT_TEST 2914 2914 - tristate "KUnit tests for CRC functions" if !KUNIT_ALL_TESTS 2915 2915 - depends on KUNIT 2916 2916 - default KUNIT_ALL_TESTS 2917 2917 - select CRC7 2918 2918 - select CRC16 2919 2919 - select CRC_T10DIF 2920 2920 - select CRC32 2921 2921 - select CRC64 2922 2922 - help 2923 2923 - Unit tests for the CRC library functions. 2924 2924 - 2925 2925 - This is intended to help people writing architecture-specific 2926 2926 - optimized versions. If unsure, say N. 2927 2927 - 2928 2928 - config CRC_BENCHMARK 2929 2929 - bool "Benchmark for the CRC functions" 2930 2930 - depends on CRC_KUNIT_TEST 2931 2931 - help 2932 2932 - Include benchmarks in the KUnit test suite for the CRC functions. 2933 2933 - 2934 2913 config SIPHASH_KUNIT_TEST 2935 2914 tristate "Perform selftest on siphash functions" if !KUNIT_ALL_TESTS 2936 2915 depends on KUNIT

+1 -31

lib/Makefile

reviewed

··· 122 122 obj-$(CONFIG_DEBUG_INFO_REDUCED) += debug_info.o 123 123 CFLAGS_debug_info.o += $(call cc-option, -femit-struct-debug-detailed=any) 124 124 125 125 - obj-y += math/ crypto/ tests/ vdso/ 125 125 + obj-y += math/ crc/ crypto/ tests/ vdso/ 126 126 127 127 obj-$(CONFIG_GENERIC_IOMAP) += iomap.o 128 128 obj-$(CONFIG_HAS_IOMEM) += iomap_copy.o devres.o ··· 148 148 obj-$(CONFIG_LINEAR_RANGES) += linear_ranges.o 149 149 obj-$(CONFIG_PACKING) += packing.o 150 150 obj-$(CONFIG_PACKING_KUNIT_TEST) += packing_test.o 151 151 - obj-$(CONFIG_CRC_CCITT) += crc-ccitt.o 152 152 - obj-$(CONFIG_CRC16) += crc16.o 153 153 - obj-$(CONFIG_CRC_T10DIF)+= crc-t10dif.o 154 154 - obj-$(CONFIG_CRC_ITU_T) += crc-itu-t.o 155 155 - obj-$(CONFIG_CRC32) += crc32.o 156 156 - obj-$(CONFIG_CRC64) += crc64.o 157 157 - obj-$(CONFIG_CRC4) += crc4.o 158 158 - obj-$(CONFIG_CRC7) += crc7.o 159 159 - obj-$(CONFIG_CRC8) += crc8.o 160 151 obj-$(CONFIG_XXHASH) += xxhash.o 161 152 obj-$(CONFIG_GENERIC_ALLOCATOR) += genalloc.o 162 153 ··· 284 293 obj-$(CONFIG_ASN1_ENCODER) += asn1_encoder.o 285 294 286 295 obj-$(CONFIG_FONT_SUPPORT) += fonts/ 287 287 - 288 288 - hostprogs := gen_crc32table 289 289 - hostprogs += gen_crc64table 290 290 - clean-files := crc32table.h 291 291 - clean-files += crc64table.h 292 292 - 293 293 - $(obj)/crc32.o: $(obj)/crc32table.h 294 294 - 295 295 - quiet_cmd_crc32 = GEN $@ 296 296 - cmd_crc32 = $< > $@ 297 297 - 298 298 - $(obj)/crc32table.h: $(obj)/gen_crc32table 299 299 - $(call cmd,crc32) 300 300 - 301 301 - $(obj)/crc64.o: $(obj)/crc64table.h 302 302 - 303 303 - quiet_cmd_crc64 = GEN $@ 304 304 - cmd_crc64 = $< > $@ 305 305 - 306 306 - $(obj)/crc64table.h: $(obj)/gen_crc64table 307 307 - $(call cmd,crc64) 308 296 309 297 # 310 298 # Build a fast OID lookip registry from include/linux/oid_registry.h

-68

lib/crc-ccitt.c

reviewed

··· 1 1 - // SPDX-License-Identifier: GPL-2.0-only 2 2 - /* 3 3 - * linux/lib/crc-ccitt.c 4 4 - */ 5 5 - 6 6 - #include <linux/types.h> 7 7 - #include <linux/module.h> 8 8 - #include <linux/crc-ccitt.h> 9 9 - 10 10 - /* 11 11 - * This mysterious table is just the CRC of each possible byte. It can be 12 12 - * computed using the standard bit-at-a-time methods. The polynomial can 13 13 - * be seen in entry 128, 0x8408. This corresponds to x^0 + x^5 + x^12. 14 14 - * Add the implicit x^16, and you have the standard CRC-CCITT. 15 15 - */ 16 16 - u16 const crc_ccitt_table[256] = { 17 17 - 0x0000, 0x1189, 0x2312, 0x329b, 0x4624, 0x57ad, 0x6536, 0x74bf, 18 18 - 0x8c48, 0x9dc1, 0xaf5a, 0xbed3, 0xca6c, 0xdbe5, 0xe97e, 0xf8f7, 19 19 - 0x1081, 0x0108, 0x3393, 0x221a, 0x56a5, 0x472c, 0x75b7, 0x643e, 20 20 - 0x9cc9, 0x8d40, 0xbfdb, 0xae52, 0xdaed, 0xcb64, 0xf9ff, 0xe876, 21 21 - 0x2102, 0x308b, 0x0210, 0x1399, 0x6726, 0x76af, 0x4434, 0x55bd, 22 22 - 0xad4a, 0xbcc3, 0x8e58, 0x9fd1, 0xeb6e, 0xfae7, 0xc87c, 0xd9f5, 23 23 - 0x3183, 0x200a, 0x1291, 0x0318, 0x77a7, 0x662e, 0x54b5, 0x453c, 24 24 - 0xbdcb, 0xac42, 0x9ed9, 0x8f50, 0xfbef, 0xea66, 0xd8fd, 0xc974, 25 25 - 0x4204, 0x538d, 0x6116, 0x709f, 0x0420, 0x15a9, 0x2732, 0x36bb, 26 26 - 0xce4c, 0xdfc5, 0xed5e, 0xfcd7, 0x8868, 0x99e1, 0xab7a, 0xbaf3, 27 27 - 0x5285, 0x430c, 0x7197, 0x601e, 0x14a1, 0x0528, 0x37b3, 0x263a, 28 28 - 0xdecd, 0xcf44, 0xfddf, 0xec56, 0x98e9, 0x8960, 0xbbfb, 0xaa72, 29 29 - 0x6306, 0x728f, 0x4014, 0x519d, 0x2522, 0x34ab, 0x0630, 0x17b9, 30 30 - 0xef4e, 0xfec7, 0xcc5c, 0xddd5, 0xa96a, 0xb8e3, 0x8a78, 0x9bf1, 31 31 - 0x7387, 0x620e, 0x5095, 0x411c, 0x35a3, 0x242a, 0x16b1, 0x0738, 32 32 - 0xffcf, 0xee46, 0xdcdd, 0xcd54, 0xb9eb, 0xa862, 0x9af9, 0x8b70, 33 33 - 0x8408, 0x9581, 0xa71a, 0xb693, 0xc22c, 0xd3a5, 0xe13e, 0xf0b7, 34 34 - 0x0840, 0x19c9, 0x2b52, 0x3adb, 0x4e64, 0x5fed, 0x6d76, 0x7cff, 35 35 - 0x9489, 0x8500, 0xb79b, 0xa612, 0xd2ad, 0xc324, 0xf1bf, 0xe036, 36 36 - 0x18c1, 0x0948, 0x3bd3, 0x2a5a, 0x5ee5, 0x4f6c, 0x7df7, 0x6c7e, 37 37 - 0xa50a, 0xb483, 0x8618, 0x9791, 0xe32e, 0xf2a7, 0xc03c, 0xd1b5, 38 38 - 0x2942, 0x38cb, 0x0a50, 0x1bd9, 0x6f66, 0x7eef, 0x4c74, 0x5dfd, 39 39 - 0xb58b, 0xa402, 0x9699, 0x8710, 0xf3af, 0xe226, 0xd0bd, 0xc134, 40 40 - 0x39c3, 0x284a, 0x1ad1, 0x0b58, 0x7fe7, 0x6e6e, 0x5cf5, 0x4d7c, 41 41 - 0xc60c, 0xd785, 0xe51e, 0xf497, 0x8028, 0x91a1, 0xa33a, 0xb2b3, 42 42 - 0x4a44, 0x5bcd, 0x6956, 0x78df, 0x0c60, 0x1de9, 0x2f72, 0x3efb, 43 43 - 0xd68d, 0xc704, 0xf59f, 0xe416, 0x90a9, 0x8120, 0xb3bb, 0xa232, 44 44 - 0x5ac5, 0x4b4c, 0x79d7, 0x685e, 0x1ce1, 0x0d68, 0x3ff3, 0x2e7a, 45 45 - 0xe70e, 0xf687, 0xc41c, 0xd595, 0xa12a, 0xb0a3, 0x8238, 0x93b1, 46 46 - 0x6b46, 0x7acf, 0x4854, 0x59dd, 0x2d62, 0x3ceb, 0x0e70, 0x1ff9, 47 47 - 0xf78f, 0xe606, 0xd49d, 0xc514, 0xb1ab, 0xa022, 0x92b9, 0x8330, 48 48 - 0x7bc7, 0x6a4e, 0x58d5, 0x495c, 0x3de3, 0x2c6a, 0x1ef1, 0x0f78 49 49 - }; 50 50 - EXPORT_SYMBOL(crc_ccitt_table); 51 51 - 52 52 - /** 53 53 - * crc_ccitt - recompute the CRC (CRC-CCITT variant) for the data 54 54 - * buffer 55 55 - * @crc: previous CRC value 56 56 - * @buffer: data pointer 57 57 - * @len: number of bytes in the buffer 58 58 - */ 59 59 - u16 crc_ccitt(u16 crc, u8 const *buffer, size_t len) 60 60 - { 61 61 - while (len--) 62 62 - crc = crc_ccitt_byte(crc, *buffer++); 63 63 - return crc; 64 64 - } 65 65 - EXPORT_SYMBOL(crc_ccitt); 66 66 - 67 67 - MODULE_DESCRIPTION("CRC-CCITT calculations"); 68 68 - MODULE_LICENSE("GPL");

-67

lib/crc-itu-t.c

reviewed

··· 1 1 - // SPDX-License-Identifier: GPL-2.0-only 2 2 - /* 3 3 - * crc-itu-t.c 4 4 - */ 5 5 - 6 6 - #include <linux/types.h> 7 7 - #include <linux/module.h> 8 8 - #include <linux/crc-itu-t.h> 9 9 - 10 10 - /* CRC table for the CRC ITU-T V.41 0x1021 (x^16 + x^12 + x^5 + 1) */ 11 11 - const u16 crc_itu_t_table[256] = { 12 12 - 0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7, 13 13 - 0x8108, 0x9129, 0xa14a, 0xb16b, 0xc18c, 0xd1ad, 0xe1ce, 0xf1ef, 14 14 - 0x1231, 0x0210, 0x3273, 0x2252, 0x52b5, 0x4294, 0x72f7, 0x62d6, 15 15 - 0x9339, 0x8318, 0xb37b, 0xa35a, 0xd3bd, 0xc39c, 0xf3ff, 0xe3de, 16 16 - 0x2462, 0x3443, 0x0420, 0x1401, 0x64e6, 0x74c7, 0x44a4, 0x5485, 17 17 - 0xa56a, 0xb54b, 0x8528, 0x9509, 0xe5ee, 0xf5cf, 0xc5ac, 0xd58d, 18 18 - 0x3653, 0x2672, 0x1611, 0x0630, 0x76d7, 0x66f6, 0x5695, 0x46b4, 19 19 - 0xb75b, 0xa77a, 0x9719, 0x8738, 0xf7df, 0xe7fe, 0xd79d, 0xc7bc, 20 20 - 0x48c4, 0x58e5, 0x6886, 0x78a7, 0x0840, 0x1861, 0x2802, 0x3823, 21 21 - 0xc9cc, 0xd9ed, 0xe98e, 0xf9af, 0x8948, 0x9969, 0xa90a, 0xb92b, 22 22 - 0x5af5, 0x4ad4, 0x7ab7, 0x6a96, 0x1a71, 0x0a50, 0x3a33, 0x2a12, 23 23 - 0xdbfd, 0xcbdc, 0xfbbf, 0xeb9e, 0x9b79, 0x8b58, 0xbb3b, 0xab1a, 24 24 - 0x6ca6, 0x7c87, 0x4ce4, 0x5cc5, 0x2c22, 0x3c03, 0x0c60, 0x1c41, 25 25 - 0xedae, 0xfd8f, 0xcdec, 0xddcd, 0xad2a, 0xbd0b, 0x8d68, 0x9d49, 26 26 - 0x7e97, 0x6eb6, 0x5ed5, 0x4ef4, 0x3e13, 0x2e32, 0x1e51, 0x0e70, 27 27 - 0xff9f, 0xefbe, 0xdfdd, 0xcffc, 0xbf1b, 0xaf3a, 0x9f59, 0x8f78, 28 28 - 0x9188, 0x81a9, 0xb1ca, 0xa1eb, 0xd10c, 0xc12d, 0xf14e, 0xe16f, 29 29 - 0x1080, 0x00a1, 0x30c2, 0x20e3, 0x5004, 0x4025, 0x7046, 0x6067, 30 30 - 0x83b9, 0x9398, 0xa3fb, 0xb3da, 0xc33d, 0xd31c, 0xe37f, 0xf35e, 31 31 - 0x02b1, 0x1290, 0x22f3, 0x32d2, 0x4235, 0x5214, 0x6277, 0x7256, 32 32 - 0xb5ea, 0xa5cb, 0x95a8, 0x8589, 0xf56e, 0xe54f, 0xd52c, 0xc50d, 33 33 - 0x34e2, 0x24c3, 0x14a0, 0x0481, 0x7466, 0x6447, 0x5424, 0x4405, 34 34 - 0xa7db, 0xb7fa, 0x8799, 0x97b8, 0xe75f, 0xf77e, 0xc71d, 0xd73c, 35 35 - 0x26d3, 0x36f2, 0x0691, 0x16b0, 0x6657, 0x7676, 0x4615, 0x5634, 36 36 - 0xd94c, 0xc96d, 0xf90e, 0xe92f, 0x99c8, 0x89e9, 0xb98a, 0xa9ab, 37 37 - 0x5844, 0x4865, 0x7806, 0x6827, 0x18c0, 0x08e1, 0x3882, 0x28a3, 38 38 - 0xcb7d, 0xdb5c, 0xeb3f, 0xfb1e, 0x8bf9, 0x9bd8, 0xabbb, 0xbb9a, 39 39 - 0x4a75, 0x5a54, 0x6a37, 0x7a16, 0x0af1, 0x1ad0, 0x2ab3, 0x3a92, 40 40 - 0xfd2e, 0xed0f, 0xdd6c, 0xcd4d, 0xbdaa, 0xad8b, 0x9de8, 0x8dc9, 41 41 - 0x7c26, 0x6c07, 0x5c64, 0x4c45, 0x3ca2, 0x2c83, 0x1ce0, 0x0cc1, 42 42 - 0xef1f, 0xff3e, 0xcf5d, 0xdf7c, 0xaf9b, 0xbfba, 0x8fd9, 0x9ff8, 43 43 - 0x6e17, 0x7e36, 0x4e55, 0x5e74, 0x2e93, 0x3eb2, 0x0ed1, 0x1ef0 44 44 - }; 45 45 - 46 46 - EXPORT_SYMBOL(crc_itu_t_table); 47 47 - 48 48 - /** 49 49 - * crc_itu_t - Compute the CRC-ITU-T for the data buffer 50 50 - * 51 51 - * @crc: previous CRC value 52 52 - * @buffer: data pointer 53 53 - * @len: number of bytes in the buffer 54 54 - * 55 55 - * Returns the updated CRC value 56 56 - */ 57 57 - u16 crc_itu_t(u16 crc, const u8 *buffer, size_t len) 58 58 - { 59 59 - while (len--) 60 60 - crc = crc_itu_t_byte(crc, *buffer++); 61 61 - return crc; 62 62 - } 63 63 - EXPORT_SYMBOL(crc_itu_t); 64 64 - 65 65 - MODULE_DESCRIPTION("CRC ITU-T V.41 calculations"); 66 66 - MODULE_LICENSE("GPL"); 67 67 -

-65

lib/crc-t10dif.c

reviewed

··· 1 1 - // SPDX-License-Identifier: GPL-2.0-only 2 2 - /* 3 3 - * T10 Data Integrity Field CRC16 calculation 4 4 - * 5 5 - * Copyright (c) 2007 Oracle Corporation. All rights reserved. 6 6 - * Written by Martin K. Petersen <martin.petersen@oracle.com> 7 7 - */ 8 8 - 9 9 - #include <linux/types.h> 10 10 - #include <linux/module.h> 11 11 - #include <linux/crc-t10dif.h> 12 12 - 13 13 - /* 14 14 - * Table generated using the following polynomial: 15 15 - * x^16 + x^15 + x^11 + x^9 + x^8 + x^7 + x^5 + x^4 + x^2 + x + 1 16 16 - * gt: 0x8bb7 17 17 - */ 18 18 - static const u16 t10_dif_crc_table[256] = { 19 19 - 0x0000, 0x8BB7, 0x9CD9, 0x176E, 0xB205, 0x39B2, 0x2EDC, 0xA56B, 20 20 - 0xEFBD, 0x640A, 0x7364, 0xF8D3, 0x5DB8, 0xD60F, 0xC161, 0x4AD6, 21 21 - 0x54CD, 0xDF7A, 0xC814, 0x43A3, 0xE6C8, 0x6D7F, 0x7A11, 0xF1A6, 22 22 - 0xBB70, 0x30C7, 0x27A9, 0xAC1E, 0x0975, 0x82C2, 0x95AC, 0x1E1B, 23 23 - 0xA99A, 0x222D, 0x3543, 0xBEF4, 0x1B9F, 0x9028, 0x8746, 0x0CF1, 24 24 - 0x4627, 0xCD90, 0xDAFE, 0x5149, 0xF422, 0x7F95, 0x68FB, 0xE34C, 25 25 - 0xFD57, 0x76E0, 0x618E, 0xEA39, 0x4F52, 0xC4E5, 0xD38B, 0x583C, 26 26 - 0x12EA, 0x995D, 0x8E33, 0x0584, 0xA0EF, 0x2B58, 0x3C36, 0xB781, 27 27 - 0xD883, 0x5334, 0x445A, 0xCFED, 0x6A86, 0xE131, 0xF65F, 0x7DE8, 28 28 - 0x373E, 0xBC89, 0xABE7, 0x2050, 0x853B, 0x0E8C, 0x19E2, 0x9255, 29 29 - 0x8C4E, 0x07F9, 0x1097, 0x9B20, 0x3E4B, 0xB5FC, 0xA292, 0x2925, 30 30 - 0x63F3, 0xE844, 0xFF2A, 0x749D, 0xD1F6, 0x5A41, 0x4D2F, 0xC698, 31 31 - 0x7119, 0xFAAE, 0xEDC0, 0x6677, 0xC31C, 0x48AB, 0x5FC5, 0xD472, 32 32 - 0x9EA4, 0x1513, 0x027D, 0x89CA, 0x2CA1, 0xA716, 0xB078, 0x3BCF, 33 33 - 0x25D4, 0xAE63, 0xB90D, 0x32BA, 0x97D1, 0x1C66, 0x0B08, 0x80BF, 34 34 - 0xCA69, 0x41DE, 0x56B0, 0xDD07, 0x786C, 0xF3DB, 0xE4B5, 0x6F02, 35 35 - 0x3AB1, 0xB106, 0xA668, 0x2DDF, 0x88B4, 0x0303, 0x146D, 0x9FDA, 36 36 - 0xD50C, 0x5EBB, 0x49D5, 0xC262, 0x6709, 0xECBE, 0xFBD0, 0x7067, 37 37 - 0x6E7C, 0xE5CB, 0xF2A5, 0x7912, 0xDC79, 0x57CE, 0x40A0, 0xCB17, 38 38 - 0x81C1, 0x0A76, 0x1D18, 0x96AF, 0x33C4, 0xB873, 0xAF1D, 0x24AA, 39 39 - 0x932B, 0x189C, 0x0FF2, 0x8445, 0x212E, 0xAA99, 0xBDF7, 0x3640, 40 40 - 0x7C96, 0xF721, 0xE04F, 0x6BF8, 0xCE93, 0x4524, 0x524A, 0xD9FD, 41 41 - 0xC7E6, 0x4C51, 0x5B3F, 0xD088, 0x75E3, 0xFE54, 0xE93A, 0x628D, 42 42 - 0x285B, 0xA3EC, 0xB482, 0x3F35, 0x9A5E, 0x11E9, 0x0687, 0x8D30, 43 43 - 0xE232, 0x6985, 0x7EEB, 0xF55C, 0x5037, 0xDB80, 0xCCEE, 0x4759, 44 44 - 0x0D8F, 0x8638, 0x9156, 0x1AE1, 0xBF8A, 0x343D, 0x2353, 0xA8E4, 45 45 - 0xB6FF, 0x3D48, 0x2A26, 0xA191, 0x04FA, 0x8F4D, 0x9823, 0x1394, 46 46 - 0x5942, 0xD2F5, 0xC59B, 0x4E2C, 0xEB47, 0x60F0, 0x779E, 0xFC29, 47 47 - 0x4BA8, 0xC01F, 0xD771, 0x5CC6, 0xF9AD, 0x721A, 0x6574, 0xEEC3, 48 48 - 0xA415, 0x2FA2, 0x38CC, 0xB37B, 0x1610, 0x9DA7, 0x8AC9, 0x017E, 49 49 - 0x1F65, 0x94D2, 0x83BC, 0x080B, 0xAD60, 0x26D7, 0x31B9, 0xBA0E, 50 50 - 0xF0D8, 0x7B6F, 0x6C01, 0xE7B6, 0x42DD, 0xC96A, 0xDE04, 0x55B3 51 51 - }; 52 52 - 53 53 - u16 crc_t10dif_generic(u16 crc, const u8 *p, size_t len) 54 54 - { 55 55 - size_t i; 56 56 - 57 57 - for (i = 0; i < len; i++) 58 58 - crc = (crc << 8) ^ t10_dif_crc_table[(crc >> 8) ^ p[i]]; 59 59 - 60 60 - return crc; 61 61 - } 62 62 - EXPORT_SYMBOL(crc_t10dif_generic); 63 63 - 64 64 - MODULE_DESCRIPTION("T10 DIF CRC calculation"); 65 65 - MODULE_LICENSE("GPL");

+5

lib/crc/.gitignore

reviewed

··· 1 1 + # SPDX-License-Identifier: GPL-2.0-only 2 2 + /crc32table.h 3 3 + /crc64table.h 4 4 + /gen_crc32table 5 5 + /gen_crc64table

+119

lib/crc/Kconfig

reviewed

··· 1 1 + # SPDX-License-Identifier: GPL-2.0-only 2 2 + 3 3 + # Kconfig for the kernel's cyclic redundancy check (CRC) library code 4 4 + 5 5 + config CRC4 6 6 + tristate 7 7 + help 8 8 + The CRC4 library functions. Select this if your module uses any of 9 9 + the functions from <linux/crc4.h>. 10 10 + 11 11 + config CRC7 12 12 + tristate 13 13 + help 14 14 + The CRC7 library functions. Select this if your module uses any of 15 15 + the functions from <linux/crc7.h>. 16 16 + 17 17 + config CRC8 18 18 + tristate 19 19 + help 20 20 + The CRC8 library functions. Select this if your module uses any of 21 21 + the functions from <linux/crc8.h>. 22 22 + 23 23 + config CRC16 24 24 + tristate 25 25 + help 26 26 + The CRC16 library functions. Select this if your module uses any of 27 27 + the functions from <linux/crc16.h>. 28 28 + 29 29 + config CRC_CCITT 30 30 + tristate 31 31 + help 32 32 + The CRC-CCITT library functions. Select this if your module uses any 33 33 + of the functions from <linux/crc-ccitt.h>. 34 34 + 35 35 + config CRC_ITU_T 36 36 + tristate 37 37 + help 38 38 + The CRC-ITU-T library functions. Select this if your module uses 39 39 + any of the functions from <linux/crc-itu-t.h>. 40 40 + 41 41 + config CRC_T10DIF 42 42 + tristate 43 43 + help 44 44 + The CRC-T10DIF library functions. Select this if your module uses 45 45 + any of the functions from <linux/crc-t10dif.h>. 46 46 + 47 47 + config CRC_T10DIF_ARCH 48 48 + bool 49 49 + depends on CRC_T10DIF && CRC_OPTIMIZATIONS 50 50 + default y if ARM && KERNEL_MODE_NEON 51 51 + default y if ARM64 && KERNEL_MODE_NEON 52 52 + default y if PPC64 && ALTIVEC 53 53 + default y if RISCV && RISCV_ISA_ZBC 54 54 + default y if X86 55 55 + 56 56 + config CRC32 57 57 + tristate 58 58 + select BITREVERSE 59 59 + help 60 60 + The CRC32 library functions. Select this if your module uses any of 61 61 + the functions from <linux/crc32.h> or <linux/crc32c.h>. 62 62 + 63 63 + config CRC32_ARCH 64 64 + bool 65 65 + depends on CRC32 && CRC_OPTIMIZATIONS 66 66 + default y if ARM && KERNEL_MODE_NEON 67 67 + default y if ARM64 68 68 + default y if LOONGARCH 69 69 + default y if MIPS && CPU_MIPSR6 70 70 + default y if PPC64 && ALTIVEC 71 71 + default y if RISCV && RISCV_ISA_ZBC 72 72 + default y if S390 73 73 + default y if SPARC64 74 74 + default y if X86 75 75 + 76 76 + config CRC64 77 77 + tristate 78 78 + help 79 79 + The CRC64 library functions. Select this if your module uses any of 80 80 + the functions from <linux/crc64.h>. 81 81 + 82 82 + config CRC64_ARCH 83 83 + bool 84 84 + depends on CRC64 && CRC_OPTIMIZATIONS 85 85 + default y if RISCV && RISCV_ISA_ZBC && 64BIT 86 86 + default y if X86_64 87 87 + 88 88 + config CRC_OPTIMIZATIONS 89 89 + bool "Enable optimized CRC implementations" if EXPERT 90 90 + depends on !UML 91 91 + default y 92 92 + help 93 93 + Disabling this option reduces code size slightly by disabling the 94 94 + architecture-optimized implementations of any CRC variants that are 95 95 + enabled. CRC checksumming performance may get much slower. 96 96 + 97 97 + Keep this enabled unless you're really trying to minimize the size of 98 98 + the kernel. 99 99 + 100 100 + config CRC_KUNIT_TEST 101 101 + tristate "KUnit tests for CRC functions" if !KUNIT_ALL_TESTS 102 102 + depends on KUNIT 103 103 + default KUNIT_ALL_TESTS 104 104 + select CRC7 105 105 + select CRC16 106 106 + select CRC_T10DIF 107 107 + select CRC32 108 108 + select CRC64 109 109 + help 110 110 + Unit tests for the CRC library functions. 111 111 + 112 112 + This is intended to help people writing architecture-specific 113 113 + optimized versions. If unsure, say N. 114 114 + 115 115 + config CRC_BENCHMARK 116 116 + bool "Benchmark for the CRC functions" 117 117 + depends on CRC_KUNIT_TEST 118 118 + help 119 119 + Include benchmarks in the KUnit test suite for the CRC functions.

+63

lib/crc/Makefile

reviewed

··· 1 1 + # SPDX-License-Identifier: GPL-2.0-only 2 2 + 3 3 + # Makefile for the kernel's cyclic redundancy check (CRC) library code 4 4 + 5 5 + obj-$(CONFIG_CRC4) += crc4.o 6 6 + obj-$(CONFIG_CRC7) += crc7.o 7 7 + obj-$(CONFIG_CRC8) += crc8.o 8 8 + obj-$(CONFIG_CRC16) += crc16.o 9 9 + obj-$(CONFIG_CRC_CCITT) += crc-ccitt.o 10 10 + obj-$(CONFIG_CRC_ITU_T) += crc-itu-t.o 11 11 + 12 12 + obj-$(CONFIG_CRC_T10DIF) += crc-t10dif.o 13 13 + crc-t10dif-y := crc-t10dif-main.o 14 14 + ifeq ($(CONFIG_CRC_T10DIF_ARCH),y) 15 15 + CFLAGS_crc-t10dif-main.o += -I$(src)/$(SRCARCH) 16 16 + crc-t10dif-$(CONFIG_ARM) += arm/crc-t10dif-core.o 17 17 + crc-t10dif-$(CONFIG_ARM64) += arm64/crc-t10dif-core.o 18 18 + crc-t10dif-$(CONFIG_PPC) += powerpc/crct10dif-vpmsum_asm.o 19 19 + crc-t10dif-$(CONFIG_RISCV) += riscv/crc16_msb.o 20 20 + crc-t10dif-$(CONFIG_X86) += x86/crc16-msb-pclmul.o 21 21 + endif 22 22 + 23 23 + obj-$(CONFIG_CRC32) += crc32.o 24 24 + crc32-y := crc32-main.o 25 25 + ifeq ($(CONFIG_CRC32_ARCH),y) 26 26 + CFLAGS_crc32-main.o += -I$(src)/$(SRCARCH) 27 27 + crc32-$(CONFIG_ARM) += arm/crc32-core.o 28 28 + crc32-$(CONFIG_ARM64) += arm64/crc32-core.o 29 29 + crc32-$(CONFIG_PPC) += powerpc/crc32c-vpmsum_asm.o 30 30 + crc32-$(CONFIG_RISCV) += riscv/crc32_lsb.o riscv/crc32_msb.o 31 31 + crc32-$(CONFIG_S390) += s390/crc32le-vx.o s390/crc32be-vx.o 32 32 + crc32-$(CONFIG_SPARC) += sparc/crc32c_asm.o 33 33 + crc32-$(CONFIG_X86) += x86/crc32-pclmul.o 34 34 + crc32-$(CONFIG_X86_64) += x86/crc32c-3way.o 35 35 + endif 36 36 + 37 37 + obj-$(CONFIG_CRC64) += crc64.o 38 38 + crc64-y := crc64-main.o 39 39 + ifeq ($(CONFIG_CRC64_ARCH),y) 40 40 + CFLAGS_crc64-main.o += -I$(src)/$(SRCARCH) 41 41 + crc64-$(CONFIG_RISCV) += riscv/crc64_lsb.o riscv/crc64_msb.o 42 42 + crc64-$(CONFIG_X86) += x86/crc64-pclmul.o 43 43 + endif 44 44 + 45 45 + obj-y += tests/ 46 46 + 47 47 + hostprogs := gen_crc32table gen_crc64table 48 48 + clean-files := crc32table.h crc64table.h 49 49 + 50 50 + $(obj)/crc32-main.o: $(obj)/crc32table.h 51 51 + $(obj)/crc64-main.o: $(obj)/crc64table.h 52 52 + 53 53 + quiet_cmd_crc32 = GEN $@ 54 54 + cmd_crc32 = $< > $@ 55 55 + 56 56 + quiet_cmd_crc64 = GEN $@ 57 57 + cmd_crc64 = $< > $@ 58 58 + 59 59 + $(obj)/crc32table.h: $(obj)/gen_crc32table 60 60 + $(call cmd,crc32) 61 61 + 62 62 + $(obj)/crc64table.h: $(obj)/gen_crc64table 63 63 + $(call cmd,crc64)

+55

lib/crc/arm/crc-t10dif.h

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0-only 2 2 + /* 3 3 + * Accelerated CRC-T10DIF using ARM NEON and Crypto Extensions instructions 4 4 + * 5 5 + * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org> 6 6 + */ 7 7 + 8 8 + #include <crypto/internal/simd.h> 9 9 + 10 10 + #include <asm/neon.h> 11 11 + #include <asm/simd.h> 12 12 + 13 13 + static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); 14 14 + static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull); 15 15 + 16 16 + #define CRC_T10DIF_PMULL_CHUNK_SIZE 16U 17 17 + 18 18 + asmlinkage u16 crc_t10dif_pmull64(u16 init_crc, const u8 *buf, size_t len); 19 19 + asmlinkage void crc_t10dif_pmull8(u16 init_crc, const u8 *buf, size_t len, 20 20 + u8 out[16]); 21 21 + 22 22 + static inline u16 crc_t10dif_arch(u16 crc, const u8 *data, size_t length) 23 23 + { 24 24 + if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE) { 25 25 + if (static_branch_likely(&have_pmull)) { 26 26 + if (crypto_simd_usable()) { 27 27 + kernel_neon_begin(); 28 28 + crc = crc_t10dif_pmull64(crc, data, length); 29 29 + kernel_neon_end(); 30 30 + return crc; 31 31 + } 32 32 + } else if (length > CRC_T10DIF_PMULL_CHUNK_SIZE && 33 33 + static_branch_likely(&have_neon) && 34 34 + crypto_simd_usable()) { 35 35 + u8 buf[16] __aligned(16); 36 36 + 37 37 + kernel_neon_begin(); 38 38 + crc_t10dif_pmull8(crc, data, length, buf); 39 39 + kernel_neon_end(); 40 40 + 41 41 + return crc_t10dif_generic(0, buf, sizeof(buf)); 42 42 + } 43 43 + } 44 44 + return crc_t10dif_generic(crc, data, length); 45 45 + } 46 46 + 47 47 + #define crc_t10dif_mod_init_arch crc_t10dif_mod_init_arch 48 48 + static inline void crc_t10dif_mod_init_arch(void) 49 49 + { 50 50 + if (elf_hwcap & HWCAP_NEON) { 51 51 + static_branch_enable(&have_neon); 52 52 + if (elf_hwcap2 & HWCAP2_PMULL) 53 53 + static_branch_enable(&have_pmull); 54 54 + } 55 55 + }

+101

lib/crc/arm/crc32.h

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0-only 2 2 + /* 3 3 + * Accelerated CRC32(C) using ARM CRC, NEON and Crypto Extensions instructions 4 4 + * 5 5 + * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org> 6 6 + */ 7 7 + 8 8 + #include <linux/cpufeature.h> 9 9 + 10 10 + #include <crypto/internal/simd.h> 11 11 + 12 12 + #include <asm/hwcap.h> 13 13 + #include <asm/neon.h> 14 14 + #include <asm/simd.h> 15 15 + 16 16 + static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_crc32); 17 17 + static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull); 18 18 + 19 19 + #define PMULL_MIN_LEN 64 /* min size of buffer for pmull functions */ 20 20 + 21 21 + asmlinkage u32 crc32_pmull_le(const u8 buf[], u32 len, u32 init_crc); 22 22 + asmlinkage u32 crc32_armv8_le(u32 init_crc, const u8 buf[], u32 len); 23 23 + 24 24 + asmlinkage u32 crc32c_pmull_le(const u8 buf[], u32 len, u32 init_crc); 25 25 + asmlinkage u32 crc32c_armv8_le(u32 init_crc, const u8 buf[], u32 len); 26 26 + 27 27 + static inline u32 crc32_le_scalar(u32 crc, const u8 *p, size_t len) 28 28 + { 29 29 + if (static_branch_likely(&have_crc32)) 30 30 + return crc32_armv8_le(crc, p, len); 31 31 + return crc32_le_base(crc, p, len); 32 32 + } 33 33 + 34 34 + static inline u32 crc32_le_arch(u32 crc, const u8 *p, size_t len) 35 35 + { 36 36 + if (len >= PMULL_MIN_LEN + 15 && 37 37 + static_branch_likely(&have_pmull) && crypto_simd_usable()) { 38 38 + size_t n = -(uintptr_t)p & 15; 39 39 + 40 40 + /* align p to 16-byte boundary */ 41 41 + if (n) { 42 42 + crc = crc32_le_scalar(crc, p, n); 43 43 + p += n; 44 44 + len -= n; 45 45 + } 46 46 + n = round_down(len, 16); 47 47 + kernel_neon_begin(); 48 48 + crc = crc32_pmull_le(p, n, crc); 49 49 + kernel_neon_end(); 50 50 + p += n; 51 51 + len -= n; 52 52 + } 53 53 + return crc32_le_scalar(crc, p, len); 54 54 + } 55 55 + 56 56 + static inline u32 crc32c_scalar(u32 crc, const u8 *p, size_t len) 57 57 + { 58 58 + if (static_branch_likely(&have_crc32)) 59 59 + return crc32c_armv8_le(crc, p, len); 60 60 + return crc32c_base(crc, p, len); 61 61 + } 62 62 + 63 63 + static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len) 64 64 + { 65 65 + if (len >= PMULL_MIN_LEN + 15 && 66 66 + static_branch_likely(&have_pmull) && crypto_simd_usable()) { 67 67 + size_t n = -(uintptr_t)p & 15; 68 68 + 69 69 + /* align p to 16-byte boundary */ 70 70 + if (n) { 71 71 + crc = crc32c_scalar(crc, p, n); 72 72 + p += n; 73 73 + len -= n; 74 74 + } 75 75 + n = round_down(len, 16); 76 76 + kernel_neon_begin(); 77 77 + crc = crc32c_pmull_le(p, n, crc); 78 78 + kernel_neon_end(); 79 79 + p += n; 80 80 + len -= n; 81 81 + } 82 82 + return crc32c_scalar(crc, p, len); 83 83 + } 84 84 + 85 85 + #define crc32_be_arch crc32_be_base /* not implemented on this arch */ 86 86 + 87 87 + #define crc32_mod_init_arch crc32_mod_init_arch 88 88 + static inline void crc32_mod_init_arch(void) 89 89 + { 90 90 + if (elf_hwcap2 & HWCAP2_CRC32) 91 91 + static_branch_enable(&have_crc32); 92 92 + if (elf_hwcap2 & HWCAP2_PMULL) 93 93 + static_branch_enable(&have_pmull); 94 94 + } 95 95 + 96 96 + static inline u32 crc32_optimizations_arch(void) 97 97 + { 98 98 + if (elf_hwcap2 & (HWCAP2_CRC32 | HWCAP2_PMULL)) 99 99 + return CRC32_LE_OPTIMIZATION | CRC32C_OPTIMIZATION; 100 100 + return 0; 101 101 + }

+57

lib/crc/arm64/crc-t10dif.h

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0-only 2 2 + /* 3 3 + * Accelerated CRC-T10DIF using arm64 NEON and Crypto Extensions instructions 4 4 + * 5 5 + * Copyright (C) 2016 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org> 6 6 + */ 7 7 + 8 8 + #include <linux/cpufeature.h> 9 9 + 10 10 + #include <crypto/internal/simd.h> 11 11 + 12 12 + #include <asm/neon.h> 13 13 + #include <asm/simd.h> 14 14 + 15 15 + static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_asimd); 16 16 + static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull); 17 17 + 18 18 + #define CRC_T10DIF_PMULL_CHUNK_SIZE 16U 19 19 + 20 20 + asmlinkage void crc_t10dif_pmull_p8(u16 init_crc, const u8 *buf, size_t len, 21 21 + u8 out[16]); 22 22 + asmlinkage u16 crc_t10dif_pmull_p64(u16 init_crc, const u8 *buf, size_t len); 23 23 + 24 24 + static inline u16 crc_t10dif_arch(u16 crc, const u8 *data, size_t length) 25 25 + { 26 26 + if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE) { 27 27 + if (static_branch_likely(&have_pmull)) { 28 28 + if (crypto_simd_usable()) { 29 29 + kernel_neon_begin(); 30 30 + crc = crc_t10dif_pmull_p64(crc, data, length); 31 31 + kernel_neon_end(); 32 32 + return crc; 33 33 + } 34 34 + } else if (length > CRC_T10DIF_PMULL_CHUNK_SIZE && 35 35 + static_branch_likely(&have_asimd) && 36 36 + crypto_simd_usable()) { 37 37 + u8 buf[16]; 38 38 + 39 39 + kernel_neon_begin(); 40 40 + crc_t10dif_pmull_p8(crc, data, length, buf); 41 41 + kernel_neon_end(); 42 42 + 43 43 + return crc_t10dif_generic(0, buf, sizeof(buf)); 44 44 + } 45 45 + } 46 46 + return crc_t10dif_generic(crc, data, length); 47 47 + } 48 48 + 49 49 + #define crc_t10dif_mod_init_arch crc_t10dif_mod_init_arch 50 50 + static inline void crc_t10dif_mod_init_arch(void) 51 51 + { 52 52 + if (cpu_have_named_feature(ASIMD)) { 53 53 + static_branch_enable(&have_asimd); 54 54 + if (cpu_have_named_feature(PMULL)) 55 55 + static_branch_enable(&have_pmull); 56 56 + } 57 57 + }

+88

lib/crc/arm64/crc32.h

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0-only 2 2 + 3 3 + #include <asm/alternative.h> 4 4 + #include <asm/cpufeature.h> 5 5 + #include <asm/neon.h> 6 6 + #include <asm/simd.h> 7 7 + 8 8 + #include <crypto/internal/simd.h> 9 9 + 10 10 + // The minimum input length to consider the 4-way interleaved code path 11 11 + static const size_t min_len = 1024; 12 12 + 13 13 + asmlinkage u32 crc32_le_arm64(u32 crc, unsigned char const *p, size_t len); 14 14 + asmlinkage u32 crc32c_le_arm64(u32 crc, unsigned char const *p, size_t len); 15 15 + asmlinkage u32 crc32_be_arm64(u32 crc, unsigned char const *p, size_t len); 16 16 + 17 17 + asmlinkage u32 crc32_le_arm64_4way(u32 crc, unsigned char const *p, size_t len); 18 18 + asmlinkage u32 crc32c_le_arm64_4way(u32 crc, unsigned char const *p, size_t len); 19 19 + asmlinkage u32 crc32_be_arm64_4way(u32 crc, unsigned char const *p, size_t len); 20 20 + 21 21 + static inline u32 crc32_le_arch(u32 crc, const u8 *p, size_t len) 22 22 + { 23 23 + if (!alternative_has_cap_likely(ARM64_HAS_CRC32)) 24 24 + return crc32_le_base(crc, p, len); 25 25 + 26 26 + if (len >= min_len && cpu_have_named_feature(PMULL) && crypto_simd_usable()) { 27 27 + kernel_neon_begin(); 28 28 + crc = crc32_le_arm64_4way(crc, p, len); 29 29 + kernel_neon_end(); 30 30 + 31 31 + p += round_down(len, 64); 32 32 + len %= 64; 33 33 + 34 34 + if (!len) 35 35 + return crc; 36 36 + } 37 37 + 38 38 + return crc32_le_arm64(crc, p, len); 39 39 + } 40 40 + 41 41 + static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len) 42 42 + { 43 43 + if (!alternative_has_cap_likely(ARM64_HAS_CRC32)) 44 44 + return crc32c_base(crc, p, len); 45 45 + 46 46 + if (len >= min_len && cpu_have_named_feature(PMULL) && crypto_simd_usable()) { 47 47 + kernel_neon_begin(); 48 48 + crc = crc32c_le_arm64_4way(crc, p, len); 49 49 + kernel_neon_end(); 50 50 + 51 51 + p += round_down(len, 64); 52 52 + len %= 64; 53 53 + 54 54 + if (!len) 55 55 + return crc; 56 56 + } 57 57 + 58 58 + return crc32c_le_arm64(crc, p, len); 59 59 + } 60 60 + 61 61 + static inline u32 crc32_be_arch(u32 crc, const u8 *p, size_t len) 62 62 + { 63 63 + if (!alternative_has_cap_likely(ARM64_HAS_CRC32)) 64 64 + return crc32_be_base(crc, p, len); 65 65 + 66 66 + if (len >= min_len && cpu_have_named_feature(PMULL) && crypto_simd_usable()) { 67 67 + kernel_neon_begin(); 68 68 + crc = crc32_be_arm64_4way(crc, p, len); 69 69 + kernel_neon_end(); 70 70 + 71 71 + p += round_down(len, 64); 72 72 + len %= 64; 73 73 + 74 74 + if (!len) 75 75 + return crc; 76 76 + } 77 77 + 78 78 + return crc32_be_arm64(crc, p, len); 79 79 + } 80 80 + 81 81 + static inline u32 crc32_optimizations_arch(void) 82 82 + { 83 83 + if (alternative_has_cap_likely(ARM64_HAS_CRC32)) 84 84 + return CRC32_LE_OPTIMIZATION | 85 85 + CRC32_BE_OPTIMIZATION | 86 86 + CRC32C_OPTIMIZATION; 87 87 + return 0; 88 88 + }

+66

lib/crc/crc-ccitt.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0-only 2 2 + 3 3 + #include <linux/crc-ccitt.h> 4 4 + #include <linux/export.h> 5 5 + #include <linux/module.h> 6 6 + #include <linux/types.h> 7 7 + 8 8 + /* 9 9 + * This mysterious table is just the CRC of each possible byte. It can be 10 10 + * computed using the standard bit-at-a-time methods. The polynomial can 11 11 + * be seen in entry 128, 0x8408. This corresponds to x^0 + x^5 + x^12. 12 12 + * Add the implicit x^16, and you have the standard CRC-CCITT. 13 13 + */ 14 14 + u16 const crc_ccitt_table[256] = { 15 15 + 0x0000, 0x1189, 0x2312, 0x329b, 0x4624, 0x57ad, 0x6536, 0x74bf, 16 16 + 0x8c48, 0x9dc1, 0xaf5a, 0xbed3, 0xca6c, 0xdbe5, 0xe97e, 0xf8f7, 17 17 + 0x1081, 0x0108, 0x3393, 0x221a, 0x56a5, 0x472c, 0x75b7, 0x643e, 18 18 + 0x9cc9, 0x8d40, 0xbfdb, 0xae52, 0xdaed, 0xcb64, 0xf9ff, 0xe876, 19 19 + 0x2102, 0x308b, 0x0210, 0x1399, 0x6726, 0x76af, 0x4434, 0x55bd, 20 20 + 0xad4a, 0xbcc3, 0x8e58, 0x9fd1, 0xeb6e, 0xfae7, 0xc87c, 0xd9f5, 21 21 + 0x3183, 0x200a, 0x1291, 0x0318, 0x77a7, 0x662e, 0x54b5, 0x453c, 22 22 + 0xbdcb, 0xac42, 0x9ed9, 0x8f50, 0xfbef, 0xea66, 0xd8fd, 0xc974, 23 23 + 0x4204, 0x538d, 0x6116, 0x709f, 0x0420, 0x15a9, 0x2732, 0x36bb, 24 24 + 0xce4c, 0xdfc5, 0xed5e, 0xfcd7, 0x8868, 0x99e1, 0xab7a, 0xbaf3, 25 25 + 0x5285, 0x430c, 0x7197, 0x601e, 0x14a1, 0x0528, 0x37b3, 0x263a, 26 26 + 0xdecd, 0xcf44, 0xfddf, 0xec56, 0x98e9, 0x8960, 0xbbfb, 0xaa72, 27 27 + 0x6306, 0x728f, 0x4014, 0x519d, 0x2522, 0x34ab, 0x0630, 0x17b9, 28 28 + 0xef4e, 0xfec7, 0xcc5c, 0xddd5, 0xa96a, 0xb8e3, 0x8a78, 0x9bf1, 29 29 + 0x7387, 0x620e, 0x5095, 0x411c, 0x35a3, 0x242a, 0x16b1, 0x0738, 30 30 + 0xffcf, 0xee46, 0xdcdd, 0xcd54, 0xb9eb, 0xa862, 0x9af9, 0x8b70, 31 31 + 0x8408, 0x9581, 0xa71a, 0xb693, 0xc22c, 0xd3a5, 0xe13e, 0xf0b7, 32 32 + 0x0840, 0x19c9, 0x2b52, 0x3adb, 0x4e64, 0x5fed, 0x6d76, 0x7cff, 33 33 + 0x9489, 0x8500, 0xb79b, 0xa612, 0xd2ad, 0xc324, 0xf1bf, 0xe036, 34 34 + 0x18c1, 0x0948, 0x3bd3, 0x2a5a, 0x5ee5, 0x4f6c, 0x7df7, 0x6c7e, 35 35 + 0xa50a, 0xb483, 0x8618, 0x9791, 0xe32e, 0xf2a7, 0xc03c, 0xd1b5, 36 36 + 0x2942, 0x38cb, 0x0a50, 0x1bd9, 0x6f66, 0x7eef, 0x4c74, 0x5dfd, 37 37 + 0xb58b, 0xa402, 0x9699, 0x8710, 0xf3af, 0xe226, 0xd0bd, 0xc134, 38 38 + 0x39c3, 0x284a, 0x1ad1, 0x0b58, 0x7fe7, 0x6e6e, 0x5cf5, 0x4d7c, 39 39 + 0xc60c, 0xd785, 0xe51e, 0xf497, 0x8028, 0x91a1, 0xa33a, 0xb2b3, 40 40 + 0x4a44, 0x5bcd, 0x6956, 0x78df, 0x0c60, 0x1de9, 0x2f72, 0x3efb, 41 41 + 0xd68d, 0xc704, 0xf59f, 0xe416, 0x90a9, 0x8120, 0xb3bb, 0xa232, 42 42 + 0x5ac5, 0x4b4c, 0x79d7, 0x685e, 0x1ce1, 0x0d68, 0x3ff3, 0x2e7a, 43 43 + 0xe70e, 0xf687, 0xc41c, 0xd595, 0xa12a, 0xb0a3, 0x8238, 0x93b1, 44 44 + 0x6b46, 0x7acf, 0x4854, 0x59dd, 0x2d62, 0x3ceb, 0x0e70, 0x1ff9, 45 45 + 0xf78f, 0xe606, 0xd49d, 0xc514, 0xb1ab, 0xa022, 0x92b9, 0x8330, 46 46 + 0x7bc7, 0x6a4e, 0x58d5, 0x495c, 0x3de3, 0x2c6a, 0x1ef1, 0x0f78 47 47 + }; 48 48 + EXPORT_SYMBOL(crc_ccitt_table); 49 49 + 50 50 + /** 51 51 + * crc_ccitt - recompute the CRC (CRC-CCITT variant) for the data 52 52 + * buffer 53 53 + * @crc: previous CRC value 54 54 + * @buffer: data pointer 55 55 + * @len: number of bytes in the buffer 56 56 + */ 57 57 + u16 crc_ccitt(u16 crc, u8 const *buffer, size_t len) 58 58 + { 59 59 + while (len--) 60 60 + crc = crc_ccitt_byte(crc, *buffer++); 61 61 + return crc; 62 62 + } 63 63 + EXPORT_SYMBOL(crc_ccitt); 64 64 + 65 65 + MODULE_DESCRIPTION("CRC-CCITT calculations"); 66 66 + MODULE_LICENSE("GPL");

+68

lib/crc/crc-itu-t.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0-only 2 2 + /* 3 3 + * crc-itu-t.c 4 4 + */ 5 5 + 6 6 + #include <linux/crc-itu-t.h> 7 7 + #include <linux/export.h> 8 8 + #include <linux/module.h> 9 9 + #include <linux/types.h> 10 10 + 11 11 + /* CRC table for the CRC ITU-T V.41 0x1021 (x^16 + x^12 + x^5 + 1) */ 12 12 + const u16 crc_itu_t_table[256] = { 13 13 + 0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7, 14 14 + 0x8108, 0x9129, 0xa14a, 0xb16b, 0xc18c, 0xd1ad, 0xe1ce, 0xf1ef, 15 15 + 0x1231, 0x0210, 0x3273, 0x2252, 0x52b5, 0x4294, 0x72f7, 0x62d6, 16 16 + 0x9339, 0x8318, 0xb37b, 0xa35a, 0xd3bd, 0xc39c, 0xf3ff, 0xe3de, 17 17 + 0x2462, 0x3443, 0x0420, 0x1401, 0x64e6, 0x74c7, 0x44a4, 0x5485, 18 18 + 0xa56a, 0xb54b, 0x8528, 0x9509, 0xe5ee, 0xf5cf, 0xc5ac, 0xd58d, 19 19 + 0x3653, 0x2672, 0x1611, 0x0630, 0x76d7, 0x66f6, 0x5695, 0x46b4, 20 20 + 0xb75b, 0xa77a, 0x9719, 0x8738, 0xf7df, 0xe7fe, 0xd79d, 0xc7bc, 21 21 + 0x48c4, 0x58e5, 0x6886, 0x78a7, 0x0840, 0x1861, 0x2802, 0x3823, 22 22 + 0xc9cc, 0xd9ed, 0xe98e, 0xf9af, 0x8948, 0x9969, 0xa90a, 0xb92b, 23 23 + 0x5af5, 0x4ad4, 0x7ab7, 0x6a96, 0x1a71, 0x0a50, 0x3a33, 0x2a12, 24 24 + 0xdbfd, 0xcbdc, 0xfbbf, 0xeb9e, 0x9b79, 0x8b58, 0xbb3b, 0xab1a, 25 25 + 0x6ca6, 0x7c87, 0x4ce4, 0x5cc5, 0x2c22, 0x3c03, 0x0c60, 0x1c41, 26 26 + 0xedae, 0xfd8f, 0xcdec, 0xddcd, 0xad2a, 0xbd0b, 0x8d68, 0x9d49, 27 27 + 0x7e97, 0x6eb6, 0x5ed5, 0x4ef4, 0x3e13, 0x2e32, 0x1e51, 0x0e70, 28 28 + 0xff9f, 0xefbe, 0xdfdd, 0xcffc, 0xbf1b, 0xaf3a, 0x9f59, 0x8f78, 29 29 + 0x9188, 0x81a9, 0xb1ca, 0xa1eb, 0xd10c, 0xc12d, 0xf14e, 0xe16f, 30 30 + 0x1080, 0x00a1, 0x30c2, 0x20e3, 0x5004, 0x4025, 0x7046, 0x6067, 31 31 + 0x83b9, 0x9398, 0xa3fb, 0xb3da, 0xc33d, 0xd31c, 0xe37f, 0xf35e, 32 32 + 0x02b1, 0x1290, 0x22f3, 0x32d2, 0x4235, 0x5214, 0x6277, 0x7256, 33 33 + 0xb5ea, 0xa5cb, 0x95a8, 0x8589, 0xf56e, 0xe54f, 0xd52c, 0xc50d, 34 34 + 0x34e2, 0x24c3, 0x14a0, 0x0481, 0x7466, 0x6447, 0x5424, 0x4405, 35 35 + 0xa7db, 0xb7fa, 0x8799, 0x97b8, 0xe75f, 0xf77e, 0xc71d, 0xd73c, 36 36 + 0x26d3, 0x36f2, 0x0691, 0x16b0, 0x6657, 0x7676, 0x4615, 0x5634, 37 37 + 0xd94c, 0xc96d, 0xf90e, 0xe92f, 0x99c8, 0x89e9, 0xb98a, 0xa9ab, 38 38 + 0x5844, 0x4865, 0x7806, 0x6827, 0x18c0, 0x08e1, 0x3882, 0x28a3, 39 39 + 0xcb7d, 0xdb5c, 0xeb3f, 0xfb1e, 0x8bf9, 0x9bd8, 0xabbb, 0xbb9a, 40 40 + 0x4a75, 0x5a54, 0x6a37, 0x7a16, 0x0af1, 0x1ad0, 0x2ab3, 0x3a92, 41 41 + 0xfd2e, 0xed0f, 0xdd6c, 0xcd4d, 0xbdaa, 0xad8b, 0x9de8, 0x8dc9, 42 42 + 0x7c26, 0x6c07, 0x5c64, 0x4c45, 0x3ca2, 0x2c83, 0x1ce0, 0x0cc1, 43 43 + 0xef1f, 0xff3e, 0xcf5d, 0xdf7c, 0xaf9b, 0xbfba, 0x8fd9, 0x9ff8, 44 44 + 0x6e17, 0x7e36, 0x4e55, 0x5e74, 0x2e93, 0x3eb2, 0x0ed1, 0x1ef0 45 45 + }; 46 46 + 47 47 + EXPORT_SYMBOL(crc_itu_t_table); 48 48 + 49 49 + /** 50 50 + * crc_itu_t - Compute the CRC-ITU-T for the data buffer 51 51 + * 52 52 + * @crc: previous CRC value 53 53 + * @buffer: data pointer 54 54 + * @len: number of bytes in the buffer 55 55 + * 56 56 + * Returns the updated CRC value 57 57 + */ 58 58 + u16 crc_itu_t(u16 crc, const u8 *buffer, size_t len) 59 59 + { 60 60 + while (len--) 61 61 + crc = crc_itu_t_byte(crc, *buffer++); 62 62 + return crc; 63 63 + } 64 64 + EXPORT_SYMBOL(crc_itu_t); 65 65 + 66 66 + MODULE_DESCRIPTION("CRC ITU-T V.41 calculations"); 67 67 + MODULE_LICENSE("GPL"); 68 68 +

+89

lib/crc/crc-t10dif-main.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0-only 2 2 + /* 3 3 + * T10 Data Integrity Field CRC16 calculation 4 4 + * 5 5 + * Copyright (c) 2007 Oracle Corporation. All rights reserved. 6 6 + * Written by Martin K. Petersen <martin.petersen@oracle.com> 7 7 + */ 8 8 + 9 9 + #include <linux/crc-t10dif.h> 10 10 + #include <linux/export.h> 11 11 + #include <linux/module.h> 12 12 + #include <linux/types.h> 13 13 + 14 14 + /* 15 15 + * Table generated using the following polynomial: 16 16 + * x^16 + x^15 + x^11 + x^9 + x^8 + x^7 + x^5 + x^4 + x^2 + x + 1 17 17 + * gt: 0x8bb7 18 18 + */ 19 19 + static const u16 t10_dif_crc_table[256] = { 20 20 + 0x0000, 0x8BB7, 0x9CD9, 0x176E, 0xB205, 0x39B2, 0x2EDC, 0xA56B, 21 21 + 0xEFBD, 0x640A, 0x7364, 0xF8D3, 0x5DB8, 0xD60F, 0xC161, 0x4AD6, 22 22 + 0x54CD, 0xDF7A, 0xC814, 0x43A3, 0xE6C8, 0x6D7F, 0x7A11, 0xF1A6, 23 23 + 0xBB70, 0x30C7, 0x27A9, 0xAC1E, 0x0975, 0x82C2, 0x95AC, 0x1E1B, 24 24 + 0xA99A, 0x222D, 0x3543, 0xBEF4, 0x1B9F, 0x9028, 0x8746, 0x0CF1, 25 25 + 0x4627, 0xCD90, 0xDAFE, 0x5149, 0xF422, 0x7F95, 0x68FB, 0xE34C, 26 26 + 0xFD57, 0x76E0, 0x618E, 0xEA39, 0x4F52, 0xC4E5, 0xD38B, 0x583C, 27 27 + 0x12EA, 0x995D, 0x8E33, 0x0584, 0xA0EF, 0x2B58, 0x3C36, 0xB781, 28 28 + 0xD883, 0x5334, 0x445A, 0xCFED, 0x6A86, 0xE131, 0xF65F, 0x7DE8, 29 29 + 0x373E, 0xBC89, 0xABE7, 0x2050, 0x853B, 0x0E8C, 0x19E2, 0x9255, 30 30 + 0x8C4E, 0x07F9, 0x1097, 0x9B20, 0x3E4B, 0xB5FC, 0xA292, 0x2925, 31 31 + 0x63F3, 0xE844, 0xFF2A, 0x749D, 0xD1F6, 0x5A41, 0x4D2F, 0xC698, 32 32 + 0x7119, 0xFAAE, 0xEDC0, 0x6677, 0xC31C, 0x48AB, 0x5FC5, 0xD472, 33 33 + 0x9EA4, 0x1513, 0x027D, 0x89CA, 0x2CA1, 0xA716, 0xB078, 0x3BCF, 34 34 + 0x25D4, 0xAE63, 0xB90D, 0x32BA, 0x97D1, 0x1C66, 0x0B08, 0x80BF, 35 35 + 0xCA69, 0x41DE, 0x56B0, 0xDD07, 0x786C, 0xF3DB, 0xE4B5, 0x6F02, 36 36 + 0x3AB1, 0xB106, 0xA668, 0x2DDF, 0x88B4, 0x0303, 0x146D, 0x9FDA, 37 37 + 0xD50C, 0x5EBB, 0x49D5, 0xC262, 0x6709, 0xECBE, 0xFBD0, 0x7067, 38 38 + 0x6E7C, 0xE5CB, 0xF2A5, 0x7912, 0xDC79, 0x57CE, 0x40A0, 0xCB17, 39 39 + 0x81C1, 0x0A76, 0x1D18, 0x96AF, 0x33C4, 0xB873, 0xAF1D, 0x24AA, 40 40 + 0x932B, 0x189C, 0x0FF2, 0x8445, 0x212E, 0xAA99, 0xBDF7, 0x3640, 41 41 + 0x7C96, 0xF721, 0xE04F, 0x6BF8, 0xCE93, 0x4524, 0x524A, 0xD9FD, 42 42 + 0xC7E6, 0x4C51, 0x5B3F, 0xD088, 0x75E3, 0xFE54, 0xE93A, 0x628D, 43 43 + 0x285B, 0xA3EC, 0xB482, 0x3F35, 0x9A5E, 0x11E9, 0x0687, 0x8D30, 44 44 + 0xE232, 0x6985, 0x7EEB, 0xF55C, 0x5037, 0xDB80, 0xCCEE, 0x4759, 45 45 + 0x0D8F, 0x8638, 0x9156, 0x1AE1, 0xBF8A, 0x343D, 0x2353, 0xA8E4, 46 46 + 0xB6FF, 0x3D48, 0x2A26, 0xA191, 0x04FA, 0x8F4D, 0x9823, 0x1394, 47 47 + 0x5942, 0xD2F5, 0xC59B, 0x4E2C, 0xEB47, 0x60F0, 0x779E, 0xFC29, 48 48 + 0x4BA8, 0xC01F, 0xD771, 0x5CC6, 0xF9AD, 0x721A, 0x6574, 0xEEC3, 49 49 + 0xA415, 0x2FA2, 0x38CC, 0xB37B, 0x1610, 0x9DA7, 0x8AC9, 0x017E, 50 50 + 0x1F65, 0x94D2, 0x83BC, 0x080B, 0xAD60, 0x26D7, 0x31B9, 0xBA0E, 51 51 + 0xF0D8, 0x7B6F, 0x6C01, 0xE7B6, 0x42DD, 0xC96A, 0xDE04, 0x55B3 52 52 + }; 53 53 + 54 54 + static inline u16 __maybe_unused 55 55 + crc_t10dif_generic(u16 crc, const u8 *p, size_t len) 56 56 + { 57 57 + while (len--) 58 58 + crc = (crc << 8) ^ t10_dif_crc_table[(crc >> 8) ^ *p++]; 59 59 + return crc; 60 60 + } 61 61 + 62 62 + #ifdef CONFIG_CRC_T10DIF_ARCH 63 63 + #include "crc-t10dif.h" /* $(SRCARCH)/crc-t10dif.h */ 64 64 + #else 65 65 + #define crc_t10dif_arch crc_t10dif_generic 66 66 + #endif 67 67 + 68 68 + u16 crc_t10dif_update(u16 crc, const u8 *p, size_t len) 69 69 + { 70 70 + return crc_t10dif_arch(crc, p, len); 71 71 + } 72 72 + EXPORT_SYMBOL(crc_t10dif_update); 73 73 + 74 74 + #ifdef crc_t10dif_mod_init_arch 75 75 + static int __init crc_t10dif_mod_init(void) 76 76 + { 77 77 + crc_t10dif_mod_init_arch(); 78 78 + return 0; 79 79 + } 80 80 + subsys_initcall(crc_t10dif_mod_init); 81 81 + 82 82 + static void __exit crc_t10dif_mod_exit(void) 83 83 + { 84 84 + } 85 85 + module_exit(crc_t10dif_mod_exit); 86 86 + #endif 87 87 + 88 88 + MODULE_DESCRIPTION("CRC-T10DIF library functions"); 89 89 + MODULE_LICENSE("GPL");

+65

lib/crc/crc16.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0-only 2 2 + /* 3 3 + * crc16.c 4 4 + */ 5 5 + 6 6 + #include <linux/crc16.h> 7 7 + #include <linux/export.h> 8 8 + #include <linux/module.h> 9 9 + #include <linux/types.h> 10 10 + 11 11 + /** CRC table for the CRC-16. The poly is 0x8005 (x^16 + x^15 + x^2 + 1) */ 12 12 + static const u16 crc16_table[256] = { 13 13 + 0x0000, 0xC0C1, 0xC181, 0x0140, 0xC301, 0x03C0, 0x0280, 0xC241, 14 14 + 0xC601, 0x06C0, 0x0780, 0xC741, 0x0500, 0xC5C1, 0xC481, 0x0440, 15 15 + 0xCC01, 0x0CC0, 0x0D80, 0xCD41, 0x0F00, 0xCFC1, 0xCE81, 0x0E40, 16 16 + 0x0A00, 0xCAC1, 0xCB81, 0x0B40, 0xC901, 0x09C0, 0x0880, 0xC841, 17 17 + 0xD801, 0x18C0, 0x1980, 0xD941, 0x1B00, 0xDBC1, 0xDA81, 0x1A40, 18 18 + 0x1E00, 0xDEC1, 0xDF81, 0x1F40, 0xDD01, 0x1DC0, 0x1C80, 0xDC41, 19 19 + 0x1400, 0xD4C1, 0xD581, 0x1540, 0xD701, 0x17C0, 0x1680, 0xD641, 20 20 + 0xD201, 0x12C0, 0x1380, 0xD341, 0x1100, 0xD1C1, 0xD081, 0x1040, 21 21 + 0xF001, 0x30C0, 0x3180, 0xF141, 0x3300, 0xF3C1, 0xF281, 0x3240, 22 22 + 0x3600, 0xF6C1, 0xF781, 0x3740, 0xF501, 0x35C0, 0x3480, 0xF441, 23 23 + 0x3C00, 0xFCC1, 0xFD81, 0x3D40, 0xFF01, 0x3FC0, 0x3E80, 0xFE41, 24 24 + 0xFA01, 0x3AC0, 0x3B80, 0xFB41, 0x3900, 0xF9C1, 0xF881, 0x3840, 25 25 + 0x2800, 0xE8C1, 0xE981, 0x2940, 0xEB01, 0x2BC0, 0x2A80, 0xEA41, 26 26 + 0xEE01, 0x2EC0, 0x2F80, 0xEF41, 0x2D00, 0xEDC1, 0xEC81, 0x2C40, 27 27 + 0xE401, 0x24C0, 0x2580, 0xE541, 0x2700, 0xE7C1, 0xE681, 0x2640, 28 28 + 0x2200, 0xE2C1, 0xE381, 0x2340, 0xE101, 0x21C0, 0x2080, 0xE041, 29 29 + 0xA001, 0x60C0, 0x6180, 0xA141, 0x6300, 0xA3C1, 0xA281, 0x6240, 30 30 + 0x6600, 0xA6C1, 0xA781, 0x6740, 0xA501, 0x65C0, 0x6480, 0xA441, 31 31 + 0x6C00, 0xACC1, 0xAD81, 0x6D40, 0xAF01, 0x6FC0, 0x6E80, 0xAE41, 32 32 + 0xAA01, 0x6AC0, 0x6B80, 0xAB41, 0x6900, 0xA9C1, 0xA881, 0x6840, 33 33 + 0x7800, 0xB8C1, 0xB981, 0x7940, 0xBB01, 0x7BC0, 0x7A80, 0xBA41, 34 34 + 0xBE01, 0x7EC0, 0x7F80, 0xBF41, 0x7D00, 0xBDC1, 0xBC81, 0x7C40, 35 35 + 0xB401, 0x74C0, 0x7580, 0xB541, 0x7700, 0xB7C1, 0xB681, 0x7640, 36 36 + 0x7200, 0xB2C1, 0xB381, 0x7340, 0xB101, 0x71C0, 0x7080, 0xB041, 37 37 + 0x5000, 0x90C1, 0x9181, 0x5140, 0x9301, 0x53C0, 0x5280, 0x9241, 38 38 + 0x9601, 0x56C0, 0x5780, 0x9741, 0x5500, 0x95C1, 0x9481, 0x5440, 39 39 + 0x9C01, 0x5CC0, 0x5D80, 0x9D41, 0x5F00, 0x9FC1, 0x9E81, 0x5E40, 40 40 + 0x5A00, 0x9AC1, 0x9B81, 0x5B40, 0x9901, 0x59C0, 0x5880, 0x9841, 41 41 + 0x8801, 0x48C0, 0x4980, 0x8941, 0x4B00, 0x8BC1, 0x8A81, 0x4A40, 42 42 + 0x4E00, 0x8EC1, 0x8F81, 0x4F40, 0x8D01, 0x4DC0, 0x4C80, 0x8C41, 43 43 + 0x4400, 0x84C1, 0x8581, 0x4540, 0x8701, 0x47C0, 0x4680, 0x8641, 44 44 + 0x8201, 0x42C0, 0x4380, 0x8341, 0x4100, 0x81C1, 0x8081, 0x4040 45 45 + }; 46 46 + 47 47 + /** 48 48 + * crc16 - compute the CRC-16 for the data buffer 49 49 + * @crc: previous CRC value 50 50 + * @p: data pointer 51 51 + * @len: number of bytes in the buffer 52 52 + * 53 53 + * Returns the updated CRC value. 54 54 + */ 55 55 + u16 crc16(u16 crc, const u8 *p, size_t len) 56 56 + { 57 57 + while (len--) 58 58 + crc = (crc >> 8) ^ crc16_table[(crc & 0xff) ^ *p++]; 59 59 + return crc; 60 60 + } 61 61 + EXPORT_SYMBOL(crc16); 62 62 + 63 63 + MODULE_DESCRIPTION("CRC16 calculations"); 64 64 + MODULE_LICENSE("GPL"); 65 65 +

+105

lib/crc/crc32-main.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0-only 2 2 + /* 3 3 + * Aug 8, 2011 Bob Pearson with help from Joakim Tjernlund and George Spelvin 4 4 + * cleaned up code to current version of sparse and added the slicing-by-8 5 5 + * algorithm to the closely similar existing slicing-by-4 algorithm. 6 6 + * 7 7 + * Oct 15, 2000 Matt Domsch <Matt_Domsch@dell.com> 8 8 + * Nicer crc32 functions/docs submitted by linux@horizon.com. Thanks! 9 9 + * Code was from the public domain, copyright abandoned. Code was 10 10 + * subsequently included in the kernel, thus was re-licensed under the 11 11 + * GNU GPL v2. 12 12 + * 13 13 + * Oct 12, 2000 Matt Domsch <Matt_Domsch@dell.com> 14 14 + * Same crc32 function was used in 5 other places in the kernel. 15 15 + * I made one version, and deleted the others. 16 16 + * There are various incantations of crc32(). Some use a seed of 0 or ~0. 17 17 + * Some xor at the end with ~0. The generic crc32() function takes 18 18 + * seed as an argument, and doesn't xor at the end. Then individual 19 19 + * users can do whatever they need. 20 20 + * drivers/net/smc9194.c uses seed ~0, doesn't xor with ~0. 21 21 + * fs/jffs2 uses seed 0, doesn't xor with ~0. 22 22 + * fs/partitions/efi.c uses seed ~0, xor's with ~0. 23 23 + */ 24 24 + 25 25 + /* see: Documentation/staging/crc32.rst for a description of algorithms */ 26 26 + 27 27 + #include <linux/crc32.h> 28 28 + #include <linux/export.h> 29 29 + #include <linux/module.h> 30 30 + #include <linux/types.h> 31 31 + 32 32 + #include "crc32table.h" 33 33 + 34 34 + static inline u32 __maybe_unused 35 35 + crc32_le_base(u32 crc, const u8 *p, size_t len) 36 36 + { 37 37 + while (len--) 38 38 + crc = (crc >> 8) ^ crc32table_le[(crc & 255) ^ *p++]; 39 39 + return crc; 40 40 + } 41 41 + 42 42 + static inline u32 __maybe_unused 43 43 + crc32_be_base(u32 crc, const u8 *p, size_t len) 44 44 + { 45 45 + while (len--) 46 46 + crc = (crc << 8) ^ crc32table_be[(crc >> 24) ^ *p++]; 47 47 + return crc; 48 48 + } 49 49 + 50 50 + static inline u32 __maybe_unused 51 51 + crc32c_base(u32 crc, const u8 *p, size_t len) 52 52 + { 53 53 + while (len--) 54 54 + crc = (crc >> 8) ^ crc32ctable_le[(crc & 255) ^ *p++]; 55 55 + return crc; 56 56 + } 57 57 + 58 58 + #ifdef CONFIG_CRC32_ARCH 59 59 + #include "crc32.h" /* $(SRCARCH)/crc32.h */ 60 60 + 61 61 + u32 crc32_optimizations(void) 62 62 + { 63 63 + return crc32_optimizations_arch(); 64 64 + } 65 65 + EXPORT_SYMBOL(crc32_optimizations); 66 66 + #else 67 67 + #define crc32_le_arch crc32_le_base 68 68 + #define crc32_be_arch crc32_be_base 69 69 + #define crc32c_arch crc32c_base 70 70 + #endif 71 71 + 72 72 + u32 crc32_le(u32 crc, const void *p, size_t len) 73 73 + { 74 74 + return crc32_le_arch(crc, p, len); 75 75 + } 76 76 + EXPORT_SYMBOL(crc32_le); 77 77 + 78 78 + u32 crc32_be(u32 crc, const void *p, size_t len) 79 79 + { 80 80 + return crc32_be_arch(crc, p, len); 81 81 + } 82 82 + EXPORT_SYMBOL(crc32_be); 83 83 + 84 84 + u32 crc32c(u32 crc, const void *p, size_t len) 85 85 + { 86 86 + return crc32c_arch(crc, p, len); 87 87 + } 88 88 + EXPORT_SYMBOL(crc32c); 89 89 + 90 90 + #ifdef crc32_mod_init_arch 91 91 + static int __init crc32_mod_init(void) 92 92 + { 93 93 + crc32_mod_init_arch(); 94 94 + return 0; 95 95 + } 96 96 + subsys_initcall(crc32_mod_init); 97 97 + 98 98 + static void __exit crc32_mod_exit(void) 99 99 + { 100 100 + } 101 101 + module_exit(crc32_mod_exit); 102 102 + #endif 103 103 + 104 104 + MODULE_DESCRIPTION("CRC32 library functions"); 105 105 + MODULE_LICENSE("GPL");

+45

lib/crc/crc4.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0-only 2 2 + /* 3 3 + * crc4.c - simple crc-4 calculations. 4 4 + */ 5 5 + 6 6 + #include <linux/crc4.h> 7 7 + #include <linux/export.h> 8 8 + #include <linux/module.h> 9 9 + 10 10 + static const uint8_t crc4_tab[] = { 11 11 + 0x0, 0x7, 0xe, 0x9, 0xb, 0xc, 0x5, 0x2, 12 12 + 0x1, 0x6, 0xf, 0x8, 0xa, 0xd, 0x4, 0x3, 13 13 + }; 14 14 + 15 15 + /** 16 16 + * crc4 - calculate the 4-bit crc of a value. 17 17 + * @c: starting crc4 18 18 + * @x: value to checksum 19 19 + * @bits: number of bits in @x to checksum 20 20 + * 21 21 + * Returns the crc4 value of @x, using polynomial 0b10111. 22 22 + * 23 23 + * The @x value is treated as left-aligned, and bits above @bits are ignored 24 24 + * in the crc calculations. 25 25 + */ 26 26 + uint8_t crc4(uint8_t c, uint64_t x, int bits) 27 27 + { 28 28 + int i; 29 29 + 30 30 + /* mask off anything above the top bit */ 31 31 + x &= (1ull << bits) - 1; 32 32 + 33 33 + /* Align to 4-bits */ 34 34 + bits = (bits + 3) & ~0x3; 35 35 + 36 36 + /* Calculate crc4 over four-bit nibbles, starting at the MSbit */ 37 37 + for (i = bits - 4; i >= 0; i -= 4) 38 38 + c = crc4_tab[c ^ ((x >> i) & 0xf)]; 39 39 + 40 40 + return c; 41 41 + } 42 42 + EXPORT_SYMBOL_GPL(crc4); 43 43 + 44 44 + MODULE_DESCRIPTION("CRC4 calculations"); 45 45 + MODULE_LICENSE("GPL");

+93

lib/crc/crc64-main.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + /* 3 3 + * Normal 64-bit CRC calculation. 4 4 + * 5 5 + * This is a basic crc64 implementation following ECMA-182 specification, 6 6 + * which can be found from, 7 7 + * https://www.ecma-international.org/publications/standards/Ecma-182.htm 8 8 + * 9 9 + * Dr. Ross N. Williams has a great document to introduce the idea of CRC 10 10 + * algorithm, here the CRC64 code is also inspired by the table-driven 11 11 + * algorithm and detail example from this paper. This paper can be found 12 12 + * from, 13 13 + * http://www.ross.net/crc/download/crc_v3.txt 14 14 + * 15 15 + * crc64table[256] is the lookup table of a table-driven 64-bit CRC 16 16 + * calculation, which is generated by gen_crc64table.c in kernel build 17 17 + * time. The polynomial of crc64 arithmetic is from ECMA-182 specification 18 18 + * as well, which is defined as, 19 19 + * 20 20 + * x^64 + x^62 + x^57 + x^55 + x^54 + x^53 + x^52 + x^47 + x^46 + x^45 + 21 21 + * x^40 + x^39 + x^38 + x^37 + x^35 + x^33 + x^32 + x^31 + x^29 + x^27 + 22 22 + * x^24 + x^23 + x^22 + x^21 + x^19 + x^17 + x^13 + x^12 + x^10 + x^9 + 23 23 + * x^7 + x^4 + x + 1 24 24 + * 25 25 + * crc64nvmetable[256] uses the CRC64 polynomial from the NVME NVM Command Set 26 26 + * Specification and uses least-significant-bit first bit order: 27 27 + * 28 28 + * x^64 + x^63 + x^61 + x^59 + x^58 + x^56 + x^55 + x^52 + x^49 + x^48 + x^47 + 29 29 + * x^46 + x^44 + x^41 + x^37 + x^36 + x^34 + x^32 + x^31 + x^28 + x^26 + x^23 + 30 30 + * x^22 + x^19 + x^16 + x^13 + x^12 + x^10 + x^9 + x^6 + x^4 + x^3 + 1 31 31 + * 32 32 + * Copyright 2018 SUSE Linux. 33 33 + * Author: Coly Li <colyli@suse.de> 34 34 + */ 35 35 + 36 36 + #include <linux/crc64.h> 37 37 + #include <linux/export.h> 38 38 + #include <linux/module.h> 39 39 + #include <linux/types.h> 40 40 + 41 41 + #include "crc64table.h" 42 42 + 43 43 + static inline u64 __maybe_unused 44 44 + crc64_be_generic(u64 crc, const u8 *p, size_t len) 45 45 + { 46 46 + while (len--) 47 47 + crc = (crc << 8) ^ crc64table[(crc >> 56) ^ *p++]; 48 48 + return crc; 49 49 + } 50 50 + 51 51 + static inline u64 __maybe_unused 52 52 + crc64_nvme_generic(u64 crc, const u8 *p, size_t len) 53 53 + { 54 54 + while (len--) 55 55 + crc = (crc >> 8) ^ crc64nvmetable[(crc & 0xff) ^ *p++]; 56 56 + return crc; 57 57 + } 58 58 + 59 59 + #ifdef CONFIG_CRC64_ARCH 60 60 + #include "crc64.h" /* $(SRCARCH)/crc64.h */ 61 61 + #else 62 62 + #define crc64_be_arch crc64_be_generic 63 63 + #define crc64_nvme_arch crc64_nvme_generic 64 64 + #endif 65 65 + 66 66 + u64 crc64_be(u64 crc, const void *p, size_t len) 67 67 + { 68 68 + return crc64_be_arch(crc, p, len); 69 69 + } 70 70 + EXPORT_SYMBOL_GPL(crc64_be); 71 71 + 72 72 + u64 crc64_nvme(u64 crc, const void *p, size_t len) 73 73 + { 74 74 + return ~crc64_nvme_arch(~crc, p, len); 75 75 + } 76 76 + EXPORT_SYMBOL_GPL(crc64_nvme); 77 77 + 78 78 + #ifdef crc64_mod_init_arch 79 79 + static int __init crc64_mod_init(void) 80 80 + { 81 81 + crc64_mod_init_arch(); 82 82 + return 0; 83 83 + } 84 84 + subsys_initcall(crc64_mod_init); 85 85 + 86 86 + static void __exit crc64_mod_exit(void) 87 87 + { 88 88 + } 89 89 + module_exit(crc64_mod_exit); 90 90 + #endif 91 91 + 92 92 + MODULE_DESCRIPTION("CRC64 library functions"); 93 93 + MODULE_LICENSE("GPL");

+73

lib/crc/crc7.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0-only 2 2 + /* 3 3 + * crc7.c 4 4 + */ 5 5 + 6 6 + #include <linux/crc7.h> 7 7 + #include <linux/export.h> 8 8 + #include <linux/module.h> 9 9 + #include <linux/types.h> 10 10 + 11 11 + /* 12 12 + * Table for CRC-7 (polynomial x^7 + x^3 + 1). 13 13 + * This is a big-endian CRC (msbit is highest power of x), 14 14 + * aligned so the msbit of the byte is the x^6 coefficient 15 15 + * and the lsbit is not used. 16 16 + */ 17 17 + static const u8 crc7_be_syndrome_table[256] = { 18 18 + 0x00, 0x12, 0x24, 0x36, 0x48, 0x5a, 0x6c, 0x7e, 19 19 + 0x90, 0x82, 0xb4, 0xa6, 0xd8, 0xca, 0xfc, 0xee, 20 20 + 0x32, 0x20, 0x16, 0x04, 0x7a, 0x68, 0x5e, 0x4c, 21 21 + 0xa2, 0xb0, 0x86, 0x94, 0xea, 0xf8, 0xce, 0xdc, 22 22 + 0x64, 0x76, 0x40, 0x52, 0x2c, 0x3e, 0x08, 0x1a, 23 23 + 0xf4, 0xe6, 0xd0, 0xc2, 0xbc, 0xae, 0x98, 0x8a, 24 24 + 0x56, 0x44, 0x72, 0x60, 0x1e, 0x0c, 0x3a, 0x28, 25 25 + 0xc6, 0xd4, 0xe2, 0xf0, 0x8e, 0x9c, 0xaa, 0xb8, 26 26 + 0xc8, 0xda, 0xec, 0xfe, 0x80, 0x92, 0xa4, 0xb6, 27 27 + 0x58, 0x4a, 0x7c, 0x6e, 0x10, 0x02, 0x34, 0x26, 28 28 + 0xfa, 0xe8, 0xde, 0xcc, 0xb2, 0xa0, 0x96, 0x84, 29 29 + 0x6a, 0x78, 0x4e, 0x5c, 0x22, 0x30, 0x06, 0x14, 30 30 + 0xac, 0xbe, 0x88, 0x9a, 0xe4, 0xf6, 0xc0, 0xd2, 31 31 + 0x3c, 0x2e, 0x18, 0x0a, 0x74, 0x66, 0x50, 0x42, 32 32 + 0x9e, 0x8c, 0xba, 0xa8, 0xd6, 0xc4, 0xf2, 0xe0, 33 33 + 0x0e, 0x1c, 0x2a, 0x38, 0x46, 0x54, 0x62, 0x70, 34 34 + 0x82, 0x90, 0xa6, 0xb4, 0xca, 0xd8, 0xee, 0xfc, 35 35 + 0x12, 0x00, 0x36, 0x24, 0x5a, 0x48, 0x7e, 0x6c, 36 36 + 0xb0, 0xa2, 0x94, 0x86, 0xf8, 0xea, 0xdc, 0xce, 37 37 + 0x20, 0x32, 0x04, 0x16, 0x68, 0x7a, 0x4c, 0x5e, 38 38 + 0xe6, 0xf4, 0xc2, 0xd0, 0xae, 0xbc, 0x8a, 0x98, 39 39 + 0x76, 0x64, 0x52, 0x40, 0x3e, 0x2c, 0x1a, 0x08, 40 40 + 0xd4, 0xc6, 0xf0, 0xe2, 0x9c, 0x8e, 0xb8, 0xaa, 41 41 + 0x44, 0x56, 0x60, 0x72, 0x0c, 0x1e, 0x28, 0x3a, 42 42 + 0x4a, 0x58, 0x6e, 0x7c, 0x02, 0x10, 0x26, 0x34, 43 43 + 0xda, 0xc8, 0xfe, 0xec, 0x92, 0x80, 0xb6, 0xa4, 44 44 + 0x78, 0x6a, 0x5c, 0x4e, 0x30, 0x22, 0x14, 0x06, 45 45 + 0xe8, 0xfa, 0xcc, 0xde, 0xa0, 0xb2, 0x84, 0x96, 46 46 + 0x2e, 0x3c, 0x0a, 0x18, 0x66, 0x74, 0x42, 0x50, 47 47 + 0xbe, 0xac, 0x9a, 0x88, 0xf6, 0xe4, 0xd2, 0xc0, 48 48 + 0x1c, 0x0e, 0x38, 0x2a, 0x54, 0x46, 0x70, 0x62, 49 49 + 0x8c, 0x9e, 0xa8, 0xba, 0xc4, 0xd6, 0xe0, 0xf2 50 50 + }; 51 51 + 52 52 + /** 53 53 + * crc7_be - update the CRC7 for the data buffer 54 54 + * @crc: previous CRC7 value 55 55 + * @buffer: data pointer 56 56 + * @len: number of bytes in the buffer 57 57 + * Context: any 58 58 + * 59 59 + * Returns the updated CRC7 value. 60 60 + * The CRC7 is left-aligned in the byte (the lsbit is always 0), as that 61 61 + * makes the computation easier, and all callers want it in that form. 62 62 + * 63 63 + */ 64 64 + u8 crc7_be(u8 crc, const u8 *buffer, size_t len) 65 65 + { 66 66 + while (len--) 67 67 + crc = crc7_be_syndrome_table[crc ^ *buffer++]; 68 68 + return crc; 69 69 + } 70 70 + EXPORT_SYMBOL(crc7_be); 71 71 + 72 72 + MODULE_DESCRIPTION("CRC7 calculations"); 73 73 + MODULE_LICENSE("GPL");

+87

lib/crc/crc8.c

reviewed

··· 1 1 + /* 2 2 + * Copyright (c) 2011 Broadcom Corporation 3 3 + * 4 4 + * Permission to use, copy, modify, and/or distribute this software for any 5 5 + * purpose with or without fee is hereby granted, provided that the above 6 6 + * copyright notice and this permission notice appear in all copies. 7 7 + * 8 8 + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 9 9 + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 10 10 + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 11 11 + * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 12 12 + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION 13 13 + * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN 14 14 + * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 15 15 + */ 16 16 + 17 17 + #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 18 18 + 19 19 + #include <linux/crc8.h> 20 20 + #include <linux/export.h> 21 21 + #include <linux/module.h> 22 22 + #include <linux/printk.h> 23 23 + 24 24 + /** 25 25 + * crc8_populate_msb - fill crc table for given polynomial in reverse bit order. 26 26 + * 27 27 + * @table: table to be filled. 28 28 + * @polynomial: polynomial for which table is to be filled. 29 29 + */ 30 30 + void crc8_populate_msb(u8 table[CRC8_TABLE_SIZE], u8 polynomial) 31 31 + { 32 32 + int i, j; 33 33 + const u8 msbit = 0x80; 34 34 + u8 t = msbit; 35 35 + 36 36 + table[0] = 0; 37 37 + 38 38 + for (i = 1; i < CRC8_TABLE_SIZE; i *= 2) { 39 39 + t = (t << 1) ^ (t & msbit ? polynomial : 0); 40 40 + for (j = 0; j < i; j++) 41 41 + table[i+j] = table[j] ^ t; 42 42 + } 43 43 + } 44 44 + EXPORT_SYMBOL(crc8_populate_msb); 45 45 + 46 46 + /** 47 47 + * crc8_populate_lsb - fill crc table for given polynomial in regular bit order. 48 48 + * 49 49 + * @table: table to be filled. 50 50 + * @polynomial: polynomial for which table is to be filled. 51 51 + */ 52 52 + void crc8_populate_lsb(u8 table[CRC8_TABLE_SIZE], u8 polynomial) 53 53 + { 54 54 + int i, j; 55 55 + u8 t = 1; 56 56 + 57 57 + table[0] = 0; 58 58 + 59 59 + for (i = (CRC8_TABLE_SIZE >> 1); i; i >>= 1) { 60 60 + t = (t >> 1) ^ (t & 1 ? polynomial : 0); 61 61 + for (j = 0; j < CRC8_TABLE_SIZE; j += 2*i) 62 62 + table[i+j] = table[j] ^ t; 63 63 + } 64 64 + } 65 65 + EXPORT_SYMBOL(crc8_populate_lsb); 66 66 + 67 67 + /** 68 68 + * crc8 - calculate a crc8 over the given input data. 69 69 + * 70 70 + * @table: crc table used for calculation. 71 71 + * @pdata: pointer to data buffer. 72 72 + * @nbytes: number of bytes in data buffer. 73 73 + * @crc: previous returned crc8 value. 74 74 + */ 75 75 + u8 crc8(const u8 table[CRC8_TABLE_SIZE], const u8 *pdata, size_t nbytes, u8 crc) 76 76 + { 77 77 + /* loop over the buffer data */ 78 78 + while (nbytes-- > 0) 79 79 + crc = table[(crc ^ *pdata++) & 0xff]; 80 80 + 81 81 + return crc; 82 82 + } 83 83 + EXPORT_SYMBOL(crc8); 84 84 + 85 85 + MODULE_DESCRIPTION("CRC8 (by Williams, Ross N.) function"); 86 86 + MODULE_AUTHOR("Broadcom Corporation"); 87 87 + MODULE_LICENSE("Dual BSD/GPL");

+89

lib/crc/gen_crc32table.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + #include <stdio.h> 3 3 + #include "../../include/linux/crc32poly.h" 4 4 + #include "../../include/generated/autoconf.h" 5 5 + #include <inttypes.h> 6 6 + 7 7 + static uint32_t crc32table_le[256]; 8 8 + static uint32_t crc32table_be[256]; 9 9 + static uint32_t crc32ctable_le[256]; 10 10 + 11 11 + /** 12 12 + * crc32init_le() - allocate and initialize LE table data 13 13 + * 14 14 + * crc is the crc of the byte i; other entries are filled in based on the 15 15 + * fact that crctable[i^j] = crctable[i] ^ crctable[j]. 16 16 + * 17 17 + */ 18 18 + static void crc32init_le_generic(const uint32_t polynomial, uint32_t tab[256]) 19 19 + { 20 20 + unsigned i, j; 21 21 + uint32_t crc = 1; 22 22 + 23 23 + tab[0] = 0; 24 24 + 25 25 + for (i = 128; i; i >>= 1) { 26 26 + crc = (crc >> 1) ^ ((crc & 1) ? polynomial : 0); 27 27 + for (j = 0; j < 256; j += 2 * i) 28 28 + tab[i + j] = crc ^ tab[j]; 29 29 + } 30 30 + } 31 31 + 32 32 + static void crc32init_le(void) 33 33 + { 34 34 + crc32init_le_generic(CRC32_POLY_LE, crc32table_le); 35 35 + } 36 36 + 37 37 + static void crc32cinit_le(void) 38 38 + { 39 39 + crc32init_le_generic(CRC32C_POLY_LE, crc32ctable_le); 40 40 + } 41 41 + 42 42 + /** 43 43 + * crc32init_be() - allocate and initialize BE table data 44 44 + */ 45 45 + static void crc32init_be(void) 46 46 + { 47 47 + unsigned i, j; 48 48 + uint32_t crc = 0x80000000; 49 49 + 50 50 + crc32table_be[0] = 0; 51 51 + 52 52 + for (i = 1; i < 256; i <<= 1) { 53 53 + crc = (crc << 1) ^ ((crc & 0x80000000) ? CRC32_POLY_BE : 0); 54 54 + for (j = 0; j < i; j++) 55 55 + crc32table_be[i + j] = crc ^ crc32table_be[j]; 56 56 + } 57 57 + } 58 58 + 59 59 + static void output_table(const uint32_t table[256]) 60 60 + { 61 61 + int i; 62 62 + 63 63 + for (i = 0; i < 256; i += 4) { 64 64 + printf("\t0x%08x, 0x%08x, 0x%08x, 0x%08x,\n", 65 65 + table[i], table[i + 1], table[i + 2], table[i + 3]); 66 66 + } 67 67 + } 68 68 + 69 69 + int main(int argc, char** argv) 70 70 + { 71 71 + printf("/* this file is generated - do not edit */\n\n"); 72 72 + 73 73 + crc32init_le(); 74 74 + printf("static const u32 ____cacheline_aligned crc32table_le[256] = {\n"); 75 75 + output_table(crc32table_le); 76 76 + printf("};\n"); 77 77 + 78 78 + crc32init_be(); 79 79 + printf("static const u32 ____cacheline_aligned crc32table_be[256] = {\n"); 80 80 + output_table(crc32table_be); 81 81 + printf("};\n"); 82 82 + 83 83 + crc32cinit_le(); 84 84 + printf("static const u32 ____cacheline_aligned crc32ctable_le[256] = {\n"); 85 85 + output_table(crc32ctable_le); 86 86 + printf("};\n"); 87 87 + 88 88 + return 0; 89 89 + }

+88

lib/crc/gen_crc64table.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + /* 3 3 + * This host program runs at kernel build time and generates the lookup tables 4 4 + * used by the generic CRC64 code. 5 5 + * 6 6 + * Copyright 2018 SUSE Linux. 7 7 + * Author: Coly Li <colyli@suse.de> 8 8 + */ 9 9 + #include <inttypes.h> 10 10 + #include <stdio.h> 11 11 + 12 12 + #define CRC64_ECMA182_POLY 0x42F0E1EBA9EA3693ULL 13 13 + #define CRC64_NVME_POLY 0x9A6C9329AC4BC9B5ULL 14 14 + 15 15 + static uint64_t crc64_table[256] = {0}; 16 16 + static uint64_t crc64_nvme_table[256] = {0}; 17 17 + 18 18 + static void generate_reflected_crc64_table(uint64_t table[256], uint64_t poly) 19 19 + { 20 20 + uint64_t i, j, c, crc; 21 21 + 22 22 + for (i = 0; i < 256; i++) { 23 23 + crc = 0ULL; 24 24 + c = i; 25 25 + 26 26 + for (j = 0; j < 8; j++) { 27 27 + if ((crc ^ (c >> j)) & 1) 28 28 + crc = (crc >> 1) ^ poly; 29 29 + else 30 30 + crc >>= 1; 31 31 + } 32 32 + table[i] = crc; 33 33 + } 34 34 + } 35 35 + 36 36 + static void generate_crc64_table(uint64_t table[256], uint64_t poly) 37 37 + { 38 38 + uint64_t i, j, c, crc; 39 39 + 40 40 + for (i = 0; i < 256; i++) { 41 41 + crc = 0; 42 42 + c = i << 56; 43 43 + 44 44 + for (j = 0; j < 8; j++) { 45 45 + if ((crc ^ c) & 0x8000000000000000ULL) 46 46 + crc = (crc << 1) ^ poly; 47 47 + else 48 48 + crc <<= 1; 49 49 + c <<= 1; 50 50 + } 51 51 + 52 52 + table[i] = crc; 53 53 + } 54 54 + } 55 55 + 56 56 + static void output_table(uint64_t table[256]) 57 57 + { 58 58 + int i; 59 59 + 60 60 + for (i = 0; i < 256; i++) { 61 61 + printf("\t0x%016" PRIx64 "ULL", table[i]); 62 62 + if (i & 0x1) 63 63 + printf(",\n"); 64 64 + else 65 65 + printf(", "); 66 66 + } 67 67 + printf("};\n"); 68 68 + } 69 69 + 70 70 + static void print_crc64_tables(void) 71 71 + { 72 72 + printf("/* this file is generated - do not edit */\n\n"); 73 73 + printf("#include <linux/types.h>\n"); 74 74 + printf("#include <linux/cache.h>\n\n"); 75 75 + printf("static const u64 ____cacheline_aligned crc64table[256] = {\n"); 76 76 + output_table(crc64_table); 77 77 + 78 78 + printf("\nstatic const u64 ____cacheline_aligned crc64nvmetable[256] = {\n"); 79 79 + output_table(crc64_nvme_table); 80 80 + } 81 81 + 82 82 + int main(int argc, char *argv[]) 83 83 + { 84 84 + generate_crc64_table(crc64_table, CRC64_ECMA182_POLY); 85 85 + generate_reflected_crc64_table(crc64_nvme_table, CRC64_NVME_POLY); 86 86 + print_crc64_tables(); 87 87 + return 0; 88 88 + }

+115

lib/crc/loongarch/crc32.h

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + /* 3 3 + * CRC32 and CRC32C using LoongArch crc* instructions 4 4 + * 5 5 + * Module based on mips/crypto/crc32-mips.c 6 6 + * 7 7 + * Copyright (C) 2014 Linaro Ltd <yazen.ghannam@linaro.org> 8 8 + * Copyright (C) 2018 MIPS Tech, LLC 9 9 + * Copyright (C) 2020-2023 Loongson Technology Corporation Limited 10 10 + */ 11 11 + 12 12 + #include <asm/cpu-features.h> 13 13 + #include <linux/unaligned.h> 14 14 + 15 15 + #define _CRC32(crc, value, size, type) \ 16 16 + do { \ 17 17 + __asm__ __volatile__( \ 18 18 + #type ".w." #size ".w" " %0, %1, %0\n\t"\ 19 19 + : "+r" (crc) \ 20 20 + : "r" (value) \ 21 21 + : "memory"); \ 22 22 + } while (0) 23 23 + 24 24 + #define CRC32(crc, value, size) _CRC32(crc, value, size, crc) 25 25 + #define CRC32C(crc, value, size) _CRC32(crc, value, size, crcc) 26 26 + 27 27 + static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_crc32); 28 28 + 29 29 + static inline u32 crc32_le_arch(u32 crc, const u8 *p, size_t len) 30 30 + { 31 31 + if (!static_branch_likely(&have_crc32)) 32 32 + return crc32_le_base(crc, p, len); 33 33 + 34 34 + while (len >= sizeof(u64)) { 35 35 + u64 value = get_unaligned_le64(p); 36 36 + 37 37 + CRC32(crc, value, d); 38 38 + p += sizeof(u64); 39 39 + len -= sizeof(u64); 40 40 + } 41 41 + 42 42 + if (len & sizeof(u32)) { 43 43 + u32 value = get_unaligned_le32(p); 44 44 + 45 45 + CRC32(crc, value, w); 46 46 + p += sizeof(u32); 47 47 + } 48 48 + 49 49 + if (len & sizeof(u16)) { 50 50 + u16 value = get_unaligned_le16(p); 51 51 + 52 52 + CRC32(crc, value, h); 53 53 + p += sizeof(u16); 54 54 + } 55 55 + 56 56 + if (len & sizeof(u8)) { 57 57 + u8 value = *p++; 58 58 + 59 59 + CRC32(crc, value, b); 60 60 + } 61 61 + 62 62 + return crc; 63 63 + } 64 64 + 65 65 + static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len) 66 66 + { 67 67 + if (!static_branch_likely(&have_crc32)) 68 68 + return crc32c_base(crc, p, len); 69 69 + 70 70 + while (len >= sizeof(u64)) { 71 71 + u64 value = get_unaligned_le64(p); 72 72 + 73 73 + CRC32C(crc, value, d); 74 74 + p += sizeof(u64); 75 75 + len -= sizeof(u64); 76 76 + } 77 77 + 78 78 + if (len & sizeof(u32)) { 79 79 + u32 value = get_unaligned_le32(p); 80 80 + 81 81 + CRC32C(crc, value, w); 82 82 + p += sizeof(u32); 83 83 + } 84 84 + 85 85 + if (len & sizeof(u16)) { 86 86 + u16 value = get_unaligned_le16(p); 87 87 + 88 88 + CRC32C(crc, value, h); 89 89 + p += sizeof(u16); 90 90 + } 91 91 + 92 92 + if (len & sizeof(u8)) { 93 93 + u8 value = *p++; 94 94 + 95 95 + CRC32C(crc, value, b); 96 96 + } 97 97 + 98 98 + return crc; 99 99 + } 100 100 + 101 101 + #define crc32_be_arch crc32_be_base /* not implemented on this arch */ 102 102 + 103 103 + #define crc32_mod_init_arch crc32_mod_init_arch 104 104 + static inline void crc32_mod_init_arch(void) 105 105 + { 106 106 + if (cpu_has_crc32) 107 107 + static_branch_enable(&have_crc32); 108 108 + } 109 109 + 110 110 + static inline u32 crc32_optimizations_arch(void) 111 111 + { 112 112 + if (static_key_enabled(&have_crc32)) 113 113 + return CRC32_LE_OPTIMIZATION | CRC32C_OPTIMIZATION; 114 114 + return 0; 115 115 + }

+162

lib/crc/mips/crc32.h

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + /* 3 3 + * crc32-mips.c - CRC32 and CRC32C using optional MIPSr6 instructions 4 4 + * 5 5 + * Module based on arm64/crypto/crc32-arm.c 6 6 + * 7 7 + * Copyright (C) 2014 Linaro Ltd <yazen.ghannam@linaro.org> 8 8 + * Copyright (C) 2018 MIPS Tech, LLC 9 9 + */ 10 10 + 11 11 + #include <linux/cpufeature.h> 12 12 + #include <asm/mipsregs.h> 13 13 + #include <linux/unaligned.h> 14 14 + 15 15 + #ifndef TOOLCHAIN_SUPPORTS_CRC 16 16 + #define _ASM_SET_CRC(OP, SZ, TYPE) \ 17 17 + _ASM_MACRO_3R(OP, rt, rs, rt2, \ 18 18 + ".ifnc \\rt, \\rt2\n\t" \ 19 19 + ".error \"invalid operands \\\"" #OP " \\rt,\\rs,\\rt2\\\"\"\n\t" \ 20 20 + ".endif\n\t" \ 21 21 + _ASM_INSN_IF_MIPS(0x7c00000f | (__rt << 16) | (__rs << 21) | \ 22 22 + ((SZ) << 6) | ((TYPE) << 8)) \ 23 23 + _ASM_INSN32_IF_MM(0x00000030 | (__rs << 16) | (__rt << 21) | \ 24 24 + ((SZ) << 14) | ((TYPE) << 3))) 25 25 + #define _ASM_UNSET_CRC(op, SZ, TYPE) ".purgem " #op "\n\t" 26 26 + #else /* !TOOLCHAIN_SUPPORTS_CRC */ 27 27 + #define _ASM_SET_CRC(op, SZ, TYPE) ".set\tcrc\n\t" 28 28 + #define _ASM_UNSET_CRC(op, SZ, TYPE) 29 29 + #endif 30 30 + 31 31 + #define __CRC32(crc, value, op, SZ, TYPE) \ 32 32 + do { \ 33 33 + __asm__ __volatile__( \ 34 34 + ".set push\n\t" \ 35 35 + _ASM_SET_CRC(op, SZ, TYPE) \ 36 36 + #op " %0, %1, %0\n\t" \ 37 37 + _ASM_UNSET_CRC(op, SZ, TYPE) \ 38 38 + ".set pop" \ 39 39 + : "+r" (crc) \ 40 40 + : "r" (value)); \ 41 41 + } while (0) 42 42 + 43 43 + #define _CRC32_crc32b(crc, value) __CRC32(crc, value, crc32b, 0, 0) 44 44 + #define _CRC32_crc32h(crc, value) __CRC32(crc, value, crc32h, 1, 0) 45 45 + #define _CRC32_crc32w(crc, value) __CRC32(crc, value, crc32w, 2, 0) 46 46 + #define _CRC32_crc32d(crc, value) __CRC32(crc, value, crc32d, 3, 0) 47 47 + #define _CRC32_crc32cb(crc, value) __CRC32(crc, value, crc32cb, 0, 1) 48 48 + #define _CRC32_crc32ch(crc, value) __CRC32(crc, value, crc32ch, 1, 1) 49 49 + #define _CRC32_crc32cw(crc, value) __CRC32(crc, value, crc32cw, 2, 1) 50 50 + #define _CRC32_crc32cd(crc, value) __CRC32(crc, value, crc32cd, 3, 1) 51 51 + 52 52 + #define _CRC32(crc, value, size, op) \ 53 53 + _CRC32_##op##size(crc, value) 54 54 + 55 55 + #define CRC32(crc, value, size) \ 56 56 + _CRC32(crc, value, size, crc32) 57 57 + 58 58 + #define CRC32C(crc, value, size) \ 59 59 + _CRC32(crc, value, size, crc32c) 60 60 + 61 61 + static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_crc32); 62 62 + 63 63 + static inline u32 crc32_le_arch(u32 crc, const u8 *p, size_t len) 64 64 + { 65 65 + if (!static_branch_likely(&have_crc32)) 66 66 + return crc32_le_base(crc, p, len); 67 67 + 68 68 + if (IS_ENABLED(CONFIG_64BIT)) { 69 69 + for (; len >= sizeof(u64); p += sizeof(u64), len -= sizeof(u64)) { 70 70 + u64 value = get_unaligned_le64(p); 71 71 + 72 72 + CRC32(crc, value, d); 73 73 + } 74 74 + 75 75 + if (len & sizeof(u32)) { 76 76 + u32 value = get_unaligned_le32(p); 77 77 + 78 78 + CRC32(crc, value, w); 79 79 + p += sizeof(u32); 80 80 + } 81 81 + } else { 82 82 + for (; len >= sizeof(u32); len -= sizeof(u32)) { 83 83 + u32 value = get_unaligned_le32(p); 84 84 + 85 85 + CRC32(crc, value, w); 86 86 + p += sizeof(u32); 87 87 + } 88 88 + } 89 89 + 90 90 + if (len & sizeof(u16)) { 91 91 + u16 value = get_unaligned_le16(p); 92 92 + 93 93 + CRC32(crc, value, h); 94 94 + p += sizeof(u16); 95 95 + } 96 96 + 97 97 + if (len & sizeof(u8)) { 98 98 + u8 value = *p++; 99 99 + 100 100 + CRC32(crc, value, b); 101 101 + } 102 102 + 103 103 + return crc; 104 104 + } 105 105 + 106 106 + static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len) 107 107 + { 108 108 + if (!static_branch_likely(&have_crc32)) 109 109 + return crc32c_base(crc, p, len); 110 110 + 111 111 + if (IS_ENABLED(CONFIG_64BIT)) { 112 112 + for (; len >= sizeof(u64); p += sizeof(u64), len -= sizeof(u64)) { 113 113 + u64 value = get_unaligned_le64(p); 114 114 + 115 115 + CRC32C(crc, value, d); 116 116 + } 117 117 + 118 118 + if (len & sizeof(u32)) { 119 119 + u32 value = get_unaligned_le32(p); 120 120 + 121 121 + CRC32C(crc, value, w); 122 122 + p += sizeof(u32); 123 123 + } 124 124 + } else { 125 125 + for (; len >= sizeof(u32); len -= sizeof(u32)) { 126 126 + u32 value = get_unaligned_le32(p); 127 127 + 128 128 + CRC32C(crc, value, w); 129 129 + p += sizeof(u32); 130 130 + } 131 131 + } 132 132 + 133 133 + if (len & sizeof(u16)) { 134 134 + u16 value = get_unaligned_le16(p); 135 135 + 136 136 + CRC32C(crc, value, h); 137 137 + p += sizeof(u16); 138 138 + } 139 139 + 140 140 + if (len & sizeof(u8)) { 141 141 + u8 value = *p++; 142 142 + 143 143 + CRC32C(crc, value, b); 144 144 + } 145 145 + return crc; 146 146 + } 147 147 + 148 148 + #define crc32_be_arch crc32_be_base /* not implemented on this arch */ 149 149 + 150 150 + #define crc32_mod_init_arch crc32_mod_init_arch 151 151 + static inline void crc32_mod_init_arch(void) 152 152 + { 153 153 + if (cpu_have_feature(cpu_feature(MIPS_CRC32))) 154 154 + static_branch_enable(&have_crc32); 155 155 + } 156 156 + 157 157 + static inline u32 crc32_optimizations_arch(void) 158 158 + { 159 159 + if (static_key_enabled(&have_crc32)) 160 160 + return CRC32_LE_OPTIMIZATION | CRC32C_OPTIMIZATION; 161 161 + return 0; 162 162 + }

+69

lib/crc/powerpc/crc-t10dif.h

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0-or-later 2 2 + /* 3 3 + * Calculate a CRC T10-DIF with vpmsum acceleration 4 4 + * 5 5 + * Copyright 2017, Daniel Axtens, IBM Corporation. 6 6 + * [based on crc32c-vpmsum_glue.c] 7 7 + */ 8 8 + 9 9 + #include <asm/switch_to.h> 10 10 + #include <crypto/internal/simd.h> 11 11 + #include <linux/cpufeature.h> 12 12 + #include <linux/jump_label.h> 13 13 + #include <linux/preempt.h> 14 14 + #include <linux/uaccess.h> 15 15 + 16 16 + #define VMX_ALIGN 16 17 17 + #define VMX_ALIGN_MASK (VMX_ALIGN-1) 18 18 + 19 19 + #define VECTOR_BREAKPOINT 64 20 20 + 21 21 + static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_vec_crypto); 22 22 + 23 23 + u32 __crct10dif_vpmsum(u32 crc, unsigned char const *p, size_t len); 24 24 + 25 25 + static inline u16 crc_t10dif_arch(u16 crci, const u8 *p, size_t len) 26 26 + { 27 27 + unsigned int prealign; 28 28 + unsigned int tail; 29 29 + u32 crc = crci; 30 30 + 31 31 + if (len < (VECTOR_BREAKPOINT + VMX_ALIGN) || 32 32 + !static_branch_likely(&have_vec_crypto) || !crypto_simd_usable()) 33 33 + return crc_t10dif_generic(crc, p, len); 34 34 + 35 35 + if ((unsigned long)p & VMX_ALIGN_MASK) { 36 36 + prealign = VMX_ALIGN - ((unsigned long)p & VMX_ALIGN_MASK); 37 37 + crc = crc_t10dif_generic(crc, p, prealign); 38 38 + len -= prealign; 39 39 + p += prealign; 40 40 + } 41 41 + 42 42 + if (len & ~VMX_ALIGN_MASK) { 43 43 + crc <<= 16; 44 44 + preempt_disable(); 45 45 + pagefault_disable(); 46 46 + enable_kernel_altivec(); 47 47 + crc = __crct10dif_vpmsum(crc, p, len & ~VMX_ALIGN_MASK); 48 48 + disable_kernel_altivec(); 49 49 + pagefault_enable(); 50 50 + preempt_enable(); 51 51 + crc >>= 16; 52 52 + } 53 53 + 54 54 + tail = len & VMX_ALIGN_MASK; 55 55 + if (tail) { 56 56 + p += len & ~VMX_ALIGN_MASK; 57 57 + crc = crc_t10dif_generic(crc, p, tail); 58 58 + } 59 59 + 60 60 + return crc & 0xffff; 61 61 + } 62 62 + 63 63 + #define crc_t10dif_mod_init_arch crc_t10dif_mod_init_arch 64 64 + static inline void crc_t10dif_mod_init_arch(void) 65 65 + { 66 66 + if (cpu_has_feature(CPU_FTR_ARCH_207S) && 67 67 + (cur_cpu_spec->cpu_user_features2 & PPC_FEATURE2_VEC_CRYPTO)) 68 68 + static_branch_enable(&have_vec_crypto); 69 69 + }

+69

lib/crc/powerpc/crc32.h

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0-only 2 2 + #include <asm/switch_to.h> 3 3 + #include <crypto/internal/simd.h> 4 4 + #include <linux/cpufeature.h> 5 5 + #include <linux/jump_label.h> 6 6 + #include <linux/preempt.h> 7 7 + #include <linux/uaccess.h> 8 8 + 9 9 + #define VMX_ALIGN 16 10 10 + #define VMX_ALIGN_MASK (VMX_ALIGN-1) 11 11 + 12 12 + #define VECTOR_BREAKPOINT 512 13 13 + 14 14 + static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_vec_crypto); 15 15 + 16 16 + #define crc32_le_arch crc32_le_base /* not implemented on this arch */ 17 17 + #define crc32_be_arch crc32_be_base /* not implemented on this arch */ 18 18 + 19 19 + u32 __crc32c_vpmsum(u32 crc, const u8 *p, size_t len); 20 20 + 21 21 + static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len) 22 22 + { 23 23 + unsigned int prealign; 24 24 + unsigned int tail; 25 25 + 26 26 + if (len < (VECTOR_BREAKPOINT + VMX_ALIGN) || 27 27 + !static_branch_likely(&have_vec_crypto) || !crypto_simd_usable()) 28 28 + return crc32c_base(crc, p, len); 29 29 + 30 30 + if ((unsigned long)p & VMX_ALIGN_MASK) { 31 31 + prealign = VMX_ALIGN - ((unsigned long)p & VMX_ALIGN_MASK); 32 32 + crc = crc32c_base(crc, p, prealign); 33 33 + len -= prealign; 34 34 + p += prealign; 35 35 + } 36 36 + 37 37 + if (len & ~VMX_ALIGN_MASK) { 38 38 + preempt_disable(); 39 39 + pagefault_disable(); 40 40 + enable_kernel_altivec(); 41 41 + crc = __crc32c_vpmsum(crc, p, len & ~VMX_ALIGN_MASK); 42 42 + disable_kernel_altivec(); 43 43 + pagefault_enable(); 44 44 + preempt_enable(); 45 45 + } 46 46 + 47 47 + tail = len & VMX_ALIGN_MASK; 48 48 + if (tail) { 49 49 + p += len & ~VMX_ALIGN_MASK; 50 50 + crc = crc32c_base(crc, p, tail); 51 51 + } 52 52 + 53 53 + return crc; 54 54 + } 55 55 + 56 56 + #define crc32_mod_init_arch crc32_mod_init_arch 57 57 + static inline void crc32_mod_init_arch(void) 58 58 + { 59 59 + if (cpu_has_feature(CPU_FTR_ARCH_207S) && 60 60 + (cur_cpu_spec->cpu_user_features2 & PPC_FEATURE2_VEC_CRYPTO)) 61 61 + static_branch_enable(&have_vec_crypto); 62 62 + } 63 63 + 64 64 + static inline u32 crc32_optimizations_arch(void) 65 65 + { 66 66 + if (static_key_enabled(&have_vec_crypto)) 67 67 + return CRC32C_OPTIMIZATION; 68 68 + return 0; 69 69 + }

+18

lib/crc/riscv/crc-t10dif.h

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0-or-later 2 2 + /* 3 3 + * RISC-V optimized CRC-T10DIF function 4 4 + * 5 5 + * Copyright 2025 Google LLC 6 6 + */ 7 7 + 8 8 + #include <asm/hwcap.h> 9 9 + #include <asm/alternative-macros.h> 10 10 + 11 11 + #include "crc-clmul.h" 12 12 + 13 13 + static inline u16 crc_t10dif_arch(u16 crc, const u8 *p, size_t len) 14 14 + { 15 15 + if (riscv_has_extension_likely(RISCV_ISA_EXT_ZBC)) 16 16 + return crc16_msb_clmul(crc, p, len, &crc16_msb_0x8bb7_consts); 17 17 + return crc_t10dif_generic(crc, p, len); 18 18 + }

+44

lib/crc/riscv/crc32.h

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0-or-later 2 2 + /* 3 3 + * RISC-V optimized CRC32 functions 4 4 + * 5 5 + * Copyright 2025 Google LLC 6 6 + */ 7 7 + 8 8 + #include <asm/hwcap.h> 9 9 + #include <asm/alternative-macros.h> 10 10 + 11 11 + #include "crc-clmul.h" 12 12 + 13 13 + static inline u32 crc32_le_arch(u32 crc, const u8 *p, size_t len) 14 14 + { 15 15 + if (riscv_has_extension_likely(RISCV_ISA_EXT_ZBC)) 16 16 + return crc32_lsb_clmul(crc, p, len, 17 17 + &crc32_lsb_0xedb88320_consts); 18 18 + return crc32_le_base(crc, p, len); 19 19 + } 20 20 + 21 21 + static inline u32 crc32_be_arch(u32 crc, const u8 *p, size_t len) 22 22 + { 23 23 + if (riscv_has_extension_likely(RISCV_ISA_EXT_ZBC)) 24 24 + return crc32_msb_clmul(crc, p, len, 25 25 + &crc32_msb_0x04c11db7_consts); 26 26 + return crc32_be_base(crc, p, len); 27 27 + } 28 28 + 29 29 + static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len) 30 30 + { 31 31 + if (riscv_has_extension_likely(RISCV_ISA_EXT_ZBC)) 32 32 + return crc32_lsb_clmul(crc, p, len, 33 33 + &crc32_lsb_0x82f63b78_consts); 34 34 + return crc32c_base(crc, p, len); 35 35 + } 36 36 + 37 37 + static inline u32 crc32_optimizations_arch(void) 38 38 + { 39 39 + if (riscv_has_extension_likely(RISCV_ISA_EXT_ZBC)) 40 40 + return CRC32_LE_OPTIMIZATION | 41 41 + CRC32_BE_OPTIMIZATION | 42 42 + CRC32C_OPTIMIZATION; 43 43 + return 0; 44 44 + }

+27

lib/crc/riscv/crc64.h

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0-or-later 2 2 + /* 3 3 + * RISC-V optimized CRC64 functions 4 4 + * 5 5 + * Copyright 2025 Google LLC 6 6 + */ 7 7 + 8 8 + #include <asm/hwcap.h> 9 9 + #include <asm/alternative-macros.h> 10 10 + 11 11 + #include "crc-clmul.h" 12 12 + 13 13 + static inline u64 crc64_be_arch(u64 crc, const u8 *p, size_t len) 14 14 + { 15 15 + if (riscv_has_extension_likely(RISCV_ISA_EXT_ZBC)) 16 16 + return crc64_msb_clmul(crc, p, len, 17 17 + &crc64_msb_0x42f0e1eba9ea3693_consts); 18 18 + return crc64_be_generic(crc, p, len); 19 19 + } 20 20 + 21 21 + static inline u64 crc64_nvme_arch(u64 crc, const u8 *p, size_t len) 22 22 + { 23 23 + if (riscv_has_extension_likely(RISCV_ISA_EXT_ZBC)) 24 24 + return crc64_lsb_clmul(crc, p, len, 25 25 + &crc64_lsb_0x9a6c9329ac4bc9b5_consts); 26 26 + return crc64_nvme_generic(crc, p, len); 27 27 + }

+67

lib/crc/s390/crc32.h

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + /* 3 3 + * CRC-32 implemented with the z/Architecture Vector Extension Facility. 4 4 + * 5 5 + * Copyright IBM Corp. 2015 6 6 + * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com> 7 7 + */ 8 8 + 9 9 + #include <linux/cpufeature.h> 10 10 + #include <asm/fpu.h> 11 11 + #include "crc32-vx.h" 12 12 + 13 13 + #define VX_MIN_LEN 64 14 14 + #define VX_ALIGNMENT 16L 15 15 + #define VX_ALIGN_MASK (VX_ALIGNMENT - 1) 16 16 + 17 17 + /* 18 18 + * DEFINE_CRC32_VX() - Define a CRC-32 function using the vector extension 19 19 + * 20 20 + * Creates a function to perform a particular CRC-32 computation. Depending 21 21 + * on the message buffer, the hardware-accelerated or software implementation 22 22 + * is used. Note that the message buffer is aligned to improve fetch 23 23 + * operations of VECTOR LOAD MULTIPLE instructions. 24 24 + */ 25 25 + #define DEFINE_CRC32_VX(___fname, ___crc32_vx, ___crc32_sw) \ 26 26 + static inline u32 ___fname(u32 crc, const u8 *data, size_t datalen) \ 27 27 + { \ 28 28 + unsigned long prealign, aligned, remaining; \ 29 29 + DECLARE_KERNEL_FPU_ONSTACK16(vxstate); \ 30 30 + \ 31 31 + if (datalen < VX_MIN_LEN + VX_ALIGN_MASK || !cpu_has_vx()) \ 32 32 + return ___crc32_sw(crc, data, datalen); \ 33 33 + \ 34 34 + if ((unsigned long)data & VX_ALIGN_MASK) { \ 35 35 + prealign = VX_ALIGNMENT - \ 36 36 + ((unsigned long)data & VX_ALIGN_MASK); \ 37 37 + datalen -= prealign; \ 38 38 + crc = ___crc32_sw(crc, data, prealign); \ 39 39 + data = (void *)((unsigned long)data + prealign); \ 40 40 + } \ 41 41 + \ 42 42 + aligned = datalen & ~VX_ALIGN_MASK; \ 43 43 + remaining = datalen & VX_ALIGN_MASK; \ 44 44 + \ 45 45 + kernel_fpu_begin(&vxstate, KERNEL_VXR_LOW); \ 46 46 + crc = ___crc32_vx(crc, data, aligned); \ 47 47 + kernel_fpu_end(&vxstate, KERNEL_VXR_LOW); \ 48 48 + \ 49 49 + if (remaining) \ 50 50 + crc = ___crc32_sw(crc, data + aligned, remaining); \ 51 51 + \ 52 52 + return crc; \ 53 53 + } 54 54 + 55 55 + DEFINE_CRC32_VX(crc32_le_arch, crc32_le_vgfm_16, crc32_le_base) 56 56 + DEFINE_CRC32_VX(crc32_be_arch, crc32_be_vgfm_16, crc32_be_base) 57 57 + DEFINE_CRC32_VX(crc32c_arch, crc32c_le_vgfm_16, crc32c_base) 58 58 + 59 59 + static inline u32 crc32_optimizations_arch(void) 60 60 + { 61 61 + if (cpu_has_vx()) { 62 62 + return CRC32_LE_OPTIMIZATION | 63 63 + CRC32_BE_OPTIMIZATION | 64 64 + CRC32C_OPTIMIZATION; 65 65 + } 66 66 + return 0; 67 67 + }

+67

lib/crc/sparc/crc32.h

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0-only 2 2 + /* CRC32c (Castagnoli), sparc64 crc32c opcode accelerated 3 3 + * 4 4 + * This is based largely upon arch/x86/crypto/crc32c-intel.c 5 5 + * 6 6 + * Copyright (C) 2008 Intel Corporation 7 7 + * Authors: Austin Zhang <austin_zhang@linux.intel.com> 8 8 + * Kent Liu <kent.liu@intel.com> 9 9 + */ 10 10 + 11 11 + #include <asm/pstate.h> 12 12 + #include <asm/elf.h> 13 13 + 14 14 + static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_crc32c_opcode); 15 15 + 16 16 + #define crc32_le_arch crc32_le_base /* not implemented on this arch */ 17 17 + #define crc32_be_arch crc32_be_base /* not implemented on this arch */ 18 18 + 19 19 + void crc32c_sparc64(u32 *crcp, const u64 *data, size_t len); 20 20 + 21 21 + static inline u32 crc32c_arch(u32 crc, const u8 *data, size_t len) 22 22 + { 23 23 + size_t n = -(uintptr_t)data & 7; 24 24 + 25 25 + if (!static_branch_likely(&have_crc32c_opcode)) 26 26 + return crc32c_base(crc, data, len); 27 27 + 28 28 + if (n) { 29 29 + /* Data isn't 8-byte aligned. Align it. */ 30 30 + n = min(n, len); 31 31 + crc = crc32c_base(crc, data, n); 32 32 + data += n; 33 33 + len -= n; 34 34 + } 35 35 + n = len & ~7U; 36 36 + if (n) { 37 37 + crc32c_sparc64(&crc, (const u64 *)data, n); 38 38 + data += n; 39 39 + len -= n; 40 40 + } 41 41 + if (len) 42 42 + crc = crc32c_base(crc, data, len); 43 43 + return crc; 44 44 + } 45 45 + 46 46 + #define crc32_mod_init_arch crc32_mod_init_arch 47 47 + static inline void crc32_mod_init_arch(void) 48 48 + { 49 49 + unsigned long cfr; 50 50 + 51 51 + if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO)) 52 52 + return; 53 53 + 54 54 + __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr)); 55 55 + if (!(cfr & CFR_CRC32C)) 56 56 + return; 57 57 + 58 58 + static_branch_enable(&have_crc32c_opcode); 59 59 + pr_info("Using sparc64 crc32c opcode optimized CRC32C implementation\n"); 60 60 + } 61 61 + 62 62 + static inline u32 crc32_optimizations_arch(void) 63 63 + { 64 64 + if (static_key_enabled(&have_crc32c_opcode)) 65 65 + return CRC32C_OPTIMIZATION; 66 66 + return 0; 67 67 + }

+2

lib/crc/tests/Makefile

reviewed

··· 1 1 + # SPDX-License-Identifier: GPL-2.0-only 2 2 + obj-$(CONFIG_CRC_KUNIT_TEST) += crc_kunit.o

+452

lib/crc/tests/crc_kunit.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0-or-later 2 2 + /* 3 3 + * Unit tests and benchmarks for the CRC library functions 4 4 + * 5 5 + * Copyright 2024 Google LLC 6 6 + * 7 7 + * Author: Eric Biggers <ebiggers@google.com> 8 8 + */ 9 9 + #include <kunit/test.h> 10 10 + #include <linux/crc7.h> 11 11 + #include <linux/crc16.h> 12 12 + #include <linux/crc-t10dif.h> 13 13 + #include <linux/crc32.h> 14 14 + #include <linux/crc32c.h> 15 15 + #include <linux/crc64.h> 16 16 + #include <linux/prandom.h> 17 17 + #include <linux/vmalloc.h> 18 18 + 19 19 + #define CRC_KUNIT_SEED 42 20 20 + #define CRC_KUNIT_MAX_LEN 16384 21 21 + #define CRC_KUNIT_NUM_TEST_ITERS 1000 22 22 + 23 23 + static struct rnd_state rng; 24 24 + static u8 *test_buffer; 25 25 + static size_t test_buflen; 26 26 + 27 27 + /** 28 28 + * struct crc_variant - describes a CRC variant 29 29 + * @bits: Number of bits in the CRC, 1 <= @bits <= 64. 30 30 + * @le: true if it's a "little endian" CRC (reversed mapping between bits and 31 31 + * polynomial coefficients in each byte), false if it's a "big endian" CRC 32 32 + * (natural mapping between bits and polynomial coefficients in each byte) 33 33 + * @poly: The generator polynomial with the highest-order term omitted. 34 34 + * Bit-reversed if @le is true. 35 35 + * @func: The function to compute a CRC. The type signature uses u64 so that it 36 36 + * can fit any CRC up to CRC-64. The CRC is passed in, and is expected 37 37 + * to be returned in, the least significant bits of the u64. The 38 38 + * function is expected to *not* invert the CRC at the beginning and end. 39 39 + */ 40 40 + struct crc_variant { 41 41 + int bits; 42 42 + bool le; 43 43 + u64 poly; 44 44 + u64 (*func)(u64 crc, const u8 *p, size_t len); 45 45 + }; 46 46 + 47 47 + static u32 rand32(void) 48 48 + { 49 49 + return prandom_u32_state(&rng); 50 50 + } 51 51 + 52 52 + static u64 rand64(void) 53 53 + { 54 54 + u32 n = rand32(); 55 55 + 56 56 + return ((u64)n << 32) | rand32(); 57 57 + } 58 58 + 59 59 + static u64 crc_mask(const struct crc_variant *v) 60 60 + { 61 61 + return (u64)-1 >> (64 - v->bits); 62 62 + } 63 63 + 64 64 + /* Reference implementation of any CRC variant */ 65 65 + static u64 crc_ref(const struct crc_variant *v, 66 66 + u64 crc, const u8 *p, size_t len) 67 67 + { 68 68 + size_t i, j; 69 69 + 70 70 + for (i = 0; i < len; i++) { 71 71 + for (j = 0; j < 8; j++) { 72 72 + if (v->le) { 73 73 + crc ^= (p[i] >> j) & 1; 74 74 + crc = (crc >> 1) ^ ((crc & 1) ? v->poly : 0); 75 75 + } else { 76 76 + crc ^= (u64)((p[i] >> (7 - j)) & 1) << 77 77 + (v->bits - 1); 78 78 + if (crc & (1ULL << (v->bits - 1))) 79 79 + crc = ((crc << 1) ^ v->poly) & 80 80 + crc_mask(v); 81 81 + else 82 82 + crc <<= 1; 83 83 + } 84 84 + } 85 85 + } 86 86 + return crc; 87 87 + } 88 88 + 89 89 + static int crc_suite_init(struct kunit_suite *suite) 90 90 + { 91 91 + /* 92 92 + * Allocate the test buffer using vmalloc() with a page-aligned length 93 93 + * so that it is immediately followed by a guard page. This allows 94 94 + * buffer overreads to be detected, even in assembly code. 95 95 + */ 96 96 + test_buflen = round_up(CRC_KUNIT_MAX_LEN, PAGE_SIZE); 97 97 + test_buffer = vmalloc(test_buflen); 98 98 + if (!test_buffer) 99 99 + return -ENOMEM; 100 100 + 101 101 + prandom_seed_state(&rng, CRC_KUNIT_SEED); 102 102 + prandom_bytes_state(&rng, test_buffer, test_buflen); 103 103 + return 0; 104 104 + } 105 105 + 106 106 + static void crc_suite_exit(struct kunit_suite *suite) 107 107 + { 108 108 + vfree(test_buffer); 109 109 + test_buffer = NULL; 110 110 + } 111 111 + 112 112 + /* Generate a random initial CRC. */ 113 113 + static u64 generate_random_initial_crc(const struct crc_variant *v) 114 114 + { 115 115 + switch (rand32() % 4) { 116 116 + case 0: 117 117 + return 0; 118 118 + case 1: 119 119 + return crc_mask(v); /* All 1 bits */ 120 120 + default: 121 121 + return rand64() & crc_mask(v); 122 122 + } 123 123 + } 124 124 + 125 125 + /* Generate a random length, preferring small lengths. */ 126 126 + static size_t generate_random_length(size_t max_length) 127 127 + { 128 128 + size_t len; 129 129 + 130 130 + switch (rand32() % 3) { 131 131 + case 0: 132 132 + len = rand32() % 128; 133 133 + break; 134 134 + case 1: 135 135 + len = rand32() % 3072; 136 136 + break; 137 137 + default: 138 138 + len = rand32(); 139 139 + break; 140 140 + } 141 141 + return len % (max_length + 1); 142 142 + } 143 143 + 144 144 + /* Test that v->func gives the same CRCs as a reference implementation. */ 145 145 + static void crc_test(struct kunit *test, const struct crc_variant *v) 146 146 + { 147 147 + size_t i; 148 148 + 149 149 + for (i = 0; i < CRC_KUNIT_NUM_TEST_ITERS; i++) { 150 150 + u64 init_crc, expected_crc, actual_crc; 151 151 + size_t len, offset; 152 152 + bool nosimd; 153 153 + 154 154 + init_crc = generate_random_initial_crc(v); 155 155 + len = generate_random_length(CRC_KUNIT_MAX_LEN); 156 156 + 157 157 + /* Generate a random offset. */ 158 158 + if (rand32() % 2 == 0) { 159 159 + /* Use a random alignment mod 64 */ 160 160 + offset = rand32() % 64; 161 161 + offset = min(offset, CRC_KUNIT_MAX_LEN - len); 162 162 + } else { 163 163 + /* Go up to the guard page, to catch buffer overreads */ 164 164 + offset = test_buflen - len; 165 165 + } 166 166 + 167 167 + if (rand32() % 8 == 0) 168 168 + /* Refresh the data occasionally. */ 169 169 + prandom_bytes_state(&rng, &test_buffer[offset], len); 170 170 + 171 171 + nosimd = rand32() % 8 == 0; 172 172 + 173 173 + /* 174 174 + * Compute the CRC, and verify that it equals the CRC computed 175 175 + * by a simple bit-at-a-time reference implementation. 176 176 + */ 177 177 + expected_crc = crc_ref(v, init_crc, &test_buffer[offset], len); 178 178 + if (nosimd) 179 179 + local_irq_disable(); 180 180 + actual_crc = v->func(init_crc, &test_buffer[offset], len); 181 181 + if (nosimd) 182 182 + local_irq_enable(); 183 183 + KUNIT_EXPECT_EQ_MSG(test, expected_crc, actual_crc, 184 184 + "Wrong result with len=%zu offset=%zu nosimd=%d", 185 185 + len, offset, nosimd); 186 186 + } 187 187 + } 188 188 + 189 189 + static __always_inline void 190 190 + crc_benchmark(struct kunit *test, 191 191 + u64 (*crc_func)(u64 crc, const u8 *p, size_t len)) 192 192 + { 193 193 + static const size_t lens_to_test[] = { 194 194 + 1, 16, 64, 127, 128, 200, 256, 511, 512, 1024, 3173, 4096, 16384, 195 195 + }; 196 196 + size_t len, i, j, num_iters; 197 197 + /* 198 198 + * The CRC value that this function computes in a series of calls to 199 199 + * crc_func is never actually used, so use volatile to ensure that the 200 200 + * computations are done as intended and don't all get optimized out. 201 201 + */ 202 202 + volatile u64 crc = 0; 203 203 + u64 t; 204 204 + 205 205 + if (!IS_ENABLED(CONFIG_CRC_BENCHMARK)) 206 206 + kunit_skip(test, "not enabled"); 207 207 + 208 208 + /* warm-up */ 209 209 + for (i = 0; i < 10000000; i += CRC_KUNIT_MAX_LEN) 210 210 + crc = crc_func(crc, test_buffer, CRC_KUNIT_MAX_LEN); 211 211 + 212 212 + for (i = 0; i < ARRAY_SIZE(lens_to_test); i++) { 213 213 + len = lens_to_test[i]; 214 214 + KUNIT_ASSERT_LE(test, len, CRC_KUNIT_MAX_LEN); 215 215 + num_iters = 10000000 / (len + 128); 216 216 + preempt_disable(); 217 217 + t = ktime_get_ns(); 218 218 + for (j = 0; j < num_iters; j++) 219 219 + crc = crc_func(crc, test_buffer, len); 220 220 + t = ktime_get_ns() - t; 221 221 + preempt_enable(); 222 222 + kunit_info(test, "len=%zu: %llu MB/s\n", 223 223 + len, div64_u64((u64)len * num_iters * 1000, t)); 224 224 + } 225 225 + } 226 226 + 227 227 + /* crc7_be */ 228 228 + 229 229 + static u64 crc7_be_wrapper(u64 crc, const u8 *p, size_t len) 230 230 + { 231 231 + /* 232 232 + * crc7_be() left-aligns the 7-bit CRC in a u8, whereas the test wants a 233 233 + * right-aligned CRC (in a u64). Convert between the conventions. 234 234 + */ 235 235 + return crc7_be(crc << 1, p, len) >> 1; 236 236 + } 237 237 + 238 238 + static const struct crc_variant crc_variant_crc7_be = { 239 239 + .bits = 7, 240 240 + .poly = 0x9, 241 241 + .func = crc7_be_wrapper, 242 242 + }; 243 243 + 244 244 + static void crc7_be_test(struct kunit *test) 245 245 + { 246 246 + crc_test(test, &crc_variant_crc7_be); 247 247 + } 248 248 + 249 249 + static void crc7_be_benchmark(struct kunit *test) 250 250 + { 251 251 + crc_benchmark(test, crc7_be_wrapper); 252 252 + } 253 253 + 254 254 + /* crc16 */ 255 255 + 256 256 + static u64 crc16_wrapper(u64 crc, const u8 *p, size_t len) 257 257 + { 258 258 + return crc16(crc, p, len); 259 259 + } 260 260 + 261 261 + static const struct crc_variant crc_variant_crc16 = { 262 262 + .bits = 16, 263 263 + .le = true, 264 264 + .poly = 0xa001, 265 265 + .func = crc16_wrapper, 266 266 + }; 267 267 + 268 268 + static void crc16_test(struct kunit *test) 269 269 + { 270 270 + crc_test(test, &crc_variant_crc16); 271 271 + } 272 272 + 273 273 + static void crc16_benchmark(struct kunit *test) 274 274 + { 275 275 + crc_benchmark(test, crc16_wrapper); 276 276 + } 277 277 + 278 278 + /* crc_t10dif */ 279 279 + 280 280 + static u64 crc_t10dif_wrapper(u64 crc, const u8 *p, size_t len) 281 281 + { 282 282 + return crc_t10dif_update(crc, p, len); 283 283 + } 284 284 + 285 285 + static const struct crc_variant crc_variant_crc_t10dif = { 286 286 + .bits = 16, 287 287 + .le = false, 288 288 + .poly = 0x8bb7, 289 289 + .func = crc_t10dif_wrapper, 290 290 + }; 291 291 + 292 292 + static void crc_t10dif_test(struct kunit *test) 293 293 + { 294 294 + crc_test(test, &crc_variant_crc_t10dif); 295 295 + } 296 296 + 297 297 + static void crc_t10dif_benchmark(struct kunit *test) 298 298 + { 299 299 + crc_benchmark(test, crc_t10dif_wrapper); 300 300 + } 301 301 + 302 302 + /* crc32_le */ 303 303 + 304 304 + static u64 crc32_le_wrapper(u64 crc, const u8 *p, size_t len) 305 305 + { 306 306 + return crc32_le(crc, p, len); 307 307 + } 308 308 + 309 309 + static const struct crc_variant crc_variant_crc32_le = { 310 310 + .bits = 32, 311 311 + .le = true, 312 312 + .poly = 0xedb88320, 313 313 + .func = crc32_le_wrapper, 314 314 + }; 315 315 + 316 316 + static void crc32_le_test(struct kunit *test) 317 317 + { 318 318 + crc_test(test, &crc_variant_crc32_le); 319 319 + } 320 320 + 321 321 + static void crc32_le_benchmark(struct kunit *test) 322 322 + { 323 323 + crc_benchmark(test, crc32_le_wrapper); 324 324 + } 325 325 + 326 326 + /* crc32_be */ 327 327 + 328 328 + static u64 crc32_be_wrapper(u64 crc, const u8 *p, size_t len) 329 329 + { 330 330 + return crc32_be(crc, p, len); 331 331 + } 332 332 + 333 333 + static const struct crc_variant crc_variant_crc32_be = { 334 334 + .bits = 32, 335 335 + .le = false, 336 336 + .poly = 0x04c11db7, 337 337 + .func = crc32_be_wrapper, 338 338 + }; 339 339 + 340 340 + static void crc32_be_test(struct kunit *test) 341 341 + { 342 342 + crc_test(test, &crc_variant_crc32_be); 343 343 + } 344 344 + 345 345 + static void crc32_be_benchmark(struct kunit *test) 346 346 + { 347 347 + crc_benchmark(test, crc32_be_wrapper); 348 348 + } 349 349 + 350 350 + /* crc32c */ 351 351 + 352 352 + static u64 crc32c_wrapper(u64 crc, const u8 *p, size_t len) 353 353 + { 354 354 + return crc32c(crc, p, len); 355 355 + } 356 356 + 357 357 + static const struct crc_variant crc_variant_crc32c = { 358 358 + .bits = 32, 359 359 + .le = true, 360 360 + .poly = 0x82f63b78, 361 361 + .func = crc32c_wrapper, 362 362 + }; 363 363 + 364 364 + static void crc32c_test(struct kunit *test) 365 365 + { 366 366 + crc_test(test, &crc_variant_crc32c); 367 367 + } 368 368 + 369 369 + static void crc32c_benchmark(struct kunit *test) 370 370 + { 371 371 + crc_benchmark(test, crc32c_wrapper); 372 372 + } 373 373 + 374 374 + /* crc64_be */ 375 375 + 376 376 + static u64 crc64_be_wrapper(u64 crc, const u8 *p, size_t len) 377 377 + { 378 378 + return crc64_be(crc, p, len); 379 379 + } 380 380 + 381 381 + static const struct crc_variant crc_variant_crc64_be = { 382 382 + .bits = 64, 383 383 + .le = false, 384 384 + .poly = 0x42f0e1eba9ea3693, 385 385 + .func = crc64_be_wrapper, 386 386 + }; 387 387 + 388 388 + static void crc64_be_test(struct kunit *test) 389 389 + { 390 390 + crc_test(test, &crc_variant_crc64_be); 391 391 + } 392 392 + 393 393 + static void crc64_be_benchmark(struct kunit *test) 394 394 + { 395 395 + crc_benchmark(test, crc64_be_wrapper); 396 396 + } 397 397 + 398 398 + /* crc64_nvme */ 399 399 + 400 400 + static u64 crc64_nvme_wrapper(u64 crc, const u8 *p, size_t len) 401 401 + { 402 402 + /* The inversions that crc64_nvme() does have to be undone here. */ 403 403 + return ~crc64_nvme(~crc, p, len); 404 404 + } 405 405 + 406 406 + static const struct crc_variant crc_variant_crc64_nvme = { 407 407 + .bits = 64, 408 408 + .le = true, 409 409 + .poly = 0x9a6c9329ac4bc9b5, 410 410 + .func = crc64_nvme_wrapper, 411 411 + }; 412 412 + 413 413 + static void crc64_nvme_test(struct kunit *test) 414 414 + { 415 415 + crc_test(test, &crc_variant_crc64_nvme); 416 416 + } 417 417 + 418 418 + static void crc64_nvme_benchmark(struct kunit *test) 419 419 + { 420 420 + crc_benchmark(test, crc64_nvme_wrapper); 421 421 + } 422 422 + 423 423 + static struct kunit_case crc_test_cases[] = { 424 424 + KUNIT_CASE(crc7_be_test), 425 425 + KUNIT_CASE(crc7_be_benchmark), 426 426 + KUNIT_CASE(crc16_test), 427 427 + KUNIT_CASE(crc16_benchmark), 428 428 + KUNIT_CASE(crc_t10dif_test), 429 429 + KUNIT_CASE(crc_t10dif_benchmark), 430 430 + KUNIT_CASE(crc32_le_test), 431 431 + KUNIT_CASE(crc32_le_benchmark), 432 432 + KUNIT_CASE(crc32_be_test), 433 433 + KUNIT_CASE(crc32_be_benchmark), 434 434 + KUNIT_CASE(crc32c_test), 435 435 + KUNIT_CASE(crc32c_benchmark), 436 436 + KUNIT_CASE(crc64_be_test), 437 437 + KUNIT_CASE(crc64_be_benchmark), 438 438 + KUNIT_CASE(crc64_nvme_test), 439 439 + KUNIT_CASE(crc64_nvme_benchmark), 440 440 + {}, 441 441 + }; 442 442 + 443 443 + static struct kunit_suite crc_test_suite = { 444 444 + .name = "crc", 445 445 + .test_cases = crc_test_cases, 446 446 + .suite_init = crc_suite_init, 447 447 + .suite_exit = crc_suite_exit, 448 448 + }; 449 449 + kunit_test_suite(crc_test_suite); 450 450 + 451 451 + MODULE_DESCRIPTION("Unit tests and benchmarks for the CRC library functions"); 452 452 + MODULE_LICENSE("GPL");

+240

lib/crc/x86/crc-pclmul-consts.h

reviewed

··· 1 1 + /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 2 + /* 3 3 + * CRC constants generated by: 4 4 + * 5 5 + * ./scripts/gen-crc-consts.py x86_pclmul crc16_msb_0x8bb7,crc32_lsb_0xedb88320,crc32_lsb_0x82f63b78,crc64_msb_0x42f0e1eba9ea3693,crc64_lsb_0x9a6c9329ac4bc9b5 6 6 + * 7 7 + * Do not edit manually. 8 8 + */ 9 9 + 10 10 + /* 11 11 + * CRC folding constants generated for most-significant-bit-first CRC-16 using 12 12 + * G(x) = x^16 + x^15 + x^11 + x^9 + x^8 + x^7 + x^5 + x^4 + x^2 + x^1 + x^0 13 13 + */ 14 14 + static const struct { 15 15 + u8 bswap_mask[16]; 16 16 + u64 fold_across_2048_bits_consts[2]; 17 17 + u64 fold_across_1024_bits_consts[2]; 18 18 + u64 fold_across_512_bits_consts[2]; 19 19 + u64 fold_across_256_bits_consts[2]; 20 20 + u64 fold_across_128_bits_consts[2]; 21 21 + u8 shuf_table[48]; 22 22 + u64 barrett_reduction_consts[2]; 23 23 + } crc16_msb_0x8bb7_consts ____cacheline_aligned __maybe_unused = { 24 24 + .bswap_mask = {15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0}, 25 25 + .fold_across_2048_bits_consts = { 26 26 + 0xdccf000000000000, /* LO64_TERMS: (x^2000 mod G) * x^48 */ 27 27 + 0x4b0b000000000000, /* HI64_TERMS: (x^2064 mod G) * x^48 */ 28 28 + }, 29 29 + .fold_across_1024_bits_consts = { 30 30 + 0x9d9d000000000000, /* LO64_TERMS: (x^976 mod G) * x^48 */ 31 31 + 0x7cf5000000000000, /* HI64_TERMS: (x^1040 mod G) * x^48 */ 32 32 + }, 33 33 + .fold_across_512_bits_consts = { 34 34 + 0x044c000000000000, /* LO64_TERMS: (x^464 mod G) * x^48 */ 35 35 + 0xe658000000000000, /* HI64_TERMS: (x^528 mod G) * x^48 */ 36 36 + }, 37 37 + .fold_across_256_bits_consts = { 38 38 + 0x6ee3000000000000, /* LO64_TERMS: (x^208 mod G) * x^48 */ 39 39 + 0xe7b5000000000000, /* HI64_TERMS: (x^272 mod G) * x^48 */ 40 40 + }, 41 41 + .fold_across_128_bits_consts = { 42 42 + 0x2d56000000000000, /* LO64_TERMS: (x^80 mod G) * x^48 */ 43 43 + 0x06df000000000000, /* HI64_TERMS: (x^144 mod G) * x^48 */ 44 44 + }, 45 45 + .shuf_table = { 46 46 + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 47 47 + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 48 48 + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 49 49 + }, 50 50 + .barrett_reduction_consts = { 51 51 + 0x8bb7000000000000, /* LO64_TERMS: (G - x^16) * x^48 */ 52 52 + 0xf65a57f81d33a48a, /* HI64_TERMS: (floor(x^79 / G) * x) - x^64 */ 53 53 + }, 54 54 + }; 55 55 + 56 56 + /* 57 57 + * CRC folding constants generated for least-significant-bit-first CRC-32 using 58 58 + * G(x) = x^32 + x^26 + x^23 + x^22 + x^16 + x^12 + x^11 + x^10 + x^8 + x^7 + 59 59 + * x^5 + x^4 + x^2 + x^1 + x^0 60 60 + */ 61 61 + static const struct { 62 62 + u64 fold_across_2048_bits_consts[2]; 63 63 + u64 fold_across_1024_bits_consts[2]; 64 64 + u64 fold_across_512_bits_consts[2]; 65 65 + u64 fold_across_256_bits_consts[2]; 66 66 + u64 fold_across_128_bits_consts[2]; 67 67 + u8 shuf_table[48]; 68 68 + u64 barrett_reduction_consts[2]; 69 69 + } crc32_lsb_0xedb88320_consts ____cacheline_aligned __maybe_unused = { 70 70 + .fold_across_2048_bits_consts = { 71 71 + 0x00000000ce3371cb, /* HI64_TERMS: (x^2079 mod G) * x^32 */ 72 72 + 0x00000000e95c1271, /* LO64_TERMS: (x^2015 mod G) * x^32 */ 73 73 + }, 74 74 + .fold_across_1024_bits_consts = { 75 75 + 0x0000000033fff533, /* HI64_TERMS: (x^1055 mod G) * x^32 */ 76 76 + 0x00000000910eeec1, /* LO64_TERMS: (x^991 mod G) * x^32 */ 77 77 + }, 78 78 + .fold_across_512_bits_consts = { 79 79 + 0x000000008f352d95, /* HI64_TERMS: (x^543 mod G) * x^32 */ 80 80 + 0x000000001d9513d7, /* LO64_TERMS: (x^479 mod G) * x^32 */ 81 81 + }, 82 82 + .fold_across_256_bits_consts = { 83 83 + 0x00000000f1da05aa, /* HI64_TERMS: (x^287 mod G) * x^32 */ 84 84 + 0x0000000081256527, /* LO64_TERMS: (x^223 mod G) * x^32 */ 85 85 + }, 86 86 + .fold_across_128_bits_consts = { 87 87 + 0x00000000ae689191, /* HI64_TERMS: (x^159 mod G) * x^32 */ 88 88 + 0x00000000ccaa009e, /* LO64_TERMS: (x^95 mod G) * x^32 */ 89 89 + }, 90 90 + .shuf_table = { 91 91 + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 92 92 + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 93 93 + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 94 94 + }, 95 95 + .barrett_reduction_consts = { 96 96 + 0xb4e5b025f7011641, /* HI64_TERMS: floor(x^95 / G) */ 97 97 + 0x00000001db710640, /* LO64_TERMS: (G - x^32) * x^31 */ 98 98 + }, 99 99 + }; 100 100 + 101 101 + /* 102 102 + * CRC folding constants generated for least-significant-bit-first CRC-32 using 103 103 + * G(x) = x^32 + x^28 + x^27 + x^26 + x^25 + x^23 + x^22 + x^20 + x^19 + x^18 + 104 104 + * x^14 + x^13 + x^11 + x^10 + x^9 + x^8 + x^6 + x^0 105 105 + */ 106 106 + static const struct { 107 107 + u64 fold_across_2048_bits_consts[2]; 108 108 + u64 fold_across_1024_bits_consts[2]; 109 109 + u64 fold_across_512_bits_consts[2]; 110 110 + u64 fold_across_256_bits_consts[2]; 111 111 + u64 fold_across_128_bits_consts[2]; 112 112 + u8 shuf_table[48]; 113 113 + u64 barrett_reduction_consts[2]; 114 114 + } crc32_lsb_0x82f63b78_consts ____cacheline_aligned __maybe_unused = { 115 115 + .fold_across_2048_bits_consts = { 116 116 + 0x00000000dcb17aa4, /* HI64_TERMS: (x^2079 mod G) * x^32 */ 117 117 + 0x00000000b9e02b86, /* LO64_TERMS: (x^2015 mod G) * x^32 */ 118 118 + }, 119 119 + .fold_across_1024_bits_consts = { 120 120 + 0x000000006992cea2, /* HI64_TERMS: (x^1055 mod G) * x^32 */ 121 121 + 0x000000000d3b6092, /* LO64_TERMS: (x^991 mod G) * x^32 */ 122 122 + }, 123 123 + .fold_across_512_bits_consts = { 124 124 + 0x00000000740eef02, /* HI64_TERMS: (x^543 mod G) * x^32 */ 125 125 + 0x000000009e4addf8, /* LO64_TERMS: (x^479 mod G) * x^32 */ 126 126 + }, 127 127 + .fold_across_256_bits_consts = { 128 128 + 0x000000003da6d0cb, /* HI64_TERMS: (x^287 mod G) * x^32 */ 129 129 + 0x00000000ba4fc28e, /* LO64_TERMS: (x^223 mod G) * x^32 */ 130 130 + }, 131 131 + .fold_across_128_bits_consts = { 132 132 + 0x00000000f20c0dfe, /* HI64_TERMS: (x^159 mod G) * x^32 */ 133 133 + 0x00000000493c7d27, /* LO64_TERMS: (x^95 mod G) * x^32 */ 134 134 + }, 135 135 + .shuf_table = { 136 136 + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 137 137 + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 138 138 + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 139 139 + }, 140 140 + .barrett_reduction_consts = { 141 141 + 0x4869ec38dea713f1, /* HI64_TERMS: floor(x^95 / G) */ 142 142 + 0x0000000105ec76f0, /* LO64_TERMS: (G - x^32) * x^31 */ 143 143 + }, 144 144 + }; 145 145 + 146 146 + /* 147 147 + * CRC folding constants generated for most-significant-bit-first CRC-64 using 148 148 + * G(x) = x^64 + x^62 + x^57 + x^55 + x^54 + x^53 + x^52 + x^47 + x^46 + x^45 + 149 149 + * x^40 + x^39 + x^38 + x^37 + x^35 + x^33 + x^32 + x^31 + x^29 + x^27 + 150 150 + * x^24 + x^23 + x^22 + x^21 + x^19 + x^17 + x^13 + x^12 + x^10 + x^9 + 151 151 + * x^7 + x^4 + x^1 + x^0 152 152 + */ 153 153 + static const struct { 154 154 + u8 bswap_mask[16]; 155 155 + u64 fold_across_2048_bits_consts[2]; 156 156 + u64 fold_across_1024_bits_consts[2]; 157 157 + u64 fold_across_512_bits_consts[2]; 158 158 + u64 fold_across_256_bits_consts[2]; 159 159 + u64 fold_across_128_bits_consts[2]; 160 160 + u8 shuf_table[48]; 161 161 + u64 barrett_reduction_consts[2]; 162 162 + } crc64_msb_0x42f0e1eba9ea3693_consts ____cacheline_aligned __maybe_unused = { 163 163 + .bswap_mask = {15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0}, 164 164 + .fold_across_2048_bits_consts = { 165 165 + 0x7f52691a60ddc70d, /* LO64_TERMS: (x^2048 mod G) * x^0 */ 166 166 + 0x7036b0389f6a0c82, /* HI64_TERMS: (x^2112 mod G) * x^0 */ 167 167 + }, 168 168 + .fold_across_1024_bits_consts = { 169 169 + 0x05cf79dea9ac37d6, /* LO64_TERMS: (x^1024 mod G) * x^0 */ 170 170 + 0x001067e571d7d5c2, /* HI64_TERMS: (x^1088 mod G) * x^0 */ 171 171 + }, 172 172 + .fold_across_512_bits_consts = { 173 173 + 0x5f6843ca540df020, /* LO64_TERMS: (x^512 mod G) * x^0 */ 174 174 + 0xddf4b6981205b83f, /* HI64_TERMS: (x^576 mod G) * x^0 */ 175 175 + }, 176 176 + .fold_across_256_bits_consts = { 177 177 + 0x571bee0a227ef92b, /* LO64_TERMS: (x^256 mod G) * x^0 */ 178 178 + 0x44bef2a201b5200c, /* HI64_TERMS: (x^320 mod G) * x^0 */ 179 179 + }, 180 180 + .fold_across_128_bits_consts = { 181 181 + 0x05f5c3c7eb52fab6, /* LO64_TERMS: (x^128 mod G) * x^0 */ 182 182 + 0x4eb938a7d257740e, /* HI64_TERMS: (x^192 mod G) * x^0 */ 183 183 + }, 184 184 + .shuf_table = { 185 185 + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 186 186 + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 187 187 + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 188 188 + }, 189 189 + .barrett_reduction_consts = { 190 190 + 0x42f0e1eba9ea3693, /* LO64_TERMS: (G - x^64) * x^0 */ 191 191 + 0x578d29d06cc4f872, /* HI64_TERMS: (floor(x^127 / G) * x) - x^64 */ 192 192 + }, 193 193 + }; 194 194 + 195 195 + /* 196 196 + * CRC folding constants generated for least-significant-bit-first CRC-64 using 197 197 + * G(x) = x^64 + x^63 + x^61 + x^59 + x^58 + x^56 + x^55 + x^52 + x^49 + x^48 + 198 198 + * x^47 + x^46 + x^44 + x^41 + x^37 + x^36 + x^34 + x^32 + x^31 + x^28 + 199 199 + * x^26 + x^23 + x^22 + x^19 + x^16 + x^13 + x^12 + x^10 + x^9 + x^6 + 200 200 + * x^4 + x^3 + x^0 201 201 + */ 202 202 + static const struct { 203 203 + u64 fold_across_2048_bits_consts[2]; 204 204 + u64 fold_across_1024_bits_consts[2]; 205 205 + u64 fold_across_512_bits_consts[2]; 206 206 + u64 fold_across_256_bits_consts[2]; 207 207 + u64 fold_across_128_bits_consts[2]; 208 208 + u8 shuf_table[48]; 209 209 + u64 barrett_reduction_consts[2]; 210 210 + } crc64_lsb_0x9a6c9329ac4bc9b5_consts ____cacheline_aligned __maybe_unused = { 211 211 + .fold_across_2048_bits_consts = { 212 212 + 0x37ccd3e14069cabc, /* HI64_TERMS: (x^2111 mod G) * x^0 */ 213 213 + 0xa043808c0f782663, /* LO64_TERMS: (x^2047 mod G) * x^0 */ 214 214 + }, 215 215 + .fold_across_1024_bits_consts = { 216 216 + 0xa1ca681e733f9c40, /* HI64_TERMS: (x^1087 mod G) * x^0 */ 217 217 + 0x5f852fb61e8d92dc, /* LO64_TERMS: (x^1023 mod G) * x^0 */ 218 218 + }, 219 219 + .fold_across_512_bits_consts = { 220 220 + 0x0c32cdb31e18a84a, /* HI64_TERMS: (x^575 mod G) * x^0 */ 221 221 + 0x62242240ace5045a, /* LO64_TERMS: (x^511 mod G) * x^0 */ 222 222 + }, 223 223 + .fold_across_256_bits_consts = { 224 224 + 0xb0bc2e589204f500, /* HI64_TERMS: (x^319 mod G) * x^0 */ 225 225 + 0xe1e0bb9d45d7a44c, /* LO64_TERMS: (x^255 mod G) * x^0 */ 226 226 + }, 227 227 + .fold_across_128_bits_consts = { 228 228 + 0xeadc41fd2ba3d420, /* HI64_TERMS: (x^191 mod G) * x^0 */ 229 229 + 0x21e9761e252621ac, /* LO64_TERMS: (x^127 mod G) * x^0 */ 230 230 + }, 231 231 + .shuf_table = { 232 232 + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 233 233 + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 234 234 + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 235 235 + }, 236 236 + .barrett_reduction_consts = { 237 237 + 0x27ecfa329aef9f77, /* HI64_TERMS: floor(x^127 / G) */ 238 238 + 0x34d926535897936a, /* LO64_TERMS: (G - x^64 - x^0) / x */ 239 239 + }, 240 240 + };

+575

lib/crc/x86/crc-pclmul-template.S

reviewed

··· 1 1 + /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 2 + // 3 3 + // Template to generate [V]PCLMULQDQ-based CRC functions for x86 4 4 + // 5 5 + // Copyright 2025 Google LLC 6 6 + // 7 7 + // Author: Eric Biggers <ebiggers@google.com> 8 8 + 9 9 + #include <linux/linkage.h> 10 10 + #include <linux/objtool.h> 11 11 + 12 12 + // Offsets within the generated constants table 13 13 + .set OFFSETOF_BSWAP_MASK, -5*16 // msb-first CRCs only 14 14 + .set OFFSETOF_FOLD_ACROSS_2048_BITS_CONSTS, -4*16 // must precede next 15 15 + .set OFFSETOF_FOLD_ACROSS_1024_BITS_CONSTS, -3*16 // must precede next 16 16 + .set OFFSETOF_FOLD_ACROSS_512_BITS_CONSTS, -2*16 // must precede next 17 17 + .set OFFSETOF_FOLD_ACROSS_256_BITS_CONSTS, -1*16 // must precede next 18 18 + .set OFFSETOF_FOLD_ACROSS_128_BITS_CONSTS, 0*16 // must be 0 19 19 + .set OFFSETOF_SHUF_TABLE, 1*16 20 20 + .set OFFSETOF_BARRETT_REDUCTION_CONSTS, 4*16 21 21 + 22 22 + // Emit a VEX (or EVEX) coded instruction if allowed, or emulate it using the 23 23 + // corresponding non-VEX instruction plus any needed moves. The supported 24 24 + // instruction formats are: 25 25 + // 26 26 + // - Two-arg [src, dst], where the non-VEX format is the same. 27 27 + // - Three-arg [src1, src2, dst] where the non-VEX format is 28 28 + // [src1, src2_and_dst]. If src2 != dst, then src1 must != dst too. 29 29 + // 30 30 + // \insn gives the instruction without a "v" prefix and including any immediate 31 31 + // argument if needed to make the instruction follow one of the above formats. 32 32 + // If \unaligned_mem_tmp is given, then the emitted non-VEX code moves \arg1 to 33 33 + // it first; this is needed when \arg1 is an unaligned mem operand. 34 34 + .macro _cond_vex insn:req, arg1:req, arg2:req, arg3, unaligned_mem_tmp 35 35 + .if AVX_LEVEL == 0 36 36 + // VEX not allowed. Emulate it. 37 37 + .ifnb \arg3 // Three-arg [src1, src2, dst] 38 38 + .ifc "\arg2", "\arg3" // src2 == dst? 39 39 + .ifnb \unaligned_mem_tmp 40 40 + movdqu \arg1, \unaligned_mem_tmp 41 41 + \insn \unaligned_mem_tmp, \arg3 42 42 + .else 43 43 + \insn \arg1, \arg3 44 44 + .endif 45 45 + .else // src2 != dst 46 46 + .ifc "\arg1", "\arg3" 47 47 + .error "Can't have src1 == dst when src2 != dst" 48 48 + .endif 49 49 + .ifnb \unaligned_mem_tmp 50 50 + movdqu \arg1, \unaligned_mem_tmp 51 51 + movdqa \arg2, \arg3 52 52 + \insn \unaligned_mem_tmp, \arg3 53 53 + .else 54 54 + movdqa \arg2, \arg3 55 55 + \insn \arg1, \arg3 56 56 + .endif 57 57 + .endif 58 58 + .else // Two-arg [src, dst] 59 59 + .ifnb \unaligned_mem_tmp 60 60 + movdqu \arg1, \unaligned_mem_tmp 61 61 + \insn \unaligned_mem_tmp, \arg2 62 62 + .else 63 63 + \insn \arg1, \arg2 64 64 + .endif 65 65 + .endif 66 66 + .else 67 67 + // VEX is allowed. Emit the desired instruction directly. 68 68 + .ifnb \arg3 69 69 + v\insn \arg1, \arg2, \arg3 70 70 + .else 71 71 + v\insn \arg1, \arg2 72 72 + .endif 73 73 + .endif 74 74 + .endm 75 75 + 76 76 + // Broadcast an aligned 128-bit mem operand to all 128-bit lanes of a vector 77 77 + // register of length VL. 78 78 + .macro _vbroadcast src, dst 79 79 + .if VL == 16 80 80 + _cond_vex movdqa, \src, \dst 81 81 + .elseif VL == 32 82 82 + vbroadcasti128 \src, \dst 83 83 + .else 84 84 + vbroadcasti32x4 \src, \dst 85 85 + .endif 86 86 + .endm 87 87 + 88 88 + // Load \vl bytes from the unaligned mem operand \src into \dst, and if the CRC 89 89 + // is msb-first use \bswap_mask to reflect the bytes within each 128-bit lane. 90 90 + .macro _load_data vl, src, bswap_mask, dst 91 91 + .if \vl < 64 92 92 + _cond_vex movdqu, "\src", \dst 93 93 + .else 94 94 + vmovdqu8 \src, \dst 95 95 + .endif 96 96 + .if !LSB_CRC 97 97 + _cond_vex pshufb, \bswap_mask, \dst, \dst 98 98 + .endif 99 99 + .endm 100 100 + 101 101 + .macro _prepare_v0 vl, v0, v1, bswap_mask 102 102 + .if LSB_CRC 103 103 + .if \vl < 64 104 104 + _cond_vex pxor, (BUF), \v0, \v0, unaligned_mem_tmp=\v1 105 105 + .else 106 106 + vpxorq (BUF), \v0, \v0 107 107 + .endif 108 108 + .else 109 109 + _load_data \vl, (BUF), \bswap_mask, \v1 110 110 + .if \vl < 64 111 111 + _cond_vex pxor, \v1, \v0, \v0 112 112 + .else 113 113 + vpxorq \v1, \v0, \v0 114 114 + .endif 115 115 + .endif 116 116 + .endm 117 117 + 118 118 + // The x^0..x^63 terms, i.e. poly128 mod x^64, i.e. the physically low qword for 119 119 + // msb-first order or the physically high qword for lsb-first order 120 120 + #define LO64_TERMS 0 121 121 + 122 122 + // The x^64..x^127 terms, i.e. floor(poly128 / x^64), i.e. the physically high 123 123 + // qword for msb-first order or the physically low qword for lsb-first order 124 124 + #define HI64_TERMS 1 125 125 + 126 126 + // Multiply the given \src1_terms of each 128-bit lane of \src1 by the given 127 127 + // \src2_terms of each 128-bit lane of \src2, and write the result(s) to \dst. 128 128 + .macro _pclmulqdq src1, src1_terms, src2, src2_terms, dst 129 129 + _cond_vex "pclmulqdq $((\src1_terms ^ LSB_CRC) << 4) ^ (\src2_terms ^ LSB_CRC),", \ 130 130 + \src1, \src2, \dst 131 131 + .endm 132 132 + 133 133 + // Fold \acc into \data and store the result back into \acc. \data can be an 134 134 + // unaligned mem operand if using VEX is allowed and the CRC is lsb-first so no 135 135 + // byte-reflection is needed; otherwise it must be a vector register. \consts 136 136 + // is a vector register containing the needed fold constants, and \tmp is a 137 137 + // temporary vector register. All arguments must be the same length. 138 138 + .macro _fold_vec acc, data, consts, tmp 139 139 + _pclmulqdq \consts, HI64_TERMS, \acc, HI64_TERMS, \tmp 140 140 + _pclmulqdq \consts, LO64_TERMS, \acc, LO64_TERMS, \acc 141 141 + .if AVX_LEVEL <= 2 142 142 + _cond_vex pxor, \data, \tmp, \tmp 143 143 + _cond_vex pxor, \tmp, \acc, \acc 144 144 + .else 145 145 + vpternlogq $0x96, \data, \tmp, \acc 146 146 + .endif 147 147 + .endm 148 148 + 149 149 + // Fold \acc into \data and store the result back into \acc. \data is an 150 150 + // unaligned mem operand, \consts is a vector register containing the needed 151 151 + // fold constants, \bswap_mask is a vector register containing the 152 152 + // byte-reflection table if the CRC is msb-first, and \tmp1 and \tmp2 are 153 153 + // temporary vector registers. All arguments must have length \vl. 154 154 + .macro _fold_vec_mem vl, acc, data, consts, bswap_mask, tmp1, tmp2 155 155 + .if AVX_LEVEL == 0 || !LSB_CRC 156 156 + _load_data \vl, \data, \bswap_mask, \tmp1 157 157 + _fold_vec \acc, \tmp1, \consts, \tmp2 158 158 + .else 159 159 + _fold_vec \acc, \data, \consts, \tmp1 160 160 + .endif 161 161 + .endm 162 162 + 163 163 + // Load the constants for folding across 2**i vectors of length VL at a time 164 164 + // into all 128-bit lanes of the vector register CONSTS. 165 165 + .macro _load_vec_folding_consts i 166 166 + _vbroadcast OFFSETOF_FOLD_ACROSS_128_BITS_CONSTS+(4-LOG2_VL-\i)*16(CONSTS_PTR), \ 167 167 + CONSTS 168 168 + .endm 169 169 + 170 170 + // Given vector registers \v0 and \v1 of length \vl, fold \v0 into \v1 and store 171 171 + // the result back into \v0. If the remaining length mod \vl is nonzero, also 172 172 + // fold \vl data bytes from BUF. For both operations the fold distance is \vl. 173 173 + // \consts must be a register of length \vl containing the fold constants. 174 174 + .macro _fold_vec_final vl, v0, v1, consts, bswap_mask, tmp1, tmp2 175 175 + _fold_vec \v0, \v1, \consts, \tmp1 176 176 + test $\vl, LEN8 177 177 + jz .Lfold_vec_final_done\@ 178 178 + _fold_vec_mem \vl, \v0, (BUF), \consts, \bswap_mask, \tmp1, \tmp2 179 179 + add $\vl, BUF 180 180 + .Lfold_vec_final_done\@: 181 181 + .endm 182 182 + 183 183 + // This macro generates the body of a CRC function with the following prototype: 184 184 + // 185 185 + // crc_t crc_func(crc_t crc, const u8 *buf, size_t len, const void *consts); 186 186 + // 187 187 + // |crc| is the initial CRC, and crc_t is a data type wide enough to hold it. 188 188 + // |buf| is the data to checksum. |len| is the data length in bytes, which must 189 189 + // be at least 16. |consts| is a pointer to the fold_across_128_bits_consts 190 190 + // field of the constants struct that was generated for the chosen CRC variant. 191 191 + // 192 192 + // Moving onto the macro parameters, \n is the number of bits in the CRC, e.g. 193 193 + // 32 for a CRC-32. Currently the supported values are 8, 16, 32, and 64. If 194 194 + // the file is compiled in i386 mode, then the maximum supported value is 32. 195 195 + // 196 196 + // \lsb_crc is 1 if the CRC processes the least significant bit of each byte 197 197 + // first, i.e. maps bit0 to x^7, bit1 to x^6, ..., bit7 to x^0. \lsb_crc is 0 198 198 + // if the CRC processes the most significant bit of each byte first, i.e. maps 199 199 + // bit0 to x^0, bit1 to x^1, bit7 to x^7. 200 200 + // 201 201 + // \vl is the maximum length of vector register to use in bytes: 16, 32, or 64. 202 202 + // 203 203 + // \avx_level is the level of AVX support to use: 0 for SSE only, 2 for AVX2, or 204 204 + // 512 for AVX512. 205 205 + // 206 206 + // If \vl == 16 && \avx_level == 0, the generated code requires: 207 207 + // PCLMULQDQ && SSE4.1. (Note: all known CPUs with PCLMULQDQ also have SSE4.1.) 208 208 + // 209 209 + // If \vl == 32 && \avx_level == 2, the generated code requires: 210 210 + // VPCLMULQDQ && AVX2. 211 211 + // 212 212 + // If \vl == 64 && \avx_level == 512, the generated code requires: 213 213 + // VPCLMULQDQ && AVX512BW && AVX512VL. 214 214 + // 215 215 + // Other \vl and \avx_level combinations are either not supported or not useful. 216 216 + .macro _crc_pclmul n, lsb_crc, vl, avx_level 217 217 + .set LSB_CRC, \lsb_crc 218 218 + .set VL, \vl 219 219 + .set AVX_LEVEL, \avx_level 220 220 + 221 221 + // Define aliases for the xmm, ymm, or zmm registers according to VL. 222 222 + .irp i, 0,1,2,3,4,5,6,7 223 223 + .if VL == 16 224 224 + .set V\i, %xmm\i 225 225 + .set LOG2_VL, 4 226 226 + .elseif VL == 32 227 227 + .set V\i, %ymm\i 228 228 + .set LOG2_VL, 5 229 229 + .elseif VL == 64 230 230 + .set V\i, %zmm\i 231 231 + .set LOG2_VL, 6 232 232 + .else 233 233 + .error "Unsupported vector length" 234 234 + .endif 235 235 + .endr 236 236 + // Define aliases for the function parameters. 237 237 + // Note: when crc_t is shorter than u32, zero-extension to 32 bits is 238 238 + // guaranteed by the ABI. Zero-extension to 64 bits is *not* guaranteed 239 239 + // when crc_t is shorter than u64. 240 240 + #ifdef __x86_64__ 241 241 + .if \n <= 32 242 242 + .set CRC, %edi 243 243 + .else 244 244 + .set CRC, %rdi 245 245 + .endif 246 246 + .set BUF, %rsi 247 247 + .set LEN, %rdx 248 248 + .set LEN32, %edx 249 249 + .set LEN8, %dl 250 250 + .set CONSTS_PTR, %rcx 251 251 + #else 252 252 + // 32-bit support, assuming -mregparm=3 and not including support for 253 253 + // CRC-64 (which would use both eax and edx to pass the crc parameter). 254 254 + .set CRC, %eax 255 255 + .set BUF, %edx 256 256 + .set LEN, %ecx 257 257 + .set LEN32, %ecx 258 258 + .set LEN8, %cl 259 259 + .set CONSTS_PTR, %ebx // Passed on stack 260 260 + #endif 261 261 + 262 262 + // Define aliases for some local variables. V0-V5 are used without 263 263 + // aliases (for accumulators, data, temporary values, etc). Staying 264 264 + // within the first 8 vector registers keeps the code 32-bit SSE 265 265 + // compatible and reduces the size of 64-bit SSE code slightly. 266 266 + .set BSWAP_MASK, V6 267 267 + .set BSWAP_MASK_YMM, %ymm6 268 268 + .set BSWAP_MASK_XMM, %xmm6 269 269 + .set CONSTS, V7 270 270 + .set CONSTS_YMM, %ymm7 271 271 + .set CONSTS_XMM, %xmm7 272 272 + 273 273 + // Use ANNOTATE_NOENDBR to suppress an objtool warning, since the 274 274 + // functions generated by this macro are called only by static_call. 275 275 + ANNOTATE_NOENDBR 276 276 + 277 277 + #ifdef __i386__ 278 278 + push CONSTS_PTR 279 279 + mov 8(%esp), CONSTS_PTR 280 280 + #endif 281 281 + 282 282 + // Create a 128-bit vector that contains the initial CRC in the end 283 283 + // representing the high-order polynomial coefficients, and the rest 0. 284 284 + // If the CRC is msb-first, also load the byte-reflection table. 285 285 + .if \n <= 32 286 286 + _cond_vex movd, CRC, %xmm0 287 287 + .else 288 288 + _cond_vex movq, CRC, %xmm0 289 289 + .endif 290 290 + .if !LSB_CRC 291 291 + _cond_vex pslldq, $(128-\n)/8, %xmm0, %xmm0 292 292 + _vbroadcast OFFSETOF_BSWAP_MASK(CONSTS_PTR), BSWAP_MASK 293 293 + .endif 294 294 + 295 295 + // Load the first vector of data and XOR the initial CRC into the 296 296 + // appropriate end of the first 128-bit lane of data. If LEN < VL, then 297 297 + // use a short vector and jump ahead to the final reduction. (LEN >= 16 298 298 + // is guaranteed here but not necessarily LEN >= VL.) 299 299 + .if VL >= 32 300 300 + cmp $VL, LEN 301 301 + jae .Lat_least_1vec\@ 302 302 + .if VL == 64 303 303 + cmp $32, LEN32 304 304 + jb .Lless_than_32bytes\@ 305 305 + _prepare_v0 32, %ymm0, %ymm1, BSWAP_MASK_YMM 306 306 + add $32, BUF 307 307 + jmp .Lreduce_256bits_to_128bits\@ 308 308 + .Lless_than_32bytes\@: 309 309 + .endif 310 310 + _prepare_v0 16, %xmm0, %xmm1, BSWAP_MASK_XMM 311 311 + add $16, BUF 312 312 + vmovdqa OFFSETOF_FOLD_ACROSS_128_BITS_CONSTS(CONSTS_PTR), CONSTS_XMM 313 313 + jmp .Lcheck_for_partial_block\@ 314 314 + .Lat_least_1vec\@: 315 315 + .endif 316 316 + _prepare_v0 VL, V0, V1, BSWAP_MASK 317 317 + 318 318 + // Handle VL <= LEN < 4*VL. 319 319 + cmp $4*VL-1, LEN 320 320 + ja .Lat_least_4vecs\@ 321 321 + add $VL, BUF 322 322 + // If VL <= LEN < 2*VL, then jump ahead to the reduction from 1 vector. 323 323 + // If VL==16 then load fold_across_128_bits_consts first, as the final 324 324 + // reduction depends on it and it won't be loaded anywhere else. 325 325 + cmp $2*VL-1, LEN32 326 326 + .if VL == 16 327 327 + _cond_vex movdqa, OFFSETOF_FOLD_ACROSS_128_BITS_CONSTS(CONSTS_PTR), CONSTS_XMM 328 328 + .endif 329 329 + jbe .Lreduce_1vec_to_128bits\@ 330 330 + // Otherwise 2*VL <= LEN < 4*VL. Load one more vector and jump ahead to 331 331 + // the reduction from 2 vectors. 332 332 + _load_data VL, (BUF), BSWAP_MASK, V1 333 333 + add $VL, BUF 334 334 + jmp .Lreduce_2vecs_to_1\@ 335 335 + 336 336 + .Lat_least_4vecs\@: 337 337 + // Load 3 more vectors of data. 338 338 + _load_data VL, 1*VL(BUF), BSWAP_MASK, V1 339 339 + _load_data VL, 2*VL(BUF), BSWAP_MASK, V2 340 340 + _load_data VL, 3*VL(BUF), BSWAP_MASK, V3 341 341 + sub $-4*VL, BUF // Shorter than 'add 4*VL' when VL=32 342 342 + add $-4*VL, LEN // Shorter than 'sub 4*VL' when VL=32 343 343 + 344 344 + // Main loop: while LEN >= 4*VL, fold the 4 vectors V0-V3 into the next 345 345 + // 4 vectors of data and write the result back to V0-V3. 346 346 + cmp $4*VL-1, LEN // Shorter than 'cmp 4*VL' when VL=32 347 347 + jbe .Lreduce_4vecs_to_2\@ 348 348 + _load_vec_folding_consts 2 349 349 + .Lfold_4vecs_loop\@: 350 350 + _fold_vec_mem VL, V0, 0*VL(BUF), CONSTS, BSWAP_MASK, V4, V5 351 351 + _fold_vec_mem VL, V1, 1*VL(BUF), CONSTS, BSWAP_MASK, V4, V5 352 352 + _fold_vec_mem VL, V2, 2*VL(BUF), CONSTS, BSWAP_MASK, V4, V5 353 353 + _fold_vec_mem VL, V3, 3*VL(BUF), CONSTS, BSWAP_MASK, V4, V5 354 354 + sub $-4*VL, BUF 355 355 + add $-4*VL, LEN 356 356 + cmp $4*VL-1, LEN 357 357 + ja .Lfold_4vecs_loop\@ 358 358 + 359 359 + // Fold V0,V1 into V2,V3 and write the result back to V0,V1. Then fold 360 360 + // two more vectors of data from BUF, if at least that much remains. 361 361 + .Lreduce_4vecs_to_2\@: 362 362 + _load_vec_folding_consts 1 363 363 + _fold_vec V0, V2, CONSTS, V4 364 364 + _fold_vec V1, V3, CONSTS, V4 365 365 + test $2*VL, LEN8 366 366 + jz .Lreduce_2vecs_to_1\@ 367 367 + _fold_vec_mem VL, V0, 0*VL(BUF), CONSTS, BSWAP_MASK, V4, V5 368 368 + _fold_vec_mem VL, V1, 1*VL(BUF), CONSTS, BSWAP_MASK, V4, V5 369 369 + sub $-2*VL, BUF 370 370 + 371 371 + // Fold V0 into V1 and write the result back to V0. Then fold one more 372 372 + // vector of data from BUF, if at least that much remains. 373 373 + .Lreduce_2vecs_to_1\@: 374 374 + _load_vec_folding_consts 0 375 375 + _fold_vec_final VL, V0, V1, CONSTS, BSWAP_MASK, V4, V5 376 376 + 377 377 + .Lreduce_1vec_to_128bits\@: 378 378 + .if VL == 64 379 379 + // Reduce 512-bit %zmm0 to 256-bit %ymm0. Then fold 256 more bits of 380 380 + // data from BUF, if at least that much remains. 381 381 + vbroadcasti128 OFFSETOF_FOLD_ACROSS_256_BITS_CONSTS(CONSTS_PTR), CONSTS_YMM 382 382 + vextracti64x4 $1, %zmm0, %ymm1 383 383 + _fold_vec_final 32, %ymm0, %ymm1, CONSTS_YMM, BSWAP_MASK_YMM, %ymm4, %ymm5 384 384 + .Lreduce_256bits_to_128bits\@: 385 385 + .endif 386 386 + .if VL >= 32 387 387 + // Reduce 256-bit %ymm0 to 128-bit %xmm0. Then fold 128 more bits of 388 388 + // data from BUF, if at least that much remains. 389 389 + vmovdqa OFFSETOF_FOLD_ACROSS_128_BITS_CONSTS(CONSTS_PTR), CONSTS_XMM 390 390 + vextracti128 $1, %ymm0, %xmm1 391 391 + _fold_vec_final 16, %xmm0, %xmm1, CONSTS_XMM, BSWAP_MASK_XMM, %xmm4, %xmm5 392 392 + .Lcheck_for_partial_block\@: 393 393 + .endif 394 394 + and $15, LEN32 395 395 + jz .Lreduce_128bits_to_crc\@ 396 396 + 397 397 + // 1 <= LEN <= 15 data bytes remain in BUF. The polynomial is now 398 398 + // A*(x^(8*LEN)) + B, where A is the 128-bit polynomial stored in %xmm0 399 399 + // and B is the polynomial of the remaining LEN data bytes. To reduce 400 400 + // this to 128 bits without needing fold constants for each possible 401 401 + // LEN, rearrange this expression into C1*(x^128) + C2, where 402 402 + // C1 = floor(A / x^(128 - 8*LEN)) and C2 = A*x^(8*LEN) + B mod x^128. 403 403 + // Then fold C1 into C2, which is just another fold across 128 bits. 404 404 + 405 405 + .if !LSB_CRC || AVX_LEVEL == 0 406 406 + // Load the last 16 data bytes. Note that originally LEN was >= 16. 407 407 + _load_data 16, "-16(BUF,LEN)", BSWAP_MASK_XMM, %xmm2 408 408 + .endif // Else will use vpblendvb mem operand later. 409 409 + .if !LSB_CRC 410 410 + neg LEN // Needed for indexing shuf_table 411 411 + .endif 412 412 + 413 413 + // tmp = A*x^(8*LEN) mod x^128 414 414 + // lsb: pshufb by [LEN, LEN+1, ..., 15, -1, -1, ..., -1] 415 415 + // i.e. right-shift by LEN bytes. 416 416 + // msb: pshufb by [-1, -1, ..., -1, 0, 1, ..., 15-LEN] 417 417 + // i.e. left-shift by LEN bytes. 418 418 + _cond_vex movdqu, "OFFSETOF_SHUF_TABLE+16(CONSTS_PTR,LEN)", %xmm3 419 419 + _cond_vex pshufb, %xmm3, %xmm0, %xmm1 420 420 + 421 421 + // C1 = floor(A / x^(128 - 8*LEN)) 422 422 + // lsb: pshufb by [-1, -1, ..., -1, 0, 1, ..., LEN-1] 423 423 + // i.e. left-shift by 16-LEN bytes. 424 424 + // msb: pshufb by [16-LEN, 16-LEN+1, ..., 15, -1, -1, ..., -1] 425 425 + // i.e. right-shift by 16-LEN bytes. 426 426 + _cond_vex pshufb, "OFFSETOF_SHUF_TABLE+32*!LSB_CRC(CONSTS_PTR,LEN)", \ 427 427 + %xmm0, %xmm0, unaligned_mem_tmp=%xmm4 428 428 + 429 429 + // C2 = tmp + B. This is just a blend of tmp with the last 16 data 430 430 + // bytes (reflected if msb-first). The blend mask is the shuffle table 431 431 + // that was used to create tmp. 0 selects tmp, and 1 last16databytes. 432 432 + .if AVX_LEVEL == 0 433 433 + movdqa %xmm0, %xmm4 434 434 + movdqa %xmm3, %xmm0 435 435 + pblendvb %xmm2, %xmm1 // uses %xmm0 as implicit operand 436 436 + movdqa %xmm4, %xmm0 437 437 + .elseif LSB_CRC 438 438 + vpblendvb %xmm3, -16(BUF,LEN), %xmm1, %xmm1 439 439 + .else 440 440 + vpblendvb %xmm3, %xmm2, %xmm1, %xmm1 441 441 + .endif 442 442 + 443 443 + // Fold C1 into C2 and store the 128-bit result in %xmm0. 444 444 + _fold_vec %xmm0, %xmm1, CONSTS_XMM, %xmm4 445 445 + 446 446 + .Lreduce_128bits_to_crc\@: 447 447 + // Compute the CRC as %xmm0 * x^n mod G. Here %xmm0 means the 128-bit 448 448 + // polynomial stored in %xmm0 (using either lsb-first or msb-first bit 449 449 + // order according to LSB_CRC), and G is the CRC's generator polynomial. 450 450 + 451 451 + // First, multiply %xmm0 by x^n and reduce the result to 64+n bits: 452 452 + // 453 453 + // t0 := (x^(64+n) mod G) * floor(%xmm0 / x^64) + 454 454 + // x^n * (%xmm0 mod x^64) 455 455 + // 456 456 + // Store t0 * x^(64-n) in %xmm0. I.e., actually do: 457 457 + // 458 458 + // %xmm0 := ((x^(64+n) mod G) * x^(64-n)) * floor(%xmm0 / x^64) + 459 459 + // x^64 * (%xmm0 mod x^64) 460 460 + // 461 461 + // The extra unreduced factor of x^(64-n) makes floor(t0 / x^n) aligned 462 462 + // to the HI64_TERMS of %xmm0 so that the next pclmulqdq can easily 463 463 + // select it. The 64-bit constant (x^(64+n) mod G) * x^(64-n) in the 464 464 + // msb-first case, or (x^(63+n) mod G) * x^(64-n) in the lsb-first case 465 465 + // (considering the extra factor of x that gets implicitly introduced by 466 466 + // each pclmulqdq when using lsb-first order), is identical to the 467 467 + // constant that was used earlier for folding the LO64_TERMS across 128 468 468 + // bits. Thus it's already available in LO64_TERMS of CONSTS_XMM. 469 469 + _pclmulqdq CONSTS_XMM, LO64_TERMS, %xmm0, HI64_TERMS, %xmm1 470 470 + .if LSB_CRC 471 471 + _cond_vex psrldq, $8, %xmm0, %xmm0 // x^64 * (%xmm0 mod x^64) 472 472 + .else 473 473 + _cond_vex pslldq, $8, %xmm0, %xmm0 // x^64 * (%xmm0 mod x^64) 474 474 + .endif 475 475 + _cond_vex pxor, %xmm1, %xmm0, %xmm0 476 476 + // The HI64_TERMS of %xmm0 now contain floor(t0 / x^n). 477 477 + // The LO64_TERMS of %xmm0 now contain (t0 mod x^n) * x^(64-n). 478 478 + 479 479 + // First step of Barrett reduction: Compute floor(t0 / G). This is the 480 480 + // polynomial by which G needs to be multiplied to cancel out the x^n 481 481 + // and higher terms of t0, i.e. to reduce t0 mod G. First do: 482 482 + // 483 483 + // t1 := floor(x^(63+n) / G) * x * floor(t0 / x^n) 484 484 + // 485 485 + // Then the desired value floor(t0 / G) is floor(t1 / x^64). The 63 in 486 486 + // x^(63+n) is the maximum degree of floor(t0 / x^n) and thus the lowest 487 487 + // value that makes enough precision be carried through the calculation. 488 488 + // 489 489 + // The '* x' makes it so the result is floor(t1 / x^64) rather than 490 490 + // floor(t1 / x^63), making it qword-aligned in HI64_TERMS so that it 491 491 + // can be extracted much more easily in the next step. In the lsb-first 492 492 + // case the '* x' happens implicitly. In the msb-first case it must be 493 493 + // done explicitly; floor(x^(63+n) / G) * x is a 65-bit constant, so the 494 494 + // constant passed to pclmulqdq is (floor(x^(63+n) / G) * x) - x^64, and 495 495 + // the multiplication by the x^64 term is handled using a pxor. The 496 496 + // pxor causes the low 64 terms of t1 to be wrong, but they are unused. 497 497 + _cond_vex movdqa, OFFSETOF_BARRETT_REDUCTION_CONSTS(CONSTS_PTR), CONSTS_XMM 498 498 + _pclmulqdq CONSTS_XMM, HI64_TERMS, %xmm0, HI64_TERMS, %xmm1 499 499 + .if !LSB_CRC 500 500 + _cond_vex pxor, %xmm0, %xmm1, %xmm1 // += x^64 * floor(t0 / x^n) 501 501 + .endif 502 502 + // The HI64_TERMS of %xmm1 now contain floor(t1 / x^64) = floor(t0 / G). 503 503 + 504 504 + // Second step of Barrett reduction: Cancel out the x^n and higher terms 505 505 + // of t0 by subtracting the needed multiple of G. This gives the CRC: 506 506 + // 507 507 + // crc := t0 - (G * floor(t0 / G)) 508 508 + // 509 509 + // But %xmm0 contains t0 * x^(64-n), so it's more convenient to do: 510 510 + // 511 511 + // crc := ((t0 * x^(64-n)) - ((G * x^(64-n)) * floor(t0 / G))) / x^(64-n) 512 512 + // 513 513 + // Furthermore, since the resulting CRC is n-bit, if mod x^n is 514 514 + // explicitly applied to it then the x^n term of G makes no difference 515 515 + // in the result and can be omitted. This helps keep the constant 516 516 + // multiplier in 64 bits in most cases. This gives the following: 517 517 + // 518 518 + // %xmm0 := %xmm0 - (((G - x^n) * x^(64-n)) * floor(t0 / G)) 519 519 + // crc := (%xmm0 / x^(64-n)) mod x^n 520 520 + // 521 521 + // In the lsb-first case, each pclmulqdq implicitly introduces 522 522 + // an extra factor of x, so in that case the constant that needs to be 523 523 + // passed to pclmulqdq is actually '(G - x^n) * x^(63-n)' when n <= 63. 524 524 + // For lsb-first CRCs where n=64, the extra factor of x cannot be as 525 525 + // easily avoided. In that case, instead pass '(G - x^n - x^0) / x' to 526 526 + // pclmulqdq and handle the x^0 term (i.e. 1) separately. (All CRC 527 527 + // polynomials have nonzero x^n and x^0 terms.) It works out as: the 528 528 + // CRC has be XORed with the physically low qword of %xmm1, representing 529 529 + // floor(t0 / G). The most efficient way to do that is to move it to 530 530 + // the physically high qword and use a ternlog to combine the two XORs. 531 531 + .if LSB_CRC && \n == 64 532 532 + _cond_vex punpcklqdq, %xmm1, %xmm2, %xmm2 533 533 + _pclmulqdq CONSTS_XMM, LO64_TERMS, %xmm1, HI64_TERMS, %xmm1 534 534 + .if AVX_LEVEL <= 2 535 535 + _cond_vex pxor, %xmm2, %xmm0, %xmm0 536 536 + _cond_vex pxor, %xmm1, %xmm0, %xmm0 537 537 + .else 538 538 + vpternlogq $0x96, %xmm2, %xmm1, %xmm0 539 539 + .endif 540 540 + _cond_vex "pextrq $1,", %xmm0, %rax // (%xmm0 / x^0) mod x^64 541 541 + .else 542 542 + _pclmulqdq CONSTS_XMM, LO64_TERMS, %xmm1, HI64_TERMS, %xmm1 543 543 + _cond_vex pxor, %xmm1, %xmm0, %xmm0 544 544 + .if \n == 8 545 545 + _cond_vex "pextrb $7 + LSB_CRC,", %xmm0, %eax // (%xmm0 / x^56) mod x^8 546 546 + .elseif \n == 16 547 547 + _cond_vex "pextrw $3 + LSB_CRC,", %xmm0, %eax // (%xmm0 / x^48) mod x^16 548 548 + .elseif \n == 32 549 549 + _cond_vex "pextrd $1 + LSB_CRC,", %xmm0, %eax // (%xmm0 / x^32) mod x^32 550 550 + .else // \n == 64 && !LSB_CRC 551 551 + _cond_vex movq, %xmm0, %rax // (%xmm0 / x^0) mod x^64 552 552 + .endif 553 553 + .endif 554 554 + 555 555 + .if VL > 16 556 556 + vzeroupper // Needed when ymm or zmm registers may have been used. 557 557 + .endif 558 558 + #ifdef __i386__ 559 559 + pop CONSTS_PTR 560 560 + #endif 561 561 + RET 562 562 + .endm 563 563 + 564 564 + #define DEFINE_CRC_PCLMUL_FUNCS(prefix, bits, lsb) \ 565 565 + SYM_FUNC_START(prefix##_pclmul_sse); \ 566 566 + _crc_pclmul n=bits, lsb_crc=lsb, vl=16, avx_level=0; \ 567 567 + SYM_FUNC_END(prefix##_pclmul_sse); \ 568 568 + \ 569 569 + SYM_FUNC_START(prefix##_vpclmul_avx2); \ 570 570 + _crc_pclmul n=bits, lsb_crc=lsb, vl=32, avx_level=2; \ 571 571 + SYM_FUNC_END(prefix##_vpclmul_avx2); \ 572 572 + \ 573 573 + SYM_FUNC_START(prefix##_vpclmul_avx512); \ 574 574 + _crc_pclmul n=bits, lsb_crc=lsb, vl=64, avx_level=512; \ 575 575 + SYM_FUNC_END(prefix##_vpclmul_avx512);

+72

lib/crc/x86/crc-pclmul-template.h

reviewed

··· 1 1 + /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 2 + /* 3 3 + * Macros for accessing the [V]PCLMULQDQ-based CRC functions that are 4 4 + * instantiated by crc-pclmul-template.S 5 5 + * 6 6 + * Copyright 2025 Google LLC 7 7 + * 8 8 + * Author: Eric Biggers <ebiggers@google.com> 9 9 + */ 10 10 + #ifndef _CRC_PCLMUL_TEMPLATE_H 11 11 + #define _CRC_PCLMUL_TEMPLATE_H 12 12 + 13 13 + #include <asm/cpufeatures.h> 14 14 + #include <asm/simd.h> 15 15 + #include <crypto/internal/simd.h> 16 16 + #include <linux/static_call.h> 17 17 + #include "crc-pclmul-consts.h" 18 18 + 19 19 + #define DECLARE_CRC_PCLMUL_FUNCS(prefix, crc_t) \ 20 20 + crc_t prefix##_pclmul_sse(crc_t crc, const u8 *p, size_t len, \ 21 21 + const void *consts_ptr); \ 22 22 + crc_t prefix##_vpclmul_avx2(crc_t crc, const u8 *p, size_t len, \ 23 23 + const void *consts_ptr); \ 24 24 + crc_t prefix##_vpclmul_avx512(crc_t crc, const u8 *p, size_t len, \ 25 25 + const void *consts_ptr); \ 26 26 + DEFINE_STATIC_CALL(prefix##_pclmul, prefix##_pclmul_sse) 27 27 + 28 28 + static inline bool have_vpclmul(void) 29 29 + { 30 30 + return boot_cpu_has(X86_FEATURE_VPCLMULQDQ) && 31 31 + boot_cpu_has(X86_FEATURE_AVX2) && 32 32 + cpu_has_xfeatures(XFEATURE_MASK_YMM, NULL); 33 33 + } 34 34 + 35 35 + static inline bool have_avx512(void) 36 36 + { 37 37 + return boot_cpu_has(X86_FEATURE_AVX512BW) && 38 38 + boot_cpu_has(X86_FEATURE_AVX512VL) && 39 39 + !boot_cpu_has(X86_FEATURE_PREFER_YMM) && 40 40 + cpu_has_xfeatures(XFEATURE_MASK_AVX512, NULL); 41 41 + } 42 42 + 43 43 + /* 44 44 + * Call a [V]PCLMULQDQ optimized CRC function if the data length is at least 16 45 45 + * bytes, the CPU has PCLMULQDQ support, and the current context may use SIMD. 46 46 + * 47 47 + * 16 bytes is the minimum length supported by the [V]PCLMULQDQ functions. 48 48 + * There is overhead associated with kernel_fpu_begin() and kernel_fpu_end(), 49 49 + * varying by CPU and factors such as which parts of the "FPU" state userspace 50 50 + * has touched, which could result in a larger cutoff being better. Indeed, a 51 51 + * larger cutoff is usually better for a *single* message. However, the 52 52 + * overhead of the FPU section gets amortized if multiple FPU sections get 53 53 + * executed before returning to userspace, since the XSAVE and XRSTOR occur only 54 54 + * once. Considering that and the fact that the [V]PCLMULQDQ code is lighter on 55 55 + * the dcache than the table-based code is, a 16-byte cutoff seems to work well. 56 56 + */ 57 57 + #define CRC_PCLMUL(crc, p, len, prefix, consts, have_pclmulqdq) \ 58 58 + do { \ 59 59 + if ((len) >= 16 && static_branch_likely(&(have_pclmulqdq)) && \ 60 60 + crypto_simd_usable()) { \ 61 61 + const void *consts_ptr; \ 62 62 + \ 63 63 + consts_ptr = (consts).fold_across_128_bits_consts; \ 64 64 + kernel_fpu_begin(); \ 65 65 + crc = static_call(prefix##_pclmul)((crc), (p), (len), \ 66 66 + consts_ptr); \ 67 67 + kernel_fpu_end(); \ 68 68 + return crc; \ 69 69 + } \ 70 70 + } while (0) 71 71 + 72 72 + #endif /* _CRC_PCLMUL_TEMPLATE_H */

+35

lib/crc/x86/crc-t10dif.h

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0-or-later 2 2 + /* 3 3 + * CRC-T10DIF using [V]PCLMULQDQ instructions 4 4 + * 5 5 + * Copyright 2024 Google LLC 6 6 + */ 7 7 + 8 8 + #include "crc-pclmul-template.h" 9 9 + 10 10 + static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pclmulqdq); 11 11 + 12 12 + DECLARE_CRC_PCLMUL_FUNCS(crc16_msb, u16); 13 13 + 14 14 + static inline u16 crc_t10dif_arch(u16 crc, const u8 *p, size_t len) 15 15 + { 16 16 + CRC_PCLMUL(crc, p, len, crc16_msb, crc16_msb_0x8bb7_consts, 17 17 + have_pclmulqdq); 18 18 + return crc_t10dif_generic(crc, p, len); 19 19 + } 20 20 + 21 21 + #define crc_t10dif_mod_init_arch crc_t10dif_mod_init_arch 22 22 + static inline void crc_t10dif_mod_init_arch(void) 23 23 + { 24 24 + if (boot_cpu_has(X86_FEATURE_PCLMULQDQ)) { 25 25 + static_branch_enable(&have_pclmulqdq); 26 26 + if (have_vpclmul()) { 27 27 + if (have_avx512()) 28 28 + static_call_update(crc16_msb_pclmul, 29 29 + crc16_msb_vpclmul_avx512); 30 30 + else 31 31 + static_call_update(crc16_msb_pclmul, 32 32 + crc16_msb_vpclmul_avx2); 33 33 + } 34 34 + } 35 35 + }

+137

lib/crc/x86/crc32.h

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0-only 2 2 + /* 3 3 + * x86-optimized CRC32 functions 4 4 + * 5 5 + * Copyright (C) 2008 Intel Corporation 6 6 + * Copyright 2012 Xyratex Technology Limited 7 7 + * Copyright 2024 Google LLC 8 8 + */ 9 9 + 10 10 + #include "crc-pclmul-template.h" 11 11 + 12 12 + static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_crc32); 13 13 + static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pclmulqdq); 14 14 + static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_vpclmul_avx512); 15 15 + 16 16 + DECLARE_CRC_PCLMUL_FUNCS(crc32_lsb, u32); 17 17 + 18 18 + static inline u32 crc32_le_arch(u32 crc, const u8 *p, size_t len) 19 19 + { 20 20 + CRC_PCLMUL(crc, p, len, crc32_lsb, crc32_lsb_0xedb88320_consts, 21 21 + have_pclmulqdq); 22 22 + return crc32_le_base(crc, p, len); 23 23 + } 24 24 + 25 25 + #ifdef CONFIG_X86_64 26 26 + #define CRC32_INST "crc32q %1, %q0" 27 27 + #else 28 28 + #define CRC32_INST "crc32l %1, %0" 29 29 + #endif 30 30 + 31 31 + /* 32 32 + * Use carryless multiply version of crc32c when buffer size is >= 512 to 33 33 + * account for FPU state save/restore overhead. 34 34 + */ 35 35 + #define CRC32C_PCLMUL_BREAKEVEN 512 36 36 + 37 37 + asmlinkage u32 crc32c_x86_3way(u32 crc, const u8 *buffer, size_t len); 38 38 + 39 39 + static inline u32 crc32c_arch(u32 crc, const u8 *p, size_t len) 40 40 + { 41 41 + size_t num_longs; 42 42 + 43 43 + if (!static_branch_likely(&have_crc32)) 44 44 + return crc32c_base(crc, p, len); 45 45 + 46 46 + if (IS_ENABLED(CONFIG_X86_64) && len >= CRC32C_PCLMUL_BREAKEVEN && 47 47 + static_branch_likely(&have_pclmulqdq) && crypto_simd_usable()) { 48 48 + /* 49 49 + * Long length, the vector registers are usable, and the CPU is 50 50 + * 64-bit and supports both CRC32 and PCLMULQDQ instructions. 51 51 + * It is worthwhile to divide the data into multiple streams, 52 52 + * CRC them independently, and combine them using PCLMULQDQ. 53 53 + * crc32c_x86_3way() does this using 3 streams, which is the 54 54 + * most that x86_64 CPUs have traditionally been capable of. 55 55 + * 56 56 + * However, due to improved VPCLMULQDQ performance on newer 57 57 + * CPUs, use crc32_lsb_vpclmul_avx512() instead of 58 58 + * crc32c_x86_3way() when the CPU supports VPCLMULQDQ and has a 59 59 + * "good" implementation of AVX-512. 60 60 + * 61 61 + * Future work: the optimal strategy on Zen 3--5 is actually to 62 62 + * use both crc32q and VPCLMULQDQ in parallel. Unfortunately, 63 63 + * different numbers of streams and vector lengths are optimal 64 64 + * on each CPU microarchitecture, making it challenging to take 65 65 + * advantage of this. (Zen 5 even supports 7 parallel crc32q, a 66 66 + * major upgrade.) For now, just choose between 67 67 + * crc32c_x86_3way() and crc32_lsb_vpclmul_avx512(). The latter 68 68 + * is needed anyway for crc32_le(), so we just reuse it here. 69 69 + */ 70 70 + kernel_fpu_begin(); 71 71 + if (static_branch_likely(&have_vpclmul_avx512)) 72 72 + crc = crc32_lsb_vpclmul_avx512(crc, p, len, 73 73 + crc32_lsb_0x82f63b78_consts.fold_across_128_bits_consts); 74 74 + else 75 75 + crc = crc32c_x86_3way(crc, p, len); 76 76 + kernel_fpu_end(); 77 77 + return crc; 78 78 + } 79 79 + 80 80 + /* 81 81 + * Short length, XMM registers unusable, or the CPU is 32-bit; but the 82 82 + * CPU supports CRC32 instructions. Just issue a single stream of CRC32 83 83 + * instructions inline. While this doesn't use the CPU's CRC32 84 84 + * throughput very well, it avoids the need to combine streams. Stream 85 85 + * combination would be inefficient here. 86 86 + */ 87 87 + 88 88 + for (num_longs = len / sizeof(unsigned long); 89 89 + num_longs != 0; num_longs--, p += sizeof(unsigned long)) 90 90 + asm(CRC32_INST : "+r" (crc) : ASM_INPUT_RM (*(unsigned long *)p)); 91 91 + 92 92 + if (sizeof(unsigned long) > 4 && (len & 4)) { 93 93 + asm("crc32l %1, %0" : "+r" (crc) : ASM_INPUT_RM (*(u32 *)p)); 94 94 + p += 4; 95 95 + } 96 96 + if (len & 2) { 97 97 + asm("crc32w %1, %0" : "+r" (crc) : ASM_INPUT_RM (*(u16 *)p)); 98 98 + p += 2; 99 99 + } 100 100 + if (len & 1) 101 101 + asm("crc32b %1, %0" : "+r" (crc) : ASM_INPUT_RM (*p)); 102 102 + 103 103 + return crc; 104 104 + } 105 105 + 106 106 + #define crc32_be_arch crc32_be_base /* not implemented on this arch */ 107 107 + 108 108 + #define crc32_mod_init_arch crc32_mod_init_arch 109 109 + static inline void crc32_mod_init_arch(void) 110 110 + { 111 111 + if (boot_cpu_has(X86_FEATURE_XMM4_2)) 112 112 + static_branch_enable(&have_crc32); 113 113 + if (boot_cpu_has(X86_FEATURE_PCLMULQDQ)) { 114 114 + static_branch_enable(&have_pclmulqdq); 115 115 + if (have_vpclmul()) { 116 116 + if (have_avx512()) { 117 117 + static_call_update(crc32_lsb_pclmul, 118 118 + crc32_lsb_vpclmul_avx512); 119 119 + static_branch_enable(&have_vpclmul_avx512); 120 120 + } else { 121 121 + static_call_update(crc32_lsb_pclmul, 122 122 + crc32_lsb_vpclmul_avx2); 123 123 + } 124 124 + } 125 125 + } 126 126 + } 127 127 + 128 128 + static inline u32 crc32_optimizations_arch(void) 129 129 + { 130 130 + u32 optimizations = 0; 131 131 + 132 132 + if (static_key_enabled(&have_crc32)) 133 133 + optimizations |= CRC32C_OPTIMIZATION; 134 134 + if (static_key_enabled(&have_pclmulqdq)) 135 135 + optimizations |= CRC32_LE_OPTIMIZATION; 136 136 + return optimizations; 137 137 + }

+48

lib/crc/x86/crc64.h

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0-or-later 2 2 + /* 3 3 + * CRC64 using [V]PCLMULQDQ instructions 4 4 + * 5 5 + * Copyright 2025 Google LLC 6 6 + */ 7 7 + 8 8 + #include "crc-pclmul-template.h" 9 9 + 10 10 + static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pclmulqdq); 11 11 + 12 12 + DECLARE_CRC_PCLMUL_FUNCS(crc64_msb, u64); 13 13 + DECLARE_CRC_PCLMUL_FUNCS(crc64_lsb, u64); 14 14 + 15 15 + static inline u64 crc64_be_arch(u64 crc, const u8 *p, size_t len) 16 16 + { 17 17 + CRC_PCLMUL(crc, p, len, crc64_msb, crc64_msb_0x42f0e1eba9ea3693_consts, 18 18 + have_pclmulqdq); 19 19 + return crc64_be_generic(crc, p, len); 20 20 + } 21 21 + 22 22 + static inline u64 crc64_nvme_arch(u64 crc, const u8 *p, size_t len) 23 23 + { 24 24 + CRC_PCLMUL(crc, p, len, crc64_lsb, crc64_lsb_0x9a6c9329ac4bc9b5_consts, 25 25 + have_pclmulqdq); 26 26 + return crc64_nvme_generic(crc, p, len); 27 27 + } 28 28 + 29 29 + #define crc64_mod_init_arch crc64_mod_init_arch 30 30 + static inline void crc64_mod_init_arch(void) 31 31 + { 32 32 + if (boot_cpu_has(X86_FEATURE_PCLMULQDQ)) { 33 33 + static_branch_enable(&have_pclmulqdq); 34 34 + if (have_vpclmul()) { 35 35 + if (have_avx512()) { 36 36 + static_call_update(crc64_msb_pclmul, 37 37 + crc64_msb_vpclmul_avx512); 38 38 + static_call_update(crc64_lsb_pclmul, 39 39 + crc64_lsb_vpclmul_avx512); 40 40 + } else { 41 41 + static_call_update(crc64_msb_pclmul, 42 42 + crc64_msb_vpclmul_avx2); 43 43 + static_call_update(crc64_lsb_pclmul, 44 44 + crc64_lsb_vpclmul_avx2); 45 45 + } 46 46 + } 47 47 + } 48 48 + }

-64

lib/crc16.c

reviewed

··· 1 1 - // SPDX-License-Identifier: GPL-2.0-only 2 2 - /* 3 3 - * crc16.c 4 4 - */ 5 5 - 6 6 - #include <linux/types.h> 7 7 - #include <linux/module.h> 8 8 - #include <linux/crc16.h> 9 9 - 10 10 - /** CRC table for the CRC-16. The poly is 0x8005 (x^16 + x^15 + x^2 + 1) */ 11 11 - static const u16 crc16_table[256] = { 12 12 - 0x0000, 0xC0C1, 0xC181, 0x0140, 0xC301, 0x03C0, 0x0280, 0xC241, 13 13 - 0xC601, 0x06C0, 0x0780, 0xC741, 0x0500, 0xC5C1, 0xC481, 0x0440, 14 14 - 0xCC01, 0x0CC0, 0x0D80, 0xCD41, 0x0F00, 0xCFC1, 0xCE81, 0x0E40, 15 15 - 0x0A00, 0xCAC1, 0xCB81, 0x0B40, 0xC901, 0x09C0, 0x0880, 0xC841, 16 16 - 0xD801, 0x18C0, 0x1980, 0xD941, 0x1B00, 0xDBC1, 0xDA81, 0x1A40, 17 17 - 0x1E00, 0xDEC1, 0xDF81, 0x1F40, 0xDD01, 0x1DC0, 0x1C80, 0xDC41, 18 18 - 0x1400, 0xD4C1, 0xD581, 0x1540, 0xD701, 0x17C0, 0x1680, 0xD641, 19 19 - 0xD201, 0x12C0, 0x1380, 0xD341, 0x1100, 0xD1C1, 0xD081, 0x1040, 20 20 - 0xF001, 0x30C0, 0x3180, 0xF141, 0x3300, 0xF3C1, 0xF281, 0x3240, 21 21 - 0x3600, 0xF6C1, 0xF781, 0x3740, 0xF501, 0x35C0, 0x3480, 0xF441, 22 22 - 0x3C00, 0xFCC1, 0xFD81, 0x3D40, 0xFF01, 0x3FC0, 0x3E80, 0xFE41, 23 23 - 0xFA01, 0x3AC0, 0x3B80, 0xFB41, 0x3900, 0xF9C1, 0xF881, 0x3840, 24 24 - 0x2800, 0xE8C1, 0xE981, 0x2940, 0xEB01, 0x2BC0, 0x2A80, 0xEA41, 25 25 - 0xEE01, 0x2EC0, 0x2F80, 0xEF41, 0x2D00, 0xEDC1, 0xEC81, 0x2C40, 26 26 - 0xE401, 0x24C0, 0x2580, 0xE541, 0x2700, 0xE7C1, 0xE681, 0x2640, 27 27 - 0x2200, 0xE2C1, 0xE381, 0x2340, 0xE101, 0x21C0, 0x2080, 0xE041, 28 28 - 0xA001, 0x60C0, 0x6180, 0xA141, 0x6300, 0xA3C1, 0xA281, 0x6240, 29 29 - 0x6600, 0xA6C1, 0xA781, 0x6740, 0xA501, 0x65C0, 0x6480, 0xA441, 30 30 - 0x6C00, 0xACC1, 0xAD81, 0x6D40, 0xAF01, 0x6FC0, 0x6E80, 0xAE41, 31 31 - 0xAA01, 0x6AC0, 0x6B80, 0xAB41, 0x6900, 0xA9C1, 0xA881, 0x6840, 32 32 - 0x7800, 0xB8C1, 0xB981, 0x7940, 0xBB01, 0x7BC0, 0x7A80, 0xBA41, 33 33 - 0xBE01, 0x7EC0, 0x7F80, 0xBF41, 0x7D00, 0xBDC1, 0xBC81, 0x7C40, 34 34 - 0xB401, 0x74C0, 0x7580, 0xB541, 0x7700, 0xB7C1, 0xB681, 0x7640, 35 35 - 0x7200, 0xB2C1, 0xB381, 0x7340, 0xB101, 0x71C0, 0x7080, 0xB041, 36 36 - 0x5000, 0x90C1, 0x9181, 0x5140, 0x9301, 0x53C0, 0x5280, 0x9241, 37 37 - 0x9601, 0x56C0, 0x5780, 0x9741, 0x5500, 0x95C1, 0x9481, 0x5440, 38 38 - 0x9C01, 0x5CC0, 0x5D80, 0x9D41, 0x5F00, 0x9FC1, 0x9E81, 0x5E40, 39 39 - 0x5A00, 0x9AC1, 0x9B81, 0x5B40, 0x9901, 0x59C0, 0x5880, 0x9841, 40 40 - 0x8801, 0x48C0, 0x4980, 0x8941, 0x4B00, 0x8BC1, 0x8A81, 0x4A40, 41 41 - 0x4E00, 0x8EC1, 0x8F81, 0x4F40, 0x8D01, 0x4DC0, 0x4C80, 0x8C41, 42 42 - 0x4400, 0x84C1, 0x8581, 0x4540, 0x8701, 0x47C0, 0x4680, 0x8641, 43 43 - 0x8201, 0x42C0, 0x4380, 0x8341, 0x4100, 0x81C1, 0x8081, 0x4040 44 44 - }; 45 45 - 46 46 - /** 47 47 - * crc16 - compute the CRC-16 for the data buffer 48 48 - * @crc: previous CRC value 49 49 - * @p: data pointer 50 50 - * @len: number of bytes in the buffer 51 51 - * 52 52 - * Returns the updated CRC value. 53 53 - */ 54 54 - u16 crc16(u16 crc, const u8 *p, size_t len) 55 55 - { 56 56 - while (len--) 57 57 - crc = (crc >> 8) ^ crc16_table[(crc & 0xff) ^ *p++]; 58 58 - return crc; 59 59 - } 60 60 - EXPORT_SYMBOL(crc16); 61 61 - 62 62 - MODULE_DESCRIPTION("CRC16 calculations"); 63 63 - MODULE_LICENSE("GPL"); 64 64 -

-126

lib/crc32.c

reviewed

··· 1 1 - // SPDX-License-Identifier: GPL-2.0-only 2 2 - /* 3 3 - * Aug 8, 2011 Bob Pearson with help from Joakim Tjernlund and George Spelvin 4 4 - * cleaned up code to current version of sparse and added the slicing-by-8 5 5 - * algorithm to the closely similar existing slicing-by-4 algorithm. 6 6 - * 7 7 - * Oct 15, 2000 Matt Domsch <Matt_Domsch@dell.com> 8 8 - * Nicer crc32 functions/docs submitted by linux@horizon.com. Thanks! 9 9 - * Code was from the public domain, copyright abandoned. Code was 10 10 - * subsequently included in the kernel, thus was re-licensed under the 11 11 - * GNU GPL v2. 12 12 - * 13 13 - * Oct 12, 2000 Matt Domsch <Matt_Domsch@dell.com> 14 14 - * Same crc32 function was used in 5 other places in the kernel. 15 15 - * I made one version, and deleted the others. 16 16 - * There are various incantations of crc32(). Some use a seed of 0 or ~0. 17 17 - * Some xor at the end with ~0. The generic crc32() function takes 18 18 - * seed as an argument, and doesn't xor at the end. Then individual 19 19 - * users can do whatever they need. 20 20 - * drivers/net/smc9194.c uses seed ~0, doesn't xor with ~0. 21 21 - * fs/jffs2 uses seed 0, doesn't xor with ~0. 22 22 - * fs/partitions/efi.c uses seed ~0, xor's with ~0. 23 23 - */ 24 24 - 25 25 - /* see: Documentation/staging/crc32.rst for a description of algorithms */ 26 26 - 27 27 - #include <linux/crc32.h> 28 28 - #include <linux/crc32poly.h> 29 29 - #include <linux/module.h> 30 30 - #include <linux/types.h> 31 31 - 32 32 - #include "crc32table.h" 33 33 - 34 34 - MODULE_AUTHOR("Matt Domsch <Matt_Domsch@dell.com>"); 35 35 - MODULE_DESCRIPTION("Various CRC32 calculations"); 36 36 - MODULE_LICENSE("GPL"); 37 37 - 38 38 - u32 crc32_le_base(u32 crc, const u8 *p, size_t len) 39 39 - { 40 40 - while (len--) 41 41 - crc = (crc >> 8) ^ crc32table_le[(crc & 255) ^ *p++]; 42 42 - return crc; 43 43 - } 44 44 - EXPORT_SYMBOL(crc32_le_base); 45 45 - 46 46 - u32 crc32c_base(u32 crc, const u8 *p, size_t len) 47 47 - { 48 48 - while (len--) 49 49 - crc = (crc >> 8) ^ crc32ctable_le[(crc & 255) ^ *p++]; 50 50 - return crc; 51 51 - } 52 52 - EXPORT_SYMBOL(crc32c_base); 53 53 - 54 54 - /* 55 55 - * This multiplies the polynomials x and y modulo the given modulus. 56 56 - * This follows the "little-endian" CRC convention that the lsbit 57 57 - * represents the highest power of x, and the msbit represents x^0. 58 58 - */ 59 59 - static u32 gf2_multiply(u32 x, u32 y, u32 modulus) 60 60 - { 61 61 - u32 product = x & 1 ? y : 0; 62 62 - int i; 63 63 - 64 64 - for (i = 0; i < 31; i++) { 65 65 - product = (product >> 1) ^ (product & 1 ? modulus : 0); 66 66 - x >>= 1; 67 67 - product ^= x & 1 ? y : 0; 68 68 - } 69 69 - 70 70 - return product; 71 71 - } 72 72 - 73 73 - /** 74 74 - * crc32_generic_shift - Append @len 0 bytes to crc, in logarithmic time 75 75 - * @crc: The original little-endian CRC (i.e. lsbit is x^31 coefficient) 76 76 - * @len: The number of bytes. @crc is multiplied by x^(8*@len) 77 77 - * @polynomial: The modulus used to reduce the result to 32 bits. 78 78 - * 79 79 - * It's possible to parallelize CRC computations by computing a CRC 80 80 - * over separate ranges of a buffer, then summing them. 81 81 - * This shifts the given CRC by 8*len bits (i.e. produces the same effect 82 82 - * as appending len bytes of zero to the data), in time proportional 83 83 - * to log(len). 84 84 - */ 85 85 - static u32 crc32_generic_shift(u32 crc, size_t len, u32 polynomial) 86 86 - { 87 87 - u32 power = polynomial; /* CRC of x^32 */ 88 88 - int i; 89 89 - 90 90 - /* Shift up to 32 bits in the simple linear way */ 91 91 - for (i = 0; i < 8 * (int)(len & 3); i++) 92 92 - crc = (crc >> 1) ^ (crc & 1 ? polynomial : 0); 93 93 - 94 94 - len >>= 2; 95 95 - if (!len) 96 96 - return crc; 97 97 - 98 98 - for (;;) { 99 99 - /* "power" is x^(2^i), modulo the polynomial */ 100 100 - if (len & 1) 101 101 - crc = gf2_multiply(crc, power, polynomial); 102 102 - 103 103 - len >>= 1; 104 104 - if (!len) 105 105 - break; 106 106 - 107 107 - /* Square power, advancing to x^(2^(i+1)) */ 108 108 - power = gf2_multiply(power, power, polynomial); 109 109 - } 110 110 - 111 111 - return crc; 112 112 - } 113 113 - 114 114 - u32 crc32_le_shift(u32 crc, size_t len) 115 115 - { 116 116 - return crc32_generic_shift(crc, len, CRC32_POLY_LE); 117 117 - } 118 118 - EXPORT_SYMBOL(crc32_le_shift); 119 119 - 120 120 - u32 crc32_be_base(u32 crc, const u8 *p, size_t len) 121 121 - { 122 122 - while (len--) 123 123 - crc = (crc << 8) ^ crc32table_be[(crc >> 24) ^ *p++]; 124 124 - return crc; 125 125 - } 126 126 - EXPORT_SYMBOL(crc32_be_base);

-44

lib/crc4.c

reviewed

··· 1 1 - // SPDX-License-Identifier: GPL-2.0-only 2 2 - /* 3 3 - * crc4.c - simple crc-4 calculations. 4 4 - */ 5 5 - 6 6 - #include <linux/crc4.h> 7 7 - #include <linux/module.h> 8 8 - 9 9 - static const uint8_t crc4_tab[] = { 10 10 - 0x0, 0x7, 0xe, 0x9, 0xb, 0xc, 0x5, 0x2, 11 11 - 0x1, 0x6, 0xf, 0x8, 0xa, 0xd, 0x4, 0x3, 12 12 - }; 13 13 - 14 14 - /** 15 15 - * crc4 - calculate the 4-bit crc of a value. 16 16 - * @c: starting crc4 17 17 - * @x: value to checksum 18 18 - * @bits: number of bits in @x to checksum 19 19 - * 20 20 - * Returns the crc4 value of @x, using polynomial 0b10111. 21 21 - * 22 22 - * The @x value is treated as left-aligned, and bits above @bits are ignored 23 23 - * in the crc calculations. 24 24 - */ 25 25 - uint8_t crc4(uint8_t c, uint64_t x, int bits) 26 26 - { 27 27 - int i; 28 28 - 29 29 - /* mask off anything above the top bit */ 30 30 - x &= (1ull << bits) - 1; 31 31 - 32 32 - /* Align to 4-bits */ 33 33 - bits = (bits + 3) & ~0x3; 34 34 - 35 35 - /* Calculate crc4 over four-bit nibbles, starting at the MSbit */ 36 36 - for (i = bits - 4; i >= 0; i -= 4) 37 37 - c = crc4_tab[c ^ ((x >> i) & 0xf)]; 38 38 - 39 39 - return c; 40 40 - } 41 41 - EXPORT_SYMBOL_GPL(crc4); 42 42 - 43 43 - MODULE_DESCRIPTION("CRC4 calculations"); 44 44 - MODULE_LICENSE("GPL");

-58

lib/crc64.c

reviewed

··· 1 1 - // SPDX-License-Identifier: GPL-2.0 2 2 - /* 3 3 - * Normal 64-bit CRC calculation. 4 4 - * 5 5 - * This is a basic crc64 implementation following ECMA-182 specification, 6 6 - * which can be found from, 7 7 - * https://www.ecma-international.org/publications/standards/Ecma-182.htm 8 8 - * 9 9 - * Dr. Ross N. Williams has a great document to introduce the idea of CRC 10 10 - * algorithm, here the CRC64 code is also inspired by the table-driven 11 11 - * algorithm and detail example from this paper. This paper can be found 12 12 - * from, 13 13 - * http://www.ross.net/crc/download/crc_v3.txt 14 14 - * 15 15 - * crc64table[256] is the lookup table of a table-driven 64-bit CRC 16 16 - * calculation, which is generated by gen_crc64table.c in kernel build 17 17 - * time. The polynomial of crc64 arithmetic is from ECMA-182 specification 18 18 - * as well, which is defined as, 19 19 - * 20 20 - * x^64 + x^62 + x^57 + x^55 + x^54 + x^53 + x^52 + x^47 + x^46 + x^45 + 21 21 - * x^40 + x^39 + x^38 + x^37 + x^35 + x^33 + x^32 + x^31 + x^29 + x^27 + 22 22 - * x^24 + x^23 + x^22 + x^21 + x^19 + x^17 + x^13 + x^12 + x^10 + x^9 + 23 23 - * x^7 + x^4 + x + 1 24 24 - * 25 25 - * crc64nvmetable[256] uses the CRC64 polynomial from the NVME NVM Command Set 26 26 - * Specification and uses least-significant-bit first bit order: 27 27 - * 28 28 - * x^64 + x^63 + x^61 + x^59 + x^58 + x^56 + x^55 + x^52 + x^49 + x^48 + x^47 + 29 29 - * x^46 + x^44 + x^41 + x^37 + x^36 + x^34 + x^32 + x^31 + x^28 + x^26 + x^23 + 30 30 - * x^22 + x^19 + x^16 + x^13 + x^12 + x^10 + x^9 + x^6 + x^4 + x^3 + 1 31 31 - * 32 32 - * Copyright 2018 SUSE Linux. 33 33 - * Author: Coly Li <colyli@suse.de> 34 34 - */ 35 35 - 36 36 - #include <linux/module.h> 37 37 - #include <linux/types.h> 38 38 - #include <linux/crc64.h> 39 39 - #include "crc64table.h" 40 40 - 41 41 - MODULE_DESCRIPTION("CRC64 calculations"); 42 42 - MODULE_LICENSE("GPL v2"); 43 43 - 44 44 - u64 crc64_be_generic(u64 crc, const u8 *p, size_t len) 45 45 - { 46 46 - while (len--) 47 47 - crc = (crc << 8) ^ crc64table[(crc >> 56) ^ *p++]; 48 48 - return crc; 49 49 - } 50 50 - EXPORT_SYMBOL_GPL(crc64_be_generic); 51 51 - 52 52 - u64 crc64_nvme_generic(u64 crc, const u8 *p, size_t len) 53 53 - { 54 54 - while (len--) 55 55 - crc = (crc >> 8) ^ crc64nvmetable[(crc & 0xff) ^ *p++]; 56 56 - return crc; 57 57 - } 58 58 - EXPORT_SYMBOL_GPL(crc64_nvme_generic);

-72

lib/crc7.c

reviewed

··· 1 1 - // SPDX-License-Identifier: GPL-2.0-only 2 2 - /* 3 3 - * crc7.c 4 4 - */ 5 5 - 6 6 - #include <linux/types.h> 7 7 - #include <linux/module.h> 8 8 - #include <linux/crc7.h> 9 9 - 10 10 - /* 11 11 - * Table for CRC-7 (polynomial x^7 + x^3 + 1). 12 12 - * This is a big-endian CRC (msbit is highest power of x), 13 13 - * aligned so the msbit of the byte is the x^6 coefficient 14 14 - * and the lsbit is not used. 15 15 - */ 16 16 - static const u8 crc7_be_syndrome_table[256] = { 17 17 - 0x00, 0x12, 0x24, 0x36, 0x48, 0x5a, 0x6c, 0x7e, 18 18 - 0x90, 0x82, 0xb4, 0xa6, 0xd8, 0xca, 0xfc, 0xee, 19 19 - 0x32, 0x20, 0x16, 0x04, 0x7a, 0x68, 0x5e, 0x4c, 20 20 - 0xa2, 0xb0, 0x86, 0x94, 0xea, 0xf8, 0xce, 0xdc, 21 21 - 0x64, 0x76, 0x40, 0x52, 0x2c, 0x3e, 0x08, 0x1a, 22 22 - 0xf4, 0xe6, 0xd0, 0xc2, 0xbc, 0xae, 0x98, 0x8a, 23 23 - 0x56, 0x44, 0x72, 0x60, 0x1e, 0x0c, 0x3a, 0x28, 24 24 - 0xc6, 0xd4, 0xe2, 0xf0, 0x8e, 0x9c, 0xaa, 0xb8, 25 25 - 0xc8, 0xda, 0xec, 0xfe, 0x80, 0x92, 0xa4, 0xb6, 26 26 - 0x58, 0x4a, 0x7c, 0x6e, 0x10, 0x02, 0x34, 0x26, 27 27 - 0xfa, 0xe8, 0xde, 0xcc, 0xb2, 0xa0, 0x96, 0x84, 28 28 - 0x6a, 0x78, 0x4e, 0x5c, 0x22, 0x30, 0x06, 0x14, 29 29 - 0xac, 0xbe, 0x88, 0x9a, 0xe4, 0xf6, 0xc0, 0xd2, 30 30 - 0x3c, 0x2e, 0x18, 0x0a, 0x74, 0x66, 0x50, 0x42, 31 31 - 0x9e, 0x8c, 0xba, 0xa8, 0xd6, 0xc4, 0xf2, 0xe0, 32 32 - 0x0e, 0x1c, 0x2a, 0x38, 0x46, 0x54, 0x62, 0x70, 33 33 - 0x82, 0x90, 0xa6, 0xb4, 0xca, 0xd8, 0xee, 0xfc, 34 34 - 0x12, 0x00, 0x36, 0x24, 0x5a, 0x48, 0x7e, 0x6c, 35 35 - 0xb0, 0xa2, 0x94, 0x86, 0xf8, 0xea, 0xdc, 0xce, 36 36 - 0x20, 0x32, 0x04, 0x16, 0x68, 0x7a, 0x4c, 0x5e, 37 37 - 0xe6, 0xf4, 0xc2, 0xd0, 0xae, 0xbc, 0x8a, 0x98, 38 38 - 0x76, 0x64, 0x52, 0x40, 0x3e, 0x2c, 0x1a, 0x08, 39 39 - 0xd4, 0xc6, 0xf0, 0xe2, 0x9c, 0x8e, 0xb8, 0xaa, 40 40 - 0x44, 0x56, 0x60, 0x72, 0x0c, 0x1e, 0x28, 0x3a, 41 41 - 0x4a, 0x58, 0x6e, 0x7c, 0x02, 0x10, 0x26, 0x34, 42 42 - 0xda, 0xc8, 0xfe, 0xec, 0x92, 0x80, 0xb6, 0xa4, 43 43 - 0x78, 0x6a, 0x5c, 0x4e, 0x30, 0x22, 0x14, 0x06, 44 44 - 0xe8, 0xfa, 0xcc, 0xde, 0xa0, 0xb2, 0x84, 0x96, 45 45 - 0x2e, 0x3c, 0x0a, 0x18, 0x66, 0x74, 0x42, 0x50, 46 46 - 0xbe, 0xac, 0x9a, 0x88, 0xf6, 0xe4, 0xd2, 0xc0, 47 47 - 0x1c, 0x0e, 0x38, 0x2a, 0x54, 0x46, 0x70, 0x62, 48 48 - 0x8c, 0x9e, 0xa8, 0xba, 0xc4, 0xd6, 0xe0, 0xf2 49 49 - }; 50 50 - 51 51 - /** 52 52 - * crc7_be - update the CRC7 for the data buffer 53 53 - * @crc: previous CRC7 value 54 54 - * @buffer: data pointer 55 55 - * @len: number of bytes in the buffer 56 56 - * Context: any 57 57 - * 58 58 - * Returns the updated CRC7 value. 59 59 - * The CRC7 is left-aligned in the byte (the lsbit is always 0), as that 60 60 - * makes the computation easier, and all callers want it in that form. 61 61 - * 62 62 - */ 63 63 - u8 crc7_be(u8 crc, const u8 *buffer, size_t len) 64 64 - { 65 65 - while (len--) 66 66 - crc = crc7_be_syndrome_table[crc ^ *buffer++]; 67 67 - return crc; 68 68 - } 69 69 - EXPORT_SYMBOL(crc7_be); 70 70 - 71 71 - MODULE_DESCRIPTION("CRC7 calculations"); 72 72 - MODULE_LICENSE("GPL");

-86

lib/crc8.c

reviewed

··· 1 1 - /* 2 2 - * Copyright (c) 2011 Broadcom Corporation 3 3 - * 4 4 - * Permission to use, copy, modify, and/or distribute this software for any 5 5 - * purpose with or without fee is hereby granted, provided that the above 6 6 - * copyright notice and this permission notice appear in all copies. 7 7 - * 8 8 - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 9 9 - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 10 10 - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 11 11 - * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 12 12 - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION 13 13 - * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN 14 14 - * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 15 15 - */ 16 16 - 17 17 - #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 18 18 - 19 19 - #include <linux/module.h> 20 20 - #include <linux/crc8.h> 21 21 - #include <linux/printk.h> 22 22 - 23 23 - /** 24 24 - * crc8_populate_msb - fill crc table for given polynomial in reverse bit order. 25 25 - * 26 26 - * @table: table to be filled. 27 27 - * @polynomial: polynomial for which table is to be filled. 28 28 - */ 29 29 - void crc8_populate_msb(u8 table[CRC8_TABLE_SIZE], u8 polynomial) 30 30 - { 31 31 - int i, j; 32 32 - const u8 msbit = 0x80; 33 33 - u8 t = msbit; 34 34 - 35 35 - table[0] = 0; 36 36 - 37 37 - for (i = 1; i < CRC8_TABLE_SIZE; i *= 2) { 38 38 - t = (t << 1) ^ (t & msbit ? polynomial : 0); 39 39 - for (j = 0; j < i; j++) 40 40 - table[i+j] = table[j] ^ t; 41 41 - } 42 42 - } 43 43 - EXPORT_SYMBOL(crc8_populate_msb); 44 44 - 45 45 - /** 46 46 - * crc8_populate_lsb - fill crc table for given polynomial in regular bit order. 47 47 - * 48 48 - * @table: table to be filled. 49 49 - * @polynomial: polynomial for which table is to be filled. 50 50 - */ 51 51 - void crc8_populate_lsb(u8 table[CRC8_TABLE_SIZE], u8 polynomial) 52 52 - { 53 53 - int i, j; 54 54 - u8 t = 1; 55 55 - 56 56 - table[0] = 0; 57 57 - 58 58 - for (i = (CRC8_TABLE_SIZE >> 1); i; i >>= 1) { 59 59 - t = (t >> 1) ^ (t & 1 ? polynomial : 0); 60 60 - for (j = 0; j < CRC8_TABLE_SIZE; j += 2*i) 61 61 - table[i+j] = table[j] ^ t; 62 62 - } 63 63 - } 64 64 - EXPORT_SYMBOL(crc8_populate_lsb); 65 65 - 66 66 - /** 67 67 - * crc8 - calculate a crc8 over the given input data. 68 68 - * 69 69 - * @table: crc table used for calculation. 70 70 - * @pdata: pointer to data buffer. 71 71 - * @nbytes: number of bytes in data buffer. 72 72 - * @crc: previous returned crc8 value. 73 73 - */ 74 74 - u8 crc8(const u8 table[CRC8_TABLE_SIZE], const u8 *pdata, size_t nbytes, u8 crc) 75 75 - { 76 76 - /* loop over the buffer data */ 77 77 - while (nbytes-- > 0) 78 78 - crc = table[(crc ^ *pdata++) & 0xff]; 79 79 - 80 80 - return crc; 81 81 - } 82 82 - EXPORT_SYMBOL(crc8); 83 83 - 84 84 - MODULE_DESCRIPTION("CRC8 (by Williams, Ross N.) function"); 85 85 - MODULE_AUTHOR("Broadcom Corporation"); 86 86 - MODULE_LICENSE("Dual BSD/GPL");

-89

lib/gen_crc32table.c

reviewed

··· 1 1 - // SPDX-License-Identifier: GPL-2.0 2 2 - #include <stdio.h> 3 3 - #include "../include/linux/crc32poly.h" 4 4 - #include "../include/generated/autoconf.h" 5 5 - #include <inttypes.h> 6 6 - 7 7 - static uint32_t crc32table_le[256]; 8 8 - static uint32_t crc32table_be[256]; 9 9 - static uint32_t crc32ctable_le[256]; 10 10 - 11 11 - /** 12 12 - * crc32init_le() - allocate and initialize LE table data 13 13 - * 14 14 - * crc is the crc of the byte i; other entries are filled in based on the 15 15 - * fact that crctable[i^j] = crctable[i] ^ crctable[j]. 16 16 - * 17 17 - */ 18 18 - static void crc32init_le_generic(const uint32_t polynomial, uint32_t tab[256]) 19 19 - { 20 20 - unsigned i, j; 21 21 - uint32_t crc = 1; 22 22 - 23 23 - tab[0] = 0; 24 24 - 25 25 - for (i = 128; i; i >>= 1) { 26 26 - crc = (crc >> 1) ^ ((crc & 1) ? polynomial : 0); 27 27 - for (j = 0; j < 256; j += 2 * i) 28 28 - tab[i + j] = crc ^ tab[j]; 29 29 - } 30 30 - } 31 31 - 32 32 - static void crc32init_le(void) 33 33 - { 34 34 - crc32init_le_generic(CRC32_POLY_LE, crc32table_le); 35 35 - } 36 36 - 37 37 - static void crc32cinit_le(void) 38 38 - { 39 39 - crc32init_le_generic(CRC32C_POLY_LE, crc32ctable_le); 40 40 - } 41 41 - 42 42 - /** 43 43 - * crc32init_be() - allocate and initialize BE table data 44 44 - */ 45 45 - static void crc32init_be(void) 46 46 - { 47 47 - unsigned i, j; 48 48 - uint32_t crc = 0x80000000; 49 49 - 50 50 - crc32table_be[0] = 0; 51 51 - 52 52 - for (i = 1; i < 256; i <<= 1) { 53 53 - crc = (crc << 1) ^ ((crc & 0x80000000) ? CRC32_POLY_BE : 0); 54 54 - for (j = 0; j < i; j++) 55 55 - crc32table_be[i + j] = crc ^ crc32table_be[j]; 56 56 - } 57 57 - } 58 58 - 59 59 - static void output_table(const uint32_t table[256]) 60 60 - { 61 61 - int i; 62 62 - 63 63 - for (i = 0; i < 256; i += 4) { 64 64 - printf("\t0x%08x, 0x%08x, 0x%08x, 0x%08x,\n", 65 65 - table[i], table[i + 1], table[i + 2], table[i + 3]); 66 66 - } 67 67 - } 68 68 - 69 69 - int main(int argc, char** argv) 70 70 - { 71 71 - printf("/* this file is generated - do not edit */\n\n"); 72 72 - 73 73 - crc32init_le(); 74 74 - printf("static const u32 ____cacheline_aligned crc32table_le[256] = {\n"); 75 75 - output_table(crc32table_le); 76 76 - printf("};\n"); 77 77 - 78 78 - crc32init_be(); 79 79 - printf("static const u32 ____cacheline_aligned crc32table_be[256] = {\n"); 80 80 - output_table(crc32table_be); 81 81 - printf("};\n"); 82 82 - 83 83 - crc32cinit_le(); 84 84 - printf("static const u32 ____cacheline_aligned crc32ctable_le[256] = {\n"); 85 85 - output_table(crc32ctable_le); 86 86 - printf("};\n"); 87 87 - 88 88 - return 0; 89 89 - }

-95

lib/gen_crc64table.c

reviewed

··· 1 1 - // SPDX-License-Identifier: GPL-2.0 2 2 - /* 3 3 - * Generate lookup table for the table-driven CRC64 calculation. 4 4 - * 5 5 - * gen_crc64table is executed in kernel build time and generates 6 6 - * lib/crc64table.h. This header is included by lib/crc64.c for 7 7 - * the table-driven CRC64 calculation. 8 8 - * 9 9 - * See lib/crc64.c for more information about which specification 10 10 - * and polynomial arithmetic that gen_crc64table.c follows to 11 11 - * generate the lookup table. 12 12 - * 13 13 - * Copyright 2018 SUSE Linux. 14 14 - * Author: Coly Li <colyli@suse.de> 15 15 - */ 16 16 - #include <inttypes.h> 17 17 - #include <stdio.h> 18 18 - 19 19 - #define CRC64_ECMA182_POLY 0x42F0E1EBA9EA3693ULL 20 20 - #define CRC64_NVME_POLY 0x9A6C9329AC4BC9B5ULL 21 21 - 22 22 - static uint64_t crc64_table[256] = {0}; 23 23 - static uint64_t crc64_nvme_table[256] = {0}; 24 24 - 25 25 - static void generate_reflected_crc64_table(uint64_t table[256], uint64_t poly) 26 26 - { 27 27 - uint64_t i, j, c, crc; 28 28 - 29 29 - for (i = 0; i < 256; i++) { 30 30 - crc = 0ULL; 31 31 - c = i; 32 32 - 33 33 - for (j = 0; j < 8; j++) { 34 34 - if ((crc ^ (c >> j)) & 1) 35 35 - crc = (crc >> 1) ^ poly; 36 36 - else 37 37 - crc >>= 1; 38 38 - } 39 39 - table[i] = crc; 40 40 - } 41 41 - } 42 42 - 43 43 - static void generate_crc64_table(uint64_t table[256], uint64_t poly) 44 44 - { 45 45 - uint64_t i, j, c, crc; 46 46 - 47 47 - for (i = 0; i < 256; i++) { 48 48 - crc = 0; 49 49 - c = i << 56; 50 50 - 51 51 - for (j = 0; j < 8; j++) { 52 52 - if ((crc ^ c) & 0x8000000000000000ULL) 53 53 - crc = (crc << 1) ^ poly; 54 54 - else 55 55 - crc <<= 1; 56 56 - c <<= 1; 57 57 - } 58 58 - 59 59 - table[i] = crc; 60 60 - } 61 61 - } 62 62 - 63 63 - static void output_table(uint64_t table[256]) 64 64 - { 65 65 - int i; 66 66 - 67 67 - for (i = 0; i < 256; i++) { 68 68 - printf("\t0x%016" PRIx64 "ULL", table[i]); 69 69 - if (i & 0x1) 70 70 - printf(",\n"); 71 71 - else 72 72 - printf(", "); 73 73 - } 74 74 - printf("};\n"); 75 75 - } 76 76 - 77 77 - static void print_crc64_tables(void) 78 78 - { 79 79 - printf("/* this file is generated - do not edit */\n\n"); 80 80 - printf("#include <linux/types.h>\n"); 81 81 - printf("#include <linux/cache.h>\n\n"); 82 82 - printf("static const u64 ____cacheline_aligned crc64table[256] = {\n"); 83 83 - output_table(crc64_table); 84 84 - 85 85 - printf("\nstatic const u64 ____cacheline_aligned crc64nvmetable[256] = {\n"); 86 86 - output_table(crc64_nvme_table); 87 87 - } 88 88 - 89 89 - int main(int argc, char *argv[]) 90 90 - { 91 91 - generate_crc64_table(crc64_table, CRC64_ECMA182_POLY); 92 92 - generate_reflected_crc64_table(crc64_nvme_table, CRC64_NVME_POLY); 93 93 - print_crc64_tables(); 94 94 - return 0; 95 95 - }

-1

lib/tests/Makefile

reviewed

··· 10 10 obj-$(CONFIG_CHECKSUM_KUNIT) += checksum_kunit.o 11 11 obj-$(CONFIG_CMDLINE_KUNIT_TEST) += cmdline_kunit.o 12 12 obj-$(CONFIG_CPUMASK_KUNIT_TEST) += cpumask_kunit.o 13 13 - obj-$(CONFIG_CRC_KUNIT_TEST) += crc_kunit.o 14 13 CFLAGS_fortify_kunit.o += $(call cc-disable-warning, unsequenced) 15 14 CFLAGS_fortify_kunit.o += $(call cc-disable-warning, stringop-overread) 16 15 CFLAGS_fortify_kunit.o += $(call cc-disable-warning, stringop-truncation)

-489

lib/tests/crc_kunit.c

reviewed

··· 1 1 - // SPDX-License-Identifier: GPL-2.0-or-later 2 2 - /* 3 3 - * Unit tests and benchmarks for the CRC library functions 4 4 - * 5 5 - * Copyright 2024 Google LLC 6 6 - * 7 7 - * Author: Eric Biggers <ebiggers@google.com> 8 8 - */ 9 9 - #include <kunit/test.h> 10 10 - #include <linux/crc7.h> 11 11 - #include <linux/crc16.h> 12 12 - #include <linux/crc-t10dif.h> 13 13 - #include <linux/crc32.h> 14 14 - #include <linux/crc32c.h> 15 15 - #include <linux/crc64.h> 16 16 - #include <linux/prandom.h> 17 17 - #include <linux/vmalloc.h> 18 18 - 19 19 - #define CRC_KUNIT_SEED 42 20 20 - #define CRC_KUNIT_MAX_LEN 16384 21 21 - #define CRC_KUNIT_NUM_TEST_ITERS 1000 22 22 - 23 23 - static struct rnd_state rng; 24 24 - static u8 *test_buffer; 25 25 - static size_t test_buflen; 26 26 - 27 27 - /** 28 28 - * struct crc_variant - describes a CRC variant 29 29 - * @bits: Number of bits in the CRC, 1 <= @bits <= 64. 30 30 - * @le: true if it's a "little endian" CRC (reversed mapping between bits and 31 31 - * polynomial coefficients in each byte), false if it's a "big endian" CRC 32 32 - * (natural mapping between bits and polynomial coefficients in each byte) 33 33 - * @poly: The generator polynomial with the highest-order term omitted. 34 34 - * Bit-reversed if @le is true. 35 35 - * @func: The function to compute a CRC. The type signature uses u64 so that it 36 36 - * can fit any CRC up to CRC-64. The CRC is passed in, and is expected 37 37 - * to be returned in, the least significant bits of the u64. The 38 38 - * function is expected to *not* invert the CRC at the beginning and end. 39 39 - * @combine_func: Optional function to combine two CRCs. 40 40 - */ 41 41 - struct crc_variant { 42 42 - int bits; 43 43 - bool le; 44 44 - u64 poly; 45 45 - u64 (*func)(u64 crc, const u8 *p, size_t len); 46 46 - u64 (*combine_func)(u64 crc1, u64 crc2, size_t len2); 47 47 - }; 48 48 - 49 49 - static u32 rand32(void) 50 50 - { 51 51 - return prandom_u32_state(&rng); 52 52 - } 53 53 - 54 54 - static u64 rand64(void) 55 55 - { 56 56 - u32 n = rand32(); 57 57 - 58 58 - return ((u64)n << 32) | rand32(); 59 59 - } 60 60 - 61 61 - static u64 crc_mask(const struct crc_variant *v) 62 62 - { 63 63 - return (u64)-1 >> (64 - v->bits); 64 64 - } 65 65 - 66 66 - /* Reference implementation of any CRC variant */ 67 67 - static u64 crc_ref(const struct crc_variant *v, 68 68 - u64 crc, const u8 *p, size_t len) 69 69 - { 70 70 - size_t i, j; 71 71 - 72 72 - for (i = 0; i < len; i++) { 73 73 - for (j = 0; j < 8; j++) { 74 74 - if (v->le) { 75 75 - crc ^= (p[i] >> j) & 1; 76 76 - crc = (crc >> 1) ^ ((crc & 1) ? v->poly : 0); 77 77 - } else { 78 78 - crc ^= (u64)((p[i] >> (7 - j)) & 1) << 79 79 - (v->bits - 1); 80 80 - if (crc & (1ULL << (v->bits - 1))) 81 81 - crc = ((crc << 1) ^ v->poly) & 82 82 - crc_mask(v); 83 83 - else 84 84 - crc <<= 1; 85 85 - } 86 86 - } 87 87 - } 88 88 - return crc; 89 89 - } 90 90 - 91 91 - static int crc_suite_init(struct kunit_suite *suite) 92 92 - { 93 93 - /* 94 94 - * Allocate the test buffer using vmalloc() with a page-aligned length 95 95 - * so that it is immediately followed by a guard page. This allows 96 96 - * buffer overreads to be detected, even in assembly code. 97 97 - */ 98 98 - test_buflen = round_up(CRC_KUNIT_MAX_LEN, PAGE_SIZE); 99 99 - test_buffer = vmalloc(test_buflen); 100 100 - if (!test_buffer) 101 101 - return -ENOMEM; 102 102 - 103 103 - prandom_seed_state(&rng, CRC_KUNIT_SEED); 104 104 - prandom_bytes_state(&rng, test_buffer, test_buflen); 105 105 - return 0; 106 106 - } 107 107 - 108 108 - static void crc_suite_exit(struct kunit_suite *suite) 109 109 - { 110 110 - vfree(test_buffer); 111 111 - test_buffer = NULL; 112 112 - } 113 113 - 114 114 - /* Generate a random initial CRC. */ 115 115 - static u64 generate_random_initial_crc(const struct crc_variant *v) 116 116 - { 117 117 - switch (rand32() % 4) { 118 118 - case 0: 119 119 - return 0; 120 120 - case 1: 121 121 - return crc_mask(v); /* All 1 bits */ 122 122 - default: 123 123 - return rand64() & crc_mask(v); 124 124 - } 125 125 - } 126 126 - 127 127 - /* Generate a random length, preferring small lengths. */ 128 128 - static size_t generate_random_length(size_t max_length) 129 129 - { 130 130 - size_t len; 131 131 - 132 132 - switch (rand32() % 3) { 133 133 - case 0: 134 134 - len = rand32() % 128; 135 135 - break; 136 136 - case 1: 137 137 - len = rand32() % 3072; 138 138 - break; 139 139 - default: 140 140 - len = rand32(); 141 141 - break; 142 142 - } 143 143 - return len % (max_length + 1); 144 144 - } 145 145 - 146 146 - /* Test that v->func gives the same CRCs as a reference implementation. */ 147 147 - static void crc_main_test(struct kunit *test, const struct crc_variant *v) 148 148 - { 149 149 - size_t i; 150 150 - 151 151 - for (i = 0; i < CRC_KUNIT_NUM_TEST_ITERS; i++) { 152 152 - u64 init_crc, expected_crc, actual_crc; 153 153 - size_t len, offset; 154 154 - bool nosimd; 155 155 - 156 156 - init_crc = generate_random_initial_crc(v); 157 157 - len = generate_random_length(CRC_KUNIT_MAX_LEN); 158 158 - 159 159 - /* Generate a random offset. */ 160 160 - if (rand32() % 2 == 0) { 161 161 - /* Use a random alignment mod 64 */ 162 162 - offset = rand32() % 64; 163 163 - offset = min(offset, CRC_KUNIT_MAX_LEN - len); 164 164 - } else { 165 165 - /* Go up to the guard page, to catch buffer overreads */ 166 166 - offset = test_buflen - len; 167 167 - } 168 168 - 169 169 - if (rand32() % 8 == 0) 170 170 - /* Refresh the data occasionally. */ 171 171 - prandom_bytes_state(&rng, &test_buffer[offset], len); 172 172 - 173 173 - nosimd = rand32() % 8 == 0; 174 174 - 175 175 - /* 176 176 - * Compute the CRC, and verify that it equals the CRC computed 177 177 - * by a simple bit-at-a-time reference implementation. 178 178 - */ 179 179 - expected_crc = crc_ref(v, init_crc, &test_buffer[offset], len); 180 180 - if (nosimd) 181 181 - local_irq_disable(); 182 182 - actual_crc = v->func(init_crc, &test_buffer[offset], len); 183 183 - if (nosimd) 184 184 - local_irq_enable(); 185 185 - KUNIT_EXPECT_EQ_MSG(test, expected_crc, actual_crc, 186 186 - "Wrong result with len=%zu offset=%zu nosimd=%d", 187 187 - len, offset, nosimd); 188 188 - } 189 189 - } 190 190 - 191 191 - /* Test that CRC(concat(A, B)) == combine_CRCs(CRC(A), CRC(B), len(B)). */ 192 192 - static void crc_combine_test(struct kunit *test, const struct crc_variant *v) 193 193 - { 194 194 - int i; 195 195 - 196 196 - for (i = 0; i < 100; i++) { 197 197 - u64 init_crc = generate_random_initial_crc(v); 198 198 - size_t len1 = generate_random_length(CRC_KUNIT_MAX_LEN); 199 199 - size_t len2 = generate_random_length(CRC_KUNIT_MAX_LEN - len1); 200 200 - u64 crc1, crc2, expected_crc, actual_crc; 201 201 - 202 202 - prandom_bytes_state(&rng, test_buffer, len1 + len2); 203 203 - crc1 = v->func(init_crc, test_buffer, len1); 204 204 - crc2 = v->func(0, &test_buffer[len1], len2); 205 205 - expected_crc = v->func(init_crc, test_buffer, len1 + len2); 206 206 - actual_crc = v->combine_func(crc1, crc2, len2); 207 207 - KUNIT_EXPECT_EQ_MSG(test, expected_crc, actual_crc, 208 208 - "CRC combination gave wrong result with len1=%zu len2=%zu\n", 209 209 - len1, len2); 210 210 - } 211 211 - } 212 212 - 213 213 - static void crc_test(struct kunit *test, const struct crc_variant *v) 214 214 - { 215 215 - crc_main_test(test, v); 216 216 - if (v->combine_func) 217 217 - crc_combine_test(test, v); 218 218 - } 219 219 - 220 220 - static __always_inline void 221 221 - crc_benchmark(struct kunit *test, 222 222 - u64 (*crc_func)(u64 crc, const u8 *p, size_t len)) 223 223 - { 224 224 - static const size_t lens_to_test[] = { 225 225 - 1, 16, 64, 127, 128, 200, 256, 511, 512, 1024, 3173, 4096, 16384, 226 226 - }; 227 227 - size_t len, i, j, num_iters; 228 228 - /* 229 229 - * The CRC value that this function computes in a series of calls to 230 230 - * crc_func is never actually used, so use volatile to ensure that the 231 231 - * computations are done as intended and don't all get optimized out. 232 232 - */ 233 233 - volatile u64 crc = 0; 234 234 - u64 t; 235 235 - 236 236 - if (!IS_ENABLED(CONFIG_CRC_BENCHMARK)) 237 237 - kunit_skip(test, "not enabled"); 238 238 - 239 239 - /* warm-up */ 240 240 - for (i = 0; i < 10000000; i += CRC_KUNIT_MAX_LEN) 241 241 - crc = crc_func(crc, test_buffer, CRC_KUNIT_MAX_LEN); 242 242 - 243 243 - for (i = 0; i < ARRAY_SIZE(lens_to_test); i++) { 244 244 - len = lens_to_test[i]; 245 245 - KUNIT_ASSERT_LE(test, len, CRC_KUNIT_MAX_LEN); 246 246 - num_iters = 10000000 / (len + 128); 247 247 - preempt_disable(); 248 248 - t = ktime_get_ns(); 249 249 - for (j = 0; j < num_iters; j++) 250 250 - crc = crc_func(crc, test_buffer, len); 251 251 - t = ktime_get_ns() - t; 252 252 - preempt_enable(); 253 253 - kunit_info(test, "len=%zu: %llu MB/s\n", 254 254 - len, div64_u64((u64)len * num_iters * 1000, t)); 255 255 - } 256 256 - } 257 257 - 258 258 - /* crc7_be */ 259 259 - 260 260 - static u64 crc7_be_wrapper(u64 crc, const u8 *p, size_t len) 261 261 - { 262 262 - /* 263 263 - * crc7_be() left-aligns the 7-bit CRC in a u8, whereas the test wants a 264 264 - * right-aligned CRC (in a u64). Convert between the conventions. 265 265 - */ 266 266 - return crc7_be(crc << 1, p, len) >> 1; 267 267 - } 268 268 - 269 269 - static const struct crc_variant crc_variant_crc7_be = { 270 270 - .bits = 7, 271 271 - .poly = 0x9, 272 272 - .func = crc7_be_wrapper, 273 273 - }; 274 274 - 275 275 - static void crc7_be_test(struct kunit *test) 276 276 - { 277 277 - crc_test(test, &crc_variant_crc7_be); 278 278 - } 279 279 - 280 280 - static void crc7_be_benchmark(struct kunit *test) 281 281 - { 282 282 - crc_benchmark(test, crc7_be_wrapper); 283 283 - } 284 284 - 285 285 - /* crc16 */ 286 286 - 287 287 - static u64 crc16_wrapper(u64 crc, const u8 *p, size_t len) 288 288 - { 289 289 - return crc16(crc, p, len); 290 290 - } 291 291 - 292 292 - static const struct crc_variant crc_variant_crc16 = { 293 293 - .bits = 16, 294 294 - .le = true, 295 295 - .poly = 0xa001, 296 296 - .func = crc16_wrapper, 297 297 - }; 298 298 - 299 299 - static void crc16_test(struct kunit *test) 300 300 - { 301 301 - crc_test(test, &crc_variant_crc16); 302 302 - } 303 303 - 304 304 - static void crc16_benchmark(struct kunit *test) 305 305 - { 306 306 - crc_benchmark(test, crc16_wrapper); 307 307 - } 308 308 - 309 309 - /* crc_t10dif */ 310 310 - 311 311 - static u64 crc_t10dif_wrapper(u64 crc, const u8 *p, size_t len) 312 312 - { 313 313 - return crc_t10dif_update(crc, p, len); 314 314 - } 315 315 - 316 316 - static const struct crc_variant crc_variant_crc_t10dif = { 317 317 - .bits = 16, 318 318 - .le = false, 319 319 - .poly = 0x8bb7, 320 320 - .func = crc_t10dif_wrapper, 321 321 - }; 322 322 - 323 323 - static void crc_t10dif_test(struct kunit *test) 324 324 - { 325 325 - crc_test(test, &crc_variant_crc_t10dif); 326 326 - } 327 327 - 328 328 - static void crc_t10dif_benchmark(struct kunit *test) 329 329 - { 330 330 - crc_benchmark(test, crc_t10dif_wrapper); 331 331 - } 332 332 - 333 333 - /* crc32_le */ 334 334 - 335 335 - static u64 crc32_le_wrapper(u64 crc, const u8 *p, size_t len) 336 336 - { 337 337 - return crc32_le(crc, p, len); 338 338 - } 339 339 - 340 340 - static u64 crc32_le_combine_wrapper(u64 crc1, u64 crc2, size_t len2) 341 341 - { 342 342 - return crc32_le_combine(crc1, crc2, len2); 343 343 - } 344 344 - 345 345 - static const struct crc_variant crc_variant_crc32_le = { 346 346 - .bits = 32, 347 347 - .le = true, 348 348 - .poly = 0xedb88320, 349 349 - .func = crc32_le_wrapper, 350 350 - .combine_func = crc32_le_combine_wrapper, 351 351 - }; 352 352 - 353 353 - static void crc32_le_test(struct kunit *test) 354 354 - { 355 355 - crc_test(test, &crc_variant_crc32_le); 356 356 - } 357 357 - 358 358 - static void crc32_le_benchmark(struct kunit *test) 359 359 - { 360 360 - crc_benchmark(test, crc32_le_wrapper); 361 361 - } 362 362 - 363 363 - /* crc32_be */ 364 364 - 365 365 - static u64 crc32_be_wrapper(u64 crc, const u8 *p, size_t len) 366 366 - { 367 367 - return crc32_be(crc, p, len); 368 368 - } 369 369 - 370 370 - static const struct crc_variant crc_variant_crc32_be = { 371 371 - .bits = 32, 372 372 - .le = false, 373 373 - .poly = 0x04c11db7, 374 374 - .func = crc32_be_wrapper, 375 375 - }; 376 376 - 377 377 - static void crc32_be_test(struct kunit *test) 378 378 - { 379 379 - crc_test(test, &crc_variant_crc32_be); 380 380 - } 381 381 - 382 382 - static void crc32_be_benchmark(struct kunit *test) 383 383 - { 384 384 - crc_benchmark(test, crc32_be_wrapper); 385 385 - } 386 386 - 387 387 - /* crc32c */ 388 388 - 389 389 - static u64 crc32c_wrapper(u64 crc, const u8 *p, size_t len) 390 390 - { 391 391 - return crc32c(crc, p, len); 392 392 - } 393 393 - 394 394 - static const struct crc_variant crc_variant_crc32c = { 395 395 - .bits = 32, 396 396 - .le = true, 397 397 - .poly = 0x82f63b78, 398 398 - .func = crc32c_wrapper, 399 399 - }; 400 400 - 401 401 - static void crc32c_test(struct kunit *test) 402 402 - { 403 403 - crc_test(test, &crc_variant_crc32c); 404 404 - } 405 405 - 406 406 - static void crc32c_benchmark(struct kunit *test) 407 407 - { 408 408 - crc_benchmark(test, crc32c_wrapper); 409 409 - } 410 410 - 411 411 - /* crc64_be */ 412 412 - 413 413 - static u64 crc64_be_wrapper(u64 crc, const u8 *p, size_t len) 414 414 - { 415 415 - return crc64_be(crc, p, len); 416 416 - } 417 417 - 418 418 - static const struct crc_variant crc_variant_crc64_be = { 419 419 - .bits = 64, 420 420 - .le = false, 421 421 - .poly = 0x42f0e1eba9ea3693, 422 422 - .func = crc64_be_wrapper, 423 423 - }; 424 424 - 425 425 - static void crc64_be_test(struct kunit *test) 426 426 - { 427 427 - crc_test(test, &crc_variant_crc64_be); 428 428 - } 429 429 - 430 430 - static void crc64_be_benchmark(struct kunit *test) 431 431 - { 432 432 - crc_benchmark(test, crc64_be_wrapper); 433 433 - } 434 434 - 435 435 - /* crc64_nvme */ 436 436 - 437 437 - static u64 crc64_nvme_wrapper(u64 crc, const u8 *p, size_t len) 438 438 - { 439 439 - /* The inversions that crc64_nvme() does have to be undone here. */ 440 440 - return ~crc64_nvme(~crc, p, len); 441 441 - } 442 442 - 443 443 - static const struct crc_variant crc_variant_crc64_nvme = { 444 444 - .bits = 64, 445 445 - .le = true, 446 446 - .poly = 0x9a6c9329ac4bc9b5, 447 447 - .func = crc64_nvme_wrapper, 448 448 - }; 449 449 - 450 450 - static void crc64_nvme_test(struct kunit *test) 451 451 - { 452 452 - crc_test(test, &crc_variant_crc64_nvme); 453 453 - } 454 454 - 455 455 - static void crc64_nvme_benchmark(struct kunit *test) 456 456 - { 457 457 - crc_benchmark(test, crc64_nvme_wrapper); 458 458 - } 459 459 - 460 460 - static struct kunit_case crc_test_cases[] = { 461 461 - KUNIT_CASE(crc7_be_test), 462 462 - KUNIT_CASE(crc7_be_benchmark), 463 463 - KUNIT_CASE(crc16_test), 464 464 - KUNIT_CASE(crc16_benchmark), 465 465 - KUNIT_CASE(crc_t10dif_test), 466 466 - KUNIT_CASE(crc_t10dif_benchmark), 467 467 - KUNIT_CASE(crc32_le_test), 468 468 - KUNIT_CASE(crc32_le_benchmark), 469 469 - KUNIT_CASE(crc32_be_test), 470 470 - KUNIT_CASE(crc32_be_benchmark), 471 471 - KUNIT_CASE(crc32c_test), 472 472 - KUNIT_CASE(crc32c_benchmark), 473 473 - KUNIT_CASE(crc64_be_test), 474 474 - KUNIT_CASE(crc64_be_benchmark), 475 475 - KUNIT_CASE(crc64_nvme_test), 476 476 - KUNIT_CASE(crc64_nvme_benchmark), 477 477 - {}, 478 478 - }; 479 479 - 480 480 - static struct kunit_suite crc_test_suite = { 481 481 - .name = "crc", 482 482 - .test_cases = crc_test_cases, 483 483 - .suite_init = crc_suite_init, 484 484 - .suite_exit = crc_suite_exit, 485 485 - }; 486 486 - kunit_test_suite(crc_test_suite); 487 487 - 488 488 - MODULE_DESCRIPTION("Unit tests and benchmarks for the CRC library functions"); 489 489 - MODULE_LICENSE("GPL");