Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
lib/sha1: use the git implementation of SHA-1
For ChromiumOS, we use SHA-1 to verify the integrity of the root
filesystem. The speed of the kernel sha-1 implementation has a major
impact on our boot performance.
To improve boot performance, we investigated using the heavily optimized
sha-1 implementation used in git. With the git sha-1 implementation, we
see a 11.7% improvement in boot time.
10 reboots, remove slowest/fastest.
Before:
Mean: 6.58 seconds Stdev: 0.14
After (with git sha-1, this patch):
Mean: 5.89 seconds Stdev: 0.07
The other cool thing about the git SHA-1 implementation is that it only
needs 64 bytes of stack for the workspace while the original kernel
implementation needed 320 bytes.
Signed-off-by: Mandeep Singh Baines <msb@chromium.org>
Cc: Ramsay Jones <ramsay@ramsay1.demon.co.uk>
Cc: Nicolas Pitre <nico@cam.org>
Cc: Herbert Xu <herbert@gondor.apana.org.au>
Cc: David S. Miller <davem@davemloft.net>
Cc: linux-crypto@vger.kernel.org
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
authored by
Mandeep Singh Baines
and committed by
Linus Torvalds
1eb19a12
de96355c
+152
-48
+1
-1
include/linux/cryptohash.h
+1
-1
include/linux/cryptohash.h
+151
-47
lib/sha1.c
+151
-47
lib/sha1.c
···
1
1
/*
2
-
* SHA transform algorithm, originally taken from code written by
3
-
* Peter Gutmann, and placed in the public domain.
2
+
* SHA1 routine optimized to do word accesses rather than byte accesses,
3
+
* and to avoid unnecessary copies into the context array.
4
+
*
5
+
* This was based on the git SHA1 implementation.
4
6
*/
5
7
6
8
#include <linux/kernel.h>
7
9
#include <linux/module.h>
8
-
#include <linux/cryptohash.h>
10
+
#include <linux/bitops.h>
11
+
#include <asm/unaligned.h>
9
12
10
-
/* The SHA f()-functions. */
13
+
/*
14
+
* If you have 32 registers or more, the compiler can (and should)
15
+
* try to change the array[] accesses into registers. However, on
16
+
* machines with less than ~25 registers, that won't really work,
17
+
* and at least gcc will make an unholy mess of it.
18
+
*
19
+
* So to avoid that mess which just slows things down, we force
20
+
* the stores to memory to actually happen (we might be better off
21
+
* with a 'W(t)=(val);asm("":"+m" (W(t))' there instead, as
22
+
* suggested by Artur Skawina - that will also make gcc unable to
23
+
* try to do the silly "optimize away loads" part because it won't
24
+
* see what the value will be).
25
+
*
26
+
* Ben Herrenschmidt reports that on PPC, the C version comes close
27
+
* to the optimized asm with this (ie on PPC you don't want that
28
+
* 'volatile', since there are lots of registers).
29
+
*
30
+
* On ARM we get the best code generation by forcing a full memory barrier
31
+
* between each SHA_ROUND, otherwise gcc happily get wild with spilling and
32
+
* the stack frame size simply explode and performance goes down the drain.
33
+
*/
11
34
12
-
#define f1(x,y,z) (z ^ (x & (y ^ z))) /* x ? y : z */
13
-
#define f2(x,y,z) (x ^ y ^ z) /* XOR */
14
-
#define f3(x,y,z) ((x & y) + (z & (x ^ y))) /* majority */
35
+
#ifdef CONFIG_X86
36
+
#define setW(x, val) (*(volatile __u32 *)&W(x) = (val))
37
+
#elif defined(CONFIG_ARM)
38
+
#define setW(x, val) do { W(x) = (val); __asm__("":::"memory"); } while (0)
39
+
#else
40
+
#define setW(x, val) (W(x) = (val))
41
+
#endif
15
42
16
-
/* The SHA Mysterious Constants */
43
+
/* This "rolls" over the 512-bit array */
44
+
#define W(x) (array[(x)&15])
17
45
18
-
#define K1 0x5A827999L /* Rounds 0-19: sqrt(2) * 2^30 */
19
-
#define K2 0x6ED9EBA1L /* Rounds 20-39: sqrt(3) * 2^30 */
20
-
#define K3 0x8F1BBCDCL /* Rounds 40-59: sqrt(5) * 2^30 */
21
-
#define K4 0xCA62C1D6L /* Rounds 60-79: sqrt(10) * 2^30 */
46
+
/*
47
+
* Where do we get the source from? The first 16 iterations get it from
48
+
* the input data, the next mix it from the 512-bit array.
49
+
*/
50
+
#define SHA_SRC(t) get_unaligned_be32((__u32 *)data + t)
51
+
#define SHA_MIX(t) rol32(W(t+13) ^ W(t+8) ^ W(t+2) ^ W(t), 1)
52
+
53
+
#define SHA_ROUND(t, input, fn, constant, A, B, C, D, E) do { \
54
+
__u32 TEMP = input(t); setW(t, TEMP); \
55
+
E += TEMP + rol32(A,5) + (fn) + (constant); \
56
+
B = ror32(B, 2); } while (0)
57
+
58
+
#define T_0_15(t, A, B, C, D, E) SHA_ROUND(t, SHA_SRC, (((C^D)&B)^D) , 0x5a827999, A, B, C, D, E )
59
+
#define T_16_19(t, A, B, C, D, E) SHA_ROUND(t, SHA_MIX, (((C^D)&B)^D) , 0x5a827999, A, B, C, D, E )
60
+
#define T_20_39(t, A, B, C, D, E) SHA_ROUND(t, SHA_MIX, (B^C^D) , 0x6ed9eba1, A, B, C, D, E )
61
+
#define T_40_59(t, A, B, C, D, E) SHA_ROUND(t, SHA_MIX, ((B&C)+(D&(B^C))) , 0x8f1bbcdc, A, B, C, D, E )
62
+
#define T_60_79(t, A, B, C, D, E) SHA_ROUND(t, SHA_MIX, (B^C^D) , 0xca62c1d6, A, B, C, D, E )
22
63
23
64
/**
24
65
* sha_transform - single block SHA1 transform
25
66
*
26
67
* @digest: 160 bit digest to update
27
68
* @data: 512 bits of data to hash
28
-
* @W: 80 words of workspace (see note)
69
+
* @array: 16 words of workspace (see note)
29
70
*
30
71
* This function generates a SHA1 digest for a single 512-bit block.
31
72
* Be warned, it does not handle padding and message digest, do not
···
77
36
* to clear the workspace. This is left to the caller to avoid
78
37
* unnecessary clears between chained hashing operations.
79
38
*/
80
-
void sha_transform(__u32 *digest, const char *in, __u32 *W)
39
+
void sha_transform(__u32 *digest, const char *data, __u32 *array)
81
40
{
82
-
__u32 a, b, c, d, e, t, i;
41
+
__u32 A, B, C, D, E;
83
42
84
-
for (i = 0; i < 16; i++)
85
-
W[i] = be32_to_cpu(((const __be32 *)in)[i]);
43
+
A = digest[0];
44
+
B = digest[1];
45
+
C = digest[2];
46
+
D = digest[3];
47
+
E = digest[4];
86
48
87
-
for (i = 0; i < 64; i++)
88
-
W[i+16] = rol32(W[i+13] ^ W[i+8] ^ W[i+2] ^ W[i], 1);
49
+
/* Round 1 - iterations 0-16 take their input from 'data' */
50
+
T_0_15( 0, A, B, C, D, E);
51
+
T_0_15( 1, E, A, B, C, D);
52
+
T_0_15( 2, D, E, A, B, C);
53
+
T_0_15( 3, C, D, E, A, B);
54
+
T_0_15( 4, B, C, D, E, A);
55
+
T_0_15( 5, A, B, C, D, E);
56
+
T_0_15( 6, E, A, B, C, D);
57
+
T_0_15( 7, D, E, A, B, C);
58
+
T_0_15( 8, C, D, E, A, B);
59
+
T_0_15( 9, B, C, D, E, A);
60
+
T_0_15(10, A, B, C, D, E);
61
+
T_0_15(11, E, A, B, C, D);
62
+
T_0_15(12, D, E, A, B, C);
63
+
T_0_15(13, C, D, E, A, B);
64
+
T_0_15(14, B, C, D, E, A);
65
+
T_0_15(15, A, B, C, D, E);
89
66
90
-
a = digest[0];
91
-
b = digest[1];
92
-
c = digest[2];
93
-
d = digest[3];
94
-
e = digest[4];
67
+
/* Round 1 - tail. Input from 512-bit mixing array */
68
+
T_16_19(16, E, A, B, C, D);
69
+
T_16_19(17, D, E, A, B, C);
70
+
T_16_19(18, C, D, E, A, B);
71
+
T_16_19(19, B, C, D, E, A);
95
72
96
-
for (i = 0; i < 20; i++) {
97
-
t = f1(b, c, d) + K1 + rol32(a, 5) + e + W[i];
98
-
e = d; d = c; c = rol32(b, 30); b = a; a = t;
99
-
}
73
+
/* Round 2 */
74
+
T_20_39(20, A, B, C, D, E);
75
+
T_20_39(21, E, A, B, C, D);
76
+
T_20_39(22, D, E, A, B, C);
77
+
T_20_39(23, C, D, E, A, B);
78
+
T_20_39(24, B, C, D, E, A);
79
+
T_20_39(25, A, B, C, D, E);
80
+
T_20_39(26, E, A, B, C, D);
81
+
T_20_39(27, D, E, A, B, C);
82
+
T_20_39(28, C, D, E, A, B);
83
+
T_20_39(29, B, C, D, E, A);
84
+
T_20_39(30, A, B, C, D, E);
85
+
T_20_39(31, E, A, B, C, D);
86
+
T_20_39(32, D, E, A, B, C);
87
+
T_20_39(33, C, D, E, A, B);
88
+
T_20_39(34, B, C, D, E, A);
89
+
T_20_39(35, A, B, C, D, E);
90
+
T_20_39(36, E, A, B, C, D);
91
+
T_20_39(37, D, E, A, B, C);
92
+
T_20_39(38, C, D, E, A, B);
93
+
T_20_39(39, B, C, D, E, A);
100
94
101
-
for (; i < 40; i ++) {
102
-
t = f2(b, c, d) + K2 + rol32(a, 5) + e + W[i];
103
-
e = d; d = c; c = rol32(b, 30); b = a; a = t;
104
-
}
95
+
/* Round 3 */
96
+
T_40_59(40, A, B, C, D, E);
97
+
T_40_59(41, E, A, B, C, D);
98
+
T_40_59(42, D, E, A, B, C);
99
+
T_40_59(43, C, D, E, A, B);
100
+
T_40_59(44, B, C, D, E, A);
101
+
T_40_59(45, A, B, C, D, E);
102
+
T_40_59(46, E, A, B, C, D);
103
+
T_40_59(47, D, E, A, B, C);
104
+
T_40_59(48, C, D, E, A, B);
105
+
T_40_59(49, B, C, D, E, A);
106
+
T_40_59(50, A, B, C, D, E);
107
+
T_40_59(51, E, A, B, C, D);
108
+
T_40_59(52, D, E, A, B, C);
109
+
T_40_59(53, C, D, E, A, B);
110
+
T_40_59(54, B, C, D, E, A);
111
+
T_40_59(55, A, B, C, D, E);
112
+
T_40_59(56, E, A, B, C, D);
113
+
T_40_59(57, D, E, A, B, C);
114
+
T_40_59(58, C, D, E, A, B);
115
+
T_40_59(59, B, C, D, E, A);
105
116
106
-
for (; i < 60; i ++) {
107
-
t = f3(b, c, d) + K3 + rol32(a, 5) + e + W[i];
108
-
e = d; d = c; c = rol32(b, 30); b = a; a = t;
109
-
}
117
+
/* Round 4 */
118
+
T_60_79(60, A, B, C, D, E);
119
+
T_60_79(61, E, A, B, C, D);
120
+
T_60_79(62, D, E, A, B, C);
121
+
T_60_79(63, C, D, E, A, B);
122
+
T_60_79(64, B, C, D, E, A);
123
+
T_60_79(65, A, B, C, D, E);
124
+
T_60_79(66, E, A, B, C, D);
125
+
T_60_79(67, D, E, A, B, C);
126
+
T_60_79(68, C, D, E, A, B);
127
+
T_60_79(69, B, C, D, E, A);
128
+
T_60_79(70, A, B, C, D, E);
129
+
T_60_79(71, E, A, B, C, D);
130
+
T_60_79(72, D, E, A, B, C);
131
+
T_60_79(73, C, D, E, A, B);
132
+
T_60_79(74, B, C, D, E, A);
133
+
T_60_79(75, A, B, C, D, E);
134
+
T_60_79(76, E, A, B, C, D);
135
+
T_60_79(77, D, E, A, B, C);
136
+
T_60_79(78, C, D, E, A, B);
137
+
T_60_79(79, B, C, D, E, A);
110
138
111
-
for (; i < 80; i ++) {
112
-
t = f2(b, c, d) + K4 + rol32(a, 5) + e + W[i];
113
-
e = d; d = c; c = rol32(b, 30); b = a; a = t;
114
-
}
115
-
116
-
digest[0] += a;
117
-
digest[1] += b;
118
-
digest[2] += c;
119
-
digest[3] += d;
120
-
digest[4] += e;
139
+
digest[0] += A;
140
+
digest[1] += B;
141
+
digest[2] += C;
142
+
digest[3] += D;
143
+
digest[4] += E;
121
144
}
122
145
EXPORT_SYMBOL(sha_transform);
123
146
···
197
92
buf[3] = 0x10325476;
198
93
buf[4] = 0xc3d2e1f0;
199
94
}
200
-