Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
powerpc/kuap: Use ASM feature fixups instead of static branches
To avoid a useless nop on top of every uaccess enable/disable and
make life easier for objtool, replace static branches by ASM feature
fixups that will nop KUAP enabling instructions out in the unlikely
case KUAP is disabled at boottime.
Leave it as is on book3s/64 for now, it will be handled later when
objtool is activated on PPC64.
Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://msgid.link/671948788024fd890ec4ed175bc332dab8664ea5.1689091022.git.christophe.leroy@csgroup.eu
authored by
Christophe Leroy
and committed by
Michael Ellerman
3a24ea0d
eb52f66f
+87
-74
+37
-9
arch/powerpc/include/asm/book3s/32/kup.h
+37
-9
arch/powerpc/include/asm/book3s/32/kup.h
···
27
27
isync(); /* Context sync required after mtsr() */
28
28
}
29
29
30
+
static __always_inline void uaccess_begin_32s(unsigned long addr)
31
+
{
32
+
unsigned long tmp;
33
+
34
+
asm volatile(ASM_MMU_FTR_IFSET(
35
+
"mfsrin %0, %1;"
36
+
"rlwinm %0, %0, 0, %2;"
37
+
"mtsrin %0, %1;"
38
+
"isync", "", %3)
39
+
: "=&r"(tmp)
40
+
: "r"(addr), "i"(~SR_KS), "i"(MMU_FTR_KUAP)
41
+
: "memory");
42
+
}
43
+
44
+
static __always_inline void uaccess_end_32s(unsigned long addr)
45
+
{
46
+
unsigned long tmp;
47
+
48
+
asm volatile(ASM_MMU_FTR_IFSET(
49
+
"mfsrin %0, %1;"
50
+
"oris %0, %0, %2;"
51
+
"mtsrin %0, %1;"
52
+
"isync", "", %3)
53
+
: "=&r"(tmp)
54
+
: "r"(addr), "i"(SR_KS >> 16), "i"(MMU_FTR_KUAP)
55
+
: "memory");
56
+
}
57
+
30
58
static __always_inline void __kuap_save_and_lock(struct pt_regs *regs)
31
59
{
32
60
unsigned long kuap = current->thread.kuap;
···
97
69
}
98
70
#define __kuap_get_and_assert_locked __kuap_get_and_assert_locked
99
71
100
-
static __always_inline void __allow_user_access(void __user *to, const void __user *from,
101
-
u32 size, unsigned long dir)
72
+
static __always_inline void allow_user_access(void __user *to, const void __user *from,
73
+
u32 size, unsigned long dir)
102
74
{
103
75
BUILD_BUG_ON(!__builtin_constant_p(dir));
104
76
···
106
78
return;
107
79
108
80
current->thread.kuap = (__force u32)to;
109
-
kuap_unlock_one((__force u32)to);
81
+
uaccess_begin_32s((__force u32)to);
110
82
}
111
83
112
-
static __always_inline void __prevent_user_access(unsigned long dir)
84
+
static __always_inline void prevent_user_access(unsigned long dir)
113
85
{
114
86
u32 kuap = current->thread.kuap;
115
87
···
119
91
return;
120
92
121
93
current->thread.kuap = KUAP_NONE;
122
-
kuap_lock_one(kuap);
94
+
uaccess_end_32s(kuap);
123
95
}
124
96
125
-
static __always_inline unsigned long __prevent_user_access_return(void)
97
+
static __always_inline unsigned long prevent_user_access_return(void)
126
98
{
127
99
unsigned long flags = current->thread.kuap;
128
100
129
101
if (flags != KUAP_NONE) {
130
102
current->thread.kuap = KUAP_NONE;
131
-
kuap_lock_one(flags);
103
+
uaccess_end_32s(flags);
132
104
}
133
105
134
106
return flags;
135
107
}
136
108
137
-
static __always_inline void __restore_user_access(unsigned long flags)
109
+
static __always_inline void restore_user_access(unsigned long flags)
138
110
{
139
111
if (flags != KUAP_NONE) {
140
112
current->thread.kuap = flags;
141
-
kuap_unlock_one(flags);
113
+
uaccess_begin_32s(flags);
142
114
}
143
115
}
144
116
+5
-40
arch/powerpc/include/asm/kup.h
+5
-40
arch/powerpc/include/asm/kup.h
···
72
72
* platforms.
73
73
*/
74
74
#ifndef CONFIG_PPC_BOOK3S_64
75
-
static __always_inline void __allow_user_access(void __user *to, const void __user *from,
76
-
unsigned long size, unsigned long dir) { }
77
-
static __always_inline void __prevent_user_access(unsigned long dir) { }
78
-
static __always_inline unsigned long __prevent_user_access_return(void) { return 0UL; }
79
-
static __always_inline void __restore_user_access(unsigned long flags) { }
75
+
static __always_inline void allow_user_access(void __user *to, const void __user *from,
76
+
unsigned long size, unsigned long dir) { }
77
+
static __always_inline void prevent_user_access(unsigned long dir) { }
78
+
static __always_inline unsigned long prevent_user_access_return(void) { return 0UL; }
79
+
static __always_inline void restore_user_access(unsigned long flags) { }
80
80
#endif /* CONFIG_PPC_BOOK3S_64 */
81
81
#endif /* CONFIG_PPC_KUAP */
82
82
···
131
131
if (IS_ENABLED(CONFIG_PPC_KUAP_DEBUG))
132
132
kuap_get_and_assert_locked();
133
133
}
134
-
135
-
#ifndef CONFIG_PPC_BOOK3S_64
136
-
static __always_inline void allow_user_access(void __user *to, const void __user *from,
137
-
unsigned long size, unsigned long dir)
138
-
{
139
-
if (kuap_is_disabled())
140
-
return;
141
-
142
-
__allow_user_access(to, from, size, dir);
143
-
}
144
-
145
-
static __always_inline void prevent_user_access(unsigned long dir)
146
-
{
147
-
if (kuap_is_disabled())
148
-
return;
149
-
150
-
__prevent_user_access(dir);
151
-
}
152
-
153
-
static __always_inline unsigned long prevent_user_access_return(void)
154
-
{
155
-
if (kuap_is_disabled())
156
-
return 0;
157
-
158
-
return __prevent_user_access_return();
159
-
}
160
-
161
-
static __always_inline void restore_user_access(unsigned long flags)
162
-
{
163
-
if (kuap_is_disabled())
164
-
return;
165
-
166
-
__restore_user_access(flags);
167
-
}
168
-
#endif /* CONFIG_PPC_BOOK3S_64 */
169
134
170
135
static __always_inline void allow_read_from_user(const void __user *from, unsigned long size)
171
136
{
+21
-9
arch/powerpc/include/asm/nohash/32/kup-8xx.h
+21
-9
arch/powerpc/include/asm/nohash/32/kup-8xx.h
···
37
37
#define __kuap_get_and_assert_locked __kuap_get_and_assert_locked
38
38
#endif
39
39
40
-
static __always_inline void __allow_user_access(void __user *to, const void __user *from,
41
-
unsigned long size, unsigned long dir)
40
+
static __always_inline void uaccess_begin_8xx(unsigned long val)
42
41
{
43
-
mtspr(SPRN_MD_AP, MD_APG_INIT);
42
+
asm(ASM_MMU_FTR_IFSET("mtspr %0, %1", "", %2) : :
43
+
"i"(SPRN_MD_AP), "r"(val), "i"(MMU_FTR_KUAP) : "memory");
44
44
}
45
45
46
-
static __always_inline void __prevent_user_access(unsigned long dir)
46
+
static __always_inline void uaccess_end_8xx(void)
47
47
{
48
-
mtspr(SPRN_MD_AP, MD_APG_KUAP);
48
+
asm(ASM_MMU_FTR_IFSET("mtspr %0, %1", "", %2) : :
49
+
"i"(SPRN_MD_AP), "r"(MD_APG_KUAP), "i"(MMU_FTR_KUAP) : "memory");
49
50
}
50
51
51
-
static __always_inline unsigned long __prevent_user_access_return(void)
52
+
static __always_inline void allow_user_access(void __user *to, const void __user *from,
53
+
unsigned long size, unsigned long dir)
54
+
{
55
+
uaccess_begin_8xx(MD_APG_INIT);
56
+
}
57
+
58
+
static __always_inline void prevent_user_access(unsigned long dir)
59
+
{
60
+
uaccess_end_8xx();
61
+
}
62
+
63
+
static __always_inline unsigned long prevent_user_access_return(void)
52
64
{
53
65
unsigned long flags;
54
66
55
67
flags = mfspr(SPRN_MD_AP);
56
68
57
-
mtspr(SPRN_MD_AP, MD_APG_KUAP);
69
+
uaccess_end_8xx();
58
70
59
71
return flags;
60
72
}
61
73
62
-
static __always_inline void __restore_user_access(unsigned long flags)
74
+
static __always_inline void restore_user_access(unsigned long flags)
63
75
{
64
-
mtspr(SPRN_MD_AP, flags);
76
+
uaccess_begin_8xx(flags);
65
77
}
66
78
67
79
static __always_inline bool
+23
-15
arch/powerpc/include/asm/nohash/kup-booke.h
+23
-15
arch/powerpc/include/asm/nohash/kup-booke.h
···
3
3
#define _ASM_POWERPC_KUP_BOOKE_H_
4
4
5
5
#include <asm/bug.h>
6
+
#include <asm/mmu.h>
6
7
7
8
#ifdef CONFIG_PPC_KUAP
8
9
···
61
60
#define __kuap_get_and_assert_locked __kuap_get_and_assert_locked
62
61
#endif
63
62
64
-
static __always_inline void __allow_user_access(void __user *to, const void __user *from,
65
-
unsigned long size, unsigned long dir)
63
+
static __always_inline void uaccess_begin_booke(unsigned long val)
66
64
{
67
-
mtspr(SPRN_PID, current->thread.pid);
68
-
isync();
65
+
asm(ASM_MMU_FTR_IFSET("mtspr %0, %1; isync", "", %2) : :
66
+
"i"(SPRN_PID), "r"(val), "i"(MMU_FTR_KUAP) : "memory");
69
67
}
70
68
71
-
static __always_inline void __prevent_user_access(unsigned long dir)
69
+
static __always_inline void uaccess_end_booke(void)
72
70
{
73
-
mtspr(SPRN_PID, 0);
74
-
isync();
71
+
asm(ASM_MMU_FTR_IFSET("mtspr %0, %1; isync", "", %2) : :
72
+
"i"(SPRN_PID), "r"(0), "i"(MMU_FTR_KUAP) : "memory");
75
73
}
76
74
77
-
static __always_inline unsigned long __prevent_user_access_return(void)
75
+
static __always_inline void allow_user_access(void __user *to, const void __user *from,
76
+
unsigned long size, unsigned long dir)
77
+
{
78
+
uaccess_begin_booke(current->thread.pid);
79
+
}
80
+
81
+
static __always_inline void prevent_user_access(unsigned long dir)
82
+
{
83
+
uaccess_end_booke();
84
+
}
85
+
86
+
static __always_inline unsigned long prevent_user_access_return(void)
78
87
{
79
88
unsigned long flags = mfspr(SPRN_PID);
80
89
81
-
mtspr(SPRN_PID, 0);
82
-
isync();
90
+
uaccess_end_booke();
83
91
84
92
return flags;
85
93
}
86
94
87
-
static __always_inline void __restore_user_access(unsigned long flags)
95
+
static __always_inline void restore_user_access(unsigned long flags)
88
96
{
89
-
if (flags) {
90
-
mtspr(SPRN_PID, current->thread.pid);
91
-
isync();
92
-
}
97
+
if (flags)
98
+
uaccess_begin_booke(current->thread.pid);
93
99
}
94
100
95
101
static __always_inline bool