Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
csky: Sync riscv mm/fault.c for easy maintenance
Sync arch/riscv/mm/fault.c into arch/csky for easy maintenance.
Here are the patches related to the modification:
cac4d1d "riscv/mm/fault: Move no context handling to no_context()"
ac416a7 "riscv/mm/fault: Move vmalloc fault handling to vmalloc_fault()"
6c11ffb "riscv/mm/fault: Move fault error handling to mm_fault_error()"
afb8c6f "riscv/mm/fault: Move access error check to function"
bda281d "riscv/mm/fault: Simplify fault error handling"
a51271d "riscv/mm/fault: Move bad area handling to bad_area()"
Signed-off-by: Guo Ren <guoren@linux.alibaba.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: Palmer Dabbelt <palmerdabbelt@google.com>
Cc: Arnd Bergmann <arnd@arndb.de>
+194
-160
+2
arch/csky/include/asm/bug.h
+2
arch/csky/include/asm/bug.h
+192
-160
arch/csky/mm/fault.c
+192
-160
arch/csky/mm/fault.c
···
1
1
// SPDX-License-Identifier: GPL-2.0
2
2
// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd.
3
3
4
-
#include <linux/signal.h>
5
-
#include <linux/module.h>
6
-
#include <linux/sched.h>
7
-
#include <linux/interrupt.h>
8
-
#include <linux/kernel.h>
9
-
#include <linux/errno.h>
10
-
#include <linux/string.h>
11
-
#include <linux/types.h>
12
-
#include <linux/ptrace.h>
13
-
#include <linux/mman.h>
14
-
#include <linux/mm.h>
15
-
#include <linux/smp.h>
16
-
#include <linux/version.h>
17
-
#include <linux/vt_kern.h>
18
4
#include <linux/extable.h>
19
-
#include <linux/uaccess.h>
20
-
#include <linux/perf_event.h>
21
5
#include <linux/kprobes.h>
22
-
23
-
#include <asm/hardirq.h>
24
-
#include <asm/mmu_context.h>
25
-
#include <asm/traps.h>
26
-
#include <asm/page.h>
6
+
#include <linux/mmu_context.h>
7
+
#include <linux/perf_event.h>
27
8
28
9
int fixup_exception(struct pt_regs *regs)
29
10
{
···
51
70
}
52
71
#endif
53
72
73
+
static inline void no_context(struct pt_regs *regs, unsigned long addr)
74
+
{
75
+
current->thread.trap_no = trap_no(regs);
76
+
77
+
/* Are we prepared to handle this kernel fault? */
78
+
if (fixup_exception(regs))
79
+
return;
80
+
81
+
/*
82
+
* Oops. The kernel tried to access some bad page. We'll have to
83
+
* terminate things with extreme prejudice.
84
+
*/
85
+
bust_spinlocks(1);
86
+
pr_alert("Unable to handle kernel paging request at virtual "
87
+
"addr 0x%08lx, pc: 0x%08lx\n", addr, regs->pc);
88
+
die(regs, "Oops");
89
+
do_exit(SIGKILL);
90
+
}
91
+
92
+
static inline void mm_fault_error(struct pt_regs *regs, unsigned long addr, vm_fault_t fault)
93
+
{
94
+
current->thread.trap_no = trap_no(regs);
95
+
96
+
if (fault & VM_FAULT_OOM) {
97
+
/*
98
+
* We ran out of memory, call the OOM killer, and return the userspace
99
+
* (which will retry the fault, or kill us if we got oom-killed).
100
+
*/
101
+
if (!user_mode(regs)) {
102
+
no_context(regs, addr);
103
+
return;
104
+
}
105
+
pagefault_out_of_memory();
106
+
return;
107
+
} else if (fault & VM_FAULT_SIGBUS) {
108
+
/* Kernel mode? Handle exceptions or die */
109
+
if (!user_mode(regs)) {
110
+
no_context(regs, addr);
111
+
return;
112
+
}
113
+
do_trap(regs, SIGBUS, BUS_ADRERR, addr);
114
+
return;
115
+
}
116
+
BUG();
117
+
}
118
+
119
+
static inline void bad_area(struct pt_regs *regs, struct mm_struct *mm, int code, unsigned long addr)
120
+
{
121
+
/*
122
+
* Something tried to access memory that isn't in our memory map.
123
+
* Fix it, but check if it's kernel or user first.
124
+
*/
125
+
mmap_read_unlock(mm);
126
+
/* User mode accesses just cause a SIGSEGV */
127
+
if (user_mode(regs)) {
128
+
do_trap(regs, SIGSEGV, code, addr);
129
+
return;
130
+
}
131
+
132
+
no_context(regs, addr);
133
+
}
134
+
135
+
static inline void vmalloc_fault(struct pt_regs *regs, int code, unsigned long addr)
136
+
{
137
+
pgd_t *pgd, *pgd_k;
138
+
pud_t *pud, *pud_k;
139
+
pmd_t *pmd, *pmd_k;
140
+
pte_t *pte_k;
141
+
int offset;
142
+
143
+
/* User mode accesses just cause a SIGSEGV */
144
+
if (user_mode(regs)) {
145
+
do_trap(regs, SIGSEGV, code, addr);
146
+
return;
147
+
}
148
+
149
+
/*
150
+
* Synchronize this task's top level page-table
151
+
* with the 'reference' page table.
152
+
*
153
+
* Do _not_ use "tsk" here. We might be inside
154
+
* an interrupt in the middle of a task switch..
155
+
*/
156
+
offset = pgd_index(addr);
157
+
158
+
pgd = get_pgd() + offset;
159
+
pgd_k = init_mm.pgd + offset;
160
+
161
+
if (!pgd_present(*pgd_k)) {
162
+
no_context(regs, addr);
163
+
return;
164
+
}
165
+
set_pgd(pgd, *pgd_k);
166
+
167
+
pud = (pud_t *)pgd;
168
+
pud_k = (pud_t *)pgd_k;
169
+
if (!pud_present(*pud_k)) {
170
+
no_context(regs, addr);
171
+
return;
172
+
}
173
+
174
+
pmd = pmd_offset(pud, addr);
175
+
pmd_k = pmd_offset(pud_k, addr);
176
+
if (!pmd_present(*pmd_k)) {
177
+
no_context(regs, addr);
178
+
return;
179
+
}
180
+
set_pmd(pmd, *pmd_k);
181
+
182
+
pte_k = pte_offset_kernel(pmd_k, addr);
183
+
if (!pte_present(*pte_k)) {
184
+
no_context(regs, addr);
185
+
return;
186
+
}
187
+
188
+
flush_tlb_one(addr);
189
+
}
190
+
191
+
static inline bool access_error(struct pt_regs *regs, struct vm_area_struct *vma)
192
+
{
193
+
if (is_write(regs)) {
194
+
if (!(vma->vm_flags & VM_WRITE))
195
+
return true;
196
+
} else {
197
+
if (unlikely(!vma_is_accessible(vma)))
198
+
return true;
199
+
}
200
+
return false;
201
+
}
202
+
54
203
/*
55
-
* This routine handles page faults. It determines the address,
56
-
* and the problem, and then passes it off to one of the appropriate
57
-
* routines.
204
+
* This routine handles page faults. It determines the address and the
205
+
* problem, and then passes it off to one of the appropriate routines.
58
206
*/
59
207
asmlinkage void do_page_fault(struct pt_regs *regs)
60
208
{
61
-
struct vm_area_struct *vma = NULL;
62
-
struct task_struct *tsk = current;
63
-
struct mm_struct *mm = tsk->mm;
209
+
struct task_struct *tsk;
210
+
struct vm_area_struct *vma;
211
+
struct mm_struct *mm;
212
+
unsigned long addr = read_mmu_entryhi() & PAGE_MASK;
64
213
unsigned int flags = FAULT_FLAG_DEFAULT;
65
-
int si_code;
66
-
int fault;
67
-
unsigned long address = read_mmu_entryhi() & PAGE_MASK;
214
+
int code = SEGV_MAPERR;
215
+
vm_fault_t fault;
216
+
217
+
tsk = current;
218
+
mm = tsk->mm;
68
219
69
220
csky_cmpxchg_fixup(regs);
70
221
71
222
if (kprobe_page_fault(regs, tsk->thread.trap_no))
72
223
return;
73
224
74
-
si_code = SEGV_MAPERR;
75
-
76
225
/*
77
-
* We fault-in kernel-space virtual memory on-demand. The
78
-
* 'reference' page table is init_mm.pgd.
226
+
* Fault-in kernel-space virtual memory on-demand.
227
+
* The 'reference' page table is init_mm.pgd.
79
228
*
80
229
* NOTE! We MUST NOT take any locks for this case. We may
81
230
* be in an interrupt or a critical region, and should
82
231
* only copy the information from the master page table,
83
232
* nothing more.
84
233
*/
85
-
if (unlikely(address >= VMALLOC_START) &&
86
-
unlikely(address <= VMALLOC_END)) {
87
-
/*
88
-
* Synchronize this task's top level page-table
89
-
* with the 'reference' page table.
90
-
*
91
-
* Do _not_ use "tsk" here. We might be inside
92
-
* an interrupt in the middle of a task switch..
93
-
*/
94
-
int offset = pgd_index(address);
95
-
pgd_t *pgd, *pgd_k;
96
-
pud_t *pud, *pud_k;
97
-
pmd_t *pmd, *pmd_k;
98
-
pte_t *pte_k;
99
-
100
-
pgd = get_pgd() + offset;
101
-
pgd_k = init_mm.pgd + offset;
102
-
103
-
if (!pgd_present(*pgd_k))
104
-
goto no_context;
105
-
set_pgd(pgd, *pgd_k);
106
-
107
-
pud = (pud_t *)pgd;
108
-
pud_k = (pud_t *)pgd_k;
109
-
if (!pud_present(*pud_k))
110
-
goto no_context;
111
-
112
-
pmd = pmd_offset(pud, address);
113
-
pmd_k = pmd_offset(pud_k, address);
114
-
if (!pmd_present(*pmd_k))
115
-
goto no_context;
116
-
set_pmd(pmd, *pmd_k);
117
-
118
-
pte_k = pte_offset_kernel(pmd_k, address);
119
-
if (!pte_present(*pte_k))
120
-
goto no_context;
121
-
122
-
flush_tlb_one(address);
123
-
234
+
if (unlikely((addr >= VMALLOC_START) && (addr <= VMALLOC_END))) {
235
+
vmalloc_fault(regs, code, addr);
124
236
return;
125
237
}
126
238
···
222
148
local_irq_enable();
223
149
224
150
/*
225
-
* If we're in an interrupt or have no user
226
-
* context, we must not take the fault..
151
+
* If we're in an interrupt, have no user context, or are running
152
+
* in an atomic region, then we must not take the fault.
227
153
*/
228
-
if (unlikely(faulthandler_disabled() || !mm))
229
-
goto bad_area_nosemaphore;
154
+
if (unlikely(faulthandler_disabled() || !mm)) {
155
+
no_context(regs, addr);
156
+
return;
157
+
}
230
158
231
159
if (user_mode(regs))
232
160
flags |= FAULT_FLAG_USER;
233
161
162
+
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr);
163
+
234
164
if (is_write(regs))
235
165
flags |= FAULT_FLAG_WRITE;
236
-
237
-
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
238
166
retry:
239
167
mmap_read_lock(mm);
240
-
vma = find_vma(mm, address);
241
-
if (!vma)
242
-
goto bad_area;
243
-
if (vma->vm_start <= address)
168
+
vma = find_vma(mm, addr);
169
+
if (unlikely(!vma)) {
170
+
bad_area(regs, mm, code, addr);
171
+
return;
172
+
}
173
+
if (likely(vma->vm_start <= addr))
244
174
goto good_area;
245
-
if (!(vma->vm_flags & VM_GROWSDOWN))
246
-
goto bad_area;
247
-
if (expand_stack(vma, address))
248
-
goto bad_area;
249
-
/*
250
-
* Ok, we have a good vm_area for this memory access, so
251
-
* we can handle it..
252
-
*/
253
-
good_area:
254
-
si_code = SEGV_ACCERR;
255
-
256
-
if (is_write(regs)) {
257
-
if (!(vma->vm_flags & VM_WRITE))
258
-
goto bad_area;
259
-
} else {
260
-
if (unlikely(!vma_is_accessible(vma)))
261
-
goto bad_area;
175
+
if (unlikely(!(vma->vm_flags & VM_GROWSDOWN))) {
176
+
bad_area(regs, mm, code, addr);
177
+
return;
178
+
}
179
+
if (unlikely(expand_stack(vma, addr))) {
180
+
bad_area(regs, mm, code, addr);
181
+
return;
262
182
}
263
183
264
184
/*
265
-
* If for any reason at all we couldn't handle the fault,
185
+
* Ok, we have a good vm_area for this memory access, so
186
+
* we can handle it.
187
+
*/
188
+
good_area:
189
+
code = SEGV_ACCERR;
190
+
191
+
if (unlikely(access_error(regs, vma))) {
192
+
bad_area(regs, mm, code, addr);
193
+
return;
194
+
}
195
+
196
+
/*
197
+
* If for any reason at all we could not handle the fault,
266
198
* make sure we exit gracefully rather than endlessly redo
267
199
* the fault.
268
200
*/
269
-
fault = handle_mm_fault(vma, address, flags, regs);
270
-
if (unlikely(fault & VM_FAULT_ERROR)) {
271
-
if (fault & VM_FAULT_OOM)
272
-
goto out_of_memory;
273
-
else if (fault & VM_FAULT_SIGBUS)
274
-
goto do_sigbus;
275
-
else if (fault & VM_FAULT_SIGSEGV)
276
-
goto bad_area;
277
-
BUG();
278
-
}
201
+
fault = handle_mm_fault(vma, addr, flags, regs);
202
+
203
+
/*
204
+
* If we need to retry but a fatal signal is pending, handle the
205
+
* signal first. We do not need to release the mmap_lock because it
206
+
* would already be released in __lock_page_or_retry in mm/filemap.c.
207
+
*/
208
+
if (fault_signal_pending(fault, regs))
209
+
return;
279
210
280
211
if (unlikely((fault & VM_FAULT_RETRY) && (flags & FAULT_FLAG_ALLOW_RETRY))) {
281
212
flags |= FAULT_FLAG_TRIED;
···
294
215
}
295
216
296
217
mmap_read_unlock(mm);
297
-
return;
298
218
299
-
/*
300
-
* Something tried to access memory that isn't in our memory map..
301
-
* Fix it, but check if it's kernel or user first..
302
-
*/
303
-
bad_area:
304
-
mmap_read_unlock(mm);
305
-
306
-
bad_area_nosemaphore:
307
-
/* User mode accesses just cause a SIGSEGV */
308
-
if (user_mode(regs)) {
309
-
tsk->thread.trap_no = trap_no(regs);
310
-
force_sig_fault(SIGSEGV, si_code, (void __user *)address);
219
+
if (unlikely(fault & VM_FAULT_ERROR)) {
220
+
mm_fault_error(regs, addr, fault);
311
221
return;
312
222
}
313
-
314
-
no_context:
315
-
tsk->thread.trap_no = trap_no(regs);
316
-
317
-
/* Are we prepared to handle this kernel fault? */
318
-
if (fixup_exception(regs))
319
-
return;
320
-
321
-
/*
322
-
* Oops. The kernel tried to access some bad page. We'll have to
323
-
* terminate things with extreme prejudice.
324
-
*/
325
-
bust_spinlocks(1);
326
-
pr_alert("Unable to handle kernel paging request at virtual "
327
-
"address 0x%08lx, pc: 0x%08lx\n", address, regs->pc);
328
-
die(regs, "Oops");
329
-
330
-
out_of_memory:
331
-
tsk->thread.trap_no = trap_no(regs);
332
-
333
-
/*
334
-
* We ran out of memory, call the OOM killer, and return the userspace
335
-
* (which will retry the fault, or kill us if we got oom-killed).
336
-
*/
337
-
pagefault_out_of_memory();
338
223
return;
339
-
340
-
do_sigbus:
341
-
tsk->thread.trap_no = trap_no(regs);
342
-
343
-
mmap_read_unlock(mm);
344
-
345
-
/* Kernel mode? Handle exceptions or die */
346
-
if (!user_mode(regs))
347
-
goto no_context;
348
-
349
-
force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)address);
350
224
}