Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
[MIPS] Fix aliasing bug in copy_to_user_page / copy_from_user_page
The current implementation uses a sequence of a cacheflush and a copy.
This is racy in case of a multithreaded debuggee and renders GDB
virtually unusable.
Aside this fixes a performance hog rendering access to /proc/cmdline very
slow and resulting in a enough cache stalls for the 34K AP/SP programming
model to make the bare metal code on the non-Linux VPE miss RT deadlines.
The main part of this patch was originally written by Ralf Baechle;
Atushi Nemoto did the the debugging.
Signed-off-by: Atsushi Nemoto <anemo@mba.ocn.ne.jp>
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
+190
-28
+160
-7
arch/mips/mm/init.c
+160
-7
arch/mips/mm/init.c
···
30
30
#include <asm/cachectl.h>
31
31
#include <asm/cpu.h>
32
32
#include <asm/dma.h>
33
+
#include <asm/kmap_types.h>
33
34
#include <asm/mmu_context.h>
34
35
#include <asm/sections.h>
35
36
#include <asm/pgtable.h>
36
37
#include <asm/pgalloc.h>
37
38
#include <asm/tlb.h>
39
+
#include <asm/fixmap.h>
40
+
41
+
/* Atomicity and interruptability */
42
+
#ifdef CONFIG_MIPS_MT_SMTC
43
+
44
+
#include <asm/mipsmtregs.h>
45
+
46
+
#define ENTER_CRITICAL(flags) \
47
+
{ \
48
+
unsigned int mvpflags; \
49
+
local_irq_save(flags);\
50
+
mvpflags = dvpe()
51
+
#define EXIT_CRITICAL(flags) \
52
+
evpe(mvpflags); \
53
+
local_irq_restore(flags); \
54
+
}
55
+
#else
56
+
57
+
#define ENTER_CRITICAL(flags) local_irq_save(flags)
58
+
#define EXIT_CRITICAL(flags) local_irq_restore(flags)
59
+
60
+
#endif /* CONFIG_MIPS_MT_SMTC */
38
61
39
62
DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
40
63
···
103
80
return 1UL << order;
104
81
}
105
82
106
-
#ifdef CONFIG_HIGHMEM
107
-
pte_t *kmap_pte;
108
-
pgprot_t kmap_prot;
83
+
/*
84
+
* These are almost like kmap_atomic / kunmap_atmic except they take an
85
+
* additional address argument as the hint.
86
+
*/
109
87
110
88
#define kmap_get_fixmap_pte(vaddr) \
111
89
pte_offset_kernel(pmd_offset(pud_offset(pgd_offset_k(vaddr), (vaddr)), (vaddr)), (vaddr))
90
+
91
+
#ifdef CONFIG_MIPS_MT_SMTC
92
+
static pte_t *kmap_coherent_pte;
93
+
static void __init kmap_coherent_init(void)
94
+
{
95
+
unsigned long vaddr;
96
+
97
+
/* cache the first coherent kmap pte */
98
+
vaddr = __fix_to_virt(FIX_CMAP_BEGIN);
99
+
kmap_coherent_pte = kmap_get_fixmap_pte(vaddr);
100
+
}
101
+
#else
102
+
static inline void kmap_coherent_init(void) {}
103
+
#endif
104
+
105
+
static inline void *kmap_coherent(struct page *page, unsigned long addr)
106
+
{
107
+
enum fixed_addresses idx;
108
+
unsigned long vaddr, flags, entrylo;
109
+
unsigned long old_ctx;
110
+
pte_t pte;
111
+
int tlbidx;
112
+
113
+
inc_preempt_count();
114
+
idx = (addr >> PAGE_SHIFT) & (FIX_N_COLOURS - 1);
115
+
#ifdef CONFIG_MIPS_MT_SMTC
116
+
idx += FIX_N_COLOURS * smp_processor_id();
117
+
#endif
118
+
vaddr = __fix_to_virt(FIX_CMAP_END - idx);
119
+
pte = mk_pte(page, PAGE_KERNEL);
120
+
#if defined(CONFIG_64BIT_PHYS_ADDR) && defined(CONFIG_CPU_MIPS32_R1)
121
+
entrylo = pte.pte_high;
122
+
#else
123
+
entrylo = pte_val(pte) >> 6;
124
+
#endif
125
+
126
+
ENTER_CRITICAL(flags);
127
+
old_ctx = read_c0_entryhi();
128
+
write_c0_entryhi(vaddr & (PAGE_MASK << 1));
129
+
write_c0_entrylo0(entrylo);
130
+
write_c0_entrylo1(entrylo);
131
+
#ifdef CONFIG_MIPS_MT_SMTC
132
+
set_pte(kmap_coherent_pte - (FIX_CMAP_END - idx), pte);
133
+
/* preload TLB instead of local_flush_tlb_one() */
134
+
mtc0_tlbw_hazard();
135
+
tlb_probe();
136
+
tlb_probe_hazard();
137
+
tlbidx = read_c0_index();
138
+
mtc0_tlbw_hazard();
139
+
if (tlbidx < 0)
140
+
tlb_write_random();
141
+
else
142
+
tlb_write_indexed();
143
+
#else
144
+
tlbidx = read_c0_wired();
145
+
write_c0_wired(tlbidx + 1);
146
+
write_c0_index(tlbidx);
147
+
mtc0_tlbw_hazard();
148
+
tlb_write_indexed();
149
+
#endif
150
+
tlbw_use_hazard();
151
+
write_c0_entryhi(old_ctx);
152
+
EXIT_CRITICAL(flags);
153
+
154
+
return (void*) vaddr;
155
+
}
156
+
157
+
#define UNIQUE_ENTRYHI(idx) (CKSEG0 + ((idx) << (PAGE_SHIFT + 1)))
158
+
159
+
static inline void kunmap_coherent(struct page *page)
160
+
{
161
+
#ifndef CONFIG_MIPS_MT_SMTC
162
+
unsigned int wired;
163
+
unsigned long flags, old_ctx;
164
+
165
+
ENTER_CRITICAL(flags);
166
+
old_ctx = read_c0_entryhi();
167
+
wired = read_c0_wired() - 1;
168
+
write_c0_wired(wired);
169
+
write_c0_index(wired);
170
+
write_c0_entryhi(UNIQUE_ENTRYHI(wired));
171
+
write_c0_entrylo0(0);
172
+
write_c0_entrylo1(0);
173
+
mtc0_tlbw_hazard();
174
+
tlb_write_indexed();
175
+
tlbw_use_hazard();
176
+
write_c0_entryhi(old_ctx);
177
+
EXIT_CRITICAL(flags);
178
+
#endif
179
+
dec_preempt_count();
180
+
preempt_check_resched();
181
+
}
182
+
183
+
void copy_to_user_page(struct vm_area_struct *vma,
184
+
struct page *page, unsigned long vaddr, void *dst, const void *src,
185
+
unsigned long len)
186
+
{
187
+
if (cpu_has_dc_aliases) {
188
+
void *vto = kmap_coherent(page, vaddr) + (vaddr & ~PAGE_MASK);
189
+
memcpy(vto, src, len);
190
+
kunmap_coherent(page);
191
+
} else
192
+
memcpy(dst, src, len);
193
+
if ((vma->vm_flags & VM_EXEC) && !cpu_has_ic_fills_f_dc)
194
+
flush_cache_page(vma, vaddr, page_to_pfn(page));
195
+
}
196
+
197
+
EXPORT_SYMBOL(copy_to_user_page);
198
+
199
+
void copy_from_user_page(struct vm_area_struct *vma,
200
+
struct page *page, unsigned long vaddr, void *dst, const void *src,
201
+
unsigned long len)
202
+
{
203
+
if (cpu_has_dc_aliases) {
204
+
void *vfrom =
205
+
kmap_coherent(page, vaddr) + (vaddr & ~PAGE_MASK);
206
+
memcpy(dst, vfrom, len);
207
+
kunmap_coherent(page);
208
+
} else
209
+
memcpy(dst, src, len);
210
+
}
211
+
212
+
EXPORT_SYMBOL(copy_from_user_page);
213
+
214
+
215
+
#ifdef CONFIG_HIGHMEM
216
+
pte_t *kmap_pte;
217
+
pgprot_t kmap_prot;
112
218
113
219
static void __init kmap_init(void)
114
220
{
···
249
97
250
98
kmap_prot = PAGE_KERNEL;
251
99
}
100
+
#endif /* CONFIG_HIGHMEM */
252
101
253
-
#ifdef CONFIG_32BIT
254
102
void __init fixrange_init(unsigned long start, unsigned long end,
255
103
pgd_t *pgd_base)
256
104
{
105
+
#if defined(CONFIG_HIGHMEM) || defined(CONFIG_MIPS_MT_SMTC)
257
106
pgd_t *pgd;
258
107
pud_t *pud;
259
108
pmd_t *pmd;
···
275
122
for (; (k < PTRS_PER_PMD) && (vaddr != end); pmd++, k++) {
276
123
if (pmd_none(*pmd)) {
277
124
pte = (pte_t *) alloc_bootmem_low_pages(PAGE_SIZE);
278
-
set_pmd(pmd, __pmd(pte));
125
+
set_pmd(pmd, __pmd((unsigned long)pte));
279
126
if (pte != pte_offset_kernel(pmd, 0))
280
127
BUG();
281
128
}
···
285
132
}
286
133
j = 0;
287
134
}
135
+
#endif
288
136
}
289
-
#endif /* CONFIG_32BIT */
290
-
#endif /* CONFIG_HIGHMEM */
291
137
292
138
#ifndef CONFIG_NEED_MULTIPLE_NODES
293
139
extern void pagetable_init(void);
···
327
175
#ifdef CONFIG_HIGHMEM
328
176
kmap_init();
329
177
#endif
178
+
kmap_coherent_init();
330
179
331
180
max_dma = virt_to_phys((char *)MAX_DMA_ADDRESS) >> PAGE_SHIFT;
332
181
low = max_low_pfn;
+4
-3
arch/mips/mm/pgtable-32.c
+4
-3
arch/mips/mm/pgtable-32.c
···
31
31
32
32
void __init pagetable_init(void)
33
33
{
34
-
#ifdef CONFIG_HIGHMEM
35
34
unsigned long vaddr;
36
-
pgd_t *pgd, *pgd_base;
35
+
pgd_t *pgd_base;
36
+
#ifdef CONFIG_HIGHMEM
37
+
pgd_t *pgd;
37
38
pud_t *pud;
38
39
pmd_t *pmd;
39
40
pte_t *pte;
···
45
44
pgd_init((unsigned long)swapper_pg_dir
46
45
+ sizeof(pgd_t) * USER_PTRS_PER_PGD);
47
46
48
-
#ifdef CONFIG_HIGHMEM
49
47
pgd_base = swapper_pg_dir;
50
48
51
49
/*
···
53
53
vaddr = __fix_to_virt(__end_of_fixed_addresses - 1) & PMD_MASK;
54
54
fixrange_init(vaddr, 0, pgd_base);
55
55
56
+
#ifdef CONFIG_HIGHMEM
56
57
/*
57
58
* Permanent kmaps:
58
59
*/
+11
arch/mips/mm/pgtable-64.c
+11
arch/mips/mm/pgtable-64.c
···
8
8
*/
9
9
#include <linux/init.h>
10
10
#include <linux/mm.h>
11
+
#include <asm/fixmap.h>
11
12
#include <asm/pgtable.h>
12
13
13
14
void pgd_init(unsigned long page)
···
53
52
54
53
void __init pagetable_init(void)
55
54
{
55
+
unsigned long vaddr;
56
+
pgd_t *pgd_base;
57
+
56
58
/* Initialize the entire pgd. */
57
59
pgd_init((unsigned long)swapper_pg_dir);
58
60
pmd_init((unsigned long)invalid_pmd_table, (unsigned long)invalid_pte_table);
61
+
62
+
pgd_base = swapper_pg_dir;
63
+
/*
64
+
* Fixed mappings:
65
+
*/
66
+
vaddr = __fix_to_virt(__end_of_fixed_addresses - 1) & PMD_MASK;
67
+
fixrange_init(vaddr, 0, pgd_base);
59
68
}
+4
-15
include/asm-mips/cacheflush.h
+4
-15
include/asm-mips/cacheflush.h
···
55
55
#define flush_cache_vmap(start, end) flush_cache_all()
56
56
#define flush_cache_vunmap(start, end) flush_cache_all()
57
57
58
-
static inline void copy_to_user_page(struct vm_area_struct *vma,
58
+
extern void copy_to_user_page(struct vm_area_struct *vma,
59
59
struct page *page, unsigned long vaddr, void *dst, const void *src,
60
-
unsigned long len)
61
-
{
62
-
if (cpu_has_dc_aliases)
63
-
flush_cache_page(vma, vaddr, page_to_pfn(page));
64
-
memcpy(dst, src, len);
65
-
__flush_icache_page(vma, page);
66
-
}
60
+
unsigned long len);
67
61
68
-
static inline void copy_from_user_page(struct vm_area_struct *vma,
62
+
extern void copy_from_user_page(struct vm_area_struct *vma,
69
63
struct page *page, unsigned long vaddr, void *dst, const void *src,
70
-
unsigned long len)
71
-
{
72
-
if (cpu_has_dc_aliases)
73
-
flush_cache_page(vma, vaddr, page_to_pfn(page));
74
-
memcpy(dst, src, len);
75
-
}
64
+
unsigned long len);
76
65
77
66
extern void (*flush_cache_sigtramp)(unsigned long addr);
78
67
extern void (*flush_icache_all)(void);
+11
-3
include/asm-mips/fixmap.h
+11
-3
include/asm-mips/fixmap.h
···
45
45
* fix-mapped?
46
46
*/
47
47
enum fixed_addresses {
48
+
#define FIX_N_COLOURS 8
49
+
FIX_CMAP_BEGIN,
50
+
#ifdef CONFIG_MIPS_MT_SMTC
51
+
FIX_CMAP_END = FIX_CMAP_BEGIN + (FIX_N_COLOURS * NR_CPUS),
52
+
#else
53
+
FIX_CMAP_END = FIX_CMAP_BEGIN + FIX_N_COLOURS,
54
+
#endif
48
55
#ifdef CONFIG_HIGHMEM
49
-
FIX_KMAP_BEGIN, /* reserved pte's for temporary kernel mappings */
56
+
/* reserved pte's for temporary kernel mappings */
57
+
FIX_KMAP_BEGIN = FIX_CMAP_END + 1,
50
58
FIX_KMAP_END = FIX_KMAP_BEGIN+(KM_TYPE_NR*NR_CPUS)-1,
51
59
#endif
52
60
__end_of_fixed_addresses
···
78
70
* at the top of mem..
79
71
*/
80
72
#if defined(CONFIG_CPU_TX39XX) || defined(CONFIG_CPU_TX49XX)
81
-
#define FIXADDR_TOP (0xff000000UL - 0x2000)
73
+
#define FIXADDR_TOP ((unsigned long)(long)(int)(0xff000000 - 0x20000))
82
74
#else
83
-
#define FIXADDR_TOP (0xffffe000UL)
75
+
#define FIXADDR_TOP ((unsigned long)(long)(int)0xfffe0000)
84
76
#endif
85
77
#define FIXADDR_SIZE (__end_of_fixed_addresses << PAGE_SHIFT)
86
78
#define FIXADDR_START (FIXADDR_TOP - FIXADDR_SIZE)