Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge branch 'tracing/mmiotrace' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip
* 'tracing/mmiotrace' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip:
x86 mmiotrace: fix race with release_kmmio_fault_page()
x86 mmiotrace: improve handling of secondary faults
x86 mmiotrace: split set_page_presence()
x86 mmiotrace: fix save/restore page table state
x86 mmiotrace: WARN_ONCE if dis/arming a page fails
x86: add far read test to testmmiotrace
x86: count errors in testmmiotrace.ko
+153
-66
+96
-53
arch/x86/mm/kmmio.c
+96
-53
arch/x86/mm/kmmio.c
···
32
32
struct list_head list;
33
33
struct kmmio_fault_page *release_next;
34
34
unsigned long page; /* location of the fault page */
35
+
bool old_presence; /* page presence prior to arming */
36
+
bool armed;
35
37
36
38
/*
37
39
* Number of times this page has been registered as a part
38
40
* of a probe. If zero, page is disarmed and this may be freed.
39
-
* Used only by writers (RCU).
41
+
* Used only by writers (RCU) and post_kmmio_handler().
42
+
* Protected by kmmio_lock, when linked into kmmio_page_table.
40
43
*/
41
44
int count;
42
45
};
···
108
105
return NULL;
109
106
}
110
107
111
-
static void set_page_present(unsigned long addr, bool present,
112
-
unsigned int *pglevel)
108
+
static void set_pmd_presence(pmd_t *pmd, bool present, bool *old)
113
109
{
114
-
pteval_t pteval;
115
-
pmdval_t pmdval;
110
+
pmdval_t v = pmd_val(*pmd);
111
+
*old = !!(v & _PAGE_PRESENT);
112
+
v &= ~_PAGE_PRESENT;
113
+
if (present)
114
+
v |= _PAGE_PRESENT;
115
+
set_pmd(pmd, __pmd(v));
116
+
}
117
+
118
+
static void set_pte_presence(pte_t *pte, bool present, bool *old)
119
+
{
120
+
pteval_t v = pte_val(*pte);
121
+
*old = !!(v & _PAGE_PRESENT);
122
+
v &= ~_PAGE_PRESENT;
123
+
if (present)
124
+
v |= _PAGE_PRESENT;
125
+
set_pte_atomic(pte, __pte(v));
126
+
}
127
+
128
+
static int set_page_presence(unsigned long addr, bool present, bool *old)
129
+
{
116
130
unsigned int level;
117
-
pmd_t *pmd;
118
131
pte_t *pte = lookup_address(addr, &level);
119
132
120
133
if (!pte) {
121
134
pr_err("kmmio: no pte for page 0x%08lx\n", addr);
122
-
return;
135
+
return -1;
123
136
}
124
-
125
-
if (pglevel)
126
-
*pglevel = level;
127
137
128
138
switch (level) {
129
139
case PG_LEVEL_2M:
130
-
pmd = (pmd_t *)pte;
131
-
pmdval = pmd_val(*pmd) & ~_PAGE_PRESENT;
132
-
if (present)
133
-
pmdval |= _PAGE_PRESENT;
134
-
set_pmd(pmd, __pmd(pmdval));
140
+
set_pmd_presence((pmd_t *)pte, present, old);
135
141
break;
136
-
137
142
case PG_LEVEL_4K:
138
-
pteval = pte_val(*pte) & ~_PAGE_PRESENT;
139
-
if (present)
140
-
pteval |= _PAGE_PRESENT;
141
-
set_pte_atomic(pte, __pte(pteval));
143
+
set_pte_presence(pte, present, old);
142
144
break;
143
-
144
145
default:
145
146
pr_err("kmmio: unexpected page level 0x%x.\n", level);
146
-
return;
147
+
return -1;
147
148
}
148
149
149
150
__flush_tlb_one(addr);
151
+
return 0;
150
152
}
151
153
152
-
/** Mark the given page as not present. Access to it will trigger a fault. */
153
-
static void arm_kmmio_fault_page(unsigned long page, unsigned int *pglevel)
154
+
/*
155
+
* Mark the given page as not present. Access to it will trigger a fault.
156
+
*
157
+
* Struct kmmio_fault_page is protected by RCU and kmmio_lock, but the
158
+
* protection is ignored here. RCU read lock is assumed held, so the struct
159
+
* will not disappear unexpectedly. Furthermore, the caller must guarantee,
160
+
* that double arming the same virtual address (page) cannot occur.
161
+
*
162
+
* Double disarming on the other hand is allowed, and may occur when a fault
163
+
* and mmiotrace shutdown happen simultaneously.
164
+
*/
165
+
static int arm_kmmio_fault_page(struct kmmio_fault_page *f)
154
166
{
155
-
set_page_present(page & PAGE_MASK, false, pglevel);
167
+
int ret;
168
+
WARN_ONCE(f->armed, KERN_ERR "kmmio page already armed.\n");
169
+
if (f->armed) {
170
+
pr_warning("kmmio double-arm: page 0x%08lx, ref %d, old %d\n",
171
+
f->page, f->count, f->old_presence);
172
+
}
173
+
ret = set_page_presence(f->page, false, &f->old_presence);
174
+
WARN_ONCE(ret < 0, KERN_ERR "kmmio arming 0x%08lx failed.\n", f->page);
175
+
f->armed = true;
176
+
return ret;
156
177
}
157
178
158
-
/** Mark the given page as present. */
159
-
static void disarm_kmmio_fault_page(unsigned long page, unsigned int *pglevel)
179
+
/** Restore the given page to saved presence state. */
180
+
static void disarm_kmmio_fault_page(struct kmmio_fault_page *f)
160
181
{
161
-
set_page_present(page & PAGE_MASK, true, pglevel);
182
+
bool tmp;
183
+
int ret = set_page_presence(f->page, f->old_presence, &tmp);
184
+
WARN_ONCE(ret < 0,
185
+
KERN_ERR "kmmio disarming 0x%08lx failed.\n", f->page);
186
+
f->armed = false;
162
187
}
163
188
164
189
/*
···
233
202
234
203
ctx = &get_cpu_var(kmmio_ctx);
235
204
if (ctx->active) {
236
-
disarm_kmmio_fault_page(faultpage->page, NULL);
237
205
if (addr == ctx->addr) {
238
206
/*
239
-
* On SMP we sometimes get recursive probe hits on the
240
-
* same address. Context is already saved, fall out.
207
+
* A second fault on the same page means some other
208
+
* condition needs handling by do_page_fault(), the
209
+
* page really not being present is the most common.
241
210
*/
242
-
pr_debug("kmmio: duplicate probe hit on CPU %d, for "
243
-
"address 0x%08lx.\n",
244
-
smp_processor_id(), addr);
245
-
ret = 1;
246
-
goto no_kmmio_ctx;
247
-
}
248
-
/*
249
-
* Prevent overwriting already in-flight context.
250
-
* This should not happen, let's hope disarming at least
251
-
* prevents a panic.
252
-
*/
253
-
pr_emerg("kmmio: recursive probe hit on CPU %d, "
211
+
pr_debug("kmmio: secondary hit for 0x%08lx CPU %d.\n",
212
+
addr, smp_processor_id());
213
+
214
+
if (!faultpage->old_presence)
215
+
pr_info("kmmio: unexpected secondary hit for "
216
+
"address 0x%08lx on CPU %d.\n", addr,
217
+
smp_processor_id());
218
+
} else {
219
+
/*
220
+
* Prevent overwriting already in-flight context.
221
+
* This should not happen, let's hope disarming at
222
+
* least prevents a panic.
223
+
*/
224
+
pr_emerg("kmmio: recursive probe hit on CPU %d, "
254
225
"for address 0x%08lx. Ignoring.\n",
255
226
smp_processor_id(), addr);
256
-
pr_emerg("kmmio: previous hit was at 0x%08lx.\n",
257
-
ctx->addr);
227
+
pr_emerg("kmmio: previous hit was at 0x%08lx.\n",
228
+
ctx->addr);
229
+
disarm_kmmio_fault_page(faultpage);
230
+
}
258
231
goto no_kmmio_ctx;
259
232
}
260
233
ctx->active++;
···
279
244
regs->flags &= ~X86_EFLAGS_IF;
280
245
281
246
/* Now we set present bit in PTE and single step. */
282
-
disarm_kmmio_fault_page(ctx->fpage->page, NULL);
247
+
disarm_kmmio_fault_page(ctx->fpage);
283
248
284
249
/*
285
250
* If another cpu accesses the same page while we are stepping,
···
310
275
struct kmmio_context *ctx = &get_cpu_var(kmmio_ctx);
311
276
312
277
if (!ctx->active) {
313
-
pr_debug("kmmio: spurious debug trap on CPU %d.\n",
278
+
pr_warning("kmmio: spurious debug trap on CPU %d.\n",
314
279
smp_processor_id());
315
280
goto out;
316
281
}
···
318
283
if (ctx->probe && ctx->probe->post_handler)
319
284
ctx->probe->post_handler(ctx->probe, condition, regs);
320
285
321
-
arm_kmmio_fault_page(ctx->fpage->page, NULL);
286
+
/* Prevent racing against release_kmmio_fault_page(). */
287
+
spin_lock(&kmmio_lock);
288
+
if (ctx->fpage->count)
289
+
arm_kmmio_fault_page(ctx->fpage);
290
+
spin_unlock(&kmmio_lock);
322
291
323
292
regs->flags &= ~X86_EFLAGS_TF;
324
293
regs->flags |= ctx->saved_flags;
···
354
315
f = get_kmmio_fault_page(page);
355
316
if (f) {
356
317
if (!f->count)
357
-
arm_kmmio_fault_page(f->page, NULL);
318
+
arm_kmmio_fault_page(f);
358
319
f->count++;
359
320
return 0;
360
321
}
361
322
362
-
f = kmalloc(sizeof(*f), GFP_ATOMIC);
323
+
f = kzalloc(sizeof(*f), GFP_ATOMIC);
363
324
if (!f)
364
325
return -1;
365
326
366
327
f->count = 1;
367
328
f->page = page;
368
-
list_add_rcu(&f->list, kmmio_page_list(f->page));
369
329
370
-
arm_kmmio_fault_page(f->page, NULL);
330
+
if (arm_kmmio_fault_page(f)) {
331
+
kfree(f);
332
+
return -1;
333
+
}
334
+
335
+
list_add_rcu(&f->list, kmmio_page_list(f->page));
371
336
372
337
return 0;
373
338
}
···
390
347
f->count--;
391
348
BUG_ON(f->count < 0);
392
349
if (!f->count) {
393
-
disarm_kmmio_fault_page(f->page, NULL);
350
+
disarm_kmmio_fault_page(f);
394
351
f->release_next = *release_list;
395
352
*release_list = f;
396
353
}
+57
-13
arch/x86/mm/testmmiotrace.c
+57
-13
arch/x86/mm/testmmiotrace.c
···
1
1
/*
2
-
* Written by Pekka Paalanen, 2008 <pq@iki.fi>
2
+
* Written by Pekka Paalanen, 2008-2009 <pq@iki.fi>
3
3
*/
4
4
#include <linux/module.h>
5
5
#include <linux/io.h>
···
9
9
10
10
static unsigned long mmio_address;
11
11
module_param(mmio_address, ulong, 0);
12
-
MODULE_PARM_DESC(mmio_address, "Start address of the mapping of 16 kB.");
12
+
MODULE_PARM_DESC(mmio_address, " Start address of the mapping of 16 kB "
13
+
"(or 8 MB if read_far is non-zero).");
14
+
15
+
static unsigned long read_far = 0x400100;
16
+
module_param(read_far, ulong, 0);
17
+
MODULE_PARM_DESC(read_far, " Offset of a 32-bit read within 8 MB "
18
+
"(default: 0x400100).");
19
+
20
+
static unsigned v16(unsigned i)
21
+
{
22
+
return i * 12 + 7;
23
+
}
24
+
25
+
static unsigned v32(unsigned i)
26
+
{
27
+
return i * 212371 + 13;
28
+
}
13
29
14
30
static void do_write_test(void __iomem *p)
15
31
{
16
32
unsigned int i;
33
+
pr_info(MODULE_NAME ": write test.\n");
17
34
mmiotrace_printk("Write test.\n");
35
+
18
36
for (i = 0; i < 256; i++)
19
37
iowrite8(i, p + i);
38
+
20
39
for (i = 1024; i < (5 * 1024); i += 2)
21
-
iowrite16(i * 12 + 7, p + i);
40
+
iowrite16(v16(i), p + i);
41
+
22
42
for (i = (5 * 1024); i < (16 * 1024); i += 4)
23
-
iowrite32(i * 212371 + 13, p + i);
43
+
iowrite32(v32(i), p + i);
24
44
}
25
45
26
46
static void do_read_test(void __iomem *p)
27
47
{
28
48
unsigned int i;
49
+
unsigned errs[3] = { 0 };
50
+
pr_info(MODULE_NAME ": read test.\n");
29
51
mmiotrace_printk("Read test.\n");
52
+
30
53
for (i = 0; i < 256; i++)
31
-
ioread8(p + i);
54
+
if (ioread8(p + i) != i)
55
+
++errs[0];
56
+
32
57
for (i = 1024; i < (5 * 1024); i += 2)
33
-
ioread16(p + i);
58
+
if (ioread16(p + i) != v16(i))
59
+
++errs[1];
60
+
34
61
for (i = (5 * 1024); i < (16 * 1024); i += 4)
35
-
ioread32(p + i);
62
+
if (ioread32(p + i) != v32(i))
63
+
++errs[2];
64
+
65
+
mmiotrace_printk("Read errors: 8-bit %d, 16-bit %d, 32-bit %d.\n",
66
+
errs[0], errs[1], errs[2]);
36
67
}
37
68
38
-
static void do_test(void)
69
+
static void do_read_far_test(void __iomem *p)
39
70
{
40
-
void __iomem *p = ioremap_nocache(mmio_address, 0x4000);
71
+
pr_info(MODULE_NAME ": read far test.\n");
72
+
mmiotrace_printk("Read far test.\n");
73
+
74
+
ioread32(p + read_far);
75
+
}
76
+
77
+
static void do_test(unsigned long size)
78
+
{
79
+
void __iomem *p = ioremap_nocache(mmio_address, size);
41
80
if (!p) {
42
81
pr_err(MODULE_NAME ": could not ioremap, aborting.\n");
43
82
return;
···
84
45
mmiotrace_printk("ioremap returned %p.\n", p);
85
46
do_write_test(p);
86
47
do_read_test(p);
48
+
if (read_far && read_far < size - 4)
49
+
do_read_far_test(p);
87
50
iounmap(p);
88
51
}
89
52
90
53
static int __init init(void)
91
54
{
55
+
unsigned long size = (read_far) ? (8 << 20) : (16 << 10);
56
+
92
57
if (mmio_address == 0) {
93
58
pr_err(MODULE_NAME ": you have to use the module argument "
94
59
"mmio_address.\n");
···
101
58
return -ENXIO;
102
59
}
103
60
104
-
pr_warning(MODULE_NAME ": WARNING: mapping 16 kB @ 0x%08lx "
105
-
"in PCI address space, and writing "
106
-
"rubbish in there.\n", mmio_address);
107
-
do_test();
61
+
pr_warning(MODULE_NAME ": WARNING: mapping %lu kB @ 0x%08lx in PCI "
62
+
"address space, and writing 16 kB of rubbish in there.\n",
63
+
size >> 10, mmio_address);
64
+
do_test(size);
65
+
pr_info(MODULE_NAME ": All done.\n");
108
66
return 0;
109
67
}
110
68