Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'mm-nonmm-stable-2022-06-05' of git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
Pull delay-accounting update from Andrew Morton:
"A single featurette for delay accounting.
Delayed a bit because, unusually, it had dependencies on both the
mm-stable and mm-nonmm-stable queues"
* tag 'mm-nonmm-stable-2022-06-05' of git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm:
delayacct: track delays from write-protect copy
+76
-3
+4
-1
Documentation/accounting/delay-accounting.rst
+4
-1
Documentation/accounting/delay-accounting.rst
···
15
15
d) memory reclaim
16
16
e) thrashing page cache
17
17
f) direct compact
18
+
g) write-protect copy
18
19
19
20
and makes these statistics available to userspace through
20
21
the taskstats interface.
···
49
48
for a description of the fields pertaining to delay accounting.
50
49
It will generally be in the form of counters returning the cumulative
51
50
delay seen for cpu, sync block I/O, swapin, memory reclaim, thrash page
52
-
cache, direct compact etc.
51
+
cache, direct compact, write-protect copy etc.
53
52
54
53
Taking the difference of two successive readings of a given
55
54
counter (say cpu_delay_total) for a task will give the delay
···
118
117
0 0 0ms
119
118
COMPACT count delay total delay average
120
119
0 0 0ms
120
+
WPCOPY count delay total delay average
121
+
0 0 0ms
121
122
122
123
Get IO accounting for pid 1, it works only with -p::
123
124
+28
include/linux/delayacct.h
+28
include/linux/delayacct.h
···
45
45
u64 compact_start;
46
46
u64 compact_delay; /* wait for memory compact */
47
47
48
+
u64 wpcopy_start;
49
+
u64 wpcopy_delay; /* wait for write-protect copy */
50
+
48
51
u32 freepages_count; /* total count of memory reclaim */
49
52
u32 thrashing_count; /* total count of thrash waits */
50
53
u32 compact_count; /* total count of memory compact */
54
+
u32 wpcopy_count; /* total count of write-protect copy */
51
55
};
52
56
#endif
53
57
···
79
75
extern void __delayacct_swapin_end(void);
80
76
extern void __delayacct_compact_start(void);
81
77
extern void __delayacct_compact_end(void);
78
+
extern void __delayacct_wpcopy_start(void);
79
+
extern void __delayacct_wpcopy_end(void);
82
80
83
81
static inline void delayacct_tsk_init(struct task_struct *tsk)
84
82
{
···
197
191
__delayacct_compact_end();
198
192
}
199
193
194
+
static inline void delayacct_wpcopy_start(void)
195
+
{
196
+
if (!static_branch_unlikely(&delayacct_key))
197
+
return;
198
+
199
+
if (current->delays)
200
+
__delayacct_wpcopy_start();
201
+
}
202
+
203
+
static inline void delayacct_wpcopy_end(void)
204
+
{
205
+
if (!static_branch_unlikely(&delayacct_key))
206
+
return;
207
+
208
+
if (current->delays)
209
+
__delayacct_wpcopy_end();
210
+
}
211
+
200
212
#else
201
213
static inline void delayacct_init(void)
202
214
{}
···
248
224
static inline void delayacct_compact_start(void)
249
225
{}
250
226
static inline void delayacct_compact_end(void)
227
+
{}
228
+
static inline void delayacct_wpcopy_start(void)
229
+
{}
230
+
static inline void delayacct_wpcopy_end(void)
251
231
{}
252
232
253
233
#endif /* CONFIG_TASK_DELAY_ACCT */
+5
-1
include/uapi/linux/taskstats.h
+5
-1
include/uapi/linux/taskstats.h
···
34
34
*/
35
35
36
36
37
-
#define TASKSTATS_VERSION 12
37
+
#define TASKSTATS_VERSION 13
38
38
#define TS_COMM_LEN 32 /* should be >= TASK_COMM_LEN
39
39
* in linux/sched.h */
40
40
···
194
194
__u64 ac_exe_dev; /* program binary device ID */
195
195
__u64 ac_exe_inode; /* program binary inode number */
196
196
/* v12 end */
197
+
198
+
/* v13: Delay waiting for write-protect copy */
199
+
__u64 wpcopy_count;
200
+
__u64 wpcopy_delay_total;
197
201
};
198
202
199
203
+16
kernel/delayacct.c
+16
kernel/delayacct.c
···
177
177
d->thrashing_delay_total = (tmp < d->thrashing_delay_total) ? 0 : tmp;
178
178
tmp = d->compact_delay_total + tsk->delays->compact_delay;
179
179
d->compact_delay_total = (tmp < d->compact_delay_total) ? 0 : tmp;
180
+
tmp = d->wpcopy_delay_total + tsk->delays->wpcopy_delay;
181
+
d->wpcopy_delay_total = (tmp < d->wpcopy_delay_total) ? 0 : tmp;
180
182
d->blkio_count += tsk->delays->blkio_count;
181
183
d->swapin_count += tsk->delays->swapin_count;
182
184
d->freepages_count += tsk->delays->freepages_count;
183
185
d->thrashing_count += tsk->delays->thrashing_count;
184
186
d->compact_count += tsk->delays->compact_count;
187
+
d->wpcopy_count += tsk->delays->wpcopy_count;
185
188
raw_spin_unlock_irqrestore(&tsk->delays->lock, flags);
186
189
187
190
return 0;
···
251
248
¤t->delays->compact_start,
252
249
¤t->delays->compact_delay,
253
250
¤t->delays->compact_count);
251
+
}
252
+
253
+
void __delayacct_wpcopy_start(void)
254
+
{
255
+
current->delays->wpcopy_start = local_clock();
256
+
}
257
+
258
+
void __delayacct_wpcopy_end(void)
259
+
{
260
+
delayacct_end(¤t->delays->lock,
261
+
¤t->delays->wpcopy_start,
262
+
¤t->delays->wpcopy_delay,
263
+
¤t->delays->wpcopy_count);
254
264
}
+8
mm/hugetlb.c
+8
mm/hugetlb.c
···
32
32
#include <linux/cma.h>
33
33
#include <linux/migrate.h>
34
34
#include <linux/nospec.h>
35
+
#include <linux/delayacct.h>
35
36
36
37
#include <asm/page.h>
37
38
#include <asm/pgalloc.h>
···
5231
5230
pte = huge_ptep_get(ptep);
5232
5231
old_page = pte_page(pte);
5233
5232
5233
+
delayacct_wpcopy_start();
5234
+
5234
5235
retry_avoidcopy:
5235
5236
/*
5236
5237
* If no-one else is actually using this page, we're the exclusive
···
5243
5240
page_move_anon_rmap(old_page, vma);
5244
5241
if (likely(!unshare))
5245
5242
set_huge_ptep_writable(vma, haddr, ptep);
5243
+
5244
+
delayacct_wpcopy_end();
5246
5245
return 0;
5247
5246
}
5248
5247
VM_BUG_ON_PAGE(PageAnon(old_page) && PageAnonExclusive(old_page),
···
5314
5309
* race occurs while re-acquiring page table
5315
5310
* lock, and our job is done.
5316
5311
*/
5312
+
delayacct_wpcopy_end();
5317
5313
return 0;
5318
5314
}
5319
5315
···
5373
5367
put_page(old_page);
5374
5368
5375
5369
spin_lock(ptl); /* Caller expects lock to be held */
5370
+
5371
+
delayacct_wpcopy_end();
5376
5372
return ret;
5377
5373
}
5378
5374
+8
mm/memory.c
+8
mm/memory.c
···
3090
3090
int page_copied = 0;
3091
3091
struct mmu_notifier_range range;
3092
3092
3093
+
delayacct_wpcopy_start();
3094
+
3093
3095
if (unlikely(anon_vma_prepare(vma)))
3094
3096
goto oom;
3095
3097
···
3116
3114
put_page(new_page);
3117
3115
if (old_page)
3118
3116
put_page(old_page);
3117
+
3118
+
delayacct_wpcopy_end();
3119
3119
return 0;
3120
3120
}
3121
3121
}
···
3224
3220
free_swap_cache(old_page);
3225
3221
put_page(old_page);
3226
3222
}
3223
+
3224
+
delayacct_wpcopy_end();
3227
3225
return (page_copied && !unshare) ? VM_FAULT_WRITE : 0;
3228
3226
oom_free_new:
3229
3227
put_page(new_page);
3230
3228
oom:
3231
3229
if (old_page)
3232
3230
put_page(old_page);
3231
+
3232
+
delayacct_wpcopy_end();
3233
3233
return VM_FAULT_OOM;
3234
3234
}
3235
3235
+7
-1
tools/accounting/getdelays.c
+7
-1
tools/accounting/getdelays.c
···
207
207
"THRASHING%12s%15s%15s\n"
208
208
" %15llu%15llu%15llums\n"
209
209
"COMPACT %12s%15s%15s\n"
210
+
" %15llu%15llu%15llums\n"
211
+
"WPCOPY %12s%15s%15s\n"
210
212
" %15llu%15llu%15llums\n",
211
213
"count", "real total", "virtual total",
212
214
"delay total", "delay average",
···
236
234
"count", "delay total", "delay average",
237
235
(unsigned long long)t->compact_count,
238
236
(unsigned long long)t->compact_delay_total,
239
-
average_ms(t->compact_delay_total, t->compact_count));
237
+
average_ms(t->compact_delay_total, t->compact_count),
238
+
"count", "delay total", "delay average",
239
+
(unsigned long long)t->wpcopy_count,
240
+
(unsigned long long)t->wpcopy_delay_total,
241
+
average_ms(t->wpcopy_delay_total, t->wpcopy_count));
240
242
}
241
243
242
244
static void task_context_switch_counts(struct taskstats *t)