Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _LINUX_VMSTAT_H
3#define _LINUX_VMSTAT_H
4
5#include <linux/types.h>
6#include <linux/percpu.h>
7#include <linux/mmzone.h>
8#include <linux/vm_event_item.h>
9#include <linux/atomic.h>
10#include <linux/static_key.h>
11#include <linux/mmdebug.h>
12
13#ifdef CONFIG_NUMA
14DECLARE_STATIC_KEY_TRUE(vm_numa_stat_key);
15#endif
16
17struct reclaim_stat {
18 unsigned nr_dirty;
19 unsigned nr_unqueued_dirty;
20 unsigned nr_congested;
21 unsigned nr_writeback;
22 unsigned nr_immediate;
23 unsigned nr_pageout;
24 unsigned nr_activate[ANON_AND_FILE];
25 unsigned nr_ref_keep;
26 unsigned nr_unmap_fail;
27 unsigned nr_lazyfree_fail;
28 unsigned nr_demoted;
29};
30
31/* Stat data for system wide items */
32enum vm_stat_item {
33 NR_DIRTY_THRESHOLD,
34 NR_DIRTY_BG_THRESHOLD,
35 NR_MEMMAP_PAGES, /* page metadata allocated through buddy allocator */
36 NR_MEMMAP_BOOT_PAGES, /* page metadata allocated through boot allocator */
37 NR_VM_STAT_ITEMS,
38};
39
40#ifdef CONFIG_VM_EVENT_COUNTERS
41/*
42 * Light weight per cpu counter implementation.
43 *
44 * Counters should only be incremented and no critical kernel component
45 * should rely on the counter values.
46 *
47 * Counters are handled completely inline. On many platforms the code
48 * generated will simply be the increment of a global address.
49 */
50
51struct vm_event_state {
52 unsigned long event[NR_VM_EVENT_ITEMS];
53};
54
55DECLARE_PER_CPU(struct vm_event_state, vm_event_states);
56
57/*
58 * vm counters are allowed to be racy. Use raw_cpu_ops to avoid the
59 * local_irq_disable overhead.
60 */
61static inline void __count_vm_event(enum vm_event_item item)
62{
63 raw_cpu_inc(vm_event_states.event[item]);
64}
65
66static inline void count_vm_event(enum vm_event_item item)
67{
68 this_cpu_inc(vm_event_states.event[item]);
69}
70
71static inline void __count_vm_events(enum vm_event_item item, long delta)
72{
73 raw_cpu_add(vm_event_states.event[item], delta);
74}
75
76static inline void count_vm_events(enum vm_event_item item, long delta)
77{
78 this_cpu_add(vm_event_states.event[item], delta);
79}
80
81extern void all_vm_events(unsigned long *);
82
83extern void vm_events_fold_cpu(int cpu);
84
85#else
86
87/* Disable counters */
88static inline void count_vm_event(enum vm_event_item item)
89{
90}
91static inline void count_vm_events(enum vm_event_item item, long delta)
92{
93}
94static inline void __count_vm_event(enum vm_event_item item)
95{
96}
97static inline void __count_vm_events(enum vm_event_item item, long delta)
98{
99}
100static inline void all_vm_events(unsigned long *ret)
101{
102}
103static inline void vm_events_fold_cpu(int cpu)
104{
105}
106
107#endif /* CONFIG_VM_EVENT_COUNTERS */
108
109#ifdef CONFIG_NUMA_BALANCING
110#define count_vm_numa_event(x) count_vm_event(x)
111#define count_vm_numa_events(x, y) count_vm_events(x, y)
112#else
113#define count_vm_numa_event(x) do {} while (0)
114#define count_vm_numa_events(x, y) do { (void)(y); } while (0)
115#endif /* CONFIG_NUMA_BALANCING */
116
117#ifdef CONFIG_DEBUG_TLBFLUSH
118#define count_vm_tlb_event(x) count_vm_event(x)
119#define count_vm_tlb_events(x, y) count_vm_events(x, y)
120#else
121#define count_vm_tlb_event(x) do {} while (0)
122#define count_vm_tlb_events(x, y) do { (void)(y); } while (0)
123#endif
124
125#ifdef CONFIG_PER_VMA_LOCK_STATS
126#define count_vm_vma_lock_event(x) count_vm_event(x)
127#else
128#define count_vm_vma_lock_event(x) do {} while (0)
129#endif
130
131#define __count_zid_vm_events(item, zid, delta) \
132 __count_vm_events(item##_NORMAL - ZONE_NORMAL + zid, delta)
133
134/*
135 * Zone and node-based page accounting with per cpu differentials.
136 */
137extern atomic_long_t vm_zone_stat[NR_VM_ZONE_STAT_ITEMS];
138extern atomic_long_t vm_node_stat[NR_VM_NODE_STAT_ITEMS];
139extern atomic_long_t vm_numa_event[NR_VM_NUMA_EVENT_ITEMS];
140
141#ifdef CONFIG_NUMA
142static inline void zone_numa_event_add(long x, struct zone *zone,
143 enum numa_stat_item item)
144{
145 atomic_long_add(x, &zone->vm_numa_event[item]);
146 atomic_long_add(x, &vm_numa_event[item]);
147}
148
149static inline unsigned long zone_numa_event_state(struct zone *zone,
150 enum numa_stat_item item)
151{
152 return atomic_long_read(&zone->vm_numa_event[item]);
153}
154
155static inline unsigned long
156global_numa_event_state(enum numa_stat_item item)
157{
158 return atomic_long_read(&vm_numa_event[item]);
159}
160#endif /* CONFIG_NUMA */
161
162static inline void zone_page_state_add(long x, struct zone *zone,
163 enum zone_stat_item item)
164{
165 atomic_long_add(x, &zone->vm_stat[item]);
166 atomic_long_add(x, &vm_zone_stat[item]);
167}
168
169static inline void node_page_state_add(long x, struct pglist_data *pgdat,
170 enum node_stat_item item)
171{
172 atomic_long_add(x, &pgdat->vm_stat[item]);
173 atomic_long_add(x, &vm_node_stat[item]);
174}
175
176static inline unsigned long global_zone_page_state(enum zone_stat_item item)
177{
178 long x = atomic_long_read(&vm_zone_stat[item]);
179#ifdef CONFIG_SMP
180 if (x < 0)
181 x = 0;
182#endif
183 return x;
184}
185
186static inline
187unsigned long global_node_page_state_pages(enum node_stat_item item)
188{
189 long x = atomic_long_read(&vm_node_stat[item]);
190#ifdef CONFIG_SMP
191 if (x < 0)
192 x = 0;
193#endif
194 return x;
195}
196
197static inline unsigned long global_node_page_state(enum node_stat_item item)
198{
199 VM_WARN_ON_ONCE(vmstat_item_in_bytes(item));
200
201 return global_node_page_state_pages(item);
202}
203
204static inline unsigned long zone_page_state(struct zone *zone,
205 enum zone_stat_item item)
206{
207 long x = atomic_long_read(&zone->vm_stat[item]);
208#ifdef CONFIG_SMP
209 if (x < 0)
210 x = 0;
211#endif
212 return x;
213}
214
215/*
216 * More accurate version that also considers the currently pending
217 * deltas. For that we need to loop over all cpus to find the current
218 * deltas. There is no synchronization so the result cannot be
219 * exactly accurate either.
220 */
221static inline unsigned long zone_page_state_snapshot(struct zone *zone,
222 enum zone_stat_item item)
223{
224 long x = atomic_long_read(&zone->vm_stat[item]);
225
226#ifdef CONFIG_SMP
227 int cpu;
228 for_each_online_cpu(cpu)
229 x += per_cpu_ptr(zone->per_cpu_zonestats, cpu)->vm_stat_diff[item];
230
231 if (x < 0)
232 x = 0;
233#endif
234 return x;
235}
236
237#ifdef CONFIG_NUMA
238/* See __count_vm_event comment on why raw_cpu_inc is used. */
239static inline void
240__count_numa_event(struct zone *zone, enum numa_stat_item item)
241{
242 struct per_cpu_zonestat __percpu *pzstats = zone->per_cpu_zonestats;
243
244 raw_cpu_inc(pzstats->vm_numa_event[item]);
245}
246
247static inline void
248__count_numa_events(struct zone *zone, enum numa_stat_item item, long delta)
249{
250 struct per_cpu_zonestat __percpu *pzstats = zone->per_cpu_zonestats;
251
252 raw_cpu_add(pzstats->vm_numa_event[item], delta);
253}
254
255extern unsigned long sum_zone_node_page_state(int node,
256 enum zone_stat_item item);
257extern unsigned long sum_zone_numa_event_state(int node, enum numa_stat_item item);
258extern unsigned long node_page_state(struct pglist_data *pgdat,
259 enum node_stat_item item);
260extern unsigned long node_page_state_pages(struct pglist_data *pgdat,
261 enum node_stat_item item);
262extern void fold_vm_numa_events(void);
263#else
264#define sum_zone_node_page_state(node, item) global_zone_page_state(item)
265#define node_page_state(node, item) global_node_page_state(item)
266#define node_page_state_pages(node, item) global_node_page_state_pages(item)
267static inline void fold_vm_numa_events(void)
268{
269}
270#endif /* CONFIG_NUMA */
271
272#ifdef CONFIG_SMP
273void __mod_zone_page_state(struct zone *, enum zone_stat_item item, long);
274void __inc_zone_page_state(struct page *, enum zone_stat_item);
275void __dec_zone_page_state(struct page *, enum zone_stat_item);
276
277void __mod_node_page_state(struct pglist_data *, enum node_stat_item item, long);
278void __inc_node_page_state(struct page *, enum node_stat_item);
279void __dec_node_page_state(struct page *, enum node_stat_item);
280
281void mod_zone_page_state(struct zone *, enum zone_stat_item, long);
282void inc_zone_page_state(struct page *, enum zone_stat_item);
283void dec_zone_page_state(struct page *, enum zone_stat_item);
284
285void mod_node_page_state(struct pglist_data *, enum node_stat_item, long);
286void inc_node_page_state(struct page *, enum node_stat_item);
287void dec_node_page_state(struct page *, enum node_stat_item);
288
289extern void __inc_zone_state(struct zone *, enum zone_stat_item);
290extern void __inc_node_state(struct pglist_data *, enum node_stat_item);
291extern void __dec_zone_state(struct zone *, enum zone_stat_item);
292extern void __dec_node_state(struct pglist_data *, enum node_stat_item);
293
294void quiet_vmstat(void);
295void cpu_vm_stats_fold(int cpu);
296void refresh_zone_stat_thresholds(void);
297
298void drain_zonestat(struct zone *zone, struct per_cpu_zonestat *);
299
300int calculate_pressure_threshold(struct zone *zone);
301int calculate_normal_threshold(struct zone *zone);
302void set_pgdat_percpu_threshold(pg_data_t *pgdat,
303 int (*calculate_pressure)(struct zone *));
304void vmstat_flush_workqueue(void);
305#else /* CONFIG_SMP */
306
307/*
308 * We do not maintain differentials in a single processor configuration.
309 * The functions directly modify the zone and global counters.
310 */
311static inline void __mod_zone_page_state(struct zone *zone,
312 enum zone_stat_item item, long delta)
313{
314 zone_page_state_add(delta, zone, item);
315}
316
317static inline void __mod_node_page_state(struct pglist_data *pgdat,
318 enum node_stat_item item, int delta)
319{
320 if (vmstat_item_in_bytes(item)) {
321 /*
322 * Only cgroups use subpage accounting right now; at
323 * the global level, these items still change in
324 * multiples of whole pages. Store them as pages
325 * internally to keep the per-cpu counters compact.
326 */
327 VM_WARN_ON_ONCE(delta & (PAGE_SIZE - 1));
328 delta >>= PAGE_SHIFT;
329 }
330
331 node_page_state_add(delta, pgdat, item);
332}
333
334static inline void __inc_zone_state(struct zone *zone, enum zone_stat_item item)
335{
336 atomic_long_inc(&zone->vm_stat[item]);
337 atomic_long_inc(&vm_zone_stat[item]);
338}
339
340static inline void __inc_node_state(struct pglist_data *pgdat, enum node_stat_item item)
341{
342 atomic_long_inc(&pgdat->vm_stat[item]);
343 atomic_long_inc(&vm_node_stat[item]);
344}
345
346static inline void __dec_zone_state(struct zone *zone, enum zone_stat_item item)
347{
348 atomic_long_dec(&zone->vm_stat[item]);
349 atomic_long_dec(&vm_zone_stat[item]);
350}
351
352static inline void __dec_node_state(struct pglist_data *pgdat, enum node_stat_item item)
353{
354 atomic_long_dec(&pgdat->vm_stat[item]);
355 atomic_long_dec(&vm_node_stat[item]);
356}
357
358static inline void __inc_zone_page_state(struct page *page,
359 enum zone_stat_item item)
360{
361 __inc_zone_state(page_zone(page), item);
362}
363
364static inline void __inc_node_page_state(struct page *page,
365 enum node_stat_item item)
366{
367 __inc_node_state(page_pgdat(page), item);
368}
369
370
371static inline void __dec_zone_page_state(struct page *page,
372 enum zone_stat_item item)
373{
374 __dec_zone_state(page_zone(page), item);
375}
376
377static inline void __dec_node_page_state(struct page *page,
378 enum node_stat_item item)
379{
380 __dec_node_state(page_pgdat(page), item);
381}
382
383
384/*
385 * We only use atomic operations to update counters. So there is no need to
386 * disable interrupts.
387 */
388#define inc_zone_page_state __inc_zone_page_state
389#define dec_zone_page_state __dec_zone_page_state
390#define mod_zone_page_state __mod_zone_page_state
391
392#define inc_node_page_state __inc_node_page_state
393#define dec_node_page_state __dec_node_page_state
394#define mod_node_page_state __mod_node_page_state
395
396#define set_pgdat_percpu_threshold(pgdat, callback) { }
397
398static inline void refresh_zone_stat_thresholds(void) { }
399static inline void cpu_vm_stats_fold(int cpu) { }
400static inline void quiet_vmstat(void) { }
401static inline void vmstat_flush_workqueue(void) { }
402
403static inline void drain_zonestat(struct zone *zone,
404 struct per_cpu_zonestat *pzstats) { }
405#endif /* CONFIG_SMP */
406
407static inline void __zone_stat_mod_folio(struct folio *folio,
408 enum zone_stat_item item, long nr)
409{
410 __mod_zone_page_state(folio_zone(folio), item, nr);
411}
412
413static inline void __zone_stat_add_folio(struct folio *folio,
414 enum zone_stat_item item)
415{
416 __mod_zone_page_state(folio_zone(folio), item, folio_nr_pages(folio));
417}
418
419static inline void __zone_stat_sub_folio(struct folio *folio,
420 enum zone_stat_item item)
421{
422 __mod_zone_page_state(folio_zone(folio), item, -folio_nr_pages(folio));
423}
424
425static inline void zone_stat_mod_folio(struct folio *folio,
426 enum zone_stat_item item, long nr)
427{
428 mod_zone_page_state(folio_zone(folio), item, nr);
429}
430
431static inline void zone_stat_add_folio(struct folio *folio,
432 enum zone_stat_item item)
433{
434 mod_zone_page_state(folio_zone(folio), item, folio_nr_pages(folio));
435}
436
437static inline void zone_stat_sub_folio(struct folio *folio,
438 enum zone_stat_item item)
439{
440 mod_zone_page_state(folio_zone(folio), item, -folio_nr_pages(folio));
441}
442
443static inline void __node_stat_mod_folio(struct folio *folio,
444 enum node_stat_item item, long nr)
445{
446 __mod_node_page_state(folio_pgdat(folio), item, nr);
447}
448
449static inline void __node_stat_add_folio(struct folio *folio,
450 enum node_stat_item item)
451{
452 __mod_node_page_state(folio_pgdat(folio), item, folio_nr_pages(folio));
453}
454
455static inline void __node_stat_sub_folio(struct folio *folio,
456 enum node_stat_item item)
457{
458 __mod_node_page_state(folio_pgdat(folio), item, -folio_nr_pages(folio));
459}
460
461static inline void node_stat_mod_folio(struct folio *folio,
462 enum node_stat_item item, long nr)
463{
464 mod_node_page_state(folio_pgdat(folio), item, nr);
465}
466
467static inline void node_stat_add_folio(struct folio *folio,
468 enum node_stat_item item)
469{
470 mod_node_page_state(folio_pgdat(folio), item, folio_nr_pages(folio));
471}
472
473static inline void node_stat_sub_folio(struct folio *folio,
474 enum node_stat_item item)
475{
476 mod_node_page_state(folio_pgdat(folio), item, -folio_nr_pages(folio));
477}
478
479extern const char * const vmstat_text[];
480
481static inline const char *zone_stat_name(enum zone_stat_item item)
482{
483 return vmstat_text[item];
484}
485
486#ifdef CONFIG_NUMA
487static inline const char *numa_stat_name(enum numa_stat_item item)
488{
489 return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
490 item];
491}
492#endif /* CONFIG_NUMA */
493
494static inline const char *node_stat_name(enum node_stat_item item)
495{
496 return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
497 NR_VM_NUMA_EVENT_ITEMS +
498 item];
499}
500
501static inline const char *lru_list_name(enum lru_list lru)
502{
503 return node_stat_name(NR_LRU_BASE + lru) + 3; // skip "nr_"
504}
505
506#if defined(CONFIG_VM_EVENT_COUNTERS)
507static inline const char *vm_event_name(enum vm_event_item item)
508{
509 return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
510 NR_VM_NUMA_EVENT_ITEMS +
511 NR_VM_NODE_STAT_ITEMS +
512 NR_VM_STAT_ITEMS +
513 item];
514}
515#endif /* CONFIG_VM_EVENT_COUNTERS */
516
517#ifdef CONFIG_MEMCG
518
519void mod_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx,
520 int val);
521
522void lruvec_stat_mod_folio(struct folio *folio,
523 enum node_stat_item idx, int val);
524
525static inline void mod_lruvec_page_state(struct page *page,
526 enum node_stat_item idx, int val)
527{
528 lruvec_stat_mod_folio(page_folio(page), idx, val);
529}
530
531#else
532
533static inline void mod_lruvec_state(struct lruvec *lruvec,
534 enum node_stat_item idx, int val)
535{
536 mod_node_page_state(lruvec_pgdat(lruvec), idx, val);
537}
538
539static inline void lruvec_stat_mod_folio(struct folio *folio,
540 enum node_stat_item idx, int val)
541{
542 mod_node_page_state(folio_pgdat(folio), idx, val);
543}
544
545static inline void mod_lruvec_page_state(struct page *page,
546 enum node_stat_item idx, int val)
547{
548 mod_node_page_state(page_pgdat(page), idx, val);
549}
550
551#endif /* CONFIG_MEMCG */
552
553static inline void lruvec_stat_add_folio(struct folio *folio,
554 enum node_stat_item idx)
555{
556 lruvec_stat_mod_folio(folio, idx, folio_nr_pages(folio));
557}
558
559static inline void lruvec_stat_sub_folio(struct folio *folio,
560 enum node_stat_item idx)
561{
562 lruvec_stat_mod_folio(folio, idx, -folio_nr_pages(folio));
563}
564
565void memmap_boot_pages_add(long delta);
566void memmap_pages_add(long delta);
567#endif /* _LINUX_VMSTAT_H */