Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * allocation tagging
4 */
5#ifndef _LINUX_ALLOC_TAG_H
6#define _LINUX_ALLOC_TAG_H
7
8#include <linux/bug.h>
9#include <linux/codetag.h>
10#include <linux/container_of.h>
11#include <linux/preempt.h>
12#include <asm/percpu.h>
13#include <linux/cpumask.h>
14#include <linux/smp.h>
15#include <linux/static_key.h>
16#include <linux/irqflags.h>
17
18struct alloc_tag_counters {
19 u64 bytes;
20 u64 calls;
21};
22
23/*
24 * An instance of this structure is created in a special ELF section at every
25 * allocation callsite. At runtime, the special section is treated as
26 * an array of these. Embedded codetag utilizes codetag framework.
27 */
28struct alloc_tag {
29 struct codetag ct;
30 struct alloc_tag_counters __percpu *counters;
31} __aligned(8);
32
33struct alloc_tag_kernel_section {
34 struct alloc_tag *first_tag;
35 unsigned long count;
36};
37
38struct alloc_tag_module_section {
39 union {
40 unsigned long start_addr;
41 struct alloc_tag *first_tag;
42 };
43 unsigned long end_addr;
44 /* used size */
45 unsigned long size;
46};
47
48#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG
49
50#define CODETAG_EMPTY ((void *)1)
51
52static inline bool is_codetag_empty(union codetag_ref *ref)
53{
54 return ref->ct == CODETAG_EMPTY;
55}
56
57static inline void set_codetag_empty(union codetag_ref *ref)
58{
59 if (ref)
60 ref->ct = CODETAG_EMPTY;
61}
62
63#else /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */
64
65static inline bool is_codetag_empty(union codetag_ref *ref) { return false; }
66
67static inline void set_codetag_empty(union codetag_ref *ref)
68{
69 if (ref)
70 ref->ct = NULL;
71}
72
73#endif /* CONFIG_MEM_ALLOC_PROFILING_DEBUG */
74
75#ifdef CONFIG_MEM_ALLOC_PROFILING
76
77#define ALLOC_TAG_SECTION_NAME "alloc_tags"
78
79struct codetag_bytes {
80 struct codetag *ct;
81 s64 bytes;
82};
83
84size_t alloc_tag_top_users(struct codetag_bytes *tags, size_t count, bool can_sleep);
85
86static inline struct alloc_tag *ct_to_alloc_tag(struct codetag *ct)
87{
88 return container_of(ct, struct alloc_tag, ct);
89}
90
91#if defined(CONFIG_ARCH_MODULE_NEEDS_WEAK_PER_CPU) && defined(MODULE)
92/*
93 * When percpu variables are required to be defined as weak, static percpu
94 * variables can't be used inside a function (see comments for DECLARE_PER_CPU_SECTION).
95 * Instead we will account all module allocations to a single counter.
96 */
97DECLARE_PER_CPU(struct alloc_tag_counters, _shared_alloc_tag);
98
99#define DEFINE_ALLOC_TAG(_alloc_tag) \
100 static struct alloc_tag _alloc_tag __used __aligned(8) \
101 __section(ALLOC_TAG_SECTION_NAME) = { \
102 .ct = CODE_TAG_INIT, \
103 .counters = &_shared_alloc_tag };
104
105#else /* CONFIG_ARCH_MODULE_NEEDS_WEAK_PER_CPU && MODULE */
106
107#ifdef MODULE
108
109#define DEFINE_ALLOC_TAG(_alloc_tag) \
110 static struct alloc_tag _alloc_tag __used __aligned(8) \
111 __section(ALLOC_TAG_SECTION_NAME) = { \
112 .ct = CODE_TAG_INIT, \
113 .counters = NULL };
114
115#else /* MODULE */
116
117#define DEFINE_ALLOC_TAG(_alloc_tag) \
118 static DEFINE_PER_CPU(struct alloc_tag_counters, _alloc_tag_cntr); \
119 static struct alloc_tag _alloc_tag __used __aligned(8) \
120 __section(ALLOC_TAG_SECTION_NAME) = { \
121 .ct = CODE_TAG_INIT, \
122 .counters = &_alloc_tag_cntr };
123
124#endif /* MODULE */
125
126#endif /* CONFIG_ARCH_MODULE_NEEDS_WEAK_PER_CPU && MODULE */
127
128DECLARE_STATIC_KEY_MAYBE(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT,
129 mem_alloc_profiling_key);
130
131static inline bool mem_alloc_profiling_enabled(void)
132{
133 return static_branch_maybe(CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT,
134 &mem_alloc_profiling_key);
135}
136
137static inline struct alloc_tag_counters alloc_tag_read(struct alloc_tag *tag)
138{
139 struct alloc_tag_counters v = { 0, 0 };
140 struct alloc_tag_counters *counter;
141 int cpu;
142
143 for_each_possible_cpu(cpu) {
144 counter = per_cpu_ptr(tag->counters, cpu);
145 v.bytes += counter->bytes;
146 v.calls += counter->calls;
147 }
148
149 return v;
150}
151
152#ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG
153static inline void alloc_tag_add_check(union codetag_ref *ref, struct alloc_tag *tag)
154{
155 WARN_ONCE(ref && ref->ct && !is_codetag_empty(ref),
156 "alloc_tag was not cleared (got tag for %s:%u)\n",
157 ref->ct->filename, ref->ct->lineno);
158
159 WARN_ONCE(!tag, "current->alloc_tag not set\n");
160}
161
162static inline void alloc_tag_sub_check(union codetag_ref *ref)
163{
164 WARN_ONCE(ref && !ref->ct, "alloc_tag was not set\n");
165}
166void alloc_tag_add_early_pfn(unsigned long pfn);
167#else
168static inline void alloc_tag_add_check(union codetag_ref *ref, struct alloc_tag *tag) {}
169static inline void alloc_tag_sub_check(union codetag_ref *ref) {}
170static inline void alloc_tag_add_early_pfn(unsigned long pfn) {}
171#endif
172
173/* Caller should verify both ref and tag to be valid */
174static inline bool __alloc_tag_ref_set(union codetag_ref *ref, struct alloc_tag *tag)
175{
176 alloc_tag_add_check(ref, tag);
177 if (!ref || !tag)
178 return false;
179
180 ref->ct = &tag->ct;
181 return true;
182}
183
184static inline bool alloc_tag_ref_set(union codetag_ref *ref, struct alloc_tag *tag)
185{
186 if (unlikely(!__alloc_tag_ref_set(ref, tag)))
187 return false;
188
189 /*
190 * We need in increment the call counter every time we have a new
191 * allocation or when we split a large allocation into smaller ones.
192 * Each new reference for every sub-allocation needs to increment call
193 * counter because when we free each part the counter will be decremented.
194 */
195 this_cpu_inc(tag->counters->calls);
196 return true;
197}
198
199static inline void alloc_tag_add(union codetag_ref *ref, struct alloc_tag *tag, size_t bytes)
200{
201 if (likely(alloc_tag_ref_set(ref, tag)))
202 this_cpu_add(tag->counters->bytes, bytes);
203}
204
205static inline void alloc_tag_sub(union codetag_ref *ref, size_t bytes)
206{
207 struct alloc_tag *tag;
208
209 alloc_tag_sub_check(ref);
210 if (!ref || !ref->ct)
211 return;
212
213 if (is_codetag_empty(ref)) {
214 ref->ct = NULL;
215 return;
216 }
217
218 tag = ct_to_alloc_tag(ref->ct);
219
220 this_cpu_sub(tag->counters->bytes, bytes);
221 this_cpu_dec(tag->counters->calls);
222
223 ref->ct = NULL;
224}
225
226static inline void alloc_tag_set_inaccurate(struct alloc_tag *tag)
227{
228 tag->ct.flags |= CODETAG_FLAG_INACCURATE;
229}
230
231static inline bool alloc_tag_is_inaccurate(struct alloc_tag *tag)
232{
233 return !!(tag->ct.flags & CODETAG_FLAG_INACCURATE);
234}
235
236#define alloc_tag_record(p) ((p) = current->alloc_tag)
237
238#else /* CONFIG_MEM_ALLOC_PROFILING */
239
240#define DEFINE_ALLOC_TAG(_alloc_tag)
241static inline bool mem_alloc_profiling_enabled(void) { return false; }
242static inline void alloc_tag_add(union codetag_ref *ref, struct alloc_tag *tag,
243 size_t bytes) {}
244static inline void alloc_tag_sub(union codetag_ref *ref, size_t bytes) {}
245static inline void alloc_tag_set_inaccurate(struct alloc_tag *tag) {}
246static inline bool alloc_tag_is_inaccurate(struct alloc_tag *tag) { return false; }
247#define alloc_tag_record(p) do {} while (0)
248
249#endif /* CONFIG_MEM_ALLOC_PROFILING */
250
251#define alloc_hooks_tag(_tag, _do_alloc) \
252({ \
253 typeof(_do_alloc) _res; \
254 if (mem_alloc_profiling_enabled()) { \
255 struct alloc_tag * __maybe_unused _old; \
256 _old = alloc_tag_save(_tag); \
257 _res = _do_alloc; \
258 alloc_tag_restore(_tag, _old); \
259 } else \
260 _res = _do_alloc; \
261 _res; \
262})
263
264#define alloc_hooks(_do_alloc) \
265({ \
266 DEFINE_ALLOC_TAG(_alloc_tag); \
267 alloc_hooks_tag(&_alloc_tag, _do_alloc); \
268})
269
270#endif /* _LINUX_ALLOC_TAG_H */