Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES
4 *
5 * Default definitions for formats that don't define these functions.
6 */
7#ifndef __GENERIC_PT_PT_FMT_DEFAULTS_H
8#define __GENERIC_PT_PT_FMT_DEFAULTS_H
9
10#include "pt_defs.h"
11#include <linux/log2.h>
12
13/* Header self-compile default defines */
14#ifndef pt_load_entry_raw
15#include "fmt/amdv1.h"
16#endif
17
18/*
19 * The format must provide PT_GRANULE_LG2SZ, PT_TABLEMEM_LG2SZ, and
20 * PT_ITEM_WORD_SIZE. They must be the same at every level excluding the top.
21 */
22#ifndef pt_table_item_lg2sz
23static inline unsigned int pt_table_item_lg2sz(const struct pt_state *pts)
24{
25 return PT_GRANULE_LG2SZ +
26 (PT_TABLEMEM_LG2SZ - ilog2(PT_ITEM_WORD_SIZE)) * pts->level;
27}
28#endif
29
30#ifndef pt_pgsz_lg2_to_level
31static inline unsigned int pt_pgsz_lg2_to_level(struct pt_common *common,
32 unsigned int pgsize_lg2)
33{
34 return ((unsigned int)(pgsize_lg2 - PT_GRANULE_LG2SZ)) /
35 (PT_TABLEMEM_LG2SZ - ilog2(PT_ITEM_WORD_SIZE));
36}
37#endif
38
39/*
40 * If not supplied by the format then contiguous pages are not supported.
41 *
42 * If contiguous pages are supported then the format must also provide
43 * pt_contig_count_lg2() if it supports a single contiguous size per level,
44 * or pt_possible_sizes() if it supports multiple sizes per level.
45 */
46#ifndef pt_entry_num_contig_lg2
47static inline unsigned int pt_entry_num_contig_lg2(const struct pt_state *pts)
48{
49 return ilog2(1);
50}
51
52/*
53 * Return the number of contiguous OA items forming an entry at this table level
54 */
55static inline unsigned short pt_contig_count_lg2(const struct pt_state *pts)
56{
57 return ilog2(1);
58}
59#endif
60
61/* If not supplied by the format then dirty tracking is not supported */
62#ifndef pt_entry_is_write_dirty
63static inline bool pt_entry_is_write_dirty(const struct pt_state *pts)
64{
65 return false;
66}
67
68static inline void pt_entry_make_write_clean(struct pt_state *pts)
69{
70}
71
72static inline bool pt_dirty_supported(struct pt_common *common)
73{
74 return false;
75}
76#else
77/* If not supplied then dirty tracking is always enabled */
78#ifndef pt_dirty_supported
79static inline bool pt_dirty_supported(struct pt_common *common)
80{
81 return true;
82}
83#endif
84#endif
85
86#ifndef pt_entry_make_write_dirty
87static inline bool pt_entry_make_write_dirty(struct pt_state *pts)
88{
89 return false;
90}
91#endif
92
93/*
94 * Format supplies either:
95 * pt_entry_oa - OA is at the start of a contiguous entry
96 * or
97 * pt_item_oa - OA is adjusted for every item in a contiguous entry
98 *
99 * Build the missing one
100 *
101 * The internal helper _pt_entry_oa_fast() allows generating
102 * an efficient pt_entry_oa_exact(), it doesn't care which
103 * option is selected.
104 */
105#ifdef pt_entry_oa
106static inline pt_oaddr_t pt_item_oa(const struct pt_state *pts)
107{
108 return pt_entry_oa(pts) |
109 log2_mul(pts->index, pt_table_item_lg2sz(pts));
110}
111#define _pt_entry_oa_fast pt_entry_oa
112#endif
113
114#ifdef pt_item_oa
115static inline pt_oaddr_t pt_entry_oa(const struct pt_state *pts)
116{
117 return log2_set_mod(pt_item_oa(pts), 0,
118 pt_entry_num_contig_lg2(pts) +
119 pt_table_item_lg2sz(pts));
120}
121#define _pt_entry_oa_fast pt_item_oa
122#endif
123
124/*
125 * If not supplied by the format then use the constant
126 * PT_MAX_OUTPUT_ADDRESS_LG2.
127 */
128#ifndef pt_max_oa_lg2
129static inline unsigned int
130pt_max_oa_lg2(const struct pt_common *common)
131{
132 return PT_MAX_OUTPUT_ADDRESS_LG2;
133}
134#endif
135
136#ifndef pt_has_system_page_size
137static inline bool pt_has_system_page_size(const struct pt_common *common)
138{
139 return PT_GRANULE_LG2SZ == PAGE_SHIFT;
140}
141#endif
142
143/*
144 * If not supplied by the format then assume only one contiguous size determined
145 * by pt_contig_count_lg2()
146 */
147#ifndef pt_possible_sizes
148static inline unsigned short pt_contig_count_lg2(const struct pt_state *pts);
149
150/* Return a bitmap of possible leaf page sizes at this level */
151static inline pt_vaddr_t pt_possible_sizes(const struct pt_state *pts)
152{
153 unsigned int isz_lg2 = pt_table_item_lg2sz(pts);
154
155 if (!pt_can_have_leaf(pts))
156 return 0;
157 return log2_to_int(isz_lg2) |
158 log2_to_int(pt_contig_count_lg2(pts) + isz_lg2);
159}
160#endif
161
162/* If not supplied by the format then use 0. */
163#ifndef pt_full_va_prefix
164static inline pt_vaddr_t pt_full_va_prefix(const struct pt_common *common)
165{
166 return 0;
167}
168#endif
169
170/* If not supplied by the format then zero fill using PT_ITEM_WORD_SIZE */
171#ifndef pt_clear_entries
172static inline void pt_clear_entries64(struct pt_state *pts,
173 unsigned int num_contig_lg2)
174{
175 u64 *tablep = pt_cur_table(pts, u64) + pts->index;
176 u64 *end = tablep + log2_to_int(num_contig_lg2);
177
178 PT_WARN_ON(log2_mod(pts->index, num_contig_lg2));
179 for (; tablep != end; tablep++)
180 WRITE_ONCE(*tablep, 0);
181}
182
183static inline void pt_clear_entries32(struct pt_state *pts,
184 unsigned int num_contig_lg2)
185{
186 u32 *tablep = pt_cur_table(pts, u32) + pts->index;
187 u32 *end = tablep + log2_to_int(num_contig_lg2);
188
189 PT_WARN_ON(log2_mod(pts->index, num_contig_lg2));
190 for (; tablep != end; tablep++)
191 WRITE_ONCE(*tablep, 0);
192}
193
194static inline void pt_clear_entries(struct pt_state *pts,
195 unsigned int num_contig_lg2)
196{
197 if (PT_ITEM_WORD_SIZE == sizeof(u32))
198 pt_clear_entries32(pts, num_contig_lg2);
199 else
200 pt_clear_entries64(pts, num_contig_lg2);
201}
202#define pt_clear_entries pt_clear_entries
203#endif
204
205/* If not supplied then SW bits are not supported */
206#ifdef pt_sw_bit
207static inline bool pt_test_sw_bit_acquire(struct pt_state *pts,
208 unsigned int bitnr)
209{
210 /* Acquire, pairs with pt_set_sw_bit_release() */
211 smp_mb();
212 /* For a contiguous entry the sw bit is only stored in the first item. */
213 return pts->entry & pt_sw_bit(bitnr);
214}
215#define pt_test_sw_bit_acquire pt_test_sw_bit_acquire
216
217static inline void pt_set_sw_bit_release(struct pt_state *pts,
218 unsigned int bitnr)
219{
220#if !IS_ENABLED(CONFIG_GENERIC_ATOMIC64)
221 if (PT_ITEM_WORD_SIZE == sizeof(u64)) {
222 u64 *entryp = pt_cur_table(pts, u64) + pts->index;
223 u64 old_entry = pts->entry;
224 u64 new_entry;
225
226 do {
227 new_entry = old_entry | pt_sw_bit(bitnr);
228 } while (!try_cmpxchg64_release(entryp, &old_entry, new_entry));
229 pts->entry = new_entry;
230 return;
231 }
232#endif
233 if (PT_ITEM_WORD_SIZE == sizeof(u32)) {
234 u32 *entryp = pt_cur_table(pts, u32) + pts->index;
235 u32 old_entry = pts->entry;
236 u32 new_entry;
237
238 do {
239 new_entry = old_entry | pt_sw_bit(bitnr);
240 } while (!try_cmpxchg_release(entryp, &old_entry, new_entry));
241 pts->entry = new_entry;
242 } else
243 BUILD_BUG();
244}
245#define pt_set_sw_bit_release pt_set_sw_bit_release
246#else
247static inline unsigned int pt_max_sw_bit(struct pt_common *common)
248{
249 return 0;
250}
251
252extern void __pt_no_sw_bit(void);
253static inline bool pt_test_sw_bit_acquire(struct pt_state *pts,
254 unsigned int bitnr)
255{
256 __pt_no_sw_bit();
257 return false;
258}
259
260static inline void pt_set_sw_bit_release(struct pt_state *pts,
261 unsigned int bitnr)
262{
263 __pt_no_sw_bit();
264}
265#endif
266
267/*
268 * Format can call in the pt_install_leaf_entry() to check the arguments are all
269 * aligned correctly.
270 */
271static inline bool pt_check_install_leaf_args(struct pt_state *pts,
272 pt_oaddr_t oa,
273 unsigned int oasz_lg2)
274{
275 unsigned int isz_lg2 = pt_table_item_lg2sz(pts);
276
277 if (PT_WARN_ON(oalog2_mod(oa, oasz_lg2)))
278 return false;
279
280#ifdef pt_possible_sizes
281 if (PT_WARN_ON(isz_lg2 > oasz_lg2 ||
282 oasz_lg2 > isz_lg2 + pt_num_items_lg2(pts)))
283 return false;
284#else
285 if (PT_WARN_ON(oasz_lg2 != isz_lg2 &&
286 oasz_lg2 != isz_lg2 + pt_contig_count_lg2(pts)))
287 return false;
288#endif
289
290 if (PT_WARN_ON(oalog2_mod(pts->index, oasz_lg2 - isz_lg2)))
291 return false;
292 return true;
293}
294
295#endif