Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES
4 *
5 * This header is included after the format. It contains definitions
6 * that build on the format definitions to create the basic format API.
7 *
8 * The format API is listed here, with kdocs. The functions without bodies are
9 * implemented in the format using the pattern:
10 * static inline FMTpt_XXX(..) {..}
11 * #define pt_XXX FMTpt_XXX
12 *
13 * If the format doesn't implement a function then pt_fmt_defaults.h can provide
14 * a generic version.
15 *
16 * The routines marked "@pts: Entry to query" operate on the entire contiguous
17 * entry and can be called with a pts->index pointing to any sub item that makes
18 * up that entry.
19 *
20 * The header order is:
21 * pt_defs.h
22 * FMT.h
23 * pt_common.h
24 */
25#ifndef __GENERIC_PT_PT_COMMON_H
26#define __GENERIC_PT_PT_COMMON_H
27
28#include "pt_defs.h"
29#include "pt_fmt_defaults.h"
30
31/**
32 * pt_attr_from_entry() - Convert the permission bits back to attrs
33 * @pts: Entry to convert from
34 * @attrs: Resulting attrs
35 *
36 * Fill in the attrs with the permission bits encoded in the current leaf entry.
37 * The attrs should be usable with pt_install_leaf_entry() to reconstruct the
38 * same entry.
39 */
40static inline void pt_attr_from_entry(const struct pt_state *pts,
41 struct pt_write_attrs *attrs);
42
43/**
44 * pt_can_have_leaf() - True if the current level can have an OA entry
45 * @pts: The current level
46 *
47 * True if the current level can support pt_install_leaf_entry(). A leaf
48 * entry produce an OA.
49 */
50static inline bool pt_can_have_leaf(const struct pt_state *pts);
51
52/**
53 * pt_can_have_table() - True if the current level can have a lower table
54 * @pts: The current level
55 *
56 * Every level except 0 is allowed to have a lower table.
57 */
58static inline bool pt_can_have_table(const struct pt_state *pts)
59{
60 /* No further tables at level 0 */
61 return pts->level > 0;
62}
63
64/**
65 * pt_clear_entries() - Make entries empty (non-present)
66 * @pts: Starting table index
67 * @num_contig_lg2: Number of contiguous items to clear
68 *
69 * Clear a run of entries. A cleared entry will load back as PT_ENTRY_EMPTY
70 * and does not have any effect on table walking. The starting index must be
71 * aligned to num_contig_lg2.
72 */
73static inline void pt_clear_entries(struct pt_state *pts,
74 unsigned int num_contig_lg2);
75
76/**
77 * pt_entry_make_write_dirty() - Make an entry dirty
78 * @pts: Table entry to change
79 *
80 * Make pt_entry_is_write_dirty() return true for this entry. This can be called
81 * asynchronously with any other table manipulation under a RCU lock and must
82 * not corrupt the table.
83 */
84static inline bool pt_entry_make_write_dirty(struct pt_state *pts);
85
86/**
87 * pt_entry_make_write_clean() - Make the entry write clean
88 * @pts: Table entry to change
89 *
90 * Modify the entry so that pt_entry_is_write_dirty() == false. The HW will
91 * eventually be notified of this change via a TLB flush, which is the point
92 * that the HW must become synchronized. Any "write dirty" prior to the TLB
93 * flush can be lost, but once the TLB flush completes all writes must make
94 * their entries write dirty.
95 *
96 * The format should alter the entry in a way that is compatible with any
97 * concurrent update from HW. The entire contiguous entry is changed.
98 */
99static inline void pt_entry_make_write_clean(struct pt_state *pts);
100
101/**
102 * pt_entry_is_write_dirty() - True if the entry has been written to
103 * @pts: Entry to query
104 *
105 * "write dirty" means that the HW has written to the OA translated
106 * by this entry. If the entry is contiguous then the consolidated
107 * "write dirty" for all the items must be returned.
108 */
109static inline bool pt_entry_is_write_dirty(const struct pt_state *pts);
110
111/**
112 * pt_dirty_supported() - True if the page table supports dirty tracking
113 * @common: Page table to query
114 */
115static inline bool pt_dirty_supported(struct pt_common *common);
116
117/**
118 * pt_entry_num_contig_lg2() - Number of contiguous items for this leaf entry
119 * @pts: Entry to query
120 *
121 * Return the number of contiguous items this leaf entry spans. If the entry
122 * is single item it returns ilog2(1).
123 */
124static inline unsigned int pt_entry_num_contig_lg2(const struct pt_state *pts);
125
126/**
127 * pt_entry_oa() - Output Address for this leaf entry
128 * @pts: Entry to query
129 *
130 * Return the output address for the start of the entry. If the entry
131 * is contiguous this returns the same value for each sub-item. I.e.::
132 *
133 * log2_mod(pt_entry_oa(), pt_entry_oa_lg2sz()) == 0
134 *
135 * See pt_item_oa(). The format should implement one of these two functions
136 * depending on how it stores the OAs in the table.
137 */
138static inline pt_oaddr_t pt_entry_oa(const struct pt_state *pts);
139
140/**
141 * pt_entry_oa_lg2sz() - Return the size of an OA entry
142 * @pts: Entry to query
143 *
144 * If the entry is not contiguous this returns pt_table_item_lg2sz(), otherwise
145 * it returns the total VA/OA size of the entire contiguous entry.
146 */
147static inline unsigned int pt_entry_oa_lg2sz(const struct pt_state *pts)
148{
149 return pt_entry_num_contig_lg2(pts) + pt_table_item_lg2sz(pts);
150}
151
152/**
153 * pt_entry_oa_exact() - Return the complete OA for an entry
154 * @pts: Entry to query
155 *
156 * During iteration the first entry could have a VA with an offset from the
157 * natural start of the entry. Return the exact OA including the pts's VA
158 * offset.
159 */
160static inline pt_oaddr_t pt_entry_oa_exact(const struct pt_state *pts)
161{
162 return _pt_entry_oa_fast(pts) |
163 log2_mod(pts->range->va, pt_entry_oa_lg2sz(pts));
164}
165
166/**
167 * pt_full_va_prefix() - The top bits of the VA
168 * @common: Page table to query
169 *
170 * This is usually 0, but some formats have their VA space going downward from
171 * PT_VADDR_MAX, and will return that instead. This value must always be
172 * adjusted by struct pt_common max_vasz_lg2.
173 */
174static inline pt_vaddr_t pt_full_va_prefix(const struct pt_common *common);
175
176/**
177 * pt_has_system_page_size() - True if level 0 can install a PAGE_SHIFT entry
178 * @common: Page table to query
179 *
180 * If true the caller can use, at level 0, pt_install_leaf_entry(PAGE_SHIFT).
181 * This is useful to create optimized paths for common cases of PAGE_SIZE
182 * mappings.
183 */
184static inline bool pt_has_system_page_size(const struct pt_common *common);
185
186/**
187 * pt_install_leaf_entry() - Write a leaf entry to the table
188 * @pts: Table index to change
189 * @oa: Output Address for this leaf
190 * @oasz_lg2: Size in VA/OA for this leaf
191 * @attrs: Attributes to modify the entry
192 *
193 * A leaf OA entry will return PT_ENTRY_OA from pt_load_entry(). It translates
194 * the VA indicated by pts to the given OA.
195 *
196 * For a single item non-contiguous entry oasz_lg2 is pt_table_item_lg2sz().
197 * For contiguous it is pt_table_item_lg2sz() + num_contig_lg2.
198 *
199 * This must not be called if pt_can_have_leaf() == false. Contiguous sizes
200 * not indicated by pt_possible_sizes() must not be specified.
201 */
202static inline void pt_install_leaf_entry(struct pt_state *pts, pt_oaddr_t oa,
203 unsigned int oasz_lg2,
204 const struct pt_write_attrs *attrs);
205
206/**
207 * pt_install_table() - Write a table entry to the table
208 * @pts: Table index to change
209 * @table_pa: CPU physical address of the lower table's memory
210 * @attrs: Attributes to modify the table index
211 *
212 * A table entry will return PT_ENTRY_TABLE from pt_load_entry(). The table_pa
213 * is the table at pts->level - 1. This is done by cmpxchg so pts must have the
214 * current entry loaded. The pts is updated with the installed entry.
215 *
216 * This must not be called if pt_can_have_table() == false.
217 *
218 * Returns: true if the table was installed successfully.
219 */
220static inline bool pt_install_table(struct pt_state *pts, pt_oaddr_t table_pa,
221 const struct pt_write_attrs *attrs);
222
223/**
224 * pt_item_oa() - Output Address for this leaf item
225 * @pts: Item to query
226 *
227 * Return the output address for this item. If the item is part of a contiguous
228 * entry it returns the value of the OA for this individual sub item.
229 *
230 * See pt_entry_oa(). The format should implement one of these two functions
231 * depending on how it stores the OA's in the table.
232 */
233static inline pt_oaddr_t pt_item_oa(const struct pt_state *pts);
234
235/**
236 * pt_load_entry_raw() - Read from the location pts points at into the pts
237 * @pts: Table index to load
238 *
239 * Return the type of entry that was loaded. pts->entry will be filled in with
240 * the entry's content. See pt_load_entry()
241 */
242static inline enum pt_entry_type pt_load_entry_raw(struct pt_state *pts);
243
244/**
245 * pt_max_oa_lg2() - Return the maximum OA the table format can hold
246 * @common: Page table to query
247 *
248 * The value oalog2_to_max_int(pt_max_oa_lg2()) is the MAX for the
249 * OA. This is the absolute maximum address the table can hold. struct pt_common
250 * max_oasz_lg2 sets a lower dynamic maximum based on HW capability.
251 */
252static inline unsigned int
253pt_max_oa_lg2(const struct pt_common *common);
254
255/**
256 * pt_num_items_lg2() - Return the number of items in this table level
257 * @pts: The current level
258 *
259 * The number of items in a table level defines the number of bits this level
260 * decodes from the VA. This function is not called for the top level,
261 * so it does not need to compute a special value for the top case. The
262 * result for the top is based on pt_common max_vasz_lg2.
263 *
264 * The value is used as part of determining the table indexes via the
265 * equation::
266 *
267 * log2_mod(log2_div(VA, pt_table_item_lg2sz()), pt_num_items_lg2())
268 */
269static inline unsigned int pt_num_items_lg2(const struct pt_state *pts);
270
271/**
272 * pt_pgsz_lg2_to_level - Return the level that maps the page size
273 * @common: Page table to query
274 * @pgsize_lg2: Log2 page size
275 *
276 * Returns the table level that will map the given page size. The page
277 * size must be part of the pt_possible_sizes() for some level.
278 */
279static inline unsigned int pt_pgsz_lg2_to_level(struct pt_common *common,
280 unsigned int pgsize_lg2);
281
282/**
283 * pt_possible_sizes() - Return a bitmap of possible output sizes at this level
284 * @pts: The current level
285 *
286 * Each level has a list of possible output sizes that can be installed as
287 * leaf entries. If pt_can_have_leaf() is false returns zero.
288 *
289 * Otherwise the bit in position pt_table_item_lg2sz() should be set indicating
290 * that a non-contiguous single item leaf entry is supported. The following
291 * pt_num_items_lg2() number of bits can be set indicating contiguous entries
292 * are supported. Bit pt_table_item_lg2sz() + pt_num_items_lg2() must not be
293 * set, contiguous entries cannot span the entire table.
294 *
295 * The OR of pt_possible_sizes() of all levels is the typical bitmask of all
296 * supported sizes in the entire table.
297 */
298static inline pt_vaddr_t pt_possible_sizes(const struct pt_state *pts);
299
300/**
301 * pt_table_item_lg2sz() - Size of a single item entry in this table level
302 * @pts: The current level
303 *
304 * The size of the item specifies how much VA and OA a single item occupies.
305 *
306 * See pt_entry_oa_lg2sz() for the same value including the effect of contiguous
307 * entries.
308 */
309static inline unsigned int pt_table_item_lg2sz(const struct pt_state *pts);
310
311/**
312 * pt_table_oa_lg2sz() - Return the VA/OA size of the entire table
313 * @pts: The current level
314 *
315 * Return the size of VA decoded by the entire table level.
316 */
317static inline unsigned int pt_table_oa_lg2sz(const struct pt_state *pts)
318{
319 if (pts->range->top_level == pts->level)
320 return pts->range->max_vasz_lg2;
321 return min_t(unsigned int, pts->range->common->max_vasz_lg2,
322 pt_num_items_lg2(pts) + pt_table_item_lg2sz(pts));
323}
324
325/**
326 * pt_table_pa() - Return the CPU physical address of the table entry
327 * @pts: Entry to query
328 *
329 * This is only ever called on PT_ENTRY_TABLE entries. Must return the same
330 * value passed to pt_install_table().
331 */
332static inline pt_oaddr_t pt_table_pa(const struct pt_state *pts);
333
334/**
335 * pt_table_ptr() - Return a CPU pointer for a table item
336 * @pts: Entry to query
337 *
338 * Same as pt_table_pa() but returns a CPU pointer.
339 */
340static inline struct pt_table_p *pt_table_ptr(const struct pt_state *pts)
341{
342 return __va(pt_table_pa(pts));
343}
344
345/**
346 * pt_max_sw_bit() - Return the maximum software bit usable for any level and
347 * entry
348 * @common: Page table
349 *
350 * The swbit can be passed as bitnr to the other sw_bit functions.
351 */
352static inline unsigned int pt_max_sw_bit(struct pt_common *common);
353
354/**
355 * pt_test_sw_bit_acquire() - Read a software bit in an item
356 * @pts: Entry to read
357 * @bitnr: Bit to read
358 *
359 * Software bits are ignored by HW and can be used for any purpose by the
360 * software. This does a test bit and acquire operation.
361 */
362static inline bool pt_test_sw_bit_acquire(struct pt_state *pts,
363 unsigned int bitnr);
364
365/**
366 * pt_set_sw_bit_release() - Set a software bit in an item
367 * @pts: Entry to set
368 * @bitnr: Bit to set
369 *
370 * Software bits are ignored by HW and can be used for any purpose by the
371 * software. This does a set bit and release operation.
372 */
373static inline void pt_set_sw_bit_release(struct pt_state *pts,
374 unsigned int bitnr);
375
376/**
377 * pt_load_entry() - Read from the location pts points at into the pts
378 * @pts: Table index to load
379 *
380 * Set the type of entry that was loaded. pts->entry and pts->table_lower
381 * will be filled in with the entry's content.
382 */
383static inline void pt_load_entry(struct pt_state *pts)
384{
385 pts->type = pt_load_entry_raw(pts);
386 if (pts->type == PT_ENTRY_TABLE)
387 pts->table_lower = pt_table_ptr(pts);
388}
389#endif