Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _LINUX_RSEQ_TYPES_H
3#define _LINUX_RSEQ_TYPES_H
4
5#include <linux/irq_work_types.h>
6#include <linux/types.h>
7#include <linux/workqueue_types.h>
8
9#ifdef CONFIG_RSEQ
10struct rseq;
11
12/**
13 * struct rseq_event - Storage for rseq related event management
14 * @all: Compound to initialize and clear the data efficiently
15 * @events: Compound to access events with a single load/store
16 * @sched_switch: True if the task was scheduled and needs update on
17 * exit to user
18 * @ids_changed: Indicator that IDs need to be updated
19 * @user_irq: True on interrupt entry from user mode
20 * @has_rseq: True if the task has a rseq pointer installed
21 * @error: Compound error code for the slow path to analyze
22 * @fatal: User space data corrupted or invalid
23 * @slowpath: Indicator that slow path processing via TIF_NOTIFY_RESUME
24 * is required
25 *
26 * @sched_switch and @ids_changed must be adjacent and the combo must be
27 * 16bit aligned to allow a single store, when both are set at the same
28 * time in the scheduler.
29 */
30struct rseq_event {
31 union {
32 u64 all;
33 struct {
34 union {
35 u32 events;
36 struct {
37 u8 sched_switch;
38 u8 ids_changed;
39 u8 user_irq;
40 };
41 };
42
43 u8 has_rseq;
44 u8 __pad;
45 union {
46 u16 error;
47 struct {
48 u8 fatal;
49 u8 slowpath;
50 };
51 };
52 };
53 };
54};
55
56/**
57 * struct rseq_ids - Cache for ids, which need to be updated
58 * @cpu_cid: Compound of @cpu_id and @mm_cid to make the
59 * compiler emit a single compare on 64-bit
60 * @cpu_id: The CPU ID which was written last to user space
61 * @mm_cid: The MM CID which was written last to user space
62 *
63 * @cpu_id and @mm_cid are updated when the data is written to user space.
64 */
65struct rseq_ids {
66 union {
67 u64 cpu_cid;
68 struct {
69 u32 cpu_id;
70 u32 mm_cid;
71 };
72 };
73};
74
75/**
76 * union rseq_slice_state - Status information for rseq time slice extension
77 * @state: Compound to access the overall state
78 * @enabled: Time slice extension is enabled for the task
79 * @granted: Time slice extension was granted to the task
80 */
81union rseq_slice_state {
82 u16 state;
83 struct {
84 u8 enabled;
85 u8 granted;
86 };
87};
88
89/**
90 * struct rseq_slice - Status information for rseq time slice extension
91 * @state: Time slice extension state
92 * @expires: The time when a grant expires
93 * @yielded: Indicator for rseq_slice_yield()
94 */
95struct rseq_slice {
96 union rseq_slice_state state;
97 u64 expires;
98 u8 yielded;
99};
100
101/**
102 * struct rseq_data - Storage for all rseq related data
103 * @usrptr: Pointer to the registered user space RSEQ memory
104 * @len: Length of the RSEQ region
105 * @sig: Signature of critical section abort IPs
106 * @event: Storage for event management
107 * @ids: Storage for cached CPU ID and MM CID
108 * @slice: Storage for time slice extension data
109 */
110struct rseq_data {
111 struct rseq __user *usrptr;
112 u32 len;
113 u32 sig;
114 struct rseq_event event;
115 struct rseq_ids ids;
116#ifdef CONFIG_RSEQ_SLICE_EXTENSION
117 struct rseq_slice slice;
118#endif
119};
120
121#else /* CONFIG_RSEQ */
122struct rseq_data { };
123#endif /* !CONFIG_RSEQ */
124
125#ifdef CONFIG_SCHED_MM_CID
126
127#define MM_CID_UNSET BIT(31)
128#define MM_CID_ONCPU BIT(30)
129#define MM_CID_TRANSIT BIT(29)
130
131/**
132 * struct sched_mm_cid - Storage for per task MM CID data
133 * @active: MM CID is active for the task
134 * @cid: The CID associated to the task either permanently or
135 * borrowed from the CPU
136 */
137struct sched_mm_cid {
138 unsigned int active;
139 unsigned int cid;
140};
141
142/**
143 * struct mm_cid_pcpu - Storage for per CPU MM_CID data
144 * @cid: The CID associated to the CPU either permanently or
145 * while a task with a CID is running
146 */
147struct mm_cid_pcpu {
148 unsigned int cid;
149}____cacheline_aligned_in_smp;
150
151/**
152 * struct mm_mm_cid - Storage for per MM CID data
153 * @pcpu: Per CPU storage for CIDs associated to a CPU
154 * @mode: Indicates per CPU and transition mode
155 * @max_cids: The exclusive maximum CID value for allocation and convergence
156 * @irq_work: irq_work to handle the affinity mode change case
157 * @work: Regular work to handle the affinity mode change case
158 * @lock: Spinlock to protect against affinity setting which can't take @mutex
159 * @mutex: Mutex to serialize forks and exits related to this mm
160 * @nr_cpus_allowed: The number of CPUs in the per MM allowed CPUs map. The map
161 * is growth only.
162 * @users: The number of tasks sharing this MM. Separate from mm::mm_users
163 * as that is modified by mmget()/mm_put() by other entities which
164 * do not actually share the MM.
165 * @pcpu_thrs: Threshold for switching back from per CPU mode
166 * @update_deferred: A deferred switch back to per task mode is pending.
167 */
168struct mm_mm_cid {
169 /* Hotpath read mostly members */
170 struct mm_cid_pcpu __percpu *pcpu;
171 unsigned int mode;
172 unsigned int max_cids;
173
174 /* Rarely used. Moves @lock and @mutex into the second cacheline */
175 struct irq_work irq_work;
176 struct work_struct work;
177
178 raw_spinlock_t lock;
179 struct mutex mutex;
180
181 /* Low frequency modified */
182 unsigned int nr_cpus_allowed;
183 unsigned int users;
184 unsigned int pcpu_thrs;
185 unsigned int update_deferred;
186}____cacheline_aligned_in_smp;
187#else /* CONFIG_SCHED_MM_CID */
188struct mm_mm_cid { };
189struct sched_mm_cid { };
190#endif /* !CONFIG_SCHED_MM_CID */
191
192#endif