Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
powerpc64/bpf: Additional NVR handling for bpf_throw
The bpf_throw() function never returns, if it has clobbered
any callee-saved register, those will remain clobbered. The
prologue must take care of saving all callee-saved registers
in the frame of exception boundary program. Later these
additional non volatile registers R14-R25 along with other
NVRs are restored back in the epilogue of exception callback.
To achieve above objective, the frame size is determined
dynamically to accommodate additional non volatile registers
in exception boundary's frame.
For non-exception boundary program, the frame size remains
optimal. The additional instructions to save & restore r14-r25
registers are emitted only during exception boundary and
exception callback program respectively.
Signed-off-by: Abhishek Dubey <adubey@linux.ibm.com>
Signed-off-by: Madhavan Srinivasan <maddy@linux.ibm.com>
Link: https://patch.msgid.link/20260124075223.6033-7-adubey@linux.ibm.com
authored by
Abhishek Dubey
and committed by
Madhavan Srinivasan
11d45eee
c1699302
+81
-8
+81
-8
arch/powerpc/net/bpf_jit_comp64.c
+81
-8
arch/powerpc/net/bpf_jit_comp64.c
···
32
32
*
33
33
* [ prev sp ] <-------------
34
34
* [ tail_call_info ] 8 |
35
-
* [ nv gpr save area ] 6*8 |
35
+
* [ nv gpr save area ] 6*8 + (12*8) |
36
36
* [ local_tmp_var ] 24 |
37
37
* fp (r31) --> [ ebpf stack space ] upto 512 |
38
38
* [ frame header ] 32/112 |
39
39
* sp (r1) ---> [ stack pointer ] --------------
40
+
*
41
+
* Additional (12*8) in 'nv gpr save area' only in case of
42
+
* exception boundary.
40
43
*/
41
44
42
45
/* for bpf JIT code internal usage */
43
46
#define BPF_PPC_STACK_LOCALS 24
47
+
/*
48
+
* for additional non volatile registers(r14-r25) to be saved
49
+
* at exception boundary
50
+
*/
51
+
#define BPF_PPC_EXC_STACK_SAVE (12*8)
52
+
44
53
/* stack frame excluding BPF stack, ensure this is quadword aligned */
45
54
#define BPF_PPC_STACKFRAME (STACK_FRAME_MIN_SIZE + \
46
55
BPF_PPC_STACK_LOCALS + \
47
56
BPF_PPC_STACK_SAVE + \
48
57
BPF_PPC_TAILCALL)
58
+
59
+
/*
60
+
* same as BPF_PPC_STACKFRAME with save area for additional
61
+
* non volatile registers saved at exception boundary.
62
+
* This is quad-word aligned.
63
+
*/
64
+
#define BPF_PPC_EXC_STACKFRAME (BPF_PPC_STACKFRAME + BPF_PPC_EXC_STACK_SAVE)
49
65
50
66
/* BPF register usage */
51
67
#define TMP_REG_1 (MAX_BPF_JIT_REG + 0)
···
104
88
* - we call other functions (kernel helpers), or
105
89
* - the bpf program uses its stack area
106
90
* The latter condition is deduced from the usage of BPF_REG_FP
91
+
*
92
+
* bpf_throw() leads to exception callback from a BPF (sub)program.
93
+
* The (sub)program is always marked as SEEN_FUNC, creating a stack
94
+
* frame. The exception callback uses the frame of the exception
95
+
* boundary, so the exception boundary program must have a frame.
107
96
*/
108
97
return ctx->seen & SEEN_FUNC ||
109
98
bpf_is_seen_register(ctx, bpf_to_ppc(BPF_REG_FP)) ||
110
-
ctx->exception_cb;
99
+
ctx->exception_cb ||
100
+
ctx->exception_boundary;
111
101
}
112
102
113
103
/*
···
125
103
* [ ... ] |
126
104
* sp (r1) ---> [ stack pointer ] --------------
127
105
* [ tail_call_info ] 8
128
-
* [ nv gpr save area ] 6*8
106
+
* [ nv gpr save area ] 6*8 + (12*8)
129
107
* [ local_tmp_var ] 24
130
108
* [ unused red zone ] 224
109
+
*
110
+
* Additional (12*8) in 'nv gpr save area' only in case of
111
+
* exception boundary.
131
112
*/
132
113
static int bpf_jit_stack_local(struct codegen_context *ctx)
133
114
{
···
139
114
return STACK_FRAME_MIN_SIZE + ctx->stack_size;
140
115
} else {
141
116
/* Stack layout with redzone */
142
-
return -(BPF_PPC_TAILCALL + BPF_PPC_STACK_SAVE + BPF_PPC_STACK_LOCALS);
117
+
return -(BPF_PPC_TAILCALL
118
+
+BPF_PPC_STACK_SAVE
119
+
+(ctx->exception_boundary || ctx->exception_cb ?
120
+
BPF_PPC_EXC_STACK_SAVE : 0)
121
+
+BPF_PPC_STACK_LOCALS
122
+
);
143
123
}
144
124
}
145
125
···
155
125
156
126
static int bpf_jit_stack_offsetof(struct codegen_context *ctx, int reg)
157
127
{
158
-
if (reg >= BPF_PPC_NVR_MIN && reg < 32)
128
+
int min_valid_nvreg = BPF_PPC_NVR_MIN;
129
+
/* Default frame size for all cases except exception boundary */
130
+
int frame_nvr_size = BPF_PPC_STACKFRAME;
131
+
132
+
/* Consider all nv regs for handling exceptions */
133
+
if (ctx->exception_boundary || ctx->exception_cb) {
134
+
min_valid_nvreg = _R14;
135
+
frame_nvr_size = BPF_PPC_EXC_STACKFRAME;
136
+
}
137
+
138
+
if (reg >= min_valid_nvreg && reg < 32)
159
139
return (bpf_has_stack_frame(ctx) ?
160
-
(BPF_PPC_STACKFRAME + ctx->stack_size) : 0)
140
+
(frame_nvr_size + ctx->stack_size) : 0)
161
141
- (8 * (32 - reg)) - BPF_PPC_TAILCALL;
162
142
163
143
pr_err("BPF JIT is asking about unknown registers");
···
176
136
177
137
void bpf_jit_realloc_regs(struct codegen_context *ctx)
178
138
{
139
+
}
140
+
141
+
/*
142
+
* For exception boundary & exception_cb progs:
143
+
* return increased size to accommodate additional NVRs.
144
+
*/
145
+
static int bpf_jit_stack_size(struct codegen_context *ctx)
146
+
{
147
+
return ctx->exception_boundary || ctx->exception_cb ?
148
+
BPF_PPC_EXC_STACKFRAME :
149
+
BPF_PPC_STACKFRAME;
179
150
}
180
151
181
152
void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx)
···
249
198
EMIT(PPC_RAW_STD(_R0, _R1, PPC_LR_STKOFF));
250
199
}
251
200
252
-
EMIT(PPC_RAW_STDU(_R1, _R1, -(BPF_PPC_STACKFRAME + ctx->stack_size)));
201
+
EMIT(PPC_RAW_STDU(_R1, _R1,
202
+
-(bpf_jit_stack_size(ctx) + ctx->stack_size)));
203
+
}
204
+
205
+
/*
206
+
* Program acting as exception boundary pushes R14..R25 in addition to
207
+
* BPF callee-saved non volatile registers. Exception callback uses
208
+
* the boundary program's stack frame, recover additionally saved
209
+
* registers in epilogue of exception callback.
210
+
*/
211
+
if (ctx->exception_boundary) {
212
+
for (i = _R14; i <= _R25; i++)
213
+
EMIT(PPC_RAW_STD(i, _R1, bpf_jit_stack_offsetof(ctx, i)));
253
214
}
254
215
255
216
if (!ctx->exception_cb) {
···
311
248
EMIT(PPC_RAW_LD(bpf_to_ppc(ARENA_VM_START), _R1,
312
249
bpf_jit_stack_offsetof(ctx, bpf_to_ppc(ARENA_VM_START))));
313
250
251
+
if (ctx->exception_cb) {
252
+
/*
253
+
* Recover additionally saved non volatile registers from stack
254
+
* frame of exception boundary program.
255
+
*/
256
+
for (i = _R14; i <= _R25; i++)
257
+
EMIT(PPC_RAW_LD(i, _R1, bpf_jit_stack_offsetof(ctx, i)));
258
+
}
259
+
314
260
/* Tear down our stack frame */
315
261
if (bpf_has_stack_frame(ctx)) {
316
-
EMIT(PPC_RAW_ADDI(_R1, _R1, BPF_PPC_STACKFRAME + ctx->stack_size));
262
+
EMIT(PPC_RAW_ADDI(_R1, _R1, bpf_jit_stack_size(ctx) + ctx->stack_size));
263
+
317
264
if (ctx->seen & SEEN_FUNC || ctx->exception_cb) {
318
265
EMIT(PPC_RAW_LD(_R0, _R1, PPC_LR_STKOFF));
319
266
EMIT(PPC_RAW_MTLR(_R0));