Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'bpf-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf
Pull bpf fixes from Alexei Starovoitov:
- Fix kCFI failures in JITed BPF code on arm64 (Sami Tolvanen, Puranjay
Mohan, Mark Rutland, Maxwell Bland)
- Disallow tail calls between BPF programs that use different cgroup
local storage maps to prevent out-of-bounds access (Daniel Borkmann)
- Fix unaligned access in flow_dissector and netfilter BPF programs
(Paul Chaignon)
- Avoid possible use of uninitialized mod_len in libbpf (Achill
Gilgenast)
* tag 'bpf-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf:
selftests/bpf: Test for unaligned flow_dissector ctx access
bpf: Improve ctx access verifier error message
bpf: Check netfilter ctx accesses are aligned
bpf: Check flow_dissector ctx accesses are aligned
arm64/cfi,bpf: Support kCFI + BPF on arm64
cfi: Move BPF CFI types and helpers to generic code
cfi: add C CFI type macro
libbpf: Avoid possible use of uninitialized mod_len
bpf: Fix oob access in cgroup local storage
bpf: Move cgroup iterator helpers to bpf.h
bpf: Move bpf map owner out of common struct
bpf: Add cookie object to bpf maps
+229
-176
+7
arch/arm64/include/asm/cfi.h
+7
arch/arm64/include/asm/cfi.h
+27
-3
arch/arm64/net/bpf_jit_comp.c
+27
-3
arch/arm64/net/bpf_jit_comp.c
···
10
10
#include <linux/arm-smccc.h>
11
11
#include <linux/bitfield.h>
12
12
#include <linux/bpf.h>
13
+
#include <linux/cfi.h>
13
14
#include <linux/filter.h>
14
15
#include <linux/memory.h>
15
16
#include <linux/printk.h>
···
115
114
ctx->idx++;
116
115
}
117
116
117
+
static inline void emit_u32_data(const u32 data, struct jit_ctx *ctx)
118
+
{
119
+
if (ctx->image != NULL && ctx->write)
120
+
ctx->image[ctx->idx] = data;
121
+
122
+
ctx->idx++;
123
+
}
124
+
118
125
static inline void emit_a64_mov_i(const int is64, const int reg,
119
126
const s32 val, struct jit_ctx *ctx)
120
127
{
···
181
172
{
182
173
if (IS_ENABLED(CONFIG_ARM64_BTI_KERNEL))
183
174
emit(insn, ctx);
175
+
}
176
+
177
+
static inline void emit_kcfi(u32 hash, struct jit_ctx *ctx)
178
+
{
179
+
if (IS_ENABLED(CONFIG_CFI_CLANG))
180
+
emit_u32_data(hash, ctx);
184
181
}
185
182
186
183
/*
···
518
503
const u8 arena_vm_base = bpf2a64[ARENA_VM_START];
519
504
const u8 priv_sp = bpf2a64[PRIVATE_SP];
520
505
void __percpu *priv_stack_ptr;
521
-
const int idx0 = ctx->idx;
522
506
int cur_offset;
523
507
524
508
/*
···
542
528
* low
543
529
*
544
530
*/
531
+
532
+
emit_kcfi(is_main_prog ? cfi_bpf_hash : cfi_bpf_subprog_hash, ctx);
533
+
const int idx0 = ctx->idx;
545
534
546
535
/* bpf function may be invoked by 3 instruction types:
547
536
* 1. bl, attached via freplace to bpf prog via short jump
···
2163
2146
jit_data->ro_header = ro_header;
2164
2147
}
2165
2148
2166
-
prog->bpf_func = (void *)ctx.ro_image;
2149
+
prog->bpf_func = (void *)ctx.ro_image + cfi_get_offset();
2167
2150
prog->jited = 1;
2168
-
prog->jited_len = prog_size;
2151
+
prog->jited_len = prog_size - cfi_get_offset();
2169
2152
2170
2153
if (!prog->is_func || extra_pass) {
2171
2154
int i;
···
2544
2527
/* return address locates above FP */
2545
2528
retaddr_off = stack_size + 8;
2546
2529
2530
+
if (flags & BPF_TRAMP_F_INDIRECT) {
2531
+
/*
2532
+
* Indirect call for bpf_struct_ops
2533
+
*/
2534
+
emit_kcfi(cfi_get_func_hash(func_addr), ctx);
2535
+
}
2547
2536
/* bpf trampoline may be invoked by 3 instruction types:
2548
2537
* 1. bl, attached to bpf prog or kernel function via short jump
2549
2538
* 2. br, attached to bpf prog or kernel function via long jump
···
3068
3045
sizeof(jit_data->header->size));
3069
3046
kfree(jit_data);
3070
3047
}
3048
+
prog->bpf_func -= cfi_get_offset();
3071
3049
hdr = bpf_jit_binary_pack_hdr(prog);
3072
3050
bpf_jit_binary_pack_free(hdr, NULL);
3073
3051
priv_stack_ptr = prog->aux->priv_stack_ptr;
-16
arch/riscv/include/asm/cfi.h
-16
arch/riscv/include/asm/cfi.h
···
14
14
#ifdef CONFIG_CFI_CLANG
15
15
enum bug_trap_type handle_cfi_failure(struct pt_regs *regs);
16
16
#define __bpfcall
17
-
static inline int cfi_get_offset(void)
18
-
{
19
-
return 4;
20
-
}
21
-
22
-
#define cfi_get_offset cfi_get_offset
23
-
extern u32 cfi_bpf_hash;
24
-
extern u32 cfi_bpf_subprog_hash;
25
-
extern u32 cfi_get_func_hash(void *func);
26
17
#else
27
18
static inline enum bug_trap_type handle_cfi_failure(struct pt_regs *regs)
28
19
{
29
20
return BUG_TRAP_TYPE_NONE;
30
-
}
31
-
32
-
#define cfi_bpf_hash 0U
33
-
#define cfi_bpf_subprog_hash 0U
34
-
static inline u32 cfi_get_func_hash(void *func)
35
-
{
36
-
return 0;
37
21
}
38
22
#endif /* CONFIG_CFI_CLANG */
39
23
-53
arch/riscv/kernel/cfi.c
-53
arch/riscv/kernel/cfi.c
···
75
75
76
76
return report_cfi_failure(regs, regs->epc, &target, type);
77
77
}
78
-
79
-
#ifdef CONFIG_CFI_CLANG
80
-
struct bpf_insn;
81
-
82
-
/* Must match bpf_func_t / DEFINE_BPF_PROG_RUN() */
83
-
extern unsigned int __bpf_prog_runX(const void *ctx,
84
-
const struct bpf_insn *insn);
85
-
86
-
/*
87
-
* Force a reference to the external symbol so the compiler generates
88
-
* __kcfi_typid.
89
-
*/
90
-
__ADDRESSABLE(__bpf_prog_runX);
91
-
92
-
/* u32 __ro_after_init cfi_bpf_hash = __kcfi_typeid___bpf_prog_runX; */
93
-
asm (
94
-
" .pushsection .data..ro_after_init,\"aw\",@progbits \n"
95
-
" .type cfi_bpf_hash,@object \n"
96
-
" .globl cfi_bpf_hash \n"
97
-
" .p2align 2, 0x0 \n"
98
-
"cfi_bpf_hash: \n"
99
-
" .word __kcfi_typeid___bpf_prog_runX \n"
100
-
" .size cfi_bpf_hash, 4 \n"
101
-
" .popsection \n"
102
-
);
103
-
104
-
/* Must match bpf_callback_t */
105
-
extern u64 __bpf_callback_fn(u64, u64, u64, u64, u64);
106
-
107
-
__ADDRESSABLE(__bpf_callback_fn);
108
-
109
-
/* u32 __ro_after_init cfi_bpf_subprog_hash = __kcfi_typeid___bpf_callback_fn; */
110
-
asm (
111
-
" .pushsection .data..ro_after_init,\"aw\",@progbits \n"
112
-
" .type cfi_bpf_subprog_hash,@object \n"
113
-
" .globl cfi_bpf_subprog_hash \n"
114
-
" .p2align 2, 0x0 \n"
115
-
"cfi_bpf_subprog_hash: \n"
116
-
" .word __kcfi_typeid___bpf_callback_fn \n"
117
-
" .size cfi_bpf_subprog_hash, 4 \n"
118
-
" .popsection \n"
119
-
);
120
-
121
-
u32 cfi_get_func_hash(void *func)
122
-
{
123
-
u32 hash;
124
-
125
-
if (get_kernel_nofault(hash, func - cfi_get_offset()))
126
-
return 0;
127
-
128
-
return hash;
129
-
}
130
-
#endif
+2
-8
arch/x86/include/asm/cfi.h
+2
-8
arch/x86/include/asm/cfi.h
···
116
116
#ifdef CONFIG_CFI_CLANG
117
117
enum bug_trap_type handle_cfi_failure(struct pt_regs *regs);
118
118
#define __bpfcall
119
-
extern u32 cfi_bpf_hash;
120
-
extern u32 cfi_bpf_subprog_hash;
121
119
122
120
static inline int cfi_get_offset(void)
123
121
{
···
133
135
#define cfi_get_offset cfi_get_offset
134
136
135
137
extern u32 cfi_get_func_hash(void *func);
138
+
#define cfi_get_func_hash cfi_get_func_hash
139
+
136
140
extern int cfi_get_func_arity(void *func);
137
141
138
142
#ifdef CONFIG_FINEIBT
···
152
152
static inline enum bug_trap_type handle_cfi_failure(struct pt_regs *regs)
153
153
{
154
154
return BUG_TRAP_TYPE_NONE;
155
-
}
156
-
#define cfi_bpf_hash 0U
157
-
#define cfi_bpf_subprog_hash 0U
158
-
static inline u32 cfi_get_func_hash(void *func)
159
-
{
160
-
return 0;
161
155
}
162
156
static inline int cfi_get_func_arity(void *func)
163
157
{
-37
arch/x86/kernel/alternative.c
-37
arch/x86/kernel/alternative.c
···
1184
1184
#endif
1185
1185
1186
1186
#ifdef CONFIG_CFI_CLANG
1187
-
struct bpf_insn;
1188
-
1189
-
/* Must match bpf_func_t / DEFINE_BPF_PROG_RUN() */
1190
-
extern unsigned int __bpf_prog_runX(const void *ctx,
1191
-
const struct bpf_insn *insn);
1192
-
1193
-
KCFI_REFERENCE(__bpf_prog_runX);
1194
-
1195
-
/* u32 __ro_after_init cfi_bpf_hash = __kcfi_typeid___bpf_prog_runX; */
1196
-
asm (
1197
-
" .pushsection .data..ro_after_init,\"aw\",@progbits \n"
1198
-
" .type cfi_bpf_hash,@object \n"
1199
-
" .globl cfi_bpf_hash \n"
1200
-
" .p2align 2, 0x0 \n"
1201
-
"cfi_bpf_hash: \n"
1202
-
" .long __kcfi_typeid___bpf_prog_runX \n"
1203
-
" .size cfi_bpf_hash, 4 \n"
1204
-
" .popsection \n"
1205
-
);
1206
-
1207
-
/* Must match bpf_callback_t */
1208
-
extern u64 __bpf_callback_fn(u64, u64, u64, u64, u64);
1209
-
1210
-
KCFI_REFERENCE(__bpf_callback_fn);
1211
-
1212
-
/* u32 __ro_after_init cfi_bpf_subprog_hash = __kcfi_typeid___bpf_callback_fn; */
1213
-
asm (
1214
-
" .pushsection .data..ro_after_init,\"aw\",@progbits \n"
1215
-
" .type cfi_bpf_subprog_hash,@object \n"
1216
-
" .globl cfi_bpf_subprog_hash \n"
1217
-
" .p2align 2, 0x0 \n"
1218
-
"cfi_bpf_subprog_hash: \n"
1219
-
" .long __kcfi_typeid___bpf_callback_fn \n"
1220
-
" .size cfi_bpf_subprog_hash, 4 \n"
1221
-
" .popsection \n"
1222
-
);
1223
-
1224
1187
u32 cfi_get_func_hash(void *func)
1225
1188
{
1226
1189
u32 hash;
-5
include/linux/bpf-cgroup.h
-5
include/linux/bpf-cgroup.h
···
77
77
extern struct static_key_false cgroup_bpf_enabled_key[MAX_CGROUP_BPF_ATTACH_TYPE];
78
78
#define cgroup_bpf_enabled(atype) static_branch_unlikely(&cgroup_bpf_enabled_key[atype])
79
79
80
-
#define for_each_cgroup_storage_type(stype) \
81
-
for (stype = 0; stype < MAX_BPF_CGROUP_STORAGE_TYPE; stype++)
82
-
83
80
struct bpf_cgroup_storage_map;
84
81
85
82
struct bpf_storage_buffer {
···
506
509
optlen, retval) ({ retval; })
507
510
#define BPF_CGROUP_RUN_PROG_SETSOCKOPT(sock, level, optname, optval, optlen, \
508
511
kernel_optval) ({ 0; })
509
-
510
-
#define for_each_cgroup_storage_type(stype) for (; false; )
511
512
512
513
#endif /* CONFIG_CGROUP_BPF */
513
514
+40
-20
include/linux/bpf.h
+40
-20
include/linux/bpf.h
···
208
208
BPF_RES_SPIN_LOCK = (1 << 12),
209
209
};
210
210
211
+
enum bpf_cgroup_storage_type {
212
+
BPF_CGROUP_STORAGE_SHARED,
213
+
BPF_CGROUP_STORAGE_PERCPU,
214
+
__BPF_CGROUP_STORAGE_MAX
215
+
#define MAX_BPF_CGROUP_STORAGE_TYPE __BPF_CGROUP_STORAGE_MAX
216
+
};
217
+
218
+
#ifdef CONFIG_CGROUP_BPF
219
+
# define for_each_cgroup_storage_type(stype) \
220
+
for (stype = 0; stype < MAX_BPF_CGROUP_STORAGE_TYPE; stype++)
221
+
#else
222
+
# define for_each_cgroup_storage_type(stype) for (; false; )
223
+
#endif /* CONFIG_CGROUP_BPF */
224
+
211
225
typedef void (*btf_dtor_kfunc_t)(void *);
212
226
213
227
struct btf_field_kptr {
···
274
260
void *owner;
275
261
} __attribute__((aligned(8)));
276
262
263
+
/* 'Ownership' of program-containing map is claimed by the first program
264
+
* that is going to use this map or by the first program which FD is
265
+
* stored in the map to make sure that all callers and callees have the
266
+
* same prog type, JITed flag and xdp_has_frags flag.
267
+
*/
268
+
struct bpf_map_owner {
269
+
enum bpf_prog_type type;
270
+
bool jited;
271
+
bool xdp_has_frags;
272
+
u64 storage_cookie[MAX_BPF_CGROUP_STORAGE_TYPE];
273
+
const struct btf_type *attach_func_proto;
274
+
};
275
+
277
276
struct bpf_map {
278
277
const struct bpf_map_ops *ops;
279
278
struct bpf_map *inner_map_meta;
···
319
292
struct rcu_head rcu;
320
293
};
321
294
atomic64_t writecnt;
322
-
/* 'Ownership' of program-containing map is claimed by the first program
323
-
* that is going to use this map or by the first program which FD is
324
-
* stored in the map to make sure that all callers and callees have the
325
-
* same prog type, JITed flag and xdp_has_frags flag.
326
-
*/
327
-
struct {
328
-
const struct btf_type *attach_func_proto;
329
-
spinlock_t lock;
330
-
enum bpf_prog_type type;
331
-
bool jited;
332
-
bool xdp_has_frags;
333
-
} owner;
295
+
spinlock_t owner_lock;
296
+
struct bpf_map_owner *owner;
334
297
bool bypass_spec_v1;
335
298
bool frozen; /* write-once; write-protected by freeze_mutex */
336
299
bool free_after_mult_rcu_gp;
337
300
bool free_after_rcu_gp;
338
301
atomic64_t sleepable_refcnt;
339
302
s64 __percpu *elem_count;
303
+
u64 cookie; /* write-once */
340
304
};
341
305
342
306
static inline const char *btf_field_type_name(enum btf_field_type type)
···
1099
1081
void *jited_image;
1100
1082
u32 jited_len;
1101
1083
};
1102
-
1103
-
enum bpf_cgroup_storage_type {
1104
-
BPF_CGROUP_STORAGE_SHARED,
1105
-
BPF_CGROUP_STORAGE_PERCPU,
1106
-
__BPF_CGROUP_STORAGE_MAX
1107
-
};
1108
-
1109
-
#define MAX_BPF_CGROUP_STORAGE_TYPE __BPF_CGROUP_STORAGE_MAX
1110
1084
1111
1085
/* The longest tracepoint has 12 args.
1112
1086
* See include/trace/bpf_probe.h
···
2116
2106
{
2117
2107
return (access_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG)) !=
2118
2108
(BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG);
2109
+
}
2110
+
2111
+
static inline struct bpf_map_owner *bpf_map_owner_alloc(struct bpf_map *map)
2112
+
{
2113
+
return kzalloc(sizeof(*map->owner), GFP_ATOMIC);
2114
+
}
2115
+
2116
+
static inline void bpf_map_owner_free(struct bpf_map *map)
2117
+
{
2118
+
kfree(map->owner);
2119
2119
}
2120
2120
2121
2121
struct bpf_event_entry {
+39
-8
include/linux/cfi.h
+39
-8
include/linux/cfi.h
···
11
11
#include <linux/module.h>
12
12
#include <asm/cfi.h>
13
13
14
+
#ifdef CONFIG_CFI_CLANG
14
15
extern bool cfi_warn;
15
16
16
-
#ifndef cfi_get_offset
17
-
static inline int cfi_get_offset(void)
18
-
{
19
-
return 0;
20
-
}
21
-
#endif
22
-
23
-
#ifdef CONFIG_CFI_CLANG
24
17
enum bug_trap_type report_cfi_failure(struct pt_regs *regs, unsigned long addr,
25
18
unsigned long *target, u32 type);
26
19
···
22
29
{
23
30
return report_cfi_failure(regs, addr, NULL, 0);
24
31
}
32
+
33
+
#ifndef cfi_get_offset
34
+
/*
35
+
* Returns the CFI prefix offset. By default, the compiler emits only
36
+
* a 4-byte CFI type hash before the function. If an architecture
37
+
* uses -fpatchable-function-entry=N,M where M>0 to change the prefix
38
+
* offset, they must override this function.
39
+
*/
40
+
static inline int cfi_get_offset(void)
41
+
{
42
+
return 4;
43
+
}
44
+
#endif
45
+
46
+
#ifndef cfi_get_func_hash
47
+
static inline u32 cfi_get_func_hash(void *func)
48
+
{
49
+
u32 hash;
50
+
51
+
if (get_kernel_nofault(hash, func - cfi_get_offset()))
52
+
return 0;
53
+
54
+
return hash;
55
+
}
56
+
#endif
57
+
58
+
/* CFI type hashes for BPF function types */
59
+
extern u32 cfi_bpf_hash;
60
+
extern u32 cfi_bpf_subprog_hash;
61
+
62
+
#else /* CONFIG_CFI_CLANG */
63
+
64
+
static inline int cfi_get_offset(void) { return 0; }
65
+
static inline u32 cfi_get_func_hash(void *func) { return 0; }
66
+
67
+
#define cfi_bpf_hash 0U
68
+
#define cfi_bpf_subprog_hash 0U
69
+
25
70
#endif /* CONFIG_CFI_CLANG */
26
71
27
72
#ifdef CONFIG_ARCH_USES_CFI_TRAPS
+23
include/linux/cfi_types.h
+23
include/linux/cfi_types.h
···
41
41
SYM_TYPED_START(name, SYM_L_GLOBAL, SYM_A_ALIGN)
42
42
#endif
43
43
44
+
#else /* __ASSEMBLY__ */
45
+
46
+
#ifdef CONFIG_CFI_CLANG
47
+
#define DEFINE_CFI_TYPE(name, func) \
48
+
/* \
49
+
* Force a reference to the function so the compiler generates \
50
+
* __kcfi_typeid_<func>. \
51
+
*/ \
52
+
__ADDRESSABLE(func); \
53
+
/* u32 name __ro_after_init = __kcfi_typeid_<func> */ \
54
+
extern u32 name; \
55
+
asm ( \
56
+
" .pushsection .data..ro_after_init,\"aw\",\%progbits \n" \
57
+
" .type " #name ",\%object \n" \
58
+
" .globl " #name " \n" \
59
+
" .p2align 2, 0x0 \n" \
60
+
#name ": \n" \
61
+
" .4byte __kcfi_typeid_" #func " \n" \
62
+
" .size " #name ", 4 \n" \
63
+
" .popsection \n" \
64
+
);
65
+
#endif
66
+
44
67
#endif /* __ASSEMBLY__ */
45
68
#endif /* _LINUX_CFI_TYPES_H */
+33
-17
kernel/bpf/core.c
+33
-17
kernel/bpf/core.c
···
2377
2377
const struct bpf_prog *fp)
2378
2378
{
2379
2379
enum bpf_prog_type prog_type = resolve_prog_type(fp);
2380
-
bool ret;
2381
2380
struct bpf_prog_aux *aux = fp->aux;
2381
+
enum bpf_cgroup_storage_type i;
2382
+
bool ret = false;
2383
+
u64 cookie;
2382
2384
2383
2385
if (fp->kprobe_override)
2384
-
return false;
2386
+
return ret;
2385
2387
2386
-
spin_lock(&map->owner.lock);
2387
-
if (!map->owner.type) {
2388
-
/* There's no owner yet where we could check for
2389
-
* compatibility.
2390
-
*/
2391
-
map->owner.type = prog_type;
2392
-
map->owner.jited = fp->jited;
2393
-
map->owner.xdp_has_frags = aux->xdp_has_frags;
2394
-
map->owner.attach_func_proto = aux->attach_func_proto;
2388
+
spin_lock(&map->owner_lock);
2389
+
/* There's no owner yet where we could check for compatibility. */
2390
+
if (!map->owner) {
2391
+
map->owner = bpf_map_owner_alloc(map);
2392
+
if (!map->owner)
2393
+
goto err;
2394
+
map->owner->type = prog_type;
2395
+
map->owner->jited = fp->jited;
2396
+
map->owner->xdp_has_frags = aux->xdp_has_frags;
2397
+
map->owner->attach_func_proto = aux->attach_func_proto;
2398
+
for_each_cgroup_storage_type(i) {
2399
+
map->owner->storage_cookie[i] =
2400
+
aux->cgroup_storage[i] ?
2401
+
aux->cgroup_storage[i]->cookie : 0;
2402
+
}
2395
2403
ret = true;
2396
2404
} else {
2397
-
ret = map->owner.type == prog_type &&
2398
-
map->owner.jited == fp->jited &&
2399
-
map->owner.xdp_has_frags == aux->xdp_has_frags;
2405
+
ret = map->owner->type == prog_type &&
2406
+
map->owner->jited == fp->jited &&
2407
+
map->owner->xdp_has_frags == aux->xdp_has_frags;
2408
+
for_each_cgroup_storage_type(i) {
2409
+
if (!ret)
2410
+
break;
2411
+
cookie = aux->cgroup_storage[i] ?
2412
+
aux->cgroup_storage[i]->cookie : 0;
2413
+
ret = map->owner->storage_cookie[i] == cookie ||
2414
+
!cookie;
2415
+
}
2400
2416
if (ret &&
2401
-
map->owner.attach_func_proto != aux->attach_func_proto) {
2417
+
map->owner->attach_func_proto != aux->attach_func_proto) {
2402
2418
switch (prog_type) {
2403
2419
case BPF_PROG_TYPE_TRACING:
2404
2420
case BPF_PROG_TYPE_LSM:
···
2427
2411
}
2428
2412
}
2429
2413
}
2430
-
spin_unlock(&map->owner.lock);
2431
-
2414
+
err:
2415
+
spin_unlock(&map->owner_lock);
2432
2416
return ret;
2433
2417
}
2434
2418
+13
-6
kernel/bpf/syscall.c
+13
-6
kernel/bpf/syscall.c
···
37
37
#include <linux/trace_events.h>
38
38
#include <linux/tracepoint.h>
39
39
#include <linux/overflow.h>
40
+
#include <linux/cookie.h>
40
41
41
42
#include <net/netfilter/nf_bpf_link.h>
42
43
#include <net/netkit.h>
···
54
53
#define BPF_OBJ_FLAG_MASK (BPF_F_RDONLY | BPF_F_WRONLY)
55
54
56
55
DEFINE_PER_CPU(int, bpf_prog_active);
56
+
DEFINE_COOKIE(bpf_map_cookie);
57
57
static DEFINE_IDR(prog_idr);
58
58
static DEFINE_SPINLOCK(prog_idr_lock);
59
59
static DEFINE_IDR(map_idr);
···
887
885
888
886
security_bpf_map_free(map);
889
887
bpf_map_release_memcg(map);
888
+
bpf_map_owner_free(map);
890
889
bpf_map_free(map);
891
890
}
892
891
···
982
979
struct bpf_map *map = filp->private_data;
983
980
u32 type = 0, jited = 0;
984
981
985
-
if (map_type_contains_progs(map)) {
986
-
spin_lock(&map->owner.lock);
987
-
type = map->owner.type;
988
-
jited = map->owner.jited;
989
-
spin_unlock(&map->owner.lock);
982
+
spin_lock(&map->owner_lock);
983
+
if (map->owner) {
984
+
type = map->owner->type;
985
+
jited = map->owner->jited;
990
986
}
987
+
spin_unlock(&map->owner_lock);
991
988
992
989
seq_printf(m,
993
990
"map_type:\t%u\n"
···
1490
1487
if (err < 0)
1491
1488
goto free_map;
1492
1489
1490
+
preempt_disable();
1491
+
map->cookie = gen_cookie_next(&bpf_map_cookie);
1492
+
preempt_enable();
1493
+
1493
1494
atomic64_set(&map->refcnt, 1);
1494
1495
atomic64_set(&map->usercnt, 1);
1495
1496
mutex_init(&map->freeze_mutex);
1496
-
spin_lock_init(&map->owner.lock);
1497
+
spin_lock_init(&map->owner_lock);
1497
1498
1498
1499
if (attr->btf_key_type_id || attr->btf_value_type_id ||
1499
1500
/* Even the map's value is a kernel's struct,
+1
-1
kernel/bpf/verifier.c
+1
-1
kernel/bpf/verifier.c
···
21445
21445
&target_size);
21446
21446
if (cnt == 0 || cnt >= INSN_BUF_SIZE ||
21447
21447
(ctx_field_size && !target_size)) {
21448
-
verifier_bug(env, "error during ctx access conversion");
21448
+
verifier_bug(env, "error during ctx access conversion (%d)", cnt);
21449
21449
return -EFAULT;
21450
21450
}
21451
21451
+15
kernel/cfi.c
+15
kernel/cfi.c
···
5
5
* Copyright (C) 2022 Google LLC
6
6
*/
7
7
8
+
#include <linux/bpf.h>
9
+
#include <linux/cfi_types.h>
8
10
#include <linux/cfi.h>
9
11
10
12
bool cfi_warn __ro_after_init = IS_ENABLED(CONFIG_CFI_PERMISSIVE);
···
28
26
29
27
return BUG_TRAP_TYPE_BUG;
30
28
}
29
+
30
+
/*
31
+
* Declare two non-existent functions with types that match bpf_func_t and
32
+
* bpf_callback_t pointers, and use DEFINE_CFI_TYPE to define type hash
33
+
* variables for each function type. The cfi_bpf_* variables are used by
34
+
* arch-specific BPF JIT implementations to ensure indirectly callable JIT
35
+
* code has matching CFI type hashes.
36
+
*/
37
+
extern typeof(*(bpf_func_t)0) __bpf_prog_runX;
38
+
DEFINE_CFI_TYPE(cfi_bpf_hash, __bpf_prog_runX);
39
+
40
+
extern typeof(*(bpf_callback_t)0) __bpf_callback_fn;
41
+
DEFINE_CFI_TYPE(cfi_bpf_subprog_hash, __bpf_callback_fn);
31
42
32
43
#ifdef CONFIG_ARCH_USES_CFI_TRAPS
33
44
static inline unsigned long trap_address(s32 *p)
+3
net/core/filter.c
+3
net/core/filter.c
+3
net/netfilter/nf_bpf_link.c
+3
net/netfilter/nf_bpf_link.c
+1
-1
tools/lib/bpf/libbpf.c
+1
-1
tools/lib/bpf/libbpf.c
+22
-1
tools/testing/selftests/bpf/progs/verifier_ctx.c
+22
-1
tools/testing/selftests/bpf/progs/verifier_ctx.c
···
1
1
// SPDX-License-Identifier: GPL-2.0
2
2
/* Converted from tools/testing/selftests/bpf/verifier/ctx.c */
3
3
4
-
#include <linux/bpf.h>
4
+
#include "vmlinux.h"
5
5
#include <bpf/bpf_helpers.h>
6
6
#include "bpf_misc.h"
7
+
8
+
#define sizeof_field(TYPE, MEMBER) sizeof((((TYPE *)0)->MEMBER))
7
9
8
10
SEC("tc")
9
11
__description("context stores via BPF_ATOMIC")
···
244
242
narrow_load("sockops", bpf_sock_ops, skb_data);
245
243
narrow_load("sockops", bpf_sock_ops, skb_data_end);
246
244
narrow_load("sockops", bpf_sock_ops, skb_hwtstamp);
245
+
246
+
#define unaligned_access(type, ctx, field) \
247
+
SEC(type) \
248
+
__description("unaligned access on field " #field " of " #ctx) \
249
+
__failure __msg("invalid bpf_context access") \
250
+
__naked void unaligned_ctx_access_##ctx##field(void) \
251
+
{ \
252
+
asm volatile (" \
253
+
r1 = *(u%[size] *)(r1 + %[off]); \
254
+
r0 = 0; \
255
+
exit;" \
256
+
: \
257
+
: __imm_const(size, sizeof_field(struct ctx, field) * 8), \
258
+
__imm_const(off, offsetof(struct ctx, field) + 1) \
259
+
: __clobber_all); \
260
+
}
261
+
262
+
unaligned_access("flow_dissector", __sk_buff, data);
263
+
unaligned_access("netfilter", bpf_nf_ctx, skb);
247
264
248
265
char _license[] SEC("license") = "GPL";