tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

io_uring/bpf-ops: implement bpf ops registration

Implement BPF struct ops registration. It's registered off the BPF
path, and can be removed by BPF as well as io_uring. To protect it,
introduce a global lock synchronising registration. ctx->uring_lock can
be nested under it. ctx->bpf_ops is write protected by both locks and
so it's safe to read it under either of them.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://patch.msgid.link/1f46bffd76008de49cbafa2ad77d348810a4f69e.1772109579.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>

authored by

Pavel Begunkov and committed by
Jens Axboe 98f37634 89081924

+104 -2
+5
include/linux/io_uring_types.h
··· 8 8 #include <linux/llist.h> 9 9 #include <uapi/linux/io_uring.h> 10 10 11 + struct iou_loop_params; 12 + struct io_uring_bpf_ops; 13 + 11 14 enum { 12 15 /* 13 16 * A hint to not wake right away but delay until there are enough of ··· 490 487 491 488 DECLARE_HASHTABLE(napi_ht, 4); 492 489 #endif 490 + 491 + struct io_uring_bpf_ops *bpf_ops; 493 492 494 493 /* 495 494 * Protection for resize vs mmap races - both the mmap and resize
+90 -2
io_uring/bpf-ops.c
··· 5 5 6 6 #include "io_uring.h" 7 7 #include "register.h" 8 + #include "loop.h" 8 9 #include "memmap.h" 9 10 #include "bpf-ops.h" 10 - #include "loop.h" 11 11 12 + static DEFINE_MUTEX(io_bpf_ctrl_mutex); 12 13 static const struct btf_type *loop_params_type; 13 14 14 15 __bpf_kfunc_start_defs(); ··· 144 143 const struct btf_member *member, 145 144 void *kdata, const void *udata) 146 145 { 146 + u32 moff = __btf_member_bit_offset(t, member) / 8; 147 + const struct io_uring_bpf_ops *uops = udata; 148 + struct io_uring_bpf_ops *ops = kdata; 149 + 150 + switch (moff) { 151 + case offsetof(struct io_uring_bpf_ops, ring_fd): 152 + ops->ring_fd = uops->ring_fd; 153 + return 1; 154 + } 155 + return 0; 156 + } 157 + 158 + static int io_install_bpf(struct io_ring_ctx *ctx, struct io_uring_bpf_ops *ops) 159 + { 160 + if (ctx->flags & (IORING_SETUP_SQPOLL | IORING_SETUP_IOPOLL)) 161 + return -EOPNOTSUPP; 162 + if (!(ctx->flags & IORING_SETUP_DEFER_TASKRUN)) 163 + return -EOPNOTSUPP; 164 + 165 + if (ctx->bpf_ops) 166 + return -EBUSY; 167 + if (WARN_ON_ONCE(!ops->loop_step)) 168 + return -EINVAL; 169 + 170 + ops->priv = ctx; 171 + ctx->bpf_ops = ops; 172 + ctx->loop_step = ops->loop_step; 147 173 return 0; 148 174 } 149 175 150 176 static int bpf_io_reg(void *kdata, struct bpf_link *link) 151 177 { 152 - return -EOPNOTSUPP; 178 + struct io_uring_bpf_ops *ops = kdata; 179 + struct io_ring_ctx *ctx; 180 + struct file *file; 181 + int ret = -EBUSY; 182 + 183 + file = io_uring_register_get_file(ops->ring_fd, false); 184 + if (IS_ERR(file)) 185 + return PTR_ERR(file); 186 + ctx = file->private_data; 187 + 188 + scoped_guard(mutex, &io_bpf_ctrl_mutex) { 189 + guard(mutex)(&ctx->uring_lock); 190 + ret = io_install_bpf(ctx, ops); 191 + } 192 + 193 + fput(file); 194 + return ret; 195 + } 196 + 197 + static void io_eject_bpf(struct io_ring_ctx *ctx) 198 + { 199 + struct io_uring_bpf_ops *ops = ctx->bpf_ops; 200 + 201 + if (WARN_ON_ONCE(!ops)) 202 + return; 203 + if (WARN_ON_ONCE(ops->priv != ctx)) 204 + return; 205 + 206 + ops->priv = NULL; 207 + ctx->bpf_ops = NULL; 208 + ctx->loop_step = NULL; 153 209 } 154 210 155 211 static void bpf_io_unreg(void *kdata, struct bpf_link *link) 156 212 { 213 + struct io_uring_bpf_ops *ops = kdata; 214 + struct io_ring_ctx *ctx; 215 + 216 + guard(mutex)(&io_bpf_ctrl_mutex); 217 + ctx = ops->priv; 218 + if (ctx) { 219 + guard(mutex)(&ctx->uring_lock); 220 + if (WARN_ON_ONCE(ctx->bpf_ops != ops)) 221 + return; 222 + 223 + io_eject_bpf(ctx); 224 + } 225 + } 226 + 227 + void io_unregister_bpf_ops(struct io_ring_ctx *ctx) 228 + { 229 + /* 230 + * ->bpf_ops is write protected by io_bpf_ctrl_mutex and uring_lock, 231 + * and read protected by either. Try to avoid taking the global lock 232 + * for rings that never had any bpf installed. 233 + */ 234 + scoped_guard(mutex, &ctx->uring_lock) { 235 + if (!ctx->bpf_ops) 236 + return; 237 + } 238 + 239 + guard(mutex)(&io_bpf_ctrl_mutex); 240 + guard(mutex)(&ctx->uring_lock); 241 + if (ctx->bpf_ops) 242 + io_eject_bpf(ctx); 157 243 } 158 244 159 245 static struct bpf_struct_ops bpf_ring_ops = {
+8
io_uring/bpf-ops.h
··· 17 17 void *priv; 18 18 }; 19 19 20 + #ifdef CONFIG_IO_URING_BPF_OPS 21 + void io_unregister_bpf_ops(struct io_ring_ctx *ctx); 22 + #else 23 + static inline void io_unregister_bpf_ops(struct io_ring_ctx *ctx) 24 + { 25 + } 26 + #endif 27 + 20 28 #endif /* IOU_BPF_OPS_H */
+1
io_uring/io_uring.c
··· 2148 2148 2149 2149 static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx) 2150 2150 { 2151 + io_unregister_bpf_ops(ctx); 2151 2152 io_sq_thread_finish(ctx); 2152 2153 2153 2154 mutex_lock(&ctx->uring_lock);