Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
dma-buf: inline spinlock for fence protection v5
Implement per-fence spinlocks, allowing implementations to not give an
external spinlock to protect the fence internal state. Instead a spinlock
embedded into the fence structure itself is used in this case.
Shared spinlocks have the problem that implementations need to guarantee
that the lock lives at least as long all fences referencing them.
Using a per-fence spinlock allows completely decoupling spinlock producer
and consumer life times, simplifying the handling in most use cases.
v2: improve naming, coverage and function documentation
v3: fix one additional locking in the selftests
v4: separate out some changes to make the patch smaller,
fix one amdgpu crash found by CI systems
v5: improve comments
Signed-off-by: Christian König <christian.koenig@amd.com>
Reviewed-by: Boris Brezillon <boris.brezillon@collabora.com>
Link: https://lore.kernel.org/r/20260219160822.1529-5-christian.koenig@amd.com
+40
-18
+16
-5
drivers/dma-buf/dma-fence.c
+16
-5
drivers/dma-buf/dma-fence.c
···
343
343
}
344
344
#endif
345
345
346
-
347
346
/**
348
347
* dma_fence_signal_timestamp_locked - signal completion of a fence
349
348
* @fence: the fence to signal
···
1069
1070
__dma_fence_init(struct dma_fence *fence, const struct dma_fence_ops *ops,
1070
1071
spinlock_t *lock, u64 context, u64 seqno, unsigned long flags)
1071
1072
{
1072
-
BUG_ON(!lock);
1073
1073
BUG_ON(!ops || !ops->get_driver_name || !ops->get_timeline_name);
1074
1074
1075
1075
kref_init(&fence->refcount);
···
1080
1082
*/
1081
1083
RCU_INIT_POINTER(fence->ops, ops);
1082
1084
INIT_LIST_HEAD(&fence->cb_list);
1083
-
fence->lock = lock;
1084
1085
fence->context = context;
1085
1086
fence->seqno = seqno;
1086
1087
fence->flags = flags | BIT(DMA_FENCE_FLAG_INITIALIZED_BIT);
1088
+
if (lock) {
1089
+
fence->extern_lock = lock;
1090
+
} else {
1091
+
spin_lock_init(&fence->inline_lock);
1092
+
fence->flags |= BIT(DMA_FENCE_FLAG_INLINE_LOCK_BIT);
1093
+
}
1087
1094
fence->error = 0;
1088
1095
1089
1096
trace_dma_fence_init(fence);
···
1098
1095
* dma_fence_init - Initialize a custom fence.
1099
1096
* @fence: the fence to initialize
1100
1097
* @ops: the dma_fence_ops for operations on this fence
1101
-
* @lock: the irqsafe spinlock to use for locking this fence
1098
+
* @lock: optional irqsafe spinlock to use for locking this fence
1102
1099
* @context: the execution context this fence is run on
1103
1100
* @seqno: a linear increasing sequence number for this context
1104
1101
*
···
1108
1105
*
1109
1106
* context and seqno are used for easy comparison between fences, allowing
1110
1107
* to check which fence is later by simply using dma_fence_later().
1108
+
*
1109
+
* It is strongly discouraged to provide an external lock because this couples
1110
+
* lock and fence life time. This is only allowed for legacy use cases when
1111
+
* multiple fences need to be prevented from signaling out of order.
1111
1112
*/
1112
1113
void
1113
1114
dma_fence_init(struct dma_fence *fence, const struct dma_fence_ops *ops,
···
1125
1118
* dma_fence_init64 - Initialize a custom fence with 64-bit seqno support.
1126
1119
* @fence: the fence to initialize
1127
1120
* @ops: the dma_fence_ops for operations on this fence
1128
-
* @lock: the irqsafe spinlock to use for locking this fence
1121
+
* @lock: optional irqsafe spinlock to use for locking this fence
1129
1122
* @context: the execution context this fence is run on
1130
1123
* @seqno: a linear increasing sequence number for this context
1131
1124
*
···
1135
1128
*
1136
1129
* Context and seqno are used for easy comparison between fences, allowing
1137
1130
* to check which fence is later by simply using dma_fence_later().
1131
+
*
1132
+
* It is strongly discouraged to provide an external lock because this couples
1133
+
* lock and fence life time. This is only allowed for legacy use cases when
1134
+
* multiple fences need to be prevented from signaling out of order.
1138
1135
*/
1139
1136
void
1140
1137
dma_fence_init64(struct dma_fence *fence, const struct dma_fence_ops *ops,
+1
-1
drivers/dma-buf/sync_debug.h
+1
-1
drivers/dma-buf/sync_debug.h
+1
-1
drivers/gpu/drm/drm_crtc.c
+1
-1
drivers/gpu/drm/drm_crtc.c
···
159
159
static struct drm_crtc *fence_to_crtc(struct dma_fence *fence)
160
160
{
161
161
BUG_ON(rcu_access_pointer(fence->ops) != &drm_crtc_fence_ops);
162
-
return container_of(fence->lock, struct drm_crtc, fence_lock);
162
+
return container_of(fence->extern_lock, struct drm_crtc, fence_lock);
163
163
}
164
164
165
165
static const char *drm_crtc_fence_get_driver_name(struct dma_fence *fence)
+1
-1
drivers/gpu/drm/drm_writeback.c
+1
-1
drivers/gpu/drm/drm_writeback.c
+2
-1
drivers/gpu/drm/nouveau/nouveau_fence.c
+2
-1
drivers/gpu/drm/nouveau/nouveau_fence.c
···
41
41
static inline struct nouveau_fence_chan *
42
42
nouveau_fctx(struct nouveau_fence *fence)
43
43
{
44
-
return container_of(fence->base.lock, struct nouveau_fence_chan, lock);
44
+
return container_of(fence->base.extern_lock, struct nouveau_fence_chan,
45
+
lock);
45
46
}
46
47
47
48
static bool
+2
-1
drivers/gpu/drm/qxl/qxl_release.c
+2
-1
drivers/gpu/drm/qxl/qxl_release.c
···
62
62
struct qxl_device *qdev;
63
63
unsigned long cur, end = jiffies + timeout;
64
64
65
-
qdev = container_of(fence->lock, struct qxl_device, release_lock);
65
+
qdev = container_of(fence->extern_lock, struct qxl_device,
66
+
release_lock);
66
67
67
68
if (!wait_event_timeout(qdev->release_event,
68
69
(dma_fence_is_signaled(fence) ||
+2
-1
drivers/gpu/drm/vmwgfx/vmwgfx_fence.c
+2
-1
drivers/gpu/drm/vmwgfx/vmwgfx_fence.c
···
47
47
static struct vmw_fence_manager *
48
48
fman_from_fence(struct vmw_fence_obj *fence)
49
49
{
50
-
return container_of(fence->base.lock, struct vmw_fence_manager, lock);
50
+
return container_of(fence->base.extern_lock, struct vmw_fence_manager,
51
+
lock);
51
52
}
52
53
53
54
static void vmw_fence_obj_destroy(struct dma_fence *f)
+2
-1
drivers/gpu/drm/xe/xe_hw_fence.c
+2
-1
drivers/gpu/drm/xe/xe_hw_fence.c
···
124
124
125
125
static struct xe_hw_fence_irq *xe_hw_fence_irq(struct xe_hw_fence *fence)
126
126
{
127
-
return container_of(fence->dma.lock, struct xe_hw_fence_irq, lock);
127
+
return container_of(fence->dma.extern_lock, struct xe_hw_fence_irq,
128
+
lock);
128
129
}
129
130
130
131
static const char *xe_hw_fence_get_driver_name(struct dma_fence *dma_fence)
+13
-6
include/linux/dma-fence.h
+13
-6
include/linux/dma-fence.h
···
34
34
* @ops: dma_fence_ops associated with this fence
35
35
* @rcu: used for releasing fence with kfree_rcu
36
36
* @cb_list: list of all callbacks to call
37
-
* @lock: spin_lock_irqsave used for locking
37
+
* @extern_lock: external spin_lock_irqsave used for locking (deprecated)
38
+
* @inline_lock: alternative internal spin_lock_irqsave used for locking
38
39
* @context: execution context this fence belongs to, returned by
39
40
* dma_fence_context_alloc()
40
41
* @seqno: the sequence number of this fence inside the execution context,
···
50
49
* of the time.
51
50
*
52
51
* DMA_FENCE_FLAG_INITIALIZED_BIT - fence was initialized
52
+
* DMA_FENCE_FLAG_INLINE_LOCK_BIT - use inline spinlock instead of external one
53
53
* DMA_FENCE_FLAG_SIGNALED_BIT - fence is already signaled
54
54
* DMA_FENCE_FLAG_TIMESTAMP_BIT - timestamp recorded for fence signaling
55
55
* DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT - enable_signaling might have been called
···
68
66
* been completed, or never called at all.
69
67
*/
70
68
struct dma_fence {
71
-
spinlock_t *lock;
69
+
union {
70
+
spinlock_t *extern_lock;
71
+
spinlock_t inline_lock;
72
+
};
72
73
const struct dma_fence_ops __rcu *ops;
73
74
/*
74
75
* We clear the callback list on kref_put so that by the time we
···
105
100
106
101
enum dma_fence_flag_bits {
107
102
DMA_FENCE_FLAG_INITIALIZED_BIT,
103
+
DMA_FENCE_FLAG_INLINE_LOCK_BIT,
108
104
DMA_FENCE_FLAG_SEQNO64_BIT,
109
105
DMA_FENCE_FLAG_SIGNALED_BIT,
110
106
DMA_FENCE_FLAG_TIMESTAMP_BIT,
···
387
381
* dma_fence_spinlock - return pointer to the spinlock protecting the fence
388
382
* @fence: the fence to get the lock from
389
383
*
390
-
* Return the pointer to the extern lock.
384
+
* Return either the pointer to the embedded or the external spin lock.
391
385
*/
392
386
static inline spinlock_t *dma_fence_spinlock(struct dma_fence *fence)
393
387
{
394
-
return fence->lock;
388
+
return test_bit(DMA_FENCE_FLAG_INLINE_LOCK_BIT, &fence->flags) ?
389
+
&fence->inline_lock : fence->extern_lock;
395
390
}
396
391
397
392
/**
···
403
396
* Lock the fence, preventing it from changing to the signaled state.
404
397
*/
405
398
#define dma_fence_lock_irqsave(fence, flags) \
406
-
spin_lock_irqsave(fence->lock, flags)
399
+
spin_lock_irqsave(dma_fence_spinlock(fence), flags)
407
400
408
401
/**
409
402
* dma_fence_unlock_irqrestore - unlock the fence and irqrestore
···
413
406
* Unlock the fence, allowing it to change it's state to signaled again.
414
407
*/
415
408
#define dma_fence_unlock_irqrestore(fence, flags) \
416
-
spin_unlock_irqrestore(fence->lock, flags)
409
+
spin_unlock_irqrestore(dma_fence_spinlock(fence), flags)
417
410
418
411
/**
419
412
* dma_fence_assert_held - lockdep assertion that fence is locked