A Modern GPGPU API & wip linux RDNA2+ Driver
rdna
driver
linux
gpu
amdgpu: implemented sync for compute & graphics queues
+192
-29
+109
-12
drivers/amdgpu/cmds.cpp
+109
-12
drivers/amdgpu/cmds.cpp
···
1
+
#include "cp_encoder.h"
1
2
#include "gpuinfo.h"
2
3
#include "kestrel/kestrel.h"
3
4
#include "impl.h"
···
62
63
}
63
64
}
64
65
65
-
SDMAAtomicOp sdma_atomic_op_map(KesSignal sig) {
66
+
AtomicOp atomic_op_map(KesSignal sig) {
66
67
switch(sig) {
67
68
case KesSignalAtomicSet:
68
-
return SDMAAtomicOp::Swap;
69
+
return AtomicOp::Swap;
69
70
case KesSignalAtomicMax:
70
-
return SDMAAtomicOp::UMax;
71
+
return AtomicOp::UMax;
71
72
case KesSignalAtomicOr:
72
-
return SDMAAtomicOp::Or;
73
+
return AtomicOp::Or;
73
74
default:
74
75
not_implemented("sdma_atomic_op_map: no mapping for {}", sig);
75
76
}
76
77
}
77
78
78
-
SDMAWaitMemOp sdma_waitmem_op_map(KesOp op) {
79
+
WaitMemOp waitmem_op_map(KesOp op) {
79
80
switch(op) {
80
81
case KesOpEqual:
81
-
return SDMAWaitMemOp::Equal;
82
+
return WaitMemOp::Equal;
82
83
default:
83
84
not_implemented("sdma_waitmem_op_map: no mapping for {}", op);
84
85
}
···
86
87
87
88
void wait_before_transfer(CommandListImpl *impl, kes_gpuptr_t ptr, uint64_t value, KesOp op, uint64_t mask) {
88
89
assert(impl->queue->type == KesQueueTypeTransfer, "wait_before_transfer: requires queue of Transfer type");
89
-
90
90
SDMAEncoder enc(impl->queue->dev->info, impl->cs);
91
91
92
-
auto func = sdma_waitmem_op_map(op);
92
+
auto func = waitmem_op_map(op);
93
93
94
-
// @todo: NOTE: this only writes the low 32-bits of value. I do not know how we should do this, it
94
+
// @todo: NOTE: this only reads the low 32-bits of value. I do not know how we should do this, it
95
+
// seems the hardware doesn't support this.
96
+
enc.wait_mem(func, ptr, value & 0xFFFFFFFF, mask & 0xFFFFFFFF);
97
+
}
98
+
99
+
void wait_before_gcs(CommandListImpl *impl, kes_gpuptr_t ptr, uint64_t value, KesOp op, uint64_t mask) {
100
+
assert(impl->queue->type == KesQueueTypeGraphics || impl->queue->type == KesQueueTypeCompute,
101
+
"wait_before_gcs: requires Graphics or Compute queue");
102
+
CPEncoder enc(impl->queue->dev->info, impl->queue->hw_ip_type, impl->cs);
103
+
104
+
auto func = waitmem_op_map(op);
105
+
106
+
// @todo: NOTE: this only reads the low 32-bits of value. I do not know how we should do this, it
95
107
// seems the hardware doesn't support this.
108
+
96
109
enc.wait_mem(func, ptr, value & 0xFFFFFFFF, mask & 0xFFFFFFFF);
97
110
}
98
111
···
104
117
case KesQueueTypeTransfer:
105
118
wait_before_transfer(cl, ptr, value, op, mask);
106
119
break;
120
+
case KesQueueTypeCompute:
121
+
case KesQueueTypeGraphics:
122
+
wait_before_gcs(cl, ptr, value, op, mask);
123
+
break;
107
124
default:
108
125
not_implemented("wait_before: not implemented for queue type: {}", cl->queue->type);
109
126
}
110
127
}
111
128
112
-
void signal_after_transfer(CommandListImpl *impl, kes_gpuptr_t ptr, uint64_t value, KesSignal sig) {
129
+
void signal_after_transfer(CommandListImpl *impl, KesStage before, kes_gpuptr_t ptr, uint64_t value, KesSignal sig) {
113
130
assert(impl->queue->type == KesQueueTypeTransfer, "signal_after_transfer: requires queue of Transfer type");
114
131
SDMAEncoder enc(impl->queue->dev->info, impl->cs);
115
132
116
-
auto op = sdma_atomic_op_map(sig);
133
+
// @todo: how do we await before? I guess in some ways, it doesn't make sense.
134
+
// MESA does a nop, as a nop ensures that all prev sdma transfers are finished.
135
+
136
+
auto op = atomic_op_map(sig);
117
137
118
138
enc.atomic(op, ptr, value);
119
139
}
120
140
141
+
void signal_after_gcs(CommandListImpl *impl, KesStage before, kes_gpuptr_t ptr, uint64_t value, KesSignal sig) {
142
+
assert(impl->queue->type == KesQueueTypeGraphics || impl->queue->type == KesQueueTypeCompute,
143
+
"signal_after_gcs: requires Graphics or Compute queue");
144
+
145
+
CPEncoder enc(impl->queue->dev->info, impl->queue->hw_ip_type, impl->cs);
146
+
147
+
// @todo: ensure this is correct and stuff.
148
+
uint32_t event_type;
149
+
if (before == KesStagePixelShader) {
150
+
event_type = V_028A90_PS_DONE;
151
+
assert(impl->queue->type == KesQueueTypeGraphics, "signal_after_gcs: PS_DONE only valid on graphics queue");
152
+
} else if (before == KesStageCompute) {
153
+
event_type = V_028A90_CS_DONE;
154
+
} else {
155
+
event_type = V_028A90_BOTTOM_OF_PIPE_TS;
156
+
}
157
+
158
+
// signaling with atomic set is more efficient and handled as an edge case
159
+
// release mem can actually set a value atomically.
160
+
if (sig == KesSignalAtomicSet) {
161
+
enc.release_mem(
162
+
event_type,
163
+
0,
164
+
EOP_DST_SEL_MEM,
165
+
EOP_INT_SEL_SEND_DATA_AFTER_WR_CONFIRM,
166
+
EOP_DATA_SEL_VALUE_32BIT,
167
+
ptr,
168
+
(uint32_t)value
169
+
);
170
+
} else {
171
+
// first wait for the barrier event, then perform atomic op.
172
+
enc.release_mem(
173
+
event_type,
174
+
0,
175
+
EOP_DST_SEL_MEM,
176
+
EOP_INT_SEL_NONE,
177
+
EOP_DATA_SEL_DISCARD,
178
+
0,
179
+
0
180
+
);
181
+
auto op = atomic_op_map(sig);
182
+
enc.atomic_mem(
183
+
op,
184
+
ATOMIC_COMMAND_LOOP,
185
+
ptr,
186
+
value,
187
+
0 // compare_data (unused for most ops)
188
+
);
189
+
}
190
+
}
191
+
121
192
void amdgpu_cmd_signal_after(KesCommandList pcl, KesStage before, kes_gpuptr_t ptr, uint64_t value, KesSignal sig) {
122
193
auto *cl = reinterpret_cast<CommandListImpl *>(pcl);
123
194
assert(cl, "signal_after: command list handle invalid: {}", (void *)pcl);
124
195
196
+
// @todo: the signaling (atomic write) on transfer queue skips over caching. On Compute/Gfx, it works differently.
197
+
// we need to ensure that we emit the proper cache flushing before writing the value.
198
+
125
199
switch(cl->queue->type) {
126
200
case KesQueueTypeTransfer:
127
-
signal_after_transfer(cl, ptr, value, sig);
201
+
signal_after_transfer(cl, before, ptr, value, sig);
202
+
break;
203
+
case KesQueueTypeCompute:
204
+
case KesQueueTypeGraphics:
205
+
signal_after_gcs(cl, before, ptr, value, sig);
128
206
break;
129
207
default:
130
208
not_implemented("wait_before: not implemented for queue type: {}", cl->queue->type);
···
138
216
enc.write_timestamp(ptr);
139
217
}
140
218
219
+
void write_timestamp_gcs(CommandListImpl *impl, kes_gpuptr_t ptr) {
220
+
assert(impl->queue->type == KesQueueTypeGraphics || impl->queue->type == KesQueueTypeCompute,
221
+
"write_timestamp_gcs: requires Graphics or Compute queue");
222
+
CPEncoder enc(impl->queue->dev->info, impl->queue->hw_ip_type, impl->cs);
223
+
224
+
// @todo: handle TopOfPipe in a special way (see mesa radv_write_timestamp).
225
+
enc.release_mem(
226
+
V_028A90_BOTTOM_OF_PIPE_TS,
227
+
0,
228
+
EOP_DST_SEL_MEM,
229
+
EOP_INT_SEL_SEND_DATA_AFTER_WR_CONFIRM,
230
+
EOP_DATA_SEL_TIMESTAMP, ptr,
231
+
0);
232
+
}
233
+
141
234
void amdgpu_cmd_write_timestamp(KesCommandList pcl, kes_gpuptr_t ptr) {
142
235
auto *cl = reinterpret_cast<CommandListImpl *>(pcl);
143
236
assert(cl, "write_timestamp: command list handle invalid: {}", (void *)pcl);
···
145
238
switch(cl->queue->type) {
146
239
case KesQueueTypeTransfer:
147
240
write_timestamp_transfer(cl, ptr);
241
+
break;
242
+
case KesQueueTypeCompute:
243
+
case KesQueueTypeGraphics:
244
+
write_timestamp_gcs(cl, ptr);
148
245
break;
149
246
default:
150
247
not_implemented("write_timestamp: not implemented for queue type: {}", cl->queue->type);
+20
-1
drivers/amdgpu/gpuinfo.h
+20
-1
drivers/amdgpu/gpuinfo.h
···
2
2
3
3
#include <cstdint>
4
4
5
+
#include "sid.h"
5
6
#include "amdgpu_drm.h"
6
7
8
+
enum class WaitMemOp {
9
+
Equal = WAIT_REG_MEM_EQUAL,
10
+
NotEqual = WAIT_REG_MEM_NOT_EQUAL,
11
+
GreaterOrEqual = WAIT_REG_MEM_GREATER_OR_EQUAL,
12
+
};
13
+
14
+
enum class AtomicOp {
15
+
Swap = 0x67,
16
+
Add = 0x6f,
17
+
Sub = 0x70,
18
+
UMin = 0x72,
19
+
UMax = 0x74,
20
+
Or = 0x76,
21
+
};
22
+
7
23
enum class GfxLevel {
8
24
GFX9,
9
25
GFX10,
10
-
GFX10_3
26
+
GFX10_3,
27
+
GFX11,
28
+
GFX11_5,
29
+
GFX12,
11
30
};
12
31
13
32
#define SDMA_VERSION_VALUE(major, minor) (((major) << 8) | (minor))
+2
-2
drivers/amdgpu/sdma_encoder.cpp
+2
-2
drivers/amdgpu/sdma_encoder.cpp
···
49
49
#define SDMA_ATOMIC_CACHE_POLICY(x) ((uint32_t)(x) << 20)
50
50
#define SDMA_ATOMIC_CPV (1u << 24)
51
51
52
-
void SDMAEncoder::atomic(SDMAAtomicOp op, uint64_t va, uint64_t value) {
52
+
void SDMAEncoder::atomic(AtomicOp op, uint64_t va, uint64_t value) {
53
53
uint32_t cache_policy = SDMA_CACHE_POLICY(SDMA_L2_POLICY_UC, SDMA_LLC_POLICY_BYPASS);
54
54
cs.emit(SDMA_PACKET(SDMA_OPCODE_ATOMIC, 0, 0) | SDMA_ATOMIC_CPV | SDMA_ATOMIC_CACHE_POLICY(cache_policy) | SDMA_ATOMIC_OP(op));
55
55
cs.emit(va);
···
61
61
cs.emit(0);
62
62
}
63
63
64
-
void SDMAEncoder::wait_mem(SDMAWaitMemOp op, uint64_t va, uint32_t ref, uint32_t mask) {
64
+
void SDMAEncoder::wait_mem(WaitMemOp op, uint64_t va, uint32_t ref, uint32_t mask) {
65
65
uint32_t cache_policy = SDMA_CACHE_POLICY(SDMA_L2_POLICY_UC, SDMA_LLC_POLICY_BYPASS);
66
66
cs.emit(
67
67
SDMA_PACKET(SDMA_OPCODE_POLL_REGMEM, 0, 0) | (uint32_t)op << 28 | SDMA_POLL_MEM
+3
-14
drivers/amdgpu/sdma_encoder.h
+3
-14
drivers/amdgpu/sdma_encoder.h
···
5
5
#include <vector>
6
6
#include <span>
7
7
8
-
enum class SDMAAtomicOp {
9
-
Swap = 0x67,
10
-
Add = 0x6f,
11
-
Sub = 0x70,
12
-
UMin = 0x72,
13
-
UMax = 0x74,
14
-
Or = 0x76,
15
-
};
16
-
17
-
enum class SDMAWaitMemOp {
18
-
Equal = 0x3
19
-
};
8
+
#include "sid.h"
20
9
21
10
class SDMAEncoder {
22
11
public:
···
25
14
void write_timestamp(uint64_t va);
26
15
void semaphore(uint64_t va);
27
16
void fence(uint64_t va, uint32_t fence);
28
-
void atomic(SDMAAtomicOp op, uint64_t va, uint64_t value);
29
-
void wait_mem(SDMAWaitMemOp op, uint64_t va, uint32_t ref, uint32_t mask);
17
+
void atomic(AtomicOp op, uint64_t va, uint64_t value);
18
+
void wait_mem(WaitMemOp op, uint64_t va, uint32_t ref, uint32_t mask);
30
19
31
20
// returns the number of bytes written; may need to be repeated.
32
21
uint64_t constant_fill(uint64_t va, uint64_t size, uint32_t value);
+58
test/test/06_hello_sync/hello_sync.cpp
+58
test/test/06_hello_sync/hello_sync.cpp
···
1
+
#include <unistd.h>
2
+
#include <kestrel/kestrel.h>
3
+
4
+
#include <stdio.h>
5
+
6
+
int main(void) {
7
+
8
+
auto dev = kes_create();
9
+
10
+
std::size_t size = 10 * 1024 * 1024;
11
+
auto x = kes_malloc(dev, size, 4, KesMemoryDefault);
12
+
auto y = kes_malloc(dev, 8, 4, KesMemoryDefault);
13
+
auto ts = kes_malloc(dev, 8 * 4, 4, KesMemoryDefault);
14
+
15
+
printf("x: %p (%p) (%llu bytes)\n", x.cpu, x.gpu, x.size);
16
+
printf("y: %p (%p) (%llu bytes)\n", y.cpu, y.gpu, y.size);
17
+
18
+
auto dma = kes_create_queue(dev, KesQueueTypeTransfer);
19
+
auto compute = kes_create_queue(dev, KesQueueTypeCompute);
20
+
21
+
auto l1 = kes_start_recording(dma);
22
+
{
23
+
kes_cmd_write_timestamp(l1, ts.gpu + 0);
24
+
kes_cmd_memset(l1, x.gpu, size, 1);
25
+
kes_cmd_write_timestamp(l1, ts.gpu + 8);
26
+
kes_cmd_signal_after(l1, KesStageTransfer, y.gpu, 1337, KesSignalAtomicMax);
27
+
}
28
+
29
+
auto l2 = kes_start_recording(compute);
30
+
{
31
+
kes_cmd_write_timestamp(l2, ts.gpu + 16);
32
+
kes_cmd_wait_before(l2, KesStageTransfer, y.gpu, 1337, KesOpEqual, KesHazardFlagsNone, ~0);
33
+
kes_cmd_write_timestamp(l2, ts.gpu + 24);
34
+
}
35
+
36
+
kes_submit(dma, l1);
37
+
kes_submit(compute, l2);
38
+
39
+
// @todo: how to wait on cpu for DMA transfer? TODO?
40
+
printf("x[0]: %u\n", ((uint32_t *)x.cpu)[0]);
41
+
sleep(1);
42
+
printf("x[0]: %u\n", ((uint32_t *)x.cpu)[0]);
43
+
sleep(1);
44
+
printf("x[0]: %u\n", ((uint32_t *)x.cpu)[0]);
45
+
sleep(1);
46
+
printf("x[0]: %u\n", ((uint32_t *)x.cpu)[0]);
47
+
48
+
printf("\n");
49
+
printf("ts0: %lu\n", ((uint64_t *)ts.cpu)[0]);
50
+
printf("ts1: %lu\n", ((uint64_t *)ts.cpu)[1]);
51
+
printf("ts2: %lu\n", ((uint64_t *)ts.cpu)[2]);
52
+
printf("ts3: %lu\n", ((uint64_t *)ts.cpu)[3]);
53
+
54
+
kes_free(dev, &x);
55
+
kes_destroy(dev);
56
+
57
+
return 0;
58
+
}