Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2023-2024, Advanced Micro Devices, Inc.
4 */
5
6#include <drm/amdxdna_accel.h>
7#include <drm/drm_cache.h>
8#include <drm/drm_device.h>
9#include <drm/drm_gem.h>
10#include <drm/drm_gem_shmem_helper.h>
11#include <drm/drm_print.h>
12#include <drm/gpu_scheduler.h>
13#include <linux/bitfield.h>
14#include <linux/errno.h>
15#include <linux/pci.h>
16#include <linux/types.h>
17#include <linux/xarray.h>
18
19#include "aie2_msg_priv.h"
20#include "aie2_pci.h"
21#include "amdxdna_ctx.h"
22#include "amdxdna_gem.h"
23#include "amdxdna_mailbox.h"
24#include "amdxdna_mailbox_helper.h"
25#include "amdxdna_pci_drv.h"
26
27#define DECLARE_AIE2_MSG(name, op) \
28 DECLARE_XDNA_MSG_COMMON(name, op, MAX_AIE2_STATUS_CODE)
29
30#define EXEC_MSG_OPS(xdna) ((xdna)->dev_handle->exec_msg_ops)
31
32static int aie2_send_mgmt_msg_wait(struct amdxdna_dev_hdl *ndev,
33 struct xdna_mailbox_msg *msg)
34{
35 struct amdxdna_dev *xdna = ndev->xdna;
36 struct xdna_notify *hdl = msg->handle;
37 int ret;
38
39 if (!ndev->mgmt_chann)
40 return -ENODEV;
41
42 ret = xdna_send_msg_wait(xdna, ndev->mgmt_chann, msg);
43 if (ret == -ETIME)
44 aie2_destroy_mgmt_chann(ndev);
45
46 if (!ret && *hdl->status != AIE2_STATUS_SUCCESS) {
47 XDNA_ERR(xdna, "command opcode 0x%x failed, status 0x%x",
48 msg->opcode, *hdl->data);
49 ret = -EINVAL;
50 }
51
52 return ret;
53}
54
55void *aie2_alloc_msg_buffer(struct amdxdna_dev_hdl *ndev, u32 *size,
56 dma_addr_t *dma_addr)
57{
58 struct amdxdna_dev *xdna = ndev->xdna;
59 void *vaddr;
60 int order;
61
62 *size = max(*size, SZ_8K);
63 order = get_order(*size);
64 if (order > MAX_PAGE_ORDER)
65 return ERR_PTR(-EINVAL);
66 *size = PAGE_SIZE << order;
67
68 if (amdxdna_iova_on(xdna))
69 return amdxdna_iommu_alloc(xdna, *size, dma_addr);
70
71 vaddr = dma_alloc_noncoherent(xdna->ddev.dev, *size, dma_addr,
72 DMA_FROM_DEVICE, GFP_KERNEL);
73 if (!vaddr)
74 return ERR_PTR(-ENOMEM);
75
76 return vaddr;
77}
78
79void aie2_free_msg_buffer(struct amdxdna_dev_hdl *ndev, size_t size,
80 void *cpu_addr, dma_addr_t dma_addr)
81{
82 struct amdxdna_dev *xdna = ndev->xdna;
83
84 if (amdxdna_iova_on(xdna)) {
85 amdxdna_iommu_free(xdna, size, cpu_addr, dma_addr);
86 return;
87 }
88
89 dma_free_noncoherent(xdna->ddev.dev, size, cpu_addr, dma_addr, DMA_FROM_DEVICE);
90}
91
92int aie2_suspend_fw(struct amdxdna_dev_hdl *ndev)
93{
94 DECLARE_AIE2_MSG(suspend, MSG_OP_SUSPEND);
95 int ret;
96
97 ret = aie2_send_mgmt_msg_wait(ndev, &msg);
98 if (ret) {
99 XDNA_ERR(ndev->xdna, "Failed to suspend fw, ret %d", ret);
100 return ret;
101 }
102
103 return aie2_psp_waitmode_poll(ndev->psp_hdl);
104}
105
106int aie2_resume_fw(struct amdxdna_dev_hdl *ndev)
107{
108 DECLARE_AIE2_MSG(suspend, MSG_OP_RESUME);
109
110 return aie2_send_mgmt_msg_wait(ndev, &msg);
111}
112
113int aie2_set_runtime_cfg(struct amdxdna_dev_hdl *ndev, u32 type, u64 value)
114{
115 DECLARE_AIE2_MSG(set_runtime_cfg, MSG_OP_SET_RUNTIME_CONFIG);
116 int ret;
117
118 req.type = type;
119 req.value = value;
120
121 ret = aie2_send_mgmt_msg_wait(ndev, &msg);
122 if (ret) {
123 XDNA_ERR(ndev->xdna, "Failed to set runtime config, ret %d", ret);
124 return ret;
125 }
126
127 return 0;
128}
129
130int aie2_get_runtime_cfg(struct amdxdna_dev_hdl *ndev, u32 type, u64 *value)
131{
132 DECLARE_AIE2_MSG(get_runtime_cfg, MSG_OP_GET_RUNTIME_CONFIG);
133 int ret;
134
135 req.type = type;
136 ret = aie2_send_mgmt_msg_wait(ndev, &msg);
137 if (ret) {
138 XDNA_ERR(ndev->xdna, "Failed to get runtime config, ret %d", ret);
139 return ret;
140 }
141
142 *value = resp.value;
143 return 0;
144}
145
146int aie2_assign_mgmt_pasid(struct amdxdna_dev_hdl *ndev, u16 pasid)
147{
148 DECLARE_AIE2_MSG(assign_mgmt_pasid, MSG_OP_ASSIGN_MGMT_PASID);
149
150 req.pasid = pasid;
151
152 return aie2_send_mgmt_msg_wait(ndev, &msg);
153}
154
155int aie2_query_aie_version(struct amdxdna_dev_hdl *ndev, struct aie_version *version)
156{
157 DECLARE_AIE2_MSG(aie_version_info, MSG_OP_QUERY_AIE_VERSION);
158 struct amdxdna_dev *xdna = ndev->xdna;
159 int ret;
160
161 ret = aie2_send_mgmt_msg_wait(ndev, &msg);
162 if (ret)
163 return ret;
164
165 XDNA_DBG(xdna, "Query AIE version - major: %u minor: %u completed",
166 resp.major, resp.minor);
167
168 version->major = resp.major;
169 version->minor = resp.minor;
170
171 return 0;
172}
173
174int aie2_query_aie_metadata(struct amdxdna_dev_hdl *ndev, struct aie_metadata *metadata)
175{
176 DECLARE_AIE2_MSG(aie_tile_info, MSG_OP_QUERY_AIE_TILE_INFO);
177 int ret;
178
179 ret = aie2_send_mgmt_msg_wait(ndev, &msg);
180 if (ret)
181 return ret;
182
183 metadata->size = resp.info.size;
184 metadata->cols = resp.info.cols;
185 metadata->rows = resp.info.rows;
186
187 metadata->version.major = resp.info.major;
188 metadata->version.minor = resp.info.minor;
189
190 metadata->core.row_count = resp.info.core_rows;
191 metadata->core.row_start = resp.info.core_row_start;
192 metadata->core.dma_channel_count = resp.info.core_dma_channels;
193 metadata->core.lock_count = resp.info.core_locks;
194 metadata->core.event_reg_count = resp.info.core_events;
195
196 metadata->mem.row_count = resp.info.mem_rows;
197 metadata->mem.row_start = resp.info.mem_row_start;
198 metadata->mem.dma_channel_count = resp.info.mem_dma_channels;
199 metadata->mem.lock_count = resp.info.mem_locks;
200 metadata->mem.event_reg_count = resp.info.mem_events;
201
202 metadata->shim.row_count = resp.info.shim_rows;
203 metadata->shim.row_start = resp.info.shim_row_start;
204 metadata->shim.dma_channel_count = resp.info.shim_dma_channels;
205 metadata->shim.lock_count = resp.info.shim_locks;
206 metadata->shim.event_reg_count = resp.info.shim_events;
207
208 return 0;
209}
210
211int aie2_query_firmware_version(struct amdxdna_dev_hdl *ndev,
212 struct amdxdna_fw_ver *fw_ver)
213{
214 DECLARE_AIE2_MSG(firmware_version, MSG_OP_GET_FIRMWARE_VERSION);
215 int ret;
216
217 ret = aie2_send_mgmt_msg_wait(ndev, &msg);
218 if (ret)
219 return ret;
220
221 fw_ver->major = resp.major;
222 fw_ver->minor = resp.minor;
223 fw_ver->sub = resp.sub;
224 fw_ver->build = resp.build;
225
226 return 0;
227}
228
229static int aie2_destroy_context_req(struct amdxdna_dev_hdl *ndev, u32 id)
230{
231 DECLARE_AIE2_MSG(destroy_ctx, MSG_OP_DESTROY_CONTEXT);
232 struct amdxdna_dev *xdna = ndev->xdna;
233 int ret;
234
235 req.context_id = id;
236 ret = aie2_send_mgmt_msg_wait(ndev, &msg);
237 if (ret && ret != -ENODEV)
238 XDNA_WARN(xdna, "Destroy context failed, ret %d", ret);
239 else if (ret == -ENODEV)
240 XDNA_DBG(xdna, "Destroy context: device already stopped");
241
242 return ret;
243}
244
245static u32 aie2_get_context_priority(struct amdxdna_dev_hdl *ndev,
246 struct amdxdna_hwctx *hwctx)
247{
248 if (!AIE2_FEATURE_ON(ndev, AIE2_PREEMPT))
249 return PRIORITY_HIGH;
250
251 switch (hwctx->qos.priority) {
252 case AMDXDNA_QOS_REALTIME_PRIORITY:
253 return PRIORITY_REALTIME;
254 case AMDXDNA_QOS_HIGH_PRIORITY:
255 return PRIORITY_HIGH;
256 case AMDXDNA_QOS_NORMAL_PRIORITY:
257 return PRIORITY_NORMAL;
258 case AMDXDNA_QOS_LOW_PRIORITY:
259 return PRIORITY_LOW;
260 default:
261 return PRIORITY_HIGH;
262 }
263}
264
265int aie2_create_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx)
266{
267 DECLARE_AIE2_MSG(create_ctx, MSG_OP_CREATE_CONTEXT);
268 struct amdxdna_dev *xdna = ndev->xdna;
269 struct xdna_mailbox_chann_res x2i;
270 struct xdna_mailbox_chann_res i2x;
271 struct cq_pair *cq_pair;
272 u32 intr_reg;
273 int ret;
274
275 req.aie_type = 1;
276 req.start_col = hwctx->start_col;
277 req.num_col = hwctx->num_col;
278 req.num_unused_col = hwctx->num_unused_col;
279 req.num_cq_pairs_requested = 1;
280 req.pasid = amdxdna_pasid_on(hwctx->client) ? hwctx->client->pasid : 0;
281 req.context_priority = aie2_get_context_priority(ndev, hwctx);
282
283 ret = aie2_send_mgmt_msg_wait(ndev, &msg);
284 if (ret)
285 return ret;
286
287 hwctx->fw_ctx_id = resp.context_id;
288 if (WARN_ON_ONCE(hwctx->fw_ctx_id == -1))
289 return -EINVAL;
290
291 if (ndev->force_preempt_enabled) {
292 ret = aie2_runtime_cfg(ndev, AIE2_RT_CFG_FORCE_PREEMPT, &hwctx->fw_ctx_id);
293 if (ret) {
294 XDNA_ERR(xdna, "failed to enable force preempt %d", ret);
295 goto del_ctx_req;
296 }
297 }
298
299 cq_pair = &resp.cq_pair[0];
300 x2i.mb_head_ptr_reg = AIE2_MBOX_OFF(ndev, cq_pair->x2i_q.head_addr);
301 x2i.mb_tail_ptr_reg = AIE2_MBOX_OFF(ndev, cq_pair->x2i_q.tail_addr);
302 x2i.rb_start_addr = AIE2_SRAM_OFF(ndev, cq_pair->x2i_q.buf_addr);
303 x2i.rb_size = cq_pair->x2i_q.buf_size;
304
305 i2x.mb_head_ptr_reg = AIE2_MBOX_OFF(ndev, cq_pair->i2x_q.head_addr);
306 i2x.mb_tail_ptr_reg = AIE2_MBOX_OFF(ndev, cq_pair->i2x_q.tail_addr);
307 i2x.rb_start_addr = AIE2_SRAM_OFF(ndev, cq_pair->i2x_q.buf_addr);
308 i2x.rb_size = cq_pair->i2x_q.buf_size;
309
310 ret = pci_irq_vector(to_pci_dev(xdna->ddev.dev), resp.msix_id);
311 if (ret == -EINVAL) {
312 XDNA_ERR(xdna, "Alloc IRQ failed %d", ret);
313 goto del_ctx_req;
314 }
315
316 intr_reg = i2x.mb_head_ptr_reg + 4;
317 hwctx->priv->mbox_chann = xdna_mailbox_alloc_channel(ndev->mbox);
318 if (!hwctx->priv->mbox_chann) {
319 XDNA_ERR(xdna, "Not able to create channel");
320 ret = -EINVAL;
321 goto del_ctx_req;
322 }
323
324 ret = xdna_mailbox_start_channel(hwctx->priv->mbox_chann, &x2i, &i2x,
325 intr_reg, ret);
326 if (ret) {
327 XDNA_ERR(xdna, "Not able to create channel");
328 ret = -EINVAL;
329 goto free_channel;
330 }
331 ndev->hwctx_num++;
332
333 XDNA_DBG(xdna, "Mailbox channel irq: %d, msix_id: %d", ret, resp.msix_id);
334 XDNA_DBG(xdna, "Created fw ctx %d pasid %d", hwctx->fw_ctx_id, hwctx->client->pasid);
335
336 return 0;
337
338free_channel:
339 xdna_mailbox_free_channel(hwctx->priv->mbox_chann);
340del_ctx_req:
341 aie2_destroy_context_req(ndev, hwctx->fw_ctx_id);
342 return ret;
343}
344
345int aie2_destroy_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx)
346{
347 struct amdxdna_dev *xdna = ndev->xdna;
348 int ret;
349
350 if (!hwctx->priv->mbox_chann)
351 return 0;
352
353 xdna_mailbox_stop_channel(hwctx->priv->mbox_chann);
354 ret = aie2_destroy_context_req(ndev, hwctx->fw_ctx_id);
355 xdna_mailbox_free_channel(hwctx->priv->mbox_chann);
356 XDNA_DBG(xdna, "Destroyed fw ctx %d", hwctx->fw_ctx_id);
357 hwctx->priv->mbox_chann = NULL;
358 hwctx->fw_ctx_id = -1;
359 ndev->hwctx_num--;
360
361 return ret;
362}
363
364int aie2_map_host_buf(struct amdxdna_dev_hdl *ndev, u32 context_id, u64 addr, u64 size)
365{
366 DECLARE_AIE2_MSG(map_host_buffer, MSG_OP_MAP_HOST_BUFFER);
367 struct amdxdna_dev *xdna = ndev->xdna;
368 int ret;
369
370 req.context_id = context_id;
371 req.buf_addr = addr;
372 req.buf_size = size;
373 ret = aie2_send_mgmt_msg_wait(ndev, &msg);
374 if (ret)
375 return ret;
376
377 XDNA_DBG(xdna, "fw ctx %d map host buf addr 0x%llx size 0x%llx",
378 context_id, addr, size);
379
380 return 0;
381}
382
383static int amdxdna_hwctx_col_map(struct amdxdna_hwctx *hwctx, void *arg)
384{
385 u32 *bitmap = arg;
386
387 *bitmap |= GENMASK(hwctx->start_col + hwctx->num_col - 1, hwctx->start_col);
388
389 return 0;
390}
391
392int aie2_query_status(struct amdxdna_dev_hdl *ndev, char __user *buf,
393 u32 size, u32 *cols_filled)
394{
395 DECLARE_AIE2_MSG(aie_column_info, MSG_OP_QUERY_COL_STATUS);
396 struct amdxdna_dev *xdna = ndev->xdna;
397 u32 buf_sz = size, aie_bitmap = 0;
398 struct amdxdna_client *client;
399 dma_addr_t dma_addr;
400 u8 *buff_addr;
401 int ret;
402
403 buff_addr = aie2_alloc_msg_buffer(ndev, &buf_sz, &dma_addr);
404 if (IS_ERR(buff_addr))
405 return PTR_ERR(buff_addr);
406
407 /* Go through each hardware context and mark the AIE columns that are active */
408 list_for_each_entry(client, &xdna->client_list, node)
409 amdxdna_hwctx_walk(client, &aie_bitmap, amdxdna_hwctx_col_map);
410
411 *cols_filled = 0;
412 req.dump_buff_addr = dma_addr;
413 req.dump_buff_size = buf_sz;
414 req.num_cols = hweight32(aie_bitmap);
415 req.aie_bitmap = aie_bitmap;
416
417 drm_clflush_virt_range(buff_addr, size); /* device can access */
418 ret = aie2_send_mgmt_msg_wait(ndev, &msg);
419 if (ret) {
420 XDNA_ERR(xdna, "Error during NPU query, status %d", ret);
421 goto fail;
422 }
423
424 XDNA_DBG(xdna, "Query NPU status completed");
425
426 if (size < resp.size) {
427 ret = -EINVAL;
428 XDNA_ERR(xdna, "Bad buffer size. Available: %u. Needs: %u", size, resp.size);
429 goto fail;
430 }
431
432 if (copy_to_user(buf, buff_addr, resp.size)) {
433 ret = -EFAULT;
434 XDNA_ERR(xdna, "Failed to copy NPU status to user space");
435 goto fail;
436 }
437
438 *cols_filled = aie_bitmap;
439
440fail:
441 aie2_free_msg_buffer(ndev, buf_sz, buff_addr, dma_addr);
442 return ret;
443}
444
445int aie2_query_telemetry(struct amdxdna_dev_hdl *ndev,
446 char __user *buf, u32 size,
447 struct amdxdna_drm_query_telemetry_header *header)
448{
449 DECLARE_AIE2_MSG(get_telemetry, MSG_OP_GET_TELEMETRY);
450 struct amdxdna_dev *xdna = ndev->xdna;
451 dma_addr_t dma_addr;
452 u32 buf_sz = size;
453 u8 *addr;
454 int ret;
455
456 if (header->type >= MAX_TELEMETRY_TYPE)
457 return -EINVAL;
458
459 addr = aie2_alloc_msg_buffer(ndev, &buf_sz, &dma_addr);
460 if (IS_ERR(addr))
461 return PTR_ERR(addr);
462
463 req.buf_addr = dma_addr;
464 req.buf_size = buf_sz;
465 req.type = header->type;
466
467 drm_clflush_virt_range(addr, size); /* device can access */
468 ret = aie2_send_mgmt_msg_wait(ndev, &msg);
469 if (ret) {
470 XDNA_ERR(xdna, "Query telemetry failed, status %d", ret);
471 goto free_buf;
472 }
473
474 if (size < resp.size) {
475 ret = -EINVAL;
476 XDNA_ERR(xdna, "Bad buffer size. Available: %u. Needs: %u", size, resp.size);
477 goto free_buf;
478 }
479
480 if (copy_to_user(buf, addr, resp.size)) {
481 ret = -EFAULT;
482 XDNA_ERR(xdna, "Failed to copy telemetry to user space");
483 goto free_buf;
484 }
485
486 header->major = resp.major;
487 header->minor = resp.minor;
488
489free_buf:
490 aie2_free_msg_buffer(ndev, buf_sz, addr, dma_addr);
491 return ret;
492}
493
494int aie2_register_asyn_event_msg(struct amdxdna_dev_hdl *ndev, dma_addr_t addr, u32 size,
495 void *handle, int (*cb)(void*, void __iomem *, size_t))
496{
497 struct async_event_msg_req req = { 0 };
498 struct xdna_mailbox_msg msg = {
499 .send_data = (u8 *)&req,
500 .send_size = sizeof(req),
501 .handle = handle,
502 .opcode = MSG_OP_REGISTER_ASYNC_EVENT_MSG,
503 .notify_cb = cb,
504 };
505
506 req.buf_addr = addr;
507 req.buf_size = size;
508
509 XDNA_DBG(ndev->xdna, "Register addr 0x%llx size 0x%x", addr, size);
510 return xdna_mailbox_send_msg(ndev->mgmt_chann, &msg, TX_TIMEOUT);
511}
512
513int aie2_config_cu(struct amdxdna_hwctx *hwctx,
514 int (*notify_cb)(void *, void __iomem *, size_t))
515{
516 struct mailbox_channel *chann = hwctx->priv->mbox_chann;
517 struct amdxdna_dev *xdna = hwctx->client->xdna;
518 u32 shift = xdna->dev_info->dev_mem_buf_shift;
519 struct config_cu_req req = { 0 };
520 struct xdna_mailbox_msg msg;
521 struct drm_gem_object *gobj;
522 struct amdxdna_gem_obj *abo;
523 int i;
524
525 if (!chann)
526 return -ENODEV;
527
528 if (!hwctx->cus)
529 return 0;
530
531 if (hwctx->cus->num_cus > MAX_NUM_CUS) {
532 XDNA_DBG(xdna, "Exceed maximum CU %d", MAX_NUM_CUS);
533 return -EINVAL;
534 }
535
536 for (i = 0; i < hwctx->cus->num_cus; i++) {
537 struct amdxdna_cu_config *cu = &hwctx->cus->cu_configs[i];
538
539 if (XDNA_MBZ_DBG(xdna, cu->pad, sizeof(cu->pad)))
540 return -EINVAL;
541
542 gobj = drm_gem_object_lookup(hwctx->client->filp, cu->cu_bo);
543 if (!gobj) {
544 XDNA_ERR(xdna, "Lookup GEM object failed");
545 return -EINVAL;
546 }
547 abo = to_xdna_obj(gobj);
548
549 if (abo->type != AMDXDNA_BO_DEV) {
550 drm_gem_object_put(gobj);
551 XDNA_ERR(xdna, "Invalid BO type");
552 return -EINVAL;
553 }
554
555 req.cfgs[i] = FIELD_PREP(AIE2_MSG_CFG_CU_PDI_ADDR,
556 amdxdna_gem_dev_addr(abo) >> shift);
557 req.cfgs[i] |= FIELD_PREP(AIE2_MSG_CFG_CU_FUNC, cu->cu_func);
558 XDNA_DBG(xdna, "CU %d full addr 0x%llx, cfg 0x%x", i,
559 amdxdna_gem_dev_addr(abo), req.cfgs[i]);
560 drm_gem_object_put(gobj);
561 }
562 req.num_cus = hwctx->cus->num_cus;
563
564 msg.send_data = (u8 *)&req;
565 msg.send_size = sizeof(req);
566 msg.handle = hwctx;
567 msg.opcode = MSG_OP_CONFIG_CU;
568 msg.notify_cb = notify_cb;
569 return xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT);
570}
571
572static int aie2_init_exec_cu_req(struct amdxdna_gem_obj *cmd_bo, void *req,
573 size_t *size, u32 *msg_op)
574{
575 struct execute_buffer_req *cu_req = req;
576 u32 cmd_len;
577 void *cmd;
578
579 cmd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len);
580 if (cmd_len > sizeof(cu_req->payload))
581 return -EINVAL;
582
583 cu_req->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo);
584 if (cu_req->cu_idx == INVALID_CU_IDX)
585 return -EINVAL;
586
587 memcpy(cu_req->payload, cmd, cmd_len);
588
589 *size = sizeof(*cu_req);
590 *msg_op = MSG_OP_EXECUTE_BUFFER_CF;
591 return 0;
592}
593
594static int aie2_init_exec_dpu_req(struct amdxdna_gem_obj *cmd_bo, void *req,
595 size_t *size, u32 *msg_op)
596{
597 struct exec_dpu_req *dpu_req = req;
598 struct amdxdna_cmd_start_npu *sn;
599 u32 cmd_len;
600
601 sn = amdxdna_cmd_get_payload(cmd_bo, &cmd_len);
602 if (cmd_len - sizeof(*sn) > sizeof(dpu_req->payload))
603 return -EINVAL;
604
605 dpu_req->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo);
606 if (dpu_req->cu_idx == INVALID_CU_IDX)
607 return -EINVAL;
608
609 dpu_req->inst_buf_addr = sn->buffer;
610 dpu_req->inst_size = sn->buffer_size;
611 dpu_req->inst_prop_cnt = sn->prop_count;
612 memcpy(dpu_req->payload, sn->prop_args, cmd_len - sizeof(*sn));
613
614 *size = sizeof(*dpu_req);
615 *msg_op = MSG_OP_EXEC_DPU;
616 return 0;
617}
618
619static void aie2_init_exec_chain_req(void *req, u64 slot_addr, size_t size, u32 cmd_cnt)
620{
621 struct cmd_chain_req *chain_req = req;
622
623 chain_req->buf_addr = slot_addr;
624 chain_req->buf_size = size;
625 chain_req->count = cmd_cnt;
626}
627
628static void aie2_init_npu_chain_req(void *req, u64 slot_addr, size_t size, u32 cmd_cnt)
629{
630 struct cmd_chain_npu_req *npu_chain_req = req;
631
632 npu_chain_req->flags = 0;
633 npu_chain_req->reserved = 0;
634 npu_chain_req->buf_addr = slot_addr;
635 npu_chain_req->buf_size = size;
636 npu_chain_req->count = cmd_cnt;
637}
638
639static int
640aie2_cmdlist_fill_cf(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size)
641{
642 struct cmd_chain_slot_execbuf_cf *cf_slot = slot;
643 u32 cmd_len;
644 void *cmd;
645
646 cmd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len);
647 if (*size < sizeof(*cf_slot) + cmd_len)
648 return -EINVAL;
649
650 cf_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo);
651 if (cf_slot->cu_idx == INVALID_CU_IDX)
652 return -EINVAL;
653
654 cf_slot->arg_cnt = cmd_len / sizeof(u32);
655 memcpy(cf_slot->args, cmd, cmd_len);
656 /* Accurate slot size to hint firmware to do necessary copy */
657 *size = sizeof(*cf_slot) + cmd_len;
658 return 0;
659}
660
661static int
662aie2_cmdlist_fill_dpu(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size)
663{
664 struct cmd_chain_slot_dpu *dpu_slot = slot;
665 struct amdxdna_cmd_start_npu *sn;
666 u32 cmd_len;
667 u32 arg_sz;
668
669 sn = amdxdna_cmd_get_payload(cmd_bo, &cmd_len);
670 arg_sz = cmd_len - sizeof(*sn);
671 if (cmd_len < sizeof(*sn) || arg_sz > MAX_DPU_ARGS_SIZE)
672 return -EINVAL;
673
674 if (*size < sizeof(*dpu_slot) + arg_sz)
675 return -EINVAL;
676
677 dpu_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo);
678 if (dpu_slot->cu_idx == INVALID_CU_IDX)
679 return -EINVAL;
680
681 dpu_slot->inst_buf_addr = sn->buffer;
682 dpu_slot->inst_size = sn->buffer_size;
683 dpu_slot->inst_prop_cnt = sn->prop_count;
684 dpu_slot->arg_cnt = arg_sz / sizeof(u32);
685 memcpy(dpu_slot->args, sn->prop_args, arg_sz);
686
687 /* Accurate slot size to hint firmware to do necessary copy */
688 *size = sizeof(*dpu_slot) + arg_sz;
689 return 0;
690}
691
692static int aie2_cmdlist_unsupp(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size)
693{
694 return -EOPNOTSUPP;
695}
696
697static u32 aie2_get_chain_msg_op(u32 cmd_op)
698{
699 switch (cmd_op) {
700 case ERT_START_CU:
701 return MSG_OP_CHAIN_EXEC_BUFFER_CF;
702 case ERT_START_NPU:
703 return MSG_OP_CHAIN_EXEC_DPU;
704 default:
705 break;
706 }
707
708 return MSG_OP_MAX_OPCODE;
709}
710
711static struct aie2_exec_msg_ops legacy_exec_message_ops = {
712 .init_cu_req = aie2_init_exec_cu_req,
713 .init_dpu_req = aie2_init_exec_dpu_req,
714 .init_chain_req = aie2_init_exec_chain_req,
715 .fill_cf_slot = aie2_cmdlist_fill_cf,
716 .fill_dpu_slot = aie2_cmdlist_fill_dpu,
717 .fill_preempt_slot = aie2_cmdlist_unsupp,
718 .fill_elf_slot = aie2_cmdlist_unsupp,
719 .get_chain_msg_op = aie2_get_chain_msg_op,
720};
721
722static int
723aie2_cmdlist_fill_npu_cf(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size)
724{
725 struct cmd_chain_slot_npu *npu_slot = slot;
726 u32 cmd_len;
727 void *cmd;
728
729 cmd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len);
730 if (*size < sizeof(*npu_slot) + cmd_len)
731 return -EINVAL;
732
733 memset(npu_slot, 0, sizeof(*npu_slot));
734 npu_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo);
735 if (npu_slot->cu_idx == INVALID_CU_IDX)
736 return -EINVAL;
737
738 npu_slot->type = EXEC_NPU_TYPE_NON_ELF;
739 npu_slot->arg_cnt = cmd_len / sizeof(u32);
740 memcpy(npu_slot->args, cmd, cmd_len);
741
742 *size = sizeof(*npu_slot) + cmd_len;
743 return 0;
744}
745
746static int
747aie2_cmdlist_fill_npu_dpu(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size)
748{
749 struct cmd_chain_slot_npu *npu_slot = slot;
750 struct amdxdna_cmd_start_npu *sn;
751 u32 cmd_len;
752 u32 arg_sz;
753
754 sn = amdxdna_cmd_get_payload(cmd_bo, &cmd_len);
755 arg_sz = cmd_len - sizeof(*sn);
756 if (cmd_len < sizeof(*sn) || arg_sz > MAX_NPU_ARGS_SIZE)
757 return -EINVAL;
758
759 if (*size < sizeof(*npu_slot) + arg_sz)
760 return -EINVAL;
761
762 memset(npu_slot, 0, sizeof(*npu_slot));
763 npu_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo);
764 if (npu_slot->cu_idx == INVALID_CU_IDX)
765 return -EINVAL;
766
767 npu_slot->type = EXEC_NPU_TYPE_PARTIAL_ELF;
768 npu_slot->inst_buf_addr = sn->buffer;
769 npu_slot->inst_size = sn->buffer_size;
770 npu_slot->inst_prop_cnt = sn->prop_count;
771 npu_slot->arg_cnt = arg_sz / sizeof(u32);
772 memcpy(npu_slot->args, sn->prop_args, arg_sz);
773
774 *size = sizeof(*npu_slot) + arg_sz;
775 return 0;
776}
777
778static int
779aie2_cmdlist_fill_npu_preempt(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size)
780{
781 struct cmd_chain_slot_npu *npu_slot = slot;
782 struct amdxdna_cmd_preempt_data *pd;
783 u32 cmd_len;
784 u32 arg_sz;
785
786 pd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len);
787 arg_sz = cmd_len - sizeof(*pd);
788 if (cmd_len < sizeof(*pd) || arg_sz > MAX_NPU_ARGS_SIZE)
789 return -EINVAL;
790
791 if (*size < sizeof(*npu_slot) + arg_sz)
792 return -EINVAL;
793
794 memset(npu_slot, 0, sizeof(*npu_slot));
795 npu_slot->cu_idx = amdxdna_cmd_get_cu_idx(cmd_bo);
796 if (npu_slot->cu_idx == INVALID_CU_IDX)
797 return -EINVAL;
798
799 npu_slot->type = EXEC_NPU_TYPE_PREEMPT;
800 npu_slot->inst_buf_addr = pd->inst_buf;
801 npu_slot->save_buf_addr = pd->save_buf;
802 npu_slot->restore_buf_addr = pd->restore_buf;
803 npu_slot->inst_size = pd->inst_size;
804 npu_slot->save_size = pd->save_size;
805 npu_slot->restore_size = pd->restore_size;
806 npu_slot->inst_prop_cnt = pd->inst_prop_cnt;
807 npu_slot->arg_cnt = arg_sz / sizeof(u32);
808 memcpy(npu_slot->args, pd->prop_args, arg_sz);
809
810 *size = sizeof(*npu_slot) + arg_sz;
811 return 0;
812}
813
814static int
815aie2_cmdlist_fill_npu_elf(struct amdxdna_gem_obj *cmd_bo, void *slot, size_t *size)
816{
817 struct cmd_chain_slot_npu *npu_slot = slot;
818 struct amdxdna_cmd_preempt_data *pd;
819 u32 cmd_len;
820 u32 arg_sz;
821
822 pd = amdxdna_cmd_get_payload(cmd_bo, &cmd_len);
823 arg_sz = cmd_len - sizeof(*pd);
824 if (cmd_len < sizeof(*pd) || arg_sz > MAX_NPU_ARGS_SIZE)
825 return -EINVAL;
826
827 if (*size < sizeof(*npu_slot) + arg_sz)
828 return -EINVAL;
829
830 memset(npu_slot, 0, sizeof(*npu_slot));
831 npu_slot->type = EXEC_NPU_TYPE_ELF;
832 npu_slot->inst_buf_addr = pd->inst_buf;
833 npu_slot->save_buf_addr = pd->save_buf;
834 npu_slot->restore_buf_addr = pd->restore_buf;
835 npu_slot->inst_size = pd->inst_size;
836 npu_slot->save_size = pd->save_size;
837 npu_slot->restore_size = pd->restore_size;
838 npu_slot->inst_prop_cnt = pd->inst_prop_cnt;
839 npu_slot->arg_cnt = 1;
840 npu_slot->args[0] = AIE2_EXEC_BUFFER_KERNEL_OP_TXN;
841
842 *size = struct_size(npu_slot, args, npu_slot->arg_cnt);
843 return 0;
844}
845
846static u32 aie2_get_npu_chain_msg_op(u32 cmd_op)
847{
848 return MSG_OP_CHAIN_EXEC_NPU;
849}
850
851static struct aie2_exec_msg_ops npu_exec_message_ops = {
852 .init_cu_req = aie2_init_exec_cu_req,
853 .init_dpu_req = aie2_init_exec_dpu_req,
854 .init_chain_req = aie2_init_npu_chain_req,
855 .fill_cf_slot = aie2_cmdlist_fill_npu_cf,
856 .fill_dpu_slot = aie2_cmdlist_fill_npu_dpu,
857 .fill_preempt_slot = aie2_cmdlist_fill_npu_preempt,
858 .fill_elf_slot = aie2_cmdlist_fill_npu_elf,
859 .get_chain_msg_op = aie2_get_npu_chain_msg_op,
860};
861
862static int aie2_init_exec_req(void *req, struct amdxdna_gem_obj *cmd_abo,
863 size_t *size, u32 *msg_op)
864{
865 struct amdxdna_dev *xdna = cmd_abo->client->xdna;
866 int ret;
867 u32 op;
868
869
870 op = amdxdna_cmd_get_op(cmd_abo);
871 switch (op) {
872 case ERT_START_CU:
873 ret = EXEC_MSG_OPS(xdna)->init_cu_req(cmd_abo, req, size, msg_op);
874 if (ret) {
875 XDNA_DBG(xdna, "Init CU req failed ret %d", ret);
876 return ret;
877 }
878 break;
879 case ERT_START_NPU:
880 ret = EXEC_MSG_OPS(xdna)->init_dpu_req(cmd_abo, req, size, msg_op);
881 if (ret) {
882 XDNA_DBG(xdna, "Init DPU req failed ret %d", ret);
883 return ret;
884 }
885
886 break;
887 default:
888 XDNA_ERR(xdna, "Unsupported op %d", op);
889 ret = -EOPNOTSUPP;
890 break;
891 }
892
893 return ret;
894}
895
896static int
897aie2_cmdlist_fill_slot(void *slot, struct amdxdna_gem_obj *cmd_abo,
898 size_t *size, u32 *cmd_op)
899{
900 struct amdxdna_dev *xdna = cmd_abo->client->xdna;
901 int ret;
902 u32 op;
903
904 op = amdxdna_cmd_get_op(cmd_abo);
905 if (*cmd_op == ERT_INVALID_CMD)
906 *cmd_op = op;
907 else if (op != *cmd_op)
908 return -EINVAL;
909
910 switch (op) {
911 case ERT_START_CU:
912 ret = EXEC_MSG_OPS(xdna)->fill_cf_slot(cmd_abo, slot, size);
913 break;
914 case ERT_START_NPU:
915 ret = EXEC_MSG_OPS(xdna)->fill_dpu_slot(cmd_abo, slot, size);
916 break;
917 case ERT_START_NPU_PREEMPT:
918 if (!AIE2_FEATURE_ON(xdna->dev_handle, AIE2_PREEMPT))
919 return -EOPNOTSUPP;
920 ret = EXEC_MSG_OPS(xdna)->fill_preempt_slot(cmd_abo, slot, size);
921 break;
922 case ERT_START_NPU_PREEMPT_ELF:
923 if (!AIE2_FEATURE_ON(xdna->dev_handle, AIE2_PREEMPT))
924 return -EOPNOTSUPP;
925 ret = EXEC_MSG_OPS(xdna)->fill_elf_slot(cmd_abo, slot, size);
926 break;
927 default:
928 XDNA_INFO(xdna, "Unsupported op %d", op);
929 ret = -EOPNOTSUPP;
930 break;
931 }
932
933 return ret;
934}
935
936void aie2_msg_init(struct amdxdna_dev_hdl *ndev)
937{
938 if (AIE2_FEATURE_ON(ndev, AIE2_NPU_COMMAND))
939 ndev->exec_msg_ops = &npu_exec_message_ops;
940 else
941 ndev->exec_msg_ops = &legacy_exec_message_ops;
942}
943
944void aie2_destroy_mgmt_chann(struct amdxdna_dev_hdl *ndev)
945{
946 struct amdxdna_dev *xdna = ndev->xdna;
947
948 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock));
949
950 if (!ndev->mgmt_chann)
951 return;
952
953 xdna_mailbox_stop_channel(ndev->mgmt_chann);
954 xdna_mailbox_free_channel(ndev->mgmt_chann);
955 ndev->mgmt_chann = NULL;
956}
957
958static inline struct amdxdna_gem_obj *
959aie2_cmdlist_get_cmd_buf(struct amdxdna_sched_job *job)
960{
961 int idx = get_job_idx(job->seq);
962
963 return job->hwctx->priv->cmd_buf[idx];
964}
965
966int aie2_execbuf(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job,
967 int (*notify_cb)(void *, void __iomem *, size_t))
968{
969 struct mailbox_channel *chann = hwctx->priv->mbox_chann;
970 struct amdxdna_dev *xdna = hwctx->client->xdna;
971 struct amdxdna_gem_obj *cmd_abo = job->cmd_bo;
972 struct xdna_mailbox_msg msg;
973 union exec_req req;
974 int ret;
975
976 if (!chann)
977 return -ENODEV;
978
979 ret = aie2_init_exec_req(&req, cmd_abo, &msg.send_size, &msg.opcode);
980 if (ret)
981 return ret;
982
983 msg.handle = job;
984 msg.notify_cb = notify_cb;
985 msg.send_data = (u8 *)&req;
986 print_hex_dump_debug("cmd: ", DUMP_PREFIX_OFFSET, 16, 4, &req,
987 0x40, false);
988
989 ret = xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT);
990 if (ret) {
991 XDNA_ERR(xdna, "Send message failed");
992 return ret;
993 }
994
995 return 0;
996}
997
998int aie2_cmdlist_multi_execbuf(struct amdxdna_hwctx *hwctx,
999 struct amdxdna_sched_job *job,
1000 int (*notify_cb)(void *, void __iomem *, size_t))
1001{
1002 struct amdxdna_gem_obj *cmdbuf_abo = aie2_cmdlist_get_cmd_buf(job);
1003 struct mailbox_channel *chann = hwctx->priv->mbox_chann;
1004 struct amdxdna_client *client = hwctx->client;
1005 struct amdxdna_gem_obj *cmd_abo = job->cmd_bo;
1006 void *cmd_buf = amdxdna_gem_vmap(cmdbuf_abo);
1007 struct amdxdna_dev *xdna = client->xdna;
1008 struct amdxdna_cmd_chain *payload;
1009 struct xdna_mailbox_msg msg;
1010 union exec_chain_req req;
1011 u32 payload_len, ccnt;
1012 u32 offset = 0;
1013 size_t size;
1014 int ret;
1015 u32 op;
1016 u32 i;
1017
1018 if (!cmd_buf)
1019 return -ENOMEM;
1020
1021 op = amdxdna_cmd_get_op(cmd_abo);
1022 payload = amdxdna_cmd_get_payload(cmd_abo, &payload_len);
1023 if (op != ERT_CMD_CHAIN) {
1024 XDNA_DBG(xdna, "Invalid op code %d", op);
1025 return -EINVAL;
1026 }
1027
1028 if (!payload) {
1029 XDNA_DBG(xdna, "Failed to get command payload");
1030 return -EINVAL;
1031 }
1032
1033 ccnt = payload->command_count;
1034 if (payload_len < struct_size(payload, data, ccnt)) {
1035 XDNA_DBG(xdna, "Invalid command count %d", ccnt);
1036 return -EINVAL;
1037 }
1038
1039 op = ERT_INVALID_CMD;
1040 for (i = 0; i < ccnt; i++) {
1041 u32 boh = (u32)(payload->data[i]);
1042 struct amdxdna_gem_obj *abo;
1043
1044 abo = amdxdna_gem_get_obj(client, boh, AMDXDNA_BO_SHARE);
1045 if (!abo) {
1046 XDNA_ERR(xdna, "Failed to find cmd BO %d", boh);
1047 return -ENOENT;
1048 }
1049
1050 size = cmdbuf_abo->mem.size - offset;
1051 ret = aie2_cmdlist_fill_slot(cmd_buf + offset, abo, &size, &op);
1052 amdxdna_gem_put_obj(abo);
1053 if (ret)
1054 return ret;
1055
1056 offset += size;
1057 }
1058
1059 XDNA_DBG(xdna, "Total %d commands:", ccnt);
1060 print_hex_dump_debug("cmdbufs: ", DUMP_PREFIX_OFFSET, 16, 4,
1061 cmd_buf, offset, false);
1062
1063 msg.opcode = EXEC_MSG_OPS(xdna)->get_chain_msg_op(op);
1064 if (msg.opcode == MSG_OP_MAX_OPCODE)
1065 return -EOPNOTSUPP;
1066
1067 /* The offset is the accumulated total size of the cmd buffer */
1068 EXEC_MSG_OPS(xdna)->init_chain_req(&req, amdxdna_gem_dev_addr(cmdbuf_abo),
1069 offset, ccnt);
1070 drm_clflush_virt_range(cmd_buf, offset);
1071
1072 msg.handle = job;
1073 msg.notify_cb = notify_cb;
1074 msg.send_data = (u8 *)&req;
1075 msg.send_size = sizeof(req);
1076 print_hex_dump_debug("cmdlist msg: ", DUMP_PREFIX_OFFSET, 16, 4,
1077 &req, msg.send_size, false);
1078 ret = xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT);
1079 if (ret) {
1080 XDNA_ERR(xdna, "Send message failed");
1081 return ret;
1082 }
1083
1084 return 0;
1085}
1086
1087int aie2_cmdlist_single_execbuf(struct amdxdna_hwctx *hwctx,
1088 struct amdxdna_sched_job *job,
1089 int (*notify_cb)(void *, void __iomem *, size_t))
1090{
1091 struct amdxdna_gem_obj *cmdbuf_abo = aie2_cmdlist_get_cmd_buf(job);
1092 struct mailbox_channel *chann = hwctx->priv->mbox_chann;
1093 struct amdxdna_dev *xdna = hwctx->client->xdna;
1094 struct amdxdna_gem_obj *cmd_abo = job->cmd_bo;
1095 void *cmd_buf = amdxdna_gem_vmap(cmdbuf_abo);
1096 struct xdna_mailbox_msg msg;
1097 union exec_chain_req req;
1098 u32 op = ERT_INVALID_CMD;
1099 size_t size;
1100 int ret;
1101
1102 if (!cmd_buf)
1103 return -ENOMEM;
1104
1105 size = cmdbuf_abo->mem.size;
1106 ret = aie2_cmdlist_fill_slot(cmd_buf, cmd_abo, &size, &op);
1107 if (ret)
1108 return ret;
1109
1110 print_hex_dump_debug("cmdbuf: ", DUMP_PREFIX_OFFSET, 16, 4, cmd_buf, size, false);
1111
1112 msg.opcode = EXEC_MSG_OPS(xdna)->get_chain_msg_op(op);
1113 if (msg.opcode == MSG_OP_MAX_OPCODE)
1114 return -EOPNOTSUPP;
1115
1116 EXEC_MSG_OPS(xdna)->init_chain_req(&req, amdxdna_gem_dev_addr(cmdbuf_abo), size, 1);
1117 drm_clflush_virt_range(cmd_buf, size);
1118
1119 msg.handle = job;
1120 msg.notify_cb = notify_cb;
1121 msg.send_data = (u8 *)&req;
1122 msg.send_size = sizeof(req);
1123 print_hex_dump_debug("cmdlist msg: ", DUMP_PREFIX_OFFSET, 16, 4,
1124 &req, msg.send_size, false);
1125 ret = xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT);
1126 if (ret) {
1127 XDNA_ERR(hwctx->client->xdna, "Send message failed");
1128 return ret;
1129 }
1130
1131 return 0;
1132}
1133
1134int aie2_sync_bo(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job,
1135 int (*notify_cb)(void *, void __iomem *, size_t))
1136{
1137 struct mailbox_channel *chann = hwctx->priv->mbox_chann;
1138 struct amdxdna_gem_obj *abo = to_xdna_obj(job->bos[0]);
1139 struct amdxdna_dev *xdna = hwctx->client->xdna;
1140 struct xdna_mailbox_msg msg;
1141 struct sync_bo_req req;
1142 int ret = 0;
1143
1144 req.src_addr = 0;
1145 req.dst_addr = amdxdna_dev_bo_offset(abo);
1146 req.size = abo->mem.size;
1147
1148 /* Device to Host */
1149 req.type = FIELD_PREP(AIE2_MSG_SYNC_BO_SRC_TYPE, SYNC_BO_DEV_MEM) |
1150 FIELD_PREP(AIE2_MSG_SYNC_BO_DST_TYPE, SYNC_BO_HOST_MEM);
1151
1152 XDNA_DBG(xdna, "sync %d bytes src(0x%llx) to dst(0x%llx) completed",
1153 req.size, req.src_addr, req.dst_addr);
1154
1155 msg.handle = job;
1156 msg.notify_cb = notify_cb;
1157 msg.send_data = (u8 *)&req;
1158 msg.send_size = sizeof(req);
1159 msg.opcode = MSG_OP_SYNC_BO;
1160
1161 ret = xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT);
1162 if (ret) {
1163 XDNA_ERR(xdna, "Send message failed");
1164 return ret;
1165 }
1166
1167 return 0;
1168}
1169
1170int aie2_config_debug_bo(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job,
1171 int (*notify_cb)(void *, void __iomem *, size_t))
1172{
1173 struct mailbox_channel *chann = hwctx->priv->mbox_chann;
1174 struct amdxdna_gem_obj *abo = to_xdna_obj(job->bos[0]);
1175 struct amdxdna_dev *xdna = hwctx->client->xdna;
1176 struct config_debug_bo_req req;
1177 struct xdna_mailbox_msg msg;
1178
1179 if (job->drv_cmd->opcode == ATTACH_DEBUG_BO)
1180 req.config = DEBUG_BO_REGISTER;
1181 else
1182 req.config = DEBUG_BO_UNREGISTER;
1183
1184 req.offset = amdxdna_dev_bo_offset(abo);
1185 req.size = abo->mem.size;
1186
1187 XDNA_DBG(xdna, "offset 0x%llx size 0x%llx config %d",
1188 req.offset, req.size, req.config);
1189
1190 msg.handle = job;
1191 msg.notify_cb = notify_cb;
1192 msg.send_data = (u8 *)&req;
1193 msg.send_size = sizeof(req);
1194 msg.opcode = MSG_OP_CONFIG_DEBUG_BO;
1195
1196 return xdna_mailbox_send_msg(chann, &msg, TX_TIMEOUT);
1197}
1198
1199int aie2_query_app_health(struct amdxdna_dev_hdl *ndev, u32 context_id,
1200 struct app_health_report *report)
1201{
1202 DECLARE_AIE2_MSG(get_app_health, MSG_OP_GET_APP_HEALTH);
1203 struct amdxdna_dev *xdna = ndev->xdna;
1204 struct app_health_report *buf;
1205 dma_addr_t dma_addr;
1206 u32 buf_size;
1207 int ret;
1208
1209 if (!AIE2_FEATURE_ON(ndev, AIE2_APP_HEALTH)) {
1210 XDNA_DBG(xdna, "App health feature not supported");
1211 return -EOPNOTSUPP;
1212 }
1213
1214 buf_size = sizeof(*report);
1215 buf = aie2_alloc_msg_buffer(ndev, &buf_size, &dma_addr);
1216 if (IS_ERR(buf)) {
1217 XDNA_ERR(xdna, "Failed to allocate buffer for app health");
1218 return PTR_ERR(buf);
1219 }
1220
1221 req.buf_addr = dma_addr;
1222 req.context_id = context_id;
1223 req.buf_size = buf_size;
1224
1225 drm_clflush_virt_range(buf, sizeof(*report));
1226 ret = aie2_send_mgmt_msg_wait(ndev, &msg);
1227 if (ret) {
1228 XDNA_ERR(xdna, "Get app health failed, ret %d status 0x%x", ret, resp.status);
1229 goto free_buf;
1230 }
1231
1232 /* Copy the report to caller's buffer */
1233 memcpy(report, buf, sizeof(*report));
1234
1235free_buf:
1236 aie2_free_msg_buffer(ndev, buf_size, buf, dma_addr);
1237 return ret;
1238}