Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
RDMA/mana_ib: Support memory windows
Implement .alloc_mw() and .dealloc_mw() for mana device.
This is just the basic infrastructure, MW is not practically usable until
additional kernel support for allowing user space to submit MW work
requests is completed.
Link: https://patch.msgid.link/r/20260331090851.2276205-1-kotaranov@linux.microsoft.com
Signed-off-by: Konstantin Taranov <kotaranov@microsoft.com>
Reviewed-by: Long Li <longli@microsoft.com>
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
authored by
Konstantin Taranov
and committed by
Jason Gunthorpe
b2105888
7244491d
+69
-1
+3
drivers/infiniband/hw/mana/device.c
+3
drivers/infiniband/hw/mana/device.c
···
17
17
.uverbs_abi_ver = MANA_IB_UVERBS_ABI_VERSION,
18
18
19
19
.add_gid = mana_ib_gd_add_gid,
20
+
.alloc_mw = mana_ib_alloc_mw,
20
21
.alloc_pd = mana_ib_alloc_pd,
21
22
.alloc_ucontext = mana_ib_alloc_ucontext,
22
23
.create_ah = mana_ib_create_ah,
···
25
24
.create_qp = mana_ib_create_qp,
26
25
.create_rwq_ind_table = mana_ib_create_rwq_ind_table,
27
26
.create_wq = mana_ib_create_wq,
27
+
.dealloc_mw = mana_ib_dealloc_mw,
28
28
.dealloc_pd = mana_ib_dealloc_pd,
29
29
.dealloc_ucontext = mana_ib_dealloc_ucontext,
30
30
.del_gid = mana_ib_gd_del_gid,
···
55
53
56
54
INIT_RDMA_OBJ_SIZE(ib_ah, mana_ib_ah, ibah),
57
55
INIT_RDMA_OBJ_SIZE(ib_cq, mana_ib_cq, ibcq),
56
+
INIT_RDMA_OBJ_SIZE(ib_mw, mana_ib_mw, ibmw),
58
57
INIT_RDMA_OBJ_SIZE(ib_pd, mana_ib_pd, ibpd),
59
58
INIT_RDMA_OBJ_SIZE(ib_qp, mana_ib_qp, ibqp),
60
59
INIT_RDMA_OBJ_SIZE(ib_ucontext, mana_ib_ucontext, ibucontext),
+8
drivers/infiniband/hw/mana/mana_ib.h
+8
drivers/infiniband/hw/mana/mana_ib.h
···
125
125
dma_addr_t dma_handle;
126
126
};
127
127
128
+
struct mana_ib_mw {
129
+
struct ib_mw ibmw;
130
+
mana_handle_t mw_handle;
131
+
};
132
+
128
133
struct mana_ib_mr {
129
134
struct ib_mr ibmr;
130
135
struct ib_umem *umem;
···
740
735
void mana_drain_gsi_sqs(struct mana_ib_dev *mdev);
741
736
int mana_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
742
737
int mana_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
738
+
739
+
int mana_ib_alloc_mw(struct ib_mw *mw, struct ib_udata *udata);
740
+
int mana_ib_dealloc_mw(struct ib_mw *mw);
743
741
744
742
struct ib_mr *mana_ib_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 start, u64 length,
745
743
u64 iova, int fd, int mr_access_flags,
+53
-1
drivers/infiniband/hw/mana/mr.c
+53
-1
drivers/infiniband/hw/mana/mr.c
···
6
6
#include "mana_ib.h"
7
7
8
8
#define VALID_MR_FLAGS (IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ |\
9
-
IB_ACCESS_REMOTE_ATOMIC | IB_ZERO_BASED)
9
+
IB_ACCESS_REMOTE_ATOMIC | IB_ACCESS_MW_BIND | IB_ZERO_BASED)
10
10
11
11
#define VALID_DMA_MR_FLAGS (IB_ACCESS_LOCAL_WRITE)
12
12
···
26
26
27
27
if (access_flags & IB_ACCESS_REMOTE_ATOMIC)
28
28
flags |= GDMA_ACCESS_FLAG_REMOTE_ATOMIC;
29
+
30
+
if (access_flags & IB_ACCESS_MW_BIND)
31
+
flags |= GDMA_ACCESS_FLAG_BIND_MW;
29
32
30
33
return flags;
31
34
}
···
288
285
err_free:
289
286
kfree(mr);
290
287
return ERR_PTR(err);
288
+
}
289
+
290
+
static int mana_ib_gd_create_mw(struct mana_ib_dev *dev, struct mana_ib_pd *pd, struct ib_mw *ibmw)
291
+
{
292
+
struct mana_ib_mw *mw = container_of(ibmw, struct mana_ib_mw, ibmw);
293
+
struct gdma_context *gc = mdev_to_gc(dev);
294
+
struct gdma_create_mr_response resp = {};
295
+
struct gdma_create_mr_request req = {};
296
+
int err;
297
+
298
+
mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_MR, sizeof(req), sizeof(resp));
299
+
req.hdr.req.msg_version = GDMA_MESSAGE_V2;
300
+
req.pd_handle = pd->pd_handle;
301
+
302
+
switch (mw->ibmw.type) {
303
+
case IB_MW_TYPE_1:
304
+
req.mr_type = GDMA_MR_TYPE_MW1;
305
+
break;
306
+
case IB_MW_TYPE_2:
307
+
req.mr_type = GDMA_MR_TYPE_MW2;
308
+
break;
309
+
default:
310
+
return -EINVAL;
311
+
}
312
+
313
+
err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
314
+
if (err)
315
+
return err;
316
+
317
+
mw->ibmw.rkey = resp.rkey;
318
+
mw->mw_handle = resp.mr_handle;
319
+
320
+
return 0;
321
+
}
322
+
323
+
int mana_ib_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata)
324
+
{
325
+
struct mana_ib_dev *mdev = container_of(ibmw->device, struct mana_ib_dev, ib_dev);
326
+
struct mana_ib_pd *pd = container_of(ibmw->pd, struct mana_ib_pd, ibpd);
327
+
328
+
return mana_ib_gd_create_mw(mdev, pd, ibmw);
329
+
}
330
+
331
+
int mana_ib_dealloc_mw(struct ib_mw *ibmw)
332
+
{
333
+
struct mana_ib_dev *dev = container_of(ibmw->device, struct mana_ib_dev, ib_dev);
334
+
struct mana_ib_mw *mw = container_of(ibmw, struct mana_ib_mw, ibmw);
335
+
336
+
return mana_ib_gd_destroy_mr(dev, mw->mw_handle);
291
337
}
292
338
293
339
int mana_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
+5
include/net/mana/gdma.h
+5
include/net/mana/gdma.h
···
778
778
GDMA_ACCESS_FLAG_REMOTE_READ = BIT_ULL(2),
779
779
GDMA_ACCESS_FLAG_REMOTE_WRITE = BIT_ULL(3),
780
780
GDMA_ACCESS_FLAG_REMOTE_ATOMIC = BIT_ULL(4),
781
+
GDMA_ACCESS_FLAG_BIND_MW = BIT_ULL(5),
781
782
};
782
783
783
784
/* GDMA_CREATE_DMA_REGION */
···
871
870
GDMA_MR_TYPE_ZBVA = 4,
872
871
/* Device address MRs */
873
872
GDMA_MR_TYPE_DM = 5,
873
+
/* Memory Window type 1 */
874
+
GDMA_MR_TYPE_MW1 = 6,
875
+
/* Memory Window type 2 */
876
+
GDMA_MR_TYPE_MW2 = 7,
874
877
};
875
878
876
879
struct gdma_create_mr_params {