Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
RDMA/irdma: Add SRQ support
Implement verb API and UAPI changes to support SRQ functionality in GEN3
devices.
Signed-off-by: Faisal Latif <faisal.latif@intel.com>
Signed-off-by: Tatyana Nikolova <tatyana.e.nikolova@intel.com>
Link: https://patch.msgid.link/20250827152545.2056-13-tatyana.e.nikolova@intel.com
Tested-by: Jacob Moroni <jmoroni@google.com>
Signed-off-by: Leon Romanovsky <leon@kernel.org>
authored by
Faisal Latif
and committed by
Leon Romanovsky
563e1feb
9a1d6878
+1103
-14
+234
-2
drivers/infiniband/hw/irdma/ctrl.c
+234
-2
drivers/infiniband/hw/irdma/ctrl.c
···
412
412
pble_obj_cnt = info->pd->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;
413
413
414
414
if ((info->virtual_map && info->sq_pa >= pble_obj_cnt) ||
415
-
(info->virtual_map && info->rq_pa >= pble_obj_cnt))
415
+
(!info->qp_uk_init_info.srq_uk &&
416
+
info->virtual_map && info->rq_pa >= pble_obj_cnt))
416
417
return -EINVAL;
417
418
418
419
qp->llp_stream_handle = (void *)(-1);
···
443
442
qp->xmit_tph_en = info->xmit_tph_en;
444
443
qp->qp_uk.first_sq_wq = info->qp_uk_init_info.first_sq_wq;
445
444
qp->qs_handle = qp->vsi->qos[qp->user_pri].qs_handle;
445
+
446
+
return 0;
447
+
}
448
+
449
+
/**
450
+
* irdma_sc_srq_init - init sc_srq structure
451
+
* @srq: srq sc struct
452
+
* @info: parameters for srq init
453
+
*/
454
+
int irdma_sc_srq_init(struct irdma_sc_srq *srq,
455
+
struct irdma_srq_init_info *info)
456
+
{
457
+
u32 srq_size_quanta;
458
+
int ret_code;
459
+
460
+
ret_code = irdma_uk_srq_init(&srq->srq_uk, &info->srq_uk_init_info);
461
+
if (ret_code)
462
+
return ret_code;
463
+
464
+
srq->dev = info->pd->dev;
465
+
srq->pd = info->pd;
466
+
srq->vsi = info->vsi;
467
+
srq->srq_pa = info->srq_pa;
468
+
srq->first_pm_pbl_idx = info->first_pm_pbl_idx;
469
+
srq->pasid = info->pasid;
470
+
srq->pasid_valid = info->pasid_valid;
471
+
srq->srq_limit = info->srq_limit;
472
+
srq->leaf_pbl_size = info->leaf_pbl_size;
473
+
srq->virtual_map = info->virtual_map;
474
+
srq->tph_en = info->tph_en;
475
+
srq->arm_limit_event = info->arm_limit_event;
476
+
srq->tph_val = info->tph_value;
477
+
srq->shadow_area_pa = info->shadow_area_pa;
478
+
479
+
/* Smallest SRQ size is 256B i.e. 8 quanta */
480
+
srq_size_quanta = max((u32)IRDMA_SRQ_MIN_QUANTA,
481
+
srq->srq_uk.srq_size *
482
+
srq->srq_uk.wqe_size_multiplier);
483
+
srq->hw_srq_size = irdma_get_encoded_wqe_size(srq_size_quanta,
484
+
IRDMA_QUEUE_TYPE_SRQ);
485
+
486
+
return 0;
487
+
}
488
+
489
+
/**
490
+
* irdma_sc_srq_create - send srq create CQP WQE
491
+
* @srq: srq sc struct
492
+
* @scratch: u64 saved to be used during cqp completion
493
+
* @post_sq: flag for cqp db to ring
494
+
*/
495
+
static int irdma_sc_srq_create(struct irdma_sc_srq *srq, u64 scratch,
496
+
bool post_sq)
497
+
{
498
+
struct irdma_sc_cqp *cqp;
499
+
__le64 *wqe;
500
+
u64 hdr;
501
+
502
+
cqp = srq->pd->dev->cqp;
503
+
if (srq->srq_uk.srq_id < cqp->dev->hw_attrs.min_hw_srq_id ||
504
+
srq->srq_uk.srq_id >
505
+
(cqp->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_SRQ].max_cnt - 1))
506
+
return -EINVAL;
507
+
508
+
wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
509
+
if (!wqe)
510
+
return -ENOMEM;
511
+
512
+
set_64bit_val(wqe, 0,
513
+
FIELD_PREP(IRDMA_CQPSQ_SRQ_SRQ_LIMIT, srq->srq_limit) |
514
+
FIELD_PREP(IRDMA_CQPSQ_SRQ_RQSIZE, srq->hw_srq_size) |
515
+
FIELD_PREP(IRDMA_CQPSQ_SRQ_RQ_WQE_SIZE, srq->srq_uk.wqe_size));
516
+
set_64bit_val(wqe, 8, (uintptr_t)srq);
517
+
set_64bit_val(wqe, 16,
518
+
FIELD_PREP(IRDMA_CQPSQ_SRQ_PD_ID, srq->pd->pd_id));
519
+
set_64bit_val(wqe, 32,
520
+
FIELD_PREP(IRDMA_CQPSQ_SRQ_PHYSICAL_BUFFER_ADDR,
521
+
srq->srq_pa >>
522
+
IRDMA_CQPSQ_SRQ_PHYSICAL_BUFFER_ADDR_S));
523
+
set_64bit_val(wqe, 40,
524
+
FIELD_PREP(IRDMA_CQPSQ_SRQ_DB_SHADOW_ADDR,
525
+
srq->shadow_area_pa >>
526
+
IRDMA_CQPSQ_SRQ_DB_SHADOW_ADDR_S));
527
+
set_64bit_val(wqe, 48,
528
+
FIELD_PREP(IRDMA_CQPSQ_SRQ_FIRST_PM_PBL_IDX,
529
+
srq->first_pm_pbl_idx));
530
+
531
+
hdr = srq->srq_uk.srq_id |
532
+
FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_SRQ) |
533
+
FIELD_PREP(IRDMA_CQPSQ_SRQ_LEAF_PBL_SIZE, srq->leaf_pbl_size) |
534
+
FIELD_PREP(IRDMA_CQPSQ_SRQ_VIRTMAP, srq->virtual_map) |
535
+
FIELD_PREP(IRDMA_CQPSQ_SRQ_ARM_LIMIT_EVENT,
536
+
srq->arm_limit_event) |
537
+
FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
538
+
539
+
dma_wmb(); /* make sure WQE is written before valid bit is set */
540
+
541
+
set_64bit_val(wqe, 24, hdr);
542
+
543
+
print_hex_dump_debug("WQE: SRQ_CREATE WQE", DUMP_PREFIX_OFFSET, 16, 8,
544
+
wqe, IRDMA_CQP_WQE_SIZE * 8, false);
545
+
if (post_sq)
546
+
irdma_sc_cqp_post_sq(cqp);
547
+
548
+
return 0;
549
+
}
550
+
551
+
/**
552
+
* irdma_sc_srq_modify - send modify_srq CQP WQE
553
+
* @srq: srq sc struct
554
+
* @info: parameters for srq modification
555
+
* @scratch: u64 saved to be used during cqp completion
556
+
* @post_sq: flag for cqp db to ring
557
+
*/
558
+
static int irdma_sc_srq_modify(struct irdma_sc_srq *srq,
559
+
struct irdma_modify_srq_info *info, u64 scratch,
560
+
bool post_sq)
561
+
{
562
+
struct irdma_sc_cqp *cqp;
563
+
__le64 *wqe;
564
+
u64 hdr;
565
+
566
+
cqp = srq->dev->cqp;
567
+
if (srq->srq_uk.srq_id < cqp->dev->hw_attrs.min_hw_srq_id ||
568
+
srq->srq_uk.srq_id >
569
+
(cqp->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_SRQ].max_cnt - 1))
570
+
return -EINVAL;
571
+
572
+
wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
573
+
if (!wqe)
574
+
return -ENOMEM;
575
+
576
+
set_64bit_val(wqe, 0,
577
+
FIELD_PREP(IRDMA_CQPSQ_SRQ_SRQ_LIMIT, info->srq_limit) |
578
+
FIELD_PREP(IRDMA_CQPSQ_SRQ_RQSIZE, srq->hw_srq_size) |
579
+
FIELD_PREP(IRDMA_CQPSQ_SRQ_RQ_WQE_SIZE, srq->srq_uk.wqe_size));
580
+
set_64bit_val(wqe, 8,
581
+
FIELD_PREP(IRDMA_CQPSQ_SRQ_SRQCTX, srq->srq_uk.srq_id));
582
+
set_64bit_val(wqe, 16,
583
+
FIELD_PREP(IRDMA_CQPSQ_SRQ_PD_ID, srq->pd->pd_id));
584
+
set_64bit_val(wqe, 32,
585
+
FIELD_PREP(IRDMA_CQPSQ_SRQ_PHYSICAL_BUFFER_ADDR,
586
+
srq->srq_pa >>
587
+
IRDMA_CQPSQ_SRQ_PHYSICAL_BUFFER_ADDR_S));
588
+
set_64bit_val(wqe, 40,
589
+
FIELD_PREP(IRDMA_CQPSQ_SRQ_DB_SHADOW_ADDR,
590
+
srq->shadow_area_pa >>
591
+
IRDMA_CQPSQ_SRQ_DB_SHADOW_ADDR_S));
592
+
set_64bit_val(wqe, 48,
593
+
FIELD_PREP(IRDMA_CQPSQ_SRQ_FIRST_PM_PBL_IDX,
594
+
srq->first_pm_pbl_idx));
595
+
596
+
hdr = srq->srq_uk.srq_id |
597
+
FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MODIFY_SRQ) |
598
+
FIELD_PREP(IRDMA_CQPSQ_SRQ_LEAF_PBL_SIZE, srq->leaf_pbl_size) |
599
+
FIELD_PREP(IRDMA_CQPSQ_SRQ_VIRTMAP, srq->virtual_map) |
600
+
FIELD_PREP(IRDMA_CQPSQ_SRQ_ARM_LIMIT_EVENT,
601
+
info->arm_limit_event) |
602
+
FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
603
+
dma_wmb(); /* make sure WQE is written before valid bit is set */
604
+
605
+
set_64bit_val(wqe, 24, hdr);
606
+
607
+
print_hex_dump_debug("WQE: SRQ_MODIFY WQE", DUMP_PREFIX_OFFSET, 16, 8,
608
+
wqe, IRDMA_CQP_WQE_SIZE * 8, false);
609
+
if (post_sq)
610
+
irdma_sc_cqp_post_sq(cqp);
611
+
612
+
return 0;
613
+
}
614
+
615
+
/**
616
+
* irdma_sc_srq_destroy - send srq_destroy CQP WQE
617
+
* @srq: srq sc struct
618
+
* @scratch: u64 saved to be used during cqp completion
619
+
* @post_sq: flag for cqp db to ring
620
+
*/
621
+
static int irdma_sc_srq_destroy(struct irdma_sc_srq *srq, u64 scratch,
622
+
bool post_sq)
623
+
{
624
+
struct irdma_sc_cqp *cqp;
625
+
__le64 *wqe;
626
+
u64 hdr;
627
+
628
+
cqp = srq->dev->cqp;
629
+
630
+
wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
631
+
if (!wqe)
632
+
return -ENOMEM;
633
+
634
+
set_64bit_val(wqe, 8, (uintptr_t)srq);
635
+
636
+
hdr = srq->srq_uk.srq_id |
637
+
FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DESTROY_SRQ) |
638
+
FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
639
+
dma_wmb(); /* make sure WQE is written before valid bit is set */
640
+
641
+
set_64bit_val(wqe, 24, hdr);
642
+
643
+
print_hex_dump_debug("WQE: SRQ_DESTROY WQE", DUMP_PREFIX_OFFSET, 16,
644
+
8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
645
+
if (post_sq)
646
+
irdma_sc_cqp_post_sq(cqp);
446
647
447
648
return 0;
448
649
}
···
1040
837
FIELD_PREP(IRDMAQPC_ISQP1, roce_info->is_qp1) |
1041
838
FIELD_PREP(IRDMAQPC_ROCE_TVER, roce_info->roce_tver) |
1042
839
FIELD_PREP(IRDMAQPC_IPV4, udp->ipv4) |
840
+
FIELD_PREP(IRDMAQPC_USE_SRQ, !qp->qp_uk.srq_uk ? 0 : 1) |
1043
841
FIELD_PREP(IRDMAQPC_INSERTVLANTAG, udp->insert_vlan_tag);
1044
842
set_64bit_val(qp_ctx, 0, qw0);
1045
843
set_64bit_val(qp_ctx, 8, qp->sq_pa);
···
1125
921
FIELD_PREP(IRDMAQPC_LOCAL_IPADDR0, udp->local_ipaddr[0]));
1126
922
set_64bit_val(qp_ctx, 200,
1127
923
FIELD_PREP(IRDMAQPC_THIGH, roce_info->t_high) |
924
+
FIELD_PREP(IRDMAQPC_SRQ_ID,
925
+
!qp->qp_uk.srq_uk ?
926
+
0 : qp->qp_uk.srq_uk->srq_id) |
1128
927
FIELD_PREP(IRDMAQPC_TLOW, roce_info->t_low));
1129
928
set_64bit_val(qp_ctx, 208, roce_info->pd_id |
1130
929
FIELD_PREP(IRDMAQPC_STAT_INDEX_GEN3, info->stats_idx) |
···
2426
2219
{
2427
2220
u8 encoded_size = 0;
2428
2221
2222
+
if (queue_type == IRDMA_QUEUE_TYPE_SRQ) {
2223
+
/* Smallest SRQ size is 256B (8 quanta) that gets
2224
+
* encoded to 0.
2225
+
*/
2226
+
encoded_size = ilog2(wqsize) - 3;
2227
+
2228
+
return encoded_size;
2229
+
}
2429
2230
/* cqp sq's hw coded value starts from 1 for size of 4
2430
2231
* while it starts from 0 for qp' wq's.
2431
2232
*/
···
4800
4585
case IRDMA_AE_SRQ_LIMIT:
4801
4586
info->srq = true;
4802
4587
/* [63:6] from CMPL_CTXT, [5:0] from WQDESCIDX. */
4803
-
info->compl_ctx = compl_ctx | info->wqe_idx;
4588
+
info->compl_ctx = compl_ctx;
4804
4589
ae_src = IRDMA_AE_SOURCE_RSVD;
4805
4590
break;
4806
4591
case IRDMA_AE_PRIV_OPERATION_DENIED:
···
6376
6161
&pcmdinfo->in.u.mc_modify.info,
6377
6162
pcmdinfo->in.u.mc_modify.scratch);
6378
6163
break;
6164
+
case IRDMA_OP_SRQ_CREATE:
6165
+
status = irdma_sc_srq_create(pcmdinfo->in.u.srq_create.srq,
6166
+
pcmdinfo->in.u.srq_create.scratch,
6167
+
pcmdinfo->post_sq);
6168
+
break;
6169
+
case IRDMA_OP_SRQ_MODIFY:
6170
+
status = irdma_sc_srq_modify(pcmdinfo->in.u.srq_modify.srq,
6171
+
&pcmdinfo->in.u.srq_modify.info,
6172
+
pcmdinfo->in.u.srq_modify.scratch,
6173
+
pcmdinfo->post_sq);
6174
+
break;
6175
+
case IRDMA_OP_SRQ_DESTROY:
6176
+
status = irdma_sc_srq_destroy(pcmdinfo->in.u.srq_destroy.srq,
6177
+
pcmdinfo->in.u.srq_destroy.scratch,
6178
+
pcmdinfo->post_sq);
6179
+
break;
6379
6180
default:
6380
6181
status = -EOPNOTSUPP;
6381
6182
break;
···
6549
6318
dev->protocol_used = info->protocol_used;
6550
6319
/* Setup the hardware limits, hmc may limit further */
6551
6320
dev->hw_attrs.min_hw_qp_id = IRDMA_MIN_IW_QP_ID;
6321
+
dev->hw_attrs.min_hw_srq_id = IRDMA_MIN_IW_SRQ_ID;
6552
6322
dev->hw_attrs.min_hw_aeq_size = IRDMA_MIN_AEQ_ENTRIES;
6553
6323
if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_3)
6554
6324
dev->hw_attrs.max_hw_aeq_size = IRDMA_MAX_AEQ_ENTRIES_GEN_3;
+35
-1
drivers/infiniband/hw/irdma/defs.h
+35
-1
drivers/infiniband/hw/irdma/defs.h
···
140
140
#define IRDMA_QP_SW_MAX_RQ_QUANTA 32768
141
141
#define IRDMA_MAX_QP_WRS(max_quanta_per_wr) \
142
142
((IRDMA_QP_SW_MAX_WQ_QUANTA - IRDMA_SQ_RSVD) / (max_quanta_per_wr))
143
+
#define IRDMA_SRQ_MIN_QUANTA 8
143
144
#define IRDMA_SRQ_MAX_QUANTA 262144
145
+
#define IRDMA_MAX_SRQ_WRS \
146
+
((IRDMA_SRQ_MAX_QUANTA - IRDMA_RQ_RSVD) / IRDMA_MAX_QUANTA_PER_WR)
147
+
144
148
#define IRDMAQP_TERM_SEND_TERM_AND_FIN 0
145
149
#define IRDMAQP_TERM_SEND_TERM_ONLY 1
146
150
#define IRDMAQP_TERM_SEND_FIN_ONLY 2
···
240
236
IRDMA_OP_ADD_LOCAL_MAC_ENTRY = 46,
241
237
IRDMA_OP_DELETE_LOCAL_MAC_ENTRY = 47,
242
238
IRDMA_OP_CQ_MODIFY = 48,
239
+
IRDMA_OP_SRQ_CREATE = 49,
240
+
IRDMA_OP_SRQ_MODIFY = 50,
241
+
IRDMA_OP_SRQ_DESTROY = 51,
243
242
244
243
/* Must be last entry*/
245
-
IRDMA_MAX_CQP_OPS = 49,
244
+
IRDMA_MAX_CQP_OPS = 52,
246
245
};
247
246
248
247
/* CQP SQ WQES */
···
255
248
#define IRDMA_CQP_OP_CREATE_CQ 0x03
256
249
#define IRDMA_CQP_OP_MODIFY_CQ 0x04
257
250
#define IRDMA_CQP_OP_DESTROY_CQ 0x05
251
+
#define IRDMA_CQP_OP_CREATE_SRQ 0x06
252
+
#define IRDMA_CQP_OP_MODIFY_SRQ 0x07
253
+
#define IRDMA_CQP_OP_DESTROY_SRQ 0x08
258
254
#define IRDMA_CQP_OP_ALLOC_STAG 0x09
259
255
#define IRDMA_CQP_OP_REG_MR 0x0a
260
256
#define IRDMA_CQP_OP_QUERY_STAG 0x0b
···
529
519
#define IRDMA_CQ_ERROR BIT_ULL(55)
530
520
#define IRDMA_CQ_SQ BIT_ULL(62)
531
521
522
+
#define IRDMA_CQ_SRQ BIT_ULL(52)
532
523
#define IRDMA_CQ_VALID BIT_ULL(63)
533
524
#define IRDMA_CQ_IMMVALID BIT_ULL(62)
534
525
#define IRDMA_CQ_UDSMACVALID BIT_ULL(61)
···
638
627
#define IRDMA_CQPSQ_QP_NEXTIWSTATE GENMASK_ULL(62, 60)
639
628
640
629
#define IRDMA_CQPSQ_QP_DBSHADOWADDR IRDMA_CQPHC_QPCTX
630
+
631
+
#define IRDMA_CQPSQ_SRQ_RQSIZE GENMASK_ULL(3, 0)
632
+
#define IRDMA_CQPSQ_SRQ_RQ_WQE_SIZE GENMASK_ULL(5, 4)
633
+
#define IRDMA_CQPSQ_SRQ_SRQ_LIMIT GENMASK_ULL(43, 32)
634
+
#define IRDMA_CQPSQ_SRQ_SRQCTX GENMASK_ULL(63, 6)
635
+
#define IRDMA_CQPSQ_SRQ_PD_ID GENMASK_ULL(39, 16)
636
+
#define IRDMA_CQPSQ_SRQ_SRQ_ID GENMASK_ULL(15, 0)
637
+
#define IRDMA_CQPSQ_SRQ_OP GENMASK_ULL(37, 32)
638
+
#define IRDMA_CQPSQ_SRQ_LEAF_PBL_SIZE GENMASK_ULL(45, 44)
639
+
#define IRDMA_CQPSQ_SRQ_VIRTMAP BIT_ULL(47)
640
+
#define IRDMA_CQPSQ_SRQ_TPH_EN BIT_ULL(60)
641
+
#define IRDMA_CQPSQ_SRQ_ARM_LIMIT_EVENT BIT_ULL(61)
642
+
#define IRDMA_CQPSQ_SRQ_FIRST_PM_PBL_IDX GENMASK_ULL(27, 0)
643
+
#define IRDMA_CQPSQ_SRQ_TPH_VALUE GENMASK_ULL(7, 0)
644
+
#define IRDMA_CQPSQ_SRQ_PHYSICAL_BUFFER_ADDR_S 8
645
+
#define IRDMA_CQPSQ_SRQ_PHYSICAL_BUFFER_ADDR GENMASK_ULL(63, 8)
646
+
#define IRDMA_CQPSQ_SRQ_DB_SHADOW_ADDR_S 6
647
+
#define IRDMA_CQPSQ_SRQ_DB_SHADOW_ADDR GENMASK_ULL(63, 6)
641
648
642
649
#define IRDMA_CQPSQ_CQ_CQSIZE GENMASK_ULL(20, 0)
643
650
#define IRDMA_CQPSQ_CQ_CQCTX GENMASK_ULL(62, 0)
···
807
778
#define IRDMAQPC_RQWQESIZE GENMASK_ULL(9, 8)
808
779
#define IRDMAQPC_INSERTL2TAG2 BIT_ULL(11)
809
780
#define IRDMAQPC_LIMIT GENMASK_ULL(13, 12)
781
+
782
+
#define IRDMAQPC_USE_SRQ BIT_ULL(10)
783
+
#define IRDMAQPC_SRQ_ID GENMASK_ULL(15, 0)
784
+
#define IRDMAQPC_PASID GENMASK_ULL(19, 0)
785
+
#define IRDMAQPC_PASID_VALID BIT_ULL(11)
810
786
811
787
#define IRDMAQPC_ECN_EN BIT_ULL(14)
812
788
#define IRDMAQPC_DROPOOOSEG BIT_ULL(15)
+19
-2
drivers/infiniband/hw/irdma/hw.c
+19
-2
drivers/infiniband/hw/irdma/hw.c
···
269
269
struct irdma_sc_qp *qp = NULL;
270
270
struct irdma_qp_host_ctx_info *ctx_info = NULL;
271
271
struct irdma_device *iwdev = rf->iwdev;
272
+
struct irdma_sc_srq *srq;
272
273
unsigned long flags;
273
274
274
275
u32 aeqcnt = 0;
···
321
320
iwqp->last_aeq = info->ae_id;
322
321
spin_unlock_irqrestore(&iwqp->lock, flags);
323
322
ctx_info = &iwqp->ctx_info;
323
+
} else if (info->srq) {
324
+
if (info->ae_id != IRDMA_AE_SRQ_LIMIT)
325
+
continue;
324
326
} else {
325
327
if (info->ae_id != IRDMA_AE_CQ_OPERATION_ERROR &&
326
328
info->ae_id != IRDMA_AE_CQP_DEFERRED_COMPLETE)
···
420
416
iwcq->ibcq.cq_context);
421
417
}
422
418
irdma_cq_rem_ref(&iwcq->ibcq);
419
+
break;
420
+
case IRDMA_AE_SRQ_LIMIT:
421
+
srq = (struct irdma_sc_srq *)(uintptr_t)info->compl_ctx;
422
+
irdma_srq_event(srq);
423
+
break;
424
+
case IRDMA_AE_SRQ_CATASTROPHIC_ERROR:
423
425
break;
424
426
case IRDMA_AE_CQP_DEFERRED_COMPLETE:
425
427
/* Remove completed CQP requests from pending list
···
1849
1839
iwdev->rf->used_qps = find_first_zero_bit(iwdev->rf->allocated_qps,
1850
1840
iwdev->rf->max_qp);
1851
1841
iwdev->rf->used_cqs = find_first_zero_bit(iwdev->rf->allocated_cqs,
1852
-
iwdev->rf->max_cq);
1842
+
iwdev->rf->max_cq);
1843
+
iwdev->rf->used_srqs = find_first_zero_bit(iwdev->rf->allocated_srqs,
1844
+
iwdev->rf->max_srq);
1853
1845
iwdev->rf->used_mrs = find_first_zero_bit(iwdev->rf->allocated_mrs,
1854
1846
iwdev->rf->max_mr);
1855
1847
}
···
2068
2056
rf->allocated_qps = (void *)(rf->mem_rsrc +
2069
2057
(sizeof(struct irdma_arp_entry) * rf->arp_table_size));
2070
2058
rf->allocated_cqs = &rf->allocated_qps[BITS_TO_LONGS(rf->max_qp)];
2071
-
rf->allocated_mrs = &rf->allocated_cqs[BITS_TO_LONGS(rf->max_cq)];
2059
+
rf->allocated_srqs = &rf->allocated_cqs[BITS_TO_LONGS(rf->max_cq)];
2060
+
rf->allocated_mrs = &rf->allocated_srqs[BITS_TO_LONGS(rf->max_srq)];
2072
2061
rf->allocated_pds = &rf->allocated_mrs[BITS_TO_LONGS(rf->max_mr)];
2073
2062
rf->allocated_ahs = &rf->allocated_pds[BITS_TO_LONGS(rf->max_pd)];
2074
2063
rf->allocated_mcgs = &rf->allocated_ahs[BITS_TO_LONGS(rf->max_ah)];
···
2097
2084
rsrc_size += sizeof(unsigned long) * BITS_TO_LONGS(rf->max_qp);
2098
2085
rsrc_size += sizeof(unsigned long) * BITS_TO_LONGS(rf->max_mr);
2099
2086
rsrc_size += sizeof(unsigned long) * BITS_TO_LONGS(rf->max_cq);
2087
+
rsrc_size += sizeof(unsigned long) * BITS_TO_LONGS(rf->max_srq);
2100
2088
rsrc_size += sizeof(unsigned long) * BITS_TO_LONGS(rf->max_pd);
2101
2089
rsrc_size += sizeof(unsigned long) * BITS_TO_LONGS(rf->arp_table_size);
2102
2090
rsrc_size += sizeof(unsigned long) * BITS_TO_LONGS(rf->max_ah);
2103
2091
rsrc_size += sizeof(unsigned long) * BITS_TO_LONGS(rf->max_mcg);
2104
2092
rsrc_size += sizeof(struct irdma_qp **) * rf->max_qp;
2105
2093
rsrc_size += sizeof(struct irdma_cq **) * rf->max_cq;
2094
+
rsrc_size += sizeof(struct irdma_srq **) * rf->max_srq;
2106
2095
2107
2096
return rsrc_size;
2108
2097
}
···
2132
2117
rf->max_qp = rf->sc_dev.hmc_info->hmc_obj[IRDMA_HMC_IW_QP].cnt;
2133
2118
rf->max_mr = rf->sc_dev.hmc_info->hmc_obj[IRDMA_HMC_IW_MR].cnt;
2134
2119
rf->max_cq = rf->sc_dev.hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].cnt;
2120
+
rf->max_srq = rf->sc_dev.hmc_info->hmc_obj[IRDMA_HMC_IW_SRQ].cnt;
2135
2121
rf->max_pd = rf->sc_dev.hw_attrs.max_hw_pds;
2136
2122
rf->arp_table_size = rf->sc_dev.hmc_info->hmc_obj[IRDMA_HMC_IW_ARP].cnt;
2137
2123
rf->max_ah = rf->sc_dev.hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt;
···
2152
2136
set_bit(0, rf->allocated_mrs);
2153
2137
set_bit(0, rf->allocated_qps);
2154
2138
set_bit(0, rf->allocated_cqs);
2139
+
set_bit(0, rf->allocated_srqs);
2155
2140
set_bit(0, rf->allocated_pds);
2156
2141
set_bit(0, rf->allocated_arps);
2157
2142
set_bit(0, rf->allocated_ahs);
+1
drivers/infiniband/hw/irdma/irdma.h
+1
drivers/infiniband/hw/irdma/irdma.h
+11
-1
drivers/infiniband/hw/irdma/main.h
+11
-1
drivers/infiniband/hw/irdma/main.h
···
274
274
u32 max_mr;
275
275
u32 max_qp;
276
276
u32 max_cq;
277
+
u32 max_srq;
278
+
u32 next_srq;
277
279
u32 max_ah;
278
280
u32 next_ah;
279
281
u32 max_mcg;
···
289
287
u32 mr_stagmask;
290
288
u32 used_pds;
291
289
u32 used_cqs;
290
+
u32 used_srqs;
292
291
u32 used_mrs;
293
292
u32 used_qps;
294
293
u32 arp_table_size;
···
301
298
unsigned long *allocated_ws_nodes;
302
299
unsigned long *allocated_qps;
303
300
unsigned long *allocated_cqs;
301
+
unsigned long *allocated_srqs;
304
302
unsigned long *allocated_mrs;
305
303
unsigned long *allocated_pds;
306
304
unsigned long *allocated_mcgs;
···
425
421
return container_of(dev, struct irdma_pci_f, sc_dev);
426
422
}
427
423
424
+
static inline struct irdma_srq *to_iwsrq(struct ib_srq *ibsrq)
425
+
{
426
+
return container_of(ibsrq, struct irdma_srq, ibsrq);
427
+
}
428
+
428
429
/**
429
430
* irdma_alloc_resource - allocate a resource
430
431
* @iwdev: device pointer
···
525
516
void irdma_cq_add_ref(struct ib_cq *ibcq);
526
517
void irdma_cq_rem_ref(struct ib_cq *ibcq);
527
518
void irdma_cq_wq_destroy(struct irdma_pci_f *rf, struct irdma_sc_cq *cq);
528
-
519
+
void irdma_srq_event(struct irdma_sc_srq *srq);
520
+
void irdma_srq_wq_destroy(struct irdma_pci_f *rf, struct irdma_sc_srq *srq);
529
521
void irdma_cleanup_pending_cqp_op(struct irdma_pci_f *rf);
530
522
int irdma_hw_modify_qp(struct irdma_device *iwdev, struct irdma_qp *iwqp,
531
523
struct irdma_modify_qp_info *info, bool wait);
+66
drivers/infiniband/hw/irdma/type.h
+66
drivers/infiniband/hw/irdma/type.h
···
242
242
enum irdma_queue_type {
243
243
IRDMA_QUEUE_TYPE_SQ_RQ = 0,
244
244
IRDMA_QUEUE_TYPE_CQP,
245
+
IRDMA_QUEUE_TYPE_SRQ,
245
246
};
246
247
247
248
struct irdma_sc_dev;
···
733
732
bool cq_resize:1;
734
733
};
735
734
735
+
struct irdma_srq_init_info {
736
+
struct irdma_sc_pd *pd;
737
+
struct irdma_sc_vsi *vsi;
738
+
u64 srq_pa;
739
+
u64 shadow_area_pa;
740
+
u32 first_pm_pbl_idx;
741
+
u32 pasid;
742
+
u32 srq_size;
743
+
u16 srq_limit;
744
+
u8 pasid_valid;
745
+
u8 wqe_size;
746
+
u8 leaf_pbl_size;
747
+
u8 virtual_map;
748
+
u8 tph_en;
749
+
u8 arm_limit_event;
750
+
u8 tph_value;
751
+
u8 pbl_chunk_size;
752
+
struct irdma_srq_uk_init_info srq_uk_init_info;
753
+
};
754
+
755
+
struct irdma_sc_srq {
756
+
struct irdma_sc_dev *dev;
757
+
struct irdma_sc_vsi *vsi;
758
+
struct irdma_sc_pd *pd;
759
+
struct irdma_srq_uk srq_uk;
760
+
void *back_srq;
761
+
u64 srq_pa;
762
+
u64 shadow_area_pa;
763
+
u32 first_pm_pbl_idx;
764
+
u32 pasid;
765
+
u32 hw_srq_size;
766
+
u16 srq_limit;
767
+
u8 pasid_valid;
768
+
u8 leaf_pbl_size;
769
+
u8 virtual_map;
770
+
u8 tph_en;
771
+
u8 arm_limit_event;
772
+
u8 tph_val;
773
+
};
774
+
775
+
struct irdma_modify_srq_info {
776
+
u16 srq_limit;
777
+
u8 arm_limit_event;
778
+
};
779
+
736
780
struct irdma_create_qp_info {
737
781
bool ord_valid:1;
738
782
bool tcp_ctx_valid:1;
···
1084
1038
};
1085
1039
u32 send_cq_num;
1086
1040
u32 rcv_cq_num;
1041
+
u32 srq_id;
1087
1042
u32 rem_endpoint_idx;
1088
1043
u16 stats_idx;
1089
1044
bool srq_valid:1;
···
1384
1337
int irdma_sc_static_hmc_pages_allocated(struct irdma_sc_cqp *cqp, u64 scratch,
1385
1338
u8 hmc_fn_id, bool post_sq,
1386
1339
bool poll_registers);
1340
+
int irdma_sc_srq_init(struct irdma_sc_srq *srq,
1341
+
struct irdma_srq_init_info *info);
1387
1342
1388
1343
void sc_vsi_update_stats(struct irdma_sc_vsi *vsi);
1389
1344
struct cqp_info {
···
1629
1580
struct irdma_dma_mem query_buff_mem;
1630
1581
u64 scratch;
1631
1582
} query_rdma;
1583
+
1584
+
struct {
1585
+
struct irdma_sc_srq *srq;
1586
+
u64 scratch;
1587
+
} srq_create;
1588
+
1589
+
struct {
1590
+
struct irdma_sc_srq *srq;
1591
+
struct irdma_modify_srq_info info;
1592
+
u64 scratch;
1593
+
} srq_modify;
1594
+
1595
+
struct {
1596
+
struct irdma_sc_srq *srq;
1597
+
u64 scratch;
1598
+
} srq_destroy;
1599
+
1632
1600
} u;
1633
1601
};
1634
1602
+157
-5
drivers/infiniband/hw/irdma/uk.c
+157
-5
drivers/infiniband/hw/irdma/uk.c
···
198
198
return wqe;
199
199
}
200
200
201
+
__le64 *irdma_srq_get_next_recv_wqe(struct irdma_srq_uk *srq, u32 *wqe_idx)
202
+
{
203
+
int ret_code;
204
+
__le64 *wqe;
205
+
206
+
if (IRDMA_RING_FULL_ERR(srq->srq_ring))
207
+
return NULL;
208
+
209
+
IRDMA_ATOMIC_RING_MOVE_HEAD(srq->srq_ring, *wqe_idx, ret_code);
210
+
if (ret_code)
211
+
return NULL;
212
+
213
+
if (!*wqe_idx)
214
+
srq->srwqe_polarity = !srq->srwqe_polarity;
215
+
/* rq_wqe_size_multiplier is no of 32 byte quanta in one rq wqe */
216
+
wqe = srq->srq_base[*wqe_idx * (srq->wqe_size_multiplier)].elem;
217
+
218
+
return wqe;
219
+
}
220
+
201
221
/**
202
222
* irdma_qp_get_next_recv_wqe - get next qp's rcv wqe
203
223
* @qp: hw qp ptr
···
333
313
334
314
if (post_sq)
335
315
irdma_uk_qp_post_wr(qp);
316
+
317
+
return 0;
318
+
}
319
+
320
+
/**
321
+
* irdma_uk_srq_post_receive - post a receive wqe to a shared rq
322
+
* @srq: shared rq ptr
323
+
* @info: post rq information
324
+
*/
325
+
int irdma_uk_srq_post_receive(struct irdma_srq_uk *srq,
326
+
struct irdma_post_rq_info *info)
327
+
{
328
+
u32 wqe_idx, i, byte_off;
329
+
u32 addl_frag_cnt;
330
+
__le64 *wqe;
331
+
u64 hdr;
332
+
333
+
if (srq->max_srq_frag_cnt < info->num_sges)
334
+
return -EINVAL;
335
+
336
+
wqe = irdma_srq_get_next_recv_wqe(srq, &wqe_idx);
337
+
if (!wqe)
338
+
return -ENOMEM;
339
+
340
+
addl_frag_cnt = info->num_sges > 1 ? info->num_sges - 1 : 0;
341
+
srq->wqe_ops.iw_set_fragment(wqe, 0, info->sg_list,
342
+
srq->srwqe_polarity);
343
+
344
+
for (i = 1, byte_off = 32; i < info->num_sges; i++) {
345
+
srq->wqe_ops.iw_set_fragment(wqe, byte_off, &info->sg_list[i],
346
+
srq->srwqe_polarity);
347
+
byte_off += 16;
348
+
}
349
+
350
+
/* if not an odd number set valid bit in next fragment */
351
+
if (srq->uk_attrs->hw_rev >= IRDMA_GEN_2 && !(info->num_sges & 0x01) &&
352
+
info->num_sges) {
353
+
srq->wqe_ops.iw_set_fragment(wqe, byte_off, NULL,
354
+
srq->srwqe_polarity);
355
+
if (srq->uk_attrs->hw_rev == IRDMA_GEN_2)
356
+
++addl_frag_cnt;
357
+
}
358
+
359
+
set_64bit_val(wqe, 16, (u64)info->wr_id);
360
+
hdr = FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) |
361
+
FIELD_PREP(IRDMAQPSQ_VALID, srq->srwqe_polarity);
362
+
363
+
dma_wmb(); /* make sure WQE is populated before valid bit is set */
364
+
365
+
set_64bit_val(wqe, 24, hdr);
366
+
367
+
set_64bit_val(srq->shadow_area, 0, (wqe_idx + 1) % srq->srq_ring.size);
336
368
337
369
return 0;
338
370
}
···
1045
973
u64 comp_ctx, qword0, qword2, qword3;
1046
974
__le64 *cqe;
1047
975
struct irdma_qp_uk *qp;
976
+
struct irdma_srq_uk *srq;
977
+
u8 is_srq;
1048
978
struct irdma_ring *pring = NULL;
1049
979
u32 wqe_idx;
1050
980
int ret_code;
···
1120
1046
}
1121
1047
1122
1048
info->q_type = (u8)FIELD_GET(IRDMA_CQ_SQ, qword3);
1049
+
is_srq = (u8)FIELD_GET(IRDMA_CQ_SRQ, qword3);
1123
1050
info->error = (bool)FIELD_GET(IRDMA_CQ_ERROR, qword3);
1124
1051
info->ipv4 = (bool)FIELD_GET(IRDMACQ_IPV4, qword3);
1052
+
get_64bit_val(cqe, 8, &comp_ctx);
1053
+
if (is_srq)
1054
+
get_64bit_val(cqe, 40, (u64 *)&qp);
1055
+
else
1056
+
qp = (struct irdma_qp_uk *)(unsigned long)comp_ctx;
1125
1057
if (info->error) {
1126
1058
info->major_err = FIELD_GET(IRDMA_CQ_MAJERR, qword3);
1127
1059
info->minor_err = FIELD_GET(IRDMA_CQ_MINERR, qword3);
···
1165
1085
info->qp_handle = (irdma_qp_handle)(unsigned long)qp;
1166
1086
info->op_type = (u8)FIELD_GET(IRDMACQ_OP, qword3);
1167
1087
1168
-
if (info->q_type == IRDMA_CQE_QTYPE_RQ) {
1088
+
if (info->q_type == IRDMA_CQE_QTYPE_RQ && is_srq) {
1089
+
srq = qp->srq_uk;
1090
+
1091
+
get_64bit_val(cqe, 8, &info->wr_id);
1092
+
info->bytes_xfered = (u32)FIELD_GET(IRDMACQ_PAYLDLEN, qword0);
1093
+
1094
+
if (qword3 & IRDMACQ_STAG) {
1095
+
info->stag_invalid_set = true;
1096
+
info->inv_stag = (u32)FIELD_GET(IRDMACQ_INVSTAG,
1097
+
qword2);
1098
+
} else {
1099
+
info->stag_invalid_set = false;
1100
+
}
1101
+
IRDMA_RING_MOVE_TAIL(srq->srq_ring);
1102
+
pring = &srq->srq_ring;
1103
+
} else if (info->q_type == IRDMA_CQE_QTYPE_RQ && !is_srq) {
1169
1104
u32 array_idx;
1170
1105
1171
1106
array_idx = wqe_idx / qp->rq_wqe_size_multiplier;
···
1305
1210
}
1306
1211
1307
1212
/**
1308
-
* irdma_qp_round_up - return round up qp wq depth
1213
+
* irdma_round_up_wq - return round up qp wq depth
1309
1214
* @wqdepth: wq depth in quanta to round up
1310
1215
*/
1311
-
static int irdma_qp_round_up(u32 wqdepth)
1216
+
static int irdma_round_up_wq(u32 wqdepth)
1312
1217
{
1313
1218
int scount = 1;
1314
1219
···
1363
1268
{
1364
1269
u32 min_size = (u32)uk_attrs->min_hw_wq_size << shift;
1365
1270
1366
-
*sqdepth = irdma_qp_round_up((sq_size << shift) + IRDMA_SQ_RSVD);
1271
+
*sqdepth = irdma_round_up_wq((sq_size << shift) + IRDMA_SQ_RSVD);
1367
1272
1368
1273
if (*sqdepth < min_size)
1369
1274
*sqdepth = min_size;
···
1385
1290
{
1386
1291
u32 min_size = (u32)uk_attrs->min_hw_wq_size << shift;
1387
1292
1388
-
*rqdepth = irdma_qp_round_up((rq_size << shift) + IRDMA_RQ_RSVD);
1293
+
*rqdepth = irdma_round_up_wq((rq_size << shift) + IRDMA_RQ_RSVD);
1389
1294
1390
1295
if (*rqdepth < min_size)
1391
1296
*rqdepth = min_size;
1392
1297
else if (*rqdepth > uk_attrs->max_hw_rq_quanta)
1298
+
return -EINVAL;
1299
+
1300
+
return 0;
1301
+
}
1302
+
1303
+
/*
1304
+
* irdma_get_srqdepth - get SRQ depth (quanta)
1305
+
* @uk_attrs: qp HW attributes
1306
+
* @srq_size: SRQ size
1307
+
* @shift: shift which determines size of WQE
1308
+
* @srqdepth: depth of SRQ
1309
+
*/
1310
+
int irdma_get_srqdepth(struct irdma_uk_attrs *uk_attrs, u32 srq_size, u8 shift,
1311
+
u32 *srqdepth)
1312
+
{
1313
+
*srqdepth = irdma_round_up_wq((srq_size << shift) + IRDMA_RQ_RSVD);
1314
+
1315
+
if (*srqdepth < ((u32)uk_attrs->min_hw_wq_size << shift))
1316
+
*srqdepth = uk_attrs->min_hw_wq_size << shift;
1317
+
else if (*srqdepth > uk_attrs->max_hw_srq_quanta)
1393
1318
return -EINVAL;
1394
1319
1395
1320
return 0;
···
1448
1333
IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->sq_ring, move_cnt);
1449
1334
IRDMA_RING_MOVE_TAIL_BY_COUNT(qp->sq_ring, move_cnt);
1450
1335
IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->initial_ring, move_cnt);
1336
+
}
1337
+
1338
+
/**
1339
+
* irdma_uk_srq_init - initialize shared qp
1340
+
* @srq: hw srq (user and kernel)
1341
+
* @info: srq initialization info
1342
+
*
1343
+
* Initializes the vars used in both user and kernel mode.
1344
+
* The size of the wqe depends on number of max fragments
1345
+
* allowed. Then size of wqe * the number of wqes should be the
1346
+
* amount of memory allocated for srq.
1347
+
*/
1348
+
int irdma_uk_srq_init(struct irdma_srq_uk *srq,
1349
+
struct irdma_srq_uk_init_info *info)
1350
+
{
1351
+
u8 rqshift;
1352
+
1353
+
srq->uk_attrs = info->uk_attrs;
1354
+
if (info->max_srq_frag_cnt > srq->uk_attrs->max_hw_wq_frags)
1355
+
return -EINVAL;
1356
+
1357
+
irdma_get_wqe_shift(srq->uk_attrs, info->max_srq_frag_cnt, 0, &rqshift);
1358
+
srq->srq_caps = info->srq_caps;
1359
+
srq->srq_base = info->srq;
1360
+
srq->shadow_area = info->shadow_area;
1361
+
srq->srq_id = info->srq_id;
1362
+
srq->srwqe_polarity = 0;
1363
+
srq->srq_size = info->srq_size;
1364
+
srq->wqe_size = rqshift;
1365
+
srq->max_srq_frag_cnt = min(srq->uk_attrs->max_hw_wq_frags,
1366
+
((u32)2 << rqshift) - 1);
1367
+
IRDMA_RING_INIT(srq->srq_ring, srq->srq_size);
1368
+
srq->wqe_size_multiplier = 1 << rqshift;
1369
+
srq->wqe_ops = iw_wqe_uk_ops;
1370
+
1371
+
return 0;
1451
1372
}
1452
1373
1453
1374
/**
···
1612
1461
qp->wqe_ops = iw_wqe_uk_ops_gen_1;
1613
1462
else
1614
1463
qp->wqe_ops = iw_wqe_uk_ops;
1464
+
qp->srq_uk = info->srq_uk;
1615
1465
return ret_code;
1616
1466
}
1617
1467
+41
drivers/infiniband/hw/irdma/user.h
+41
drivers/infiniband/hw/irdma/user.h
···
60
60
IRDMA_GATHER_STATS_BUF_SIZE = 1024,
61
61
IRDMA_MIN_IW_QP_ID = 0,
62
62
IRDMA_MAX_IW_QP_ID = 262143,
63
+
IRDMA_MIN_IW_SRQ_ID = 0,
63
64
IRDMA_MIN_CEQID = 0,
64
65
IRDMA_MAX_CEQID = 1023,
65
66
IRDMA_CEQ_MAX_COUNT = IRDMA_MAX_CEQID + 1,
···
149
148
IRDMA_PUSH_MODE = 8,
150
149
};
151
150
151
+
struct irdma_srq_uk;
152
+
struct irdma_srq_uk_init_info;
152
153
struct irdma_qp_uk;
153
154
struct irdma_cq_uk;
154
155
struct irdma_qp_uk_init_info;
···
304
301
u32 *sq_depth, u8 *sq_shift);
305
302
int irdma_uk_calc_depth_shift_rq(struct irdma_qp_uk_init_info *ukinfo,
306
303
u32 *rq_depth, u8 *rq_shift);
304
+
int irdma_uk_srq_init(struct irdma_srq_uk *srq,
305
+
struct irdma_srq_uk_init_info *info);
306
+
int irdma_uk_srq_post_receive(struct irdma_srq_uk *srq,
307
+
struct irdma_post_rq_info *info);
308
+
309
+
struct irdma_srq_uk {
310
+
u32 srq_caps;
311
+
struct irdma_qp_quanta *srq_base;
312
+
struct irdma_uk_attrs *uk_attrs;
313
+
__le64 *shadow_area;
314
+
struct irdma_ring srq_ring;
315
+
struct irdma_ring initial_ring;
316
+
u32 srq_id;
317
+
u32 srq_size;
318
+
u32 max_srq_frag_cnt;
319
+
struct irdma_wqe_uk_ops wqe_ops;
320
+
u8 srwqe_polarity;
321
+
u8 wqe_size;
322
+
u8 wqe_size_multiplier;
323
+
u8 deferred_flag;
324
+
};
325
+
326
+
struct irdma_srq_uk_init_info {
327
+
struct irdma_qp_quanta *srq;
328
+
struct irdma_uk_attrs *uk_attrs;
329
+
__le64 *shadow_area;
330
+
u64 *srq_wrid_array;
331
+
u32 srq_id;
332
+
u32 srq_caps;
333
+
u32 srq_size;
334
+
u32 max_srq_frag_cnt;
335
+
};
336
+
307
337
struct irdma_sq_uk_wr_trk_info {
308
338
u64 wrid;
309
339
u32 wr_len;
···
381
345
bool destroy_pending:1; /* Indicates the QP is being destroyed */
382
346
void *back_qp;
383
347
u8 dbg_rq_flushed;
348
+
struct irdma_srq_uk *srq_uk;
384
349
u8 sq_flush_seen;
385
350
u8 rq_flush_seen;
386
351
};
···
421
384
u8 rq_shift;
422
385
int abi_ver;
423
386
bool legacy_mode;
387
+
struct irdma_srq_uk *srq_uk;
424
388
};
425
389
426
390
struct irdma_cq_uk_init_info {
···
437
399
__le64 *irdma_qp_get_next_send_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx,
438
400
u16 quanta, u32 total_size,
439
401
struct irdma_post_sq_info *info);
402
+
__le64 *irdma_srq_get_next_recv_wqe(struct irdma_srq_uk *srq, u32 *wqe_idx);
440
403
__le64 *irdma_qp_get_next_recv_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx);
441
404
void irdma_uk_clean_cq(void *q, struct irdma_cq_uk *cq);
442
405
int irdma_nop(struct irdma_qp_uk *qp, u64 wr_id, bool signaled, bool post_sq);
···
449
410
u32 *wqdepth);
450
411
int irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs, u32 rq_size, u8 shift,
451
412
u32 *wqdepth);
413
+
int irdma_get_srqdepth(struct irdma_uk_attrs *uk_attrs, u32 srq_size, u8 shift,
414
+
u32 *srqdepth);
452
415
void irdma_clr_wqes(struct irdma_qp_uk *qp, u32 qp_wqe_idx);
453
416
#endif /* IRDMA_USER_H */
+27
drivers/infiniband/hw/irdma/utils.c
+27
drivers/infiniband/hw/irdma/utils.c
···
697
697
[IRDMA_OP_ADD_LOCAL_MAC_ENTRY] = "Add Local MAC Entry Cmd",
698
698
[IRDMA_OP_DELETE_LOCAL_MAC_ENTRY] = "Delete Local MAC Entry Cmd",
699
699
[IRDMA_OP_CQ_MODIFY] = "CQ Modify Cmd",
700
+
[IRDMA_OP_SRQ_CREATE] = "Create SRQ Cmd",
701
+
[IRDMA_OP_SRQ_MODIFY] = "Modify SRQ Cmd",
702
+
[IRDMA_OP_SRQ_DESTROY] = "Destroy SRQ Cmd",
700
703
};
701
704
702
705
static const struct irdma_cqp_err_info irdma_noncrit_err_list[] = {
···
1168
1165
iwqp->kqp.dma_mem.va = NULL;
1169
1166
kfree(iwqp->kqp.sq_wrid_mem);
1170
1167
kfree(iwqp->kqp.rq_wrid_mem);
1168
+
}
1169
+
1170
+
/**
1171
+
* irdma_srq_wq_destroy - send srq destroy cqp
1172
+
* @rf: RDMA PCI function
1173
+
* @srq: hardware control srq
1174
+
*/
1175
+
void irdma_srq_wq_destroy(struct irdma_pci_f *rf, struct irdma_sc_srq *srq)
1176
+
{
1177
+
struct irdma_cqp_request *cqp_request;
1178
+
struct cqp_cmds_info *cqp_info;
1179
+
1180
+
cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
1181
+
if (!cqp_request)
1182
+
return;
1183
+
1184
+
cqp_info = &cqp_request->info;
1185
+
cqp_info->cqp_cmd = IRDMA_OP_SRQ_DESTROY;
1186
+
cqp_info->post_sq = 1;
1187
+
cqp_info->in.u.srq_destroy.srq = srq;
1188
+
cqp_info->in.u.srq_destroy.scratch = (uintptr_t)cqp_request;
1189
+
1190
+
irdma_handle_cqp_op(rf, cqp_request);
1191
+
irdma_put_cqp_request(&rf->cqp, cqp_request);
1171
1192
}
1172
1193
1173
1194
/**
+473
-2
drivers/infiniband/hw/irdma/verbs.c
+473
-2
drivers/infiniband/hw/irdma/verbs.c
···
59
59
#define HCA_CLOCK_TIMESTAMP_MASK 0x1ffff
60
60
if (hw_attrs->uk_attrs.hw_rev >= IRDMA_GEN_2)
61
61
props->timestamp_mask = HCA_CLOCK_TIMESTAMP_MASK;
62
+
props->max_srq = rf->max_srq - rf->used_srqs;
63
+
props->max_srq_wr = IRDMA_MAX_SRQ_WRS;
64
+
props->max_srq_sge = hw_attrs->uk_attrs.max_hw_wq_frags;
62
65
63
66
return 0;
64
67
}
···
339
336
uresp.comp_mask |= IRDMA_ALLOC_UCTX_USE_RAW_ATTR;
340
337
uresp.min_hw_wq_size = uk_attrs->min_hw_wq_size;
341
338
uresp.comp_mask |= IRDMA_ALLOC_UCTX_MIN_HW_WQ_SIZE;
339
+
uresp.max_hw_srq_quanta = uk_attrs->max_hw_srq_quanta;
340
+
uresp.comp_mask |= IRDMA_ALLOC_UCTX_MAX_HW_SRQ_QUANTA;
342
341
if (ib_copy_to_udata(udata, &uresp,
343
342
min(sizeof(uresp), udata->outlen))) {
344
343
rdma_user_mmap_entry_remove(ucontext->db_mmap_entry);
···
352
347
spin_lock_init(&ucontext->cq_reg_mem_list_lock);
353
348
INIT_LIST_HEAD(&ucontext->qp_reg_mem_list);
354
349
spin_lock_init(&ucontext->qp_reg_mem_list_lock);
350
+
INIT_LIST_HEAD(&ucontext->srq_reg_mem_list);
351
+
spin_lock_init(&ucontext->srq_reg_mem_list_lock);
355
352
356
353
return 0;
357
354
···
578
571
if (iwpbl->pbl_allocated) {
579
572
init_info->virtual_map = true;
580
573
init_info->sq_pa = qpmr->sq_pbl.idx;
581
-
init_info->rq_pa = qpmr->rq_pbl.idx;
574
+
/* Need to use contiguous buffer for RQ of QP
575
+
* in case it is associated with SRQ.
576
+
*/
577
+
init_info->rq_pa = init_info->qp_uk_init_info.srq_uk ?
578
+
qpmr->rq_pa : qpmr->rq_pbl.idx;
582
579
} else {
583
580
init_info->sq_pa = qpmr->sq_pbl.addr;
584
581
init_info->rq_pa = qpmr->rq_pbl.addr;
···
951
940
struct irdma_uk_attrs *uk_attrs = &dev->hw_attrs.uk_attrs;
952
941
struct irdma_qp_init_info init_info = {};
953
942
struct irdma_qp_host_ctx_info *ctx_info;
943
+
struct irdma_srq *iwsrq;
944
+
bool srq_valid = false;
945
+
u32 srq_id = 0;
946
+
947
+
if (init_attr->srq) {
948
+
iwsrq = to_iwsrq(init_attr->srq);
949
+
srq_valid = true;
950
+
srq_id = iwsrq->srq_num;
951
+
init_attr->cap.max_recv_sge = uk_attrs->max_hw_wq_frags;
952
+
init_attr->cap.max_recv_wr = 4;
953
+
init_info.qp_uk_init_info.srq_uk = &iwsrq->sc_srq.srq_uk;
954
+
}
954
955
955
956
err_code = irdma_validate_qp_attrs(init_attr, iwdev);
956
957
if (err_code)
···
1069
1046
}
1070
1047
1071
1048
ctx_info = &iwqp->ctx_info;
1049
+
ctx_info->srq_valid = srq_valid;
1050
+
ctx_info->srq_id = srq_id;
1072
1051
ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id;
1073
1052
ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id;
1074
1053
···
1196
1171
init_attr->qp_context = iwqp->ibqp.qp_context;
1197
1172
init_attr->send_cq = iwqp->ibqp.send_cq;
1198
1173
init_attr->recv_cq = iwqp->ibqp.recv_cq;
1174
+
init_attr->srq = iwqp->ibqp.srq;
1199
1175
init_attr->cap = attr->cap;
1200
1176
1201
1177
return 0;
···
1860
1834
}
1861
1835
1862
1836
/**
1837
+
* irdma_srq_free_rsrc - free up resources for srq
1838
+
* @rf: RDMA PCI function
1839
+
* @iwsrq: srq ptr
1840
+
*/
1841
+
static void irdma_srq_free_rsrc(struct irdma_pci_f *rf, struct irdma_srq *iwsrq)
1842
+
{
1843
+
struct irdma_sc_srq *srq = &iwsrq->sc_srq;
1844
+
1845
+
if (!iwsrq->user_mode) {
1846
+
dma_free_coherent(rf->sc_dev.hw->device, iwsrq->kmem.size,
1847
+
iwsrq->kmem.va, iwsrq->kmem.pa);
1848
+
iwsrq->kmem.va = NULL;
1849
+
}
1850
+
1851
+
irdma_free_rsrc(rf, rf->allocated_srqs, srq->srq_uk.srq_id);
1852
+
}
1853
+
1854
+
/**
1863
1855
* irdma_cq_free_rsrc - free up resources for cq
1864
1856
* @rf: RDMA PCI function
1865
1857
* @iwcq: cq ptr
···
1938
1894
}
1939
1895
1940
1896
return cnt;
1897
+
}
1898
+
1899
+
/**
1900
+
* irdma_destroy_srq - destroy srq
1901
+
* @ibsrq: srq pointer
1902
+
* @udata: user data
1903
+
*/
1904
+
static int irdma_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
1905
+
{
1906
+
struct irdma_device *iwdev = to_iwdev(ibsrq->device);
1907
+
struct irdma_srq *iwsrq = to_iwsrq(ibsrq);
1908
+
struct irdma_sc_srq *srq = &iwsrq->sc_srq;
1909
+
1910
+
irdma_srq_wq_destroy(iwdev->rf, srq);
1911
+
irdma_srq_free_rsrc(iwdev->rf, iwsrq);
1912
+
return 0;
1941
1913
}
1942
1914
1943
1915
/**
···
2142
2082
kfree(cq_buf);
2143
2083
2144
2084
return ret;
2085
+
}
2086
+
2087
+
/**
2088
+
* irdma_srq_event - event notification for srq limit
2089
+
* @srq: shared srq struct
2090
+
*/
2091
+
void irdma_srq_event(struct irdma_sc_srq *srq)
2092
+
{
2093
+
struct irdma_srq *iwsrq = container_of(srq, struct irdma_srq, sc_srq);
2094
+
struct ib_srq *ibsrq = &iwsrq->ibsrq;
2095
+
struct ib_event event;
2096
+
2097
+
srq->srq_limit = 0;
2098
+
2099
+
if (!ibsrq->event_handler)
2100
+
return;
2101
+
2102
+
event.device = ibsrq->device;
2103
+
event.element.port_num = 1;
2104
+
event.element.srq = ibsrq;
2105
+
event.event = IB_EVENT_SRQ_LIMIT_REACHED;
2106
+
ibsrq->event_handler(&event, ibsrq->srq_context);
2107
+
}
2108
+
2109
+
/**
2110
+
* irdma_modify_srq - modify srq request
2111
+
* @ibsrq: srq's pointer for modify
2112
+
* @attr: access attributes
2113
+
* @attr_mask: state mask
2114
+
* @udata: user data
2115
+
*/
2116
+
static int irdma_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
2117
+
enum ib_srq_attr_mask attr_mask,
2118
+
struct ib_udata *udata)
2119
+
{
2120
+
struct irdma_device *iwdev = to_iwdev(ibsrq->device);
2121
+
struct irdma_srq *iwsrq = to_iwsrq(ibsrq);
2122
+
struct irdma_cqp_request *cqp_request;
2123
+
struct irdma_pci_f *rf = iwdev->rf;
2124
+
struct irdma_modify_srq_info *info;
2125
+
struct cqp_cmds_info *cqp_info;
2126
+
int status;
2127
+
2128
+
if (attr_mask & IB_SRQ_MAX_WR)
2129
+
return -EINVAL;
2130
+
2131
+
if (!(attr_mask & IB_SRQ_LIMIT))
2132
+
return 0;
2133
+
2134
+
if (attr->srq_limit > iwsrq->sc_srq.srq_uk.srq_size)
2135
+
return -EINVAL;
2136
+
2137
+
/* Execute this cqp op synchronously, so we can update srq_limit
2138
+
* upon successful completion.
2139
+
*/
2140
+
cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
2141
+
if (!cqp_request)
2142
+
return -ENOMEM;
2143
+
2144
+
cqp_info = &cqp_request->info;
2145
+
info = &cqp_info->in.u.srq_modify.info;
2146
+
info->srq_limit = attr->srq_limit;
2147
+
if (info->srq_limit > 0xFFF)
2148
+
info->srq_limit = 0xFFF;
2149
+
info->arm_limit_event = 1;
2150
+
2151
+
cqp_info->cqp_cmd = IRDMA_OP_SRQ_MODIFY;
2152
+
cqp_info->post_sq = 1;
2153
+
cqp_info->in.u.srq_modify.srq = &iwsrq->sc_srq;
2154
+
cqp_info->in.u.srq_modify.scratch = (uintptr_t)cqp_request;
2155
+
status = irdma_handle_cqp_op(rf, cqp_request);
2156
+
irdma_put_cqp_request(&rf->cqp, cqp_request);
2157
+
if (status)
2158
+
return status;
2159
+
2160
+
iwsrq->sc_srq.srq_limit = info->srq_limit;
2161
+
2162
+
return 0;
2163
+
}
2164
+
2165
+
static int irdma_setup_umode_srq(struct irdma_device *iwdev,
2166
+
struct irdma_srq *iwsrq,
2167
+
struct irdma_srq_init_info *info,
2168
+
struct ib_udata *udata)
2169
+
{
2170
+
#define IRDMA_CREATE_SRQ_MIN_REQ_LEN \
2171
+
offsetofend(struct irdma_create_srq_req, user_shadow_area)
2172
+
struct irdma_create_srq_req req = {};
2173
+
struct irdma_ucontext *ucontext;
2174
+
struct irdma_srq_mr *srqmr;
2175
+
struct irdma_pbl *iwpbl;
2176
+
unsigned long flags;
2177
+
2178
+
iwsrq->user_mode = true;
2179
+
ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext,
2180
+
ibucontext);
2181
+
2182
+
if (udata->inlen < IRDMA_CREATE_SRQ_MIN_REQ_LEN)
2183
+
return -EINVAL;
2184
+
2185
+
if (ib_copy_from_udata(&req, udata,
2186
+
min(sizeof(req), udata->inlen)))
2187
+
return -EFAULT;
2188
+
2189
+
spin_lock_irqsave(&ucontext->srq_reg_mem_list_lock, flags);
2190
+
iwpbl = irdma_get_pbl((unsigned long)req.user_srq_buf,
2191
+
&ucontext->srq_reg_mem_list);
2192
+
spin_unlock_irqrestore(&ucontext->srq_reg_mem_list_lock, flags);
2193
+
if (!iwpbl)
2194
+
return -EPROTO;
2195
+
2196
+
iwsrq->iwpbl = iwpbl;
2197
+
srqmr = &iwpbl->srq_mr;
2198
+
2199
+
if (iwpbl->pbl_allocated) {
2200
+
info->virtual_map = true;
2201
+
info->pbl_chunk_size = 1;
2202
+
info->first_pm_pbl_idx = srqmr->srq_pbl.idx;
2203
+
info->leaf_pbl_size = 1;
2204
+
} else {
2205
+
info->srq_pa = srqmr->srq_pbl.addr;
2206
+
}
2207
+
info->shadow_area_pa = srqmr->shadow;
2208
+
2209
+
return 0;
2210
+
}
2211
+
2212
+
static int irdma_setup_kmode_srq(struct irdma_device *iwdev,
2213
+
struct irdma_srq *iwsrq,
2214
+
struct irdma_srq_init_info *info, u32 depth,
2215
+
u8 shift)
2216
+
{
2217
+
struct irdma_srq_uk_init_info *ukinfo = &info->srq_uk_init_info;
2218
+
struct irdma_dma_mem *mem = &iwsrq->kmem;
2219
+
u32 size, ring_size;
2220
+
2221
+
ring_size = depth * IRDMA_QP_WQE_MIN_SIZE;
2222
+
size = ring_size + (IRDMA_SHADOW_AREA_SIZE << 3);
2223
+
2224
+
mem->size = ALIGN(size, 256);
2225
+
mem->va = dma_alloc_coherent(iwdev->rf->hw.device, mem->size,
2226
+
&mem->pa, GFP_KERNEL);
2227
+
if (!mem->va)
2228
+
return -ENOMEM;
2229
+
2230
+
ukinfo->srq = mem->va;
2231
+
ukinfo->srq_size = depth >> shift;
2232
+
ukinfo->shadow_area = mem->va + ring_size;
2233
+
2234
+
info->shadow_area_pa = info->srq_pa + ring_size;
2235
+
info->srq_pa = mem->pa;
2236
+
2237
+
return 0;
2238
+
}
2239
+
2240
+
/**
2241
+
* irdma_create_srq - create srq
2242
+
* @ibsrq: ib's srq pointer
2243
+
* @initattrs: attributes for srq
2244
+
* @udata: user data for create srq
2245
+
*/
2246
+
static int irdma_create_srq(struct ib_srq *ibsrq,
2247
+
struct ib_srq_init_attr *initattrs,
2248
+
struct ib_udata *udata)
2249
+
{
2250
+
struct irdma_device *iwdev = to_iwdev(ibsrq->device);
2251
+
struct ib_srq_attr *attr = &initattrs->attr;
2252
+
struct irdma_pd *iwpd = to_iwpd(ibsrq->pd);
2253
+
struct irdma_srq *iwsrq = to_iwsrq(ibsrq);
2254
+
struct irdma_srq_uk_init_info *ukinfo;
2255
+
struct irdma_cqp_request *cqp_request;
2256
+
struct irdma_srq_init_info info = {};
2257
+
struct irdma_pci_f *rf = iwdev->rf;
2258
+
struct irdma_uk_attrs *uk_attrs;
2259
+
struct cqp_cmds_info *cqp_info;
2260
+
int err_code = 0;
2261
+
u32 depth;
2262
+
u8 shift;
2263
+
2264
+
uk_attrs = &rf->sc_dev.hw_attrs.uk_attrs;
2265
+
ukinfo = &info.srq_uk_init_info;
2266
+
2267
+
if (initattrs->srq_type != IB_SRQT_BASIC)
2268
+
return -EOPNOTSUPP;
2269
+
2270
+
if (!(uk_attrs->feature_flags & IRDMA_FEATURE_SRQ) ||
2271
+
attr->max_sge > uk_attrs->max_hw_wq_frags)
2272
+
return -EINVAL;
2273
+
2274
+
refcount_set(&iwsrq->refcnt, 1);
2275
+
spin_lock_init(&iwsrq->lock);
2276
+
err_code = irdma_alloc_rsrc(rf, rf->allocated_srqs, rf->max_srq,
2277
+
&iwsrq->srq_num, &rf->next_srq);
2278
+
if (err_code)
2279
+
return err_code;
2280
+
2281
+
ukinfo->max_srq_frag_cnt = attr->max_sge;
2282
+
ukinfo->uk_attrs = uk_attrs;
2283
+
ukinfo->srq_id = iwsrq->srq_num;
2284
+
2285
+
irdma_get_wqe_shift(ukinfo->uk_attrs, ukinfo->max_srq_frag_cnt, 0,
2286
+
&shift);
2287
+
2288
+
err_code = irdma_get_srqdepth(ukinfo->uk_attrs, attr->max_wr,
2289
+
shift, &depth);
2290
+
if (err_code)
2291
+
return err_code;
2292
+
2293
+
/* Actual SRQ size in WRs for ring and HW */
2294
+
ukinfo->srq_size = depth >> shift;
2295
+
2296
+
/* Max postable WRs to SRQ */
2297
+
iwsrq->max_wr = (depth - IRDMA_RQ_RSVD) >> shift;
2298
+
attr->max_wr = iwsrq->max_wr;
2299
+
2300
+
if (udata)
2301
+
err_code = irdma_setup_umode_srq(iwdev, iwsrq, &info, udata);
2302
+
else
2303
+
err_code = irdma_setup_kmode_srq(iwdev, iwsrq, &info, depth,
2304
+
shift);
2305
+
2306
+
if (err_code)
2307
+
goto free_rsrc;
2308
+
2309
+
info.vsi = &iwdev->vsi;
2310
+
info.pd = &iwpd->sc_pd;
2311
+
2312
+
err_code = irdma_sc_srq_init(&iwsrq->sc_srq, &info);
2313
+
if (err_code)
2314
+
goto free_dmem;
2315
+
2316
+
cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
2317
+
if (!cqp_request) {
2318
+
err_code = -ENOMEM;
2319
+
goto free_dmem;
2320
+
}
2321
+
2322
+
cqp_info = &cqp_request->info;
2323
+
cqp_info->cqp_cmd = IRDMA_OP_SRQ_CREATE;
2324
+
cqp_info->post_sq = 1;
2325
+
cqp_info->in.u.srq_create.srq = &iwsrq->sc_srq;
2326
+
cqp_info->in.u.srq_create.scratch = (uintptr_t)cqp_request;
2327
+
err_code = irdma_handle_cqp_op(rf, cqp_request);
2328
+
irdma_put_cqp_request(&rf->cqp, cqp_request);
2329
+
if (err_code)
2330
+
goto free_dmem;
2331
+
2332
+
if (udata) {
2333
+
struct irdma_create_srq_resp resp = {};
2334
+
2335
+
resp.srq_id = iwsrq->srq_num;
2336
+
resp.srq_size = ukinfo->srq_size;
2337
+
if (ib_copy_to_udata(udata, &resp,
2338
+
min(sizeof(resp), udata->outlen))) {
2339
+
err_code = -EPROTO;
2340
+
goto srq_destroy;
2341
+
}
2342
+
}
2343
+
2344
+
return 0;
2345
+
2346
+
srq_destroy:
2347
+
irdma_srq_wq_destroy(rf, &iwsrq->sc_srq);
2348
+
2349
+
free_dmem:
2350
+
if (!iwsrq->user_mode)
2351
+
dma_free_coherent(rf->hw.device, iwsrq->kmem.size,
2352
+
iwsrq->kmem.va, iwsrq->kmem.pa);
2353
+
free_rsrc:
2354
+
irdma_free_rsrc(rf, rf->allocated_srqs, iwsrq->srq_num);
2355
+
return err_code;
2356
+
}
2357
+
2358
+
/**
2359
+
* irdma_query_srq - get SRQ attributes
2360
+
* @ibsrq: the SRQ to query
2361
+
* @attr: the attributes of the SRQ
2362
+
*/
2363
+
static int irdma_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr)
2364
+
{
2365
+
struct irdma_srq *iwsrq = to_iwsrq(ibsrq);
2366
+
2367
+
attr->max_wr = iwsrq->max_wr;
2368
+
attr->max_sge = iwsrq->sc_srq.srq_uk.max_srq_frag_cnt;
2369
+
attr->srq_limit = iwsrq->sc_srq.srq_limit;
2370
+
2371
+
return 0;
2145
2372
}
2146
2373
2147
2374
static inline int cq_validate_flags(u32 flags, u8 hw_rev)
···
2883
2536
struct irdma_mr *iwmr = iwpbl->iwmr;
2884
2537
struct irdma_qp_mr *qpmr = &iwpbl->qp_mr;
2885
2538
struct irdma_cq_mr *cqmr = &iwpbl->cq_mr;
2539
+
struct irdma_srq_mr *srqmr = &iwpbl->srq_mr;
2886
2540
struct irdma_hmc_pble *hmc_p;
2887
2541
u64 *arr = iwmr->pgaddrmem;
2888
2542
u32 pg_size, total;
···
2903
2555
total = req->sq_pages + req->rq_pages;
2904
2556
hmc_p = &qpmr->sq_pbl;
2905
2557
qpmr->shadow = (dma_addr_t)arr[total];
2906
-
2558
+
/* Need to use physical address for RQ of QP
2559
+
* in case it is associated with SRQ.
2560
+
*/
2561
+
qpmr->rq_pa = (dma_addr_t)arr[req->sq_pages];
2907
2562
if (lvl) {
2908
2563
ret = irdma_check_mem_contiguous(arr, req->sq_pages,
2909
2564
pg_size);
···
2926
2575
hmc_p->addr = arr[req->sq_pages];
2927
2576
}
2928
2577
break;
2578
+
case IRDMA_MEMREG_TYPE_SRQ:
2579
+
hmc_p = &srqmr->srq_pbl;
2580
+
srqmr->shadow = (dma_addr_t)arr[req->rq_pages];
2581
+
if (lvl)
2582
+
ret = irdma_check_mem_contiguous(arr, req->rq_pages,
2583
+
pg_size);
2584
+
2585
+
if (!ret)
2586
+
hmc_p->idx = palloc->level1.idx;
2587
+
else
2588
+
hmc_p->addr = arr[0];
2589
+
break;
2929
2590
case IRDMA_MEMREG_TYPE_CQ:
2930
2591
hmc_p = &cqmr->cq_pbl;
2931
2592
···
3408
3045
return 0;
3409
3046
}
3410
3047
3048
+
static int irdma_reg_user_mr_type_srq(struct irdma_mem_reg_req req,
3049
+
struct ib_udata *udata,
3050
+
struct irdma_mr *iwmr)
3051
+
{
3052
+
struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device);
3053
+
struct irdma_pbl *iwpbl = &iwmr->iwpbl;
3054
+
struct irdma_ucontext *ucontext;
3055
+
unsigned long flags;
3056
+
u32 total;
3057
+
int err;
3058
+
u8 lvl;
3059
+
3060
+
total = req.rq_pages + IRDMA_SHADOW_PGCNT;
3061
+
if (total > iwmr->page_cnt)
3062
+
return -EINVAL;
3063
+
3064
+
lvl = req.rq_pages > 1 ? PBLE_LEVEL_1 : PBLE_LEVEL_0;
3065
+
err = irdma_handle_q_mem(iwdev, &req, iwpbl, lvl);
3066
+
if (err)
3067
+
return err;
3068
+
3069
+
ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext,
3070
+
ibucontext);
3071
+
spin_lock_irqsave(&ucontext->srq_reg_mem_list_lock, flags);
3072
+
list_add_tail(&iwpbl->list, &ucontext->srq_reg_mem_list);
3073
+
iwpbl->on_list = true;
3074
+
spin_unlock_irqrestore(&ucontext->srq_reg_mem_list_lock, flags);
3075
+
3076
+
return 0;
3077
+
}
3078
+
3411
3079
static int irdma_reg_user_mr_type_cq(struct irdma_mem_reg_req req,
3412
3080
struct ib_udata *udata,
3413
3081
struct irdma_mr *iwmr)
···
3526
3132
switch (req.reg_type) {
3527
3133
case IRDMA_MEMREG_TYPE_QP:
3528
3134
err = irdma_reg_user_mr_type_qp(req, udata, iwmr);
3135
+
if (err)
3136
+
goto error;
3137
+
3138
+
break;
3139
+
case IRDMA_MEMREG_TYPE_SRQ:
3140
+
err = irdma_reg_user_mr_type_srq(req, udata, iwmr);
3529
3141
if (err)
3530
3142
goto error;
3531
3143
···
3855
3455
}
3856
3456
spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags);
3857
3457
break;
3458
+
case IRDMA_MEMREG_TYPE_SRQ:
3459
+
spin_lock_irqsave(&ucontext->srq_reg_mem_list_lock, flags);
3460
+
if (iwpbl->on_list) {
3461
+
iwpbl->on_list = false;
3462
+
list_del(&iwpbl->list);
3463
+
}
3464
+
spin_unlock_irqrestore(&ucontext->srq_reg_mem_list_lock, flags);
3465
+
break;
3858
3466
default:
3859
3467
break;
3860
3468
}
···
4082
3674
}
4083
3675
4084
3676
/**
3677
+
* irdma_post_srq_recv - post receive wr for kernel application
3678
+
* @ibsrq: ib srq pointer
3679
+
* @ib_wr: work request for receive
3680
+
* @bad_wr: bad wr caused an error
3681
+
*/
3682
+
static int irdma_post_srq_recv(struct ib_srq *ibsrq,
3683
+
const struct ib_recv_wr *ib_wr,
3684
+
const struct ib_recv_wr **bad_wr)
3685
+
{
3686
+
struct irdma_srq *iwsrq = to_iwsrq(ibsrq);
3687
+
struct irdma_srq_uk *uksrq = &iwsrq->sc_srq.srq_uk;
3688
+
struct irdma_post_rq_info post_recv = {};
3689
+
unsigned long flags;
3690
+
int err = 0;
3691
+
3692
+
spin_lock_irqsave(&iwsrq->lock, flags);
3693
+
while (ib_wr) {
3694
+
if (ib_wr->num_sge > uksrq->max_srq_frag_cnt) {
3695
+
err = -EINVAL;
3696
+
goto out;
3697
+
}
3698
+
post_recv.num_sges = ib_wr->num_sge;
3699
+
post_recv.wr_id = ib_wr->wr_id;
3700
+
post_recv.sg_list = ib_wr->sg_list;
3701
+
err = irdma_uk_srq_post_receive(uksrq, &post_recv);
3702
+
if (err)
3703
+
goto out;
3704
+
3705
+
ib_wr = ib_wr->next;
3706
+
}
3707
+
3708
+
out:
3709
+
spin_unlock_irqrestore(&iwsrq->lock, flags);
3710
+
3711
+
if (err)
3712
+
*bad_wr = ib_wr;
3713
+
3714
+
return err;
3715
+
}
3716
+
3717
+
/**
4085
3718
* irdma_post_recv - post receive wr for kernel application
4086
3719
* @ibqp: ib qp pointer
4087
3720
* @ib_wr: work request for receive
···
4140
3691
4141
3692
iwqp = to_iwqp(ibqp);
4142
3693
ukqp = &iwqp->sc_qp.qp_uk;
3694
+
3695
+
if (ukqp->srq_uk) {
3696
+
*bad_wr = ib_wr;
3697
+
return -EINVAL;
3698
+
}
4143
3699
4144
3700
spin_lock_irqsave(&iwqp->lock, flags);
4145
3701
while (ib_wr) {
···
5234
4780
return IB_LINK_LAYER_ETHERNET;
5235
4781
}
5236
4782
4783
+
static const struct ib_device_ops irdma_gen1_dev_ops = {
4784
+
.dealloc_driver = irdma_ib_dealloc_device,
4785
+
};
4786
+
4787
+
static const struct ib_device_ops irdma_gen3_dev_ops = {
4788
+
.create_srq = irdma_create_srq,
4789
+
.destroy_srq = irdma_destroy_srq,
4790
+
.modify_srq = irdma_modify_srq,
4791
+
.post_srq_recv = irdma_post_srq_recv,
4792
+
.query_srq = irdma_query_srq,
4793
+
};
4794
+
5237
4795
static const struct ib_device_ops irdma_roce_dev_ops = {
5238
4796
.attach_mcast = irdma_attach_mcast,
5239
4797
.create_ah = irdma_create_ah,
···
5316
4850
INIT_RDMA_OBJ_SIZE(ib_cq, irdma_cq, ibcq),
5317
4851
INIT_RDMA_OBJ_SIZE(ib_mw, irdma_mr, ibmw),
5318
4852
INIT_RDMA_OBJ_SIZE(ib_qp, irdma_qp, ibqp),
4853
+
INIT_RDMA_OBJ_SIZE(ib_srq, irdma_srq, ibsrq),
5319
4854
};
5320
4855
5321
4856
/**
···
5364
4897
iwdev->ibdev.num_comp_vectors = iwdev->rf->ceqs_count;
5365
4898
iwdev->ibdev.dev.parent = &pcidev->dev;
5366
4899
ib_set_device_ops(&iwdev->ibdev, &irdma_dev_ops);
4900
+
if (iwdev->rf->rdma_ver == IRDMA_GEN_1)
4901
+
ib_set_device_ops(&iwdev->ibdev, &irdma_gen1_dev_ops);
4902
+
if (iwdev->rf->rdma_ver >= IRDMA_GEN_3)
4903
+
ib_set_device_ops(&iwdev->ibdev, &irdma_gen3_dev_ops);
5367
4904
}
5368
4905
5369
4906
/**
+25
drivers/infiniband/hw/irdma/verbs.h
+25
drivers/infiniband/hw/irdma/verbs.h
···
8
8
9
9
#define IRDMA_PKEY_TBL_SZ 1
10
10
#define IRDMA_DEFAULT_PKEY 0xFFFF
11
+
#define IRDMA_SHADOW_PGCNT 1
11
12
12
13
struct irdma_ucontext {
13
14
struct ib_ucontext ibucontext;
···
18
17
spinlock_t cq_reg_mem_list_lock; /* protect CQ memory list */
19
18
struct list_head qp_reg_mem_list;
20
19
spinlock_t qp_reg_mem_list_lock; /* protect QP memory list */
20
+
struct list_head srq_reg_mem_list;
21
+
spinlock_t srq_reg_mem_list_lock; /* protect SRQ memory list */
21
22
int abi_ver;
22
23
u8 legacy_mode : 1;
23
24
u8 use_raw_attrs : 1;
···
68
65
bool split;
69
66
};
70
67
68
+
struct irdma_srq_mr {
69
+
struct irdma_hmc_pble srq_pbl;
70
+
dma_addr_t shadow;
71
+
};
72
+
71
73
struct irdma_qp_mr {
72
74
struct irdma_hmc_pble sq_pbl;
73
75
struct irdma_hmc_pble rq_pbl;
74
76
dma_addr_t shadow;
77
+
dma_addr_t rq_pa;
75
78
struct page *sq_page;
76
79
};
77
80
···
94
85
union {
95
86
struct irdma_qp_mr qp_mr;
96
87
struct irdma_cq_mr cq_mr;
88
+
struct irdma_srq_mr srq_mr;
97
89
};
98
90
99
91
bool pbl_allocated:1;
···
120
110
u64 len;
121
111
u64 pgaddrmem[IRDMA_MAX_SAVED_PHY_PGADDR];
122
112
struct irdma_pbl iwpbl;
113
+
};
114
+
115
+
struct irdma_srq {
116
+
struct ib_srq ibsrq;
117
+
struct irdma_sc_srq sc_srq __aligned(64);
118
+
struct irdma_dma_mem kmem;
119
+
u64 *srq_wrid_mem;
120
+
refcount_t refcnt;
121
+
spinlock_t lock; /* for poll srq */
122
+
struct irdma_pbl *iwpbl;
123
+
struct irdma_sge *sg_list;
124
+
u16 srq_head;
125
+
u32 srq_num;
126
+
u32 max_wr;
127
+
bool user_mode:1;
123
128
};
124
129
125
130
struct irdma_cq {
+14
-1
include/uapi/rdma/irdma-abi.h
+14
-1
include/uapi/rdma/irdma-abi.h
···
20
20
IRDMA_MEMREG_TYPE_MEM = 0,
21
21
IRDMA_MEMREG_TYPE_QP = 1,
22
22
IRDMA_MEMREG_TYPE_CQ = 2,
23
+
IRDMA_MEMREG_TYPE_SRQ = 3,
23
24
};
24
25
25
26
enum {
26
27
IRDMA_ALLOC_UCTX_USE_RAW_ATTR = 1 << 0,
27
28
IRDMA_ALLOC_UCTX_MIN_HW_WQ_SIZE = 1 << 1,
29
+
IRDMA_ALLOC_UCTX_MAX_HW_SRQ_QUANTA = 1 << 2,
28
30
IRDMA_SUPPORT_WQE_FORMAT_V2 = 1 << 3,
29
31
};
30
32
···
57
55
__u8 rsvd2;
58
56
__aligned_u64 comp_mask;
59
57
__u16 min_hw_wq_size;
60
-
__u8 rsvd3[6];
58
+
__u32 max_hw_srq_quanta;
59
+
__u8 rsvd3[2];
61
60
};
62
61
63
62
struct irdma_alloc_pd_resp {
···
73
70
struct irdma_create_cq_req {
74
71
__aligned_u64 user_cq_buf;
75
72
__aligned_u64 user_shadow_area;
73
+
};
74
+
75
+
struct irdma_create_srq_req {
76
+
__aligned_u64 user_srq_buf;
77
+
__aligned_u64 user_shadow_area;
78
+
};
79
+
80
+
struct irdma_create_srq_resp {
81
+
__u32 srq_id;
82
+
__u32 srq_size;
76
83
};
77
84
78
85
struct irdma_create_qp_req {