tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

RDMA/erdma: Allocate doorbell records from dma pool

Currently, the 8 byte doorbell record is allocated along with the queue
buffer, which may result in waste of dma space when the queue buffer is
page aligned. To address this issue, we introduce a dma pool named
db_pool and allocate doorbell record from it.

Reviewed-by: Cheng Xu <chengyou@linux.alibaba.com>
Signed-off-by: Boshi Yu <boshiyu@linux.alibaba.com>
Link: https://lore.kernel.org/r/20240311113821.22482-2-boshiyu@alibaba-inc.com
Signed-off-by: Leon Romanovsky <leon@kernel.org>

authored by

Boshi Yu and committed by
Leon Romanovsky f0697bf0 39cd87c4

+167 -101
+4 -3
drivers/infiniband/hw/erdma/erdma.h
··· 34 34 35 35 void __iomem *db; 36 36 u64 *db_record; 37 + dma_addr_t db_record_dma_addr; 37 38 }; 38 39 39 40 struct erdma_cmdq_sq { ··· 50 49 u16 wqebb_cnt; 51 50 52 51 u64 *db_record; 52 + dma_addr_t db_record_dma_addr; 53 53 }; 54 54 55 55 struct erdma_cmdq_cq { ··· 64 62 u32 cmdsn; 65 63 66 64 u64 *db_record; 65 + dma_addr_t db_record_dma_addr; 67 66 68 67 atomic64_t armed_num; 69 68 }; ··· 180 177 ERDMA_RES_CNT = 2, 181 178 }; 182 179 183 - #define ERDMA_EXTRA_BUFFER_SIZE ERDMA_DB_SIZE 184 - #define WARPPED_BUFSIZE(size) ((size) + ERDMA_EXTRA_BUFFER_SIZE) 185 - 186 180 struct erdma_dev { 187 181 struct ib_device ibdev; 188 182 struct net_device *netdev; ··· 213 213 atomic_t num_ctx; 214 214 struct list_head cep_list; 215 215 216 + struct dma_pool *db_pool; 216 217 struct dma_pool *resp_pool; 217 218 }; 218 219
+63 -39
drivers/infiniband/hw/erdma/erdma_cmdq.c
··· 89 89 { 90 90 struct erdma_cmdq *cmdq = &dev->cmdq; 91 91 struct erdma_cmdq_sq *sq = &cmdq->sq; 92 - u32 buf_size; 93 92 94 93 sq->wqebb_cnt = SQEBB_COUNT(ERDMA_CMDQ_SQE_SIZE); 95 94 sq->depth = cmdq->max_outstandings * sq->wqebb_cnt; 96 95 97 - buf_size = sq->depth << SQEBB_SHIFT; 98 - 99 - sq->qbuf = 100 - dma_alloc_coherent(&dev->pdev->dev, WARPPED_BUFSIZE(buf_size), 101 - &sq->qbuf_dma_addr, GFP_KERNEL); 96 + sq->qbuf = dma_alloc_coherent(&dev->pdev->dev, sq->depth << SQEBB_SHIFT, 97 + &sq->qbuf_dma_addr, GFP_KERNEL); 102 98 if (!sq->qbuf) 103 99 return -ENOMEM; 104 100 105 - sq->db_record = (u64 *)(sq->qbuf + buf_size); 101 + sq->db_record = dma_pool_zalloc(dev->db_pool, GFP_KERNEL, 102 + &sq->db_record_dma_addr); 103 + if (!sq->db_record) 104 + goto err_out; 106 105 107 106 spin_lock_init(&sq->lock); 108 107 ··· 111 112 lower_32_bits(sq->qbuf_dma_addr)); 112 113 erdma_reg_write32(dev, ERDMA_REGS_CMDQ_DEPTH_REG, sq->depth); 113 114 erdma_reg_write64(dev, ERDMA_CMDQ_SQ_DB_HOST_ADDR_REG, 114 - sq->qbuf_dma_addr + buf_size); 115 + sq->db_record_dma_addr); 115 116 116 117 return 0; 118 + 119 + err_out: 120 + dma_free_coherent(&dev->pdev->dev, sq->depth << SQEBB_SHIFT, 121 + sq->qbuf, sq->qbuf_dma_addr); 122 + 123 + return -ENOMEM; 117 124 } 118 125 119 126 static int erdma_cmdq_cq_init(struct erdma_dev *dev) 120 127 { 121 128 struct erdma_cmdq *cmdq = &dev->cmdq; 122 129 struct erdma_cmdq_cq *cq = &cmdq->cq; 123 - u32 buf_size; 124 130 125 131 cq->depth = cmdq->sq.depth; 126 - buf_size = cq->depth << CQE_SHIFT; 127 - 128 - cq->qbuf = 129 - dma_alloc_coherent(&dev->pdev->dev, WARPPED_BUFSIZE(buf_size), 130 - &cq->qbuf_dma_addr, GFP_KERNEL | __GFP_ZERO); 132 + cq->qbuf = dma_alloc_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT, 133 + &cq->qbuf_dma_addr, 134 + GFP_KERNEL | __GFP_ZERO); 131 135 if (!cq->qbuf) 132 136 return -ENOMEM; 133 137 134 138 spin_lock_init(&cq->lock); 135 139 136 - cq->db_record = (u64 *)(cq->qbuf + buf_size); 140 + cq->db_record = dma_pool_zalloc(dev->db_pool, GFP_KERNEL, 141 + &cq->db_record_dma_addr); 142 + if (!cq->db_record) 143 + goto err_out; 137 144 138 145 atomic64_set(&cq->armed_num, 0); 139 146 ··· 148 143 erdma_reg_write32(dev, ERDMA_REGS_CMDQ_CQ_ADDR_L_REG, 149 144 lower_32_bits(cq->qbuf_dma_addr)); 150 145 erdma_reg_write64(dev, ERDMA_CMDQ_CQ_DB_HOST_ADDR_REG, 151 - cq->qbuf_dma_addr + buf_size); 146 + cq->db_record_dma_addr); 152 147 153 148 return 0; 149 + 150 + err_out: 151 + dma_free_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT, cq->qbuf, 152 + cq->qbuf_dma_addr); 153 + 154 + return -ENOMEM; 154 155 } 155 156 156 157 static int erdma_cmdq_eq_init(struct erdma_dev *dev) 157 158 { 158 159 struct erdma_cmdq *cmdq = &dev->cmdq; 159 160 struct erdma_eq *eq = &cmdq->eq; 160 - u32 buf_size; 161 161 162 162 eq->depth = cmdq->max_outstandings; 163 - buf_size = eq->depth << EQE_SHIFT; 164 - 165 - eq->qbuf = 166 - dma_alloc_coherent(&dev->pdev->dev, WARPPED_BUFSIZE(buf_size), 167 - &eq->qbuf_dma_addr, GFP_KERNEL | __GFP_ZERO); 163 + eq->qbuf = dma_alloc_coherent(&dev->pdev->dev, eq->depth << EQE_SHIFT, 164 + &eq->qbuf_dma_addr, 165 + GFP_KERNEL | __GFP_ZERO); 168 166 if (!eq->qbuf) 169 167 return -ENOMEM; 170 168 ··· 175 167 atomic64_set(&eq->event_num, 0); 176 168 177 169 eq->db = dev->func_bar + ERDMA_REGS_CEQ_DB_BASE_REG; 178 - eq->db_record = (u64 *)(eq->qbuf + buf_size); 170 + eq->db_record = dma_pool_zalloc(dev->db_pool, GFP_KERNEL, 171 + &eq->db_record_dma_addr); 172 + if (!eq->db_record) 173 + goto err_out; 179 174 180 175 erdma_reg_write32(dev, ERDMA_REGS_CMDQ_EQ_ADDR_H_REG, 181 176 upper_32_bits(eq->qbuf_dma_addr)); ··· 186 175 lower_32_bits(eq->qbuf_dma_addr)); 187 176 erdma_reg_write32(dev, ERDMA_REGS_CMDQ_EQ_DEPTH_REG, eq->depth); 188 177 erdma_reg_write64(dev, ERDMA_CMDQ_EQ_DB_HOST_ADDR_REG, 189 - eq->qbuf_dma_addr + buf_size); 178 + eq->db_record_dma_addr); 190 179 191 180 return 0; 181 + 182 + err_out: 183 + dma_free_coherent(&dev->pdev->dev, eq->depth << EQE_SHIFT, eq->qbuf, 184 + eq->qbuf_dma_addr); 185 + 186 + return -ENOMEM; 192 187 } 193 188 194 189 int erdma_cmdq_init(struct erdma_dev *dev) ··· 228 211 return 0; 229 212 230 213 err_destroy_cq: 231 - dma_free_coherent(&dev->pdev->dev, 232 - (cmdq->cq.depth << CQE_SHIFT) + 233 - ERDMA_EXTRA_BUFFER_SIZE, 214 + dma_free_coherent(&dev->pdev->dev, cmdq->cq.depth << CQE_SHIFT, 234 215 cmdq->cq.qbuf, cmdq->cq.qbuf_dma_addr); 235 216 217 + dma_pool_free(dev->db_pool, cmdq->cq.db_record, 218 + cmdq->cq.db_record_dma_addr); 219 + 236 220 err_destroy_sq: 237 - dma_free_coherent(&dev->pdev->dev, 238 - (cmdq->sq.depth << SQEBB_SHIFT) + 239 - ERDMA_EXTRA_BUFFER_SIZE, 221 + dma_free_coherent(&dev->pdev->dev, cmdq->sq.depth << SQEBB_SHIFT, 240 222 cmdq->sq.qbuf, cmdq->sq.qbuf_dma_addr); 223 + 224 + dma_pool_free(dev->db_pool, cmdq->sq.db_record, 225 + cmdq->sq.db_record_dma_addr); 241 226 242 227 return err; 243 228 } ··· 257 238 258 239 clear_bit(ERDMA_CMDQ_STATE_OK_BIT, &cmdq->state); 259 240 260 - dma_free_coherent(&dev->pdev->dev, 261 - (cmdq->eq.depth << EQE_SHIFT) + 262 - ERDMA_EXTRA_BUFFER_SIZE, 241 + dma_free_coherent(&dev->pdev->dev, cmdq->eq.depth << EQE_SHIFT, 263 242 cmdq->eq.qbuf, cmdq->eq.qbuf_dma_addr); 264 - dma_free_coherent(&dev->pdev->dev, 265 - (cmdq->sq.depth << SQEBB_SHIFT) + 266 - ERDMA_EXTRA_BUFFER_SIZE, 243 + 244 + dma_pool_free(dev->db_pool, cmdq->eq.db_record, 245 + cmdq->eq.db_record_dma_addr); 246 + 247 + dma_free_coherent(&dev->pdev->dev, cmdq->sq.depth << SQEBB_SHIFT, 267 248 cmdq->sq.qbuf, cmdq->sq.qbuf_dma_addr); 268 - dma_free_coherent(&dev->pdev->dev, 269 - (cmdq->cq.depth << CQE_SHIFT) + 270 - ERDMA_EXTRA_BUFFER_SIZE, 249 + 250 + dma_pool_free(dev->db_pool, cmdq->sq.db_record, 251 + cmdq->sq.db_record_dma_addr); 252 + 253 + dma_free_coherent(&dev->pdev->dev, cmdq->cq.depth << CQE_SHIFT, 271 254 cmdq->cq.qbuf, cmdq->cq.qbuf_dma_addr); 255 + 256 + dma_pool_free(dev->db_pool, cmdq->cq.db_record, 257 + cmdq->cq.db_record_dma_addr); 272 258 } 273 259 274 260 static void *get_next_valid_cmdq_cqe(struct erdma_cmdq *cmdq)
+34 -21
drivers/infiniband/hw/erdma/erdma_eq.c
··· 83 83 int erdma_aeq_init(struct erdma_dev *dev) 84 84 { 85 85 struct erdma_eq *eq = &dev->aeq; 86 - u32 buf_size; 87 86 88 87 eq->depth = ERDMA_DEFAULT_EQ_DEPTH; 89 - buf_size = eq->depth << EQE_SHIFT; 90 88 91 - eq->qbuf = 92 - dma_alloc_coherent(&dev->pdev->dev, WARPPED_BUFSIZE(buf_size), 93 - &eq->qbuf_dma_addr, GFP_KERNEL | __GFP_ZERO); 89 + eq->qbuf = dma_alloc_coherent(&dev->pdev->dev, eq->depth << EQE_SHIFT, 90 + &eq->qbuf_dma_addr, 91 + GFP_KERNEL | __GFP_ZERO); 94 92 if (!eq->qbuf) 95 93 return -ENOMEM; 96 94 ··· 97 99 atomic64_set(&eq->notify_num, 0); 98 100 99 101 eq->db = dev->func_bar + ERDMA_REGS_AEQ_DB_REG; 100 - eq->db_record = (u64 *)(eq->qbuf + buf_size); 102 + eq->db_record = dma_pool_zalloc(dev->db_pool, GFP_KERNEL, 103 + &eq->db_record_dma_addr); 104 + if (!eq->db_record) 105 + goto err_out; 101 106 102 107 erdma_reg_write32(dev, ERDMA_REGS_AEQ_ADDR_H_REG, 103 108 upper_32_bits(eq->qbuf_dma_addr)); ··· 108 107 lower_32_bits(eq->qbuf_dma_addr)); 109 108 erdma_reg_write32(dev, ERDMA_REGS_AEQ_DEPTH_REG, eq->depth); 110 109 erdma_reg_write64(dev, ERDMA_AEQ_DB_HOST_ADDR_REG, 111 - eq->qbuf_dma_addr + buf_size); 110 + eq->db_record_dma_addr); 112 111 113 112 return 0; 113 + 114 + err_out: 115 + dma_free_coherent(&dev->pdev->dev, eq->depth << EQE_SHIFT, eq->qbuf, 116 + eq->qbuf_dma_addr); 117 + 118 + return -ENOMEM; 114 119 } 115 120 116 121 void erdma_aeq_destroy(struct erdma_dev *dev) 117 122 { 118 123 struct erdma_eq *eq = &dev->aeq; 119 124 120 - dma_free_coherent(&dev->pdev->dev, 121 - WARPPED_BUFSIZE(eq->depth << EQE_SHIFT), eq->qbuf, 125 + dma_free_coherent(&dev->pdev->dev, eq->depth << EQE_SHIFT, eq->qbuf, 122 126 eq->qbuf_dma_addr); 127 + 128 + dma_pool_free(dev->db_pool, eq->db_record, eq->db_record_dma_addr); 123 129 } 124 130 125 131 void erdma_ceq_completion_handler(struct erdma_eq_cb *ceq_cb) ··· 217 209 static int create_eq_cmd(struct erdma_dev *dev, u32 eqn, struct erdma_eq *eq) 218 210 { 219 211 struct erdma_cmdq_create_eq_req req; 220 - dma_addr_t db_info_dma_addr; 221 212 222 213 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON, 223 214 CMDQ_OPCODE_CREATE_EQ); ··· 226 219 req.qtype = ERDMA_EQ_TYPE_CEQ; 227 220 /* Vector index is the same as EQN. */ 228 221 req.vector_idx = eqn; 229 - db_info_dma_addr = eq->qbuf_dma_addr + (eq->depth << EQE_SHIFT); 230 - req.db_dma_addr_l = lower_32_bits(db_info_dma_addr); 231 - req.db_dma_addr_h = upper_32_bits(db_info_dma_addr); 222 + req.db_dma_addr_l = lower_32_bits(eq->db_record_dma_addr); 223 + req.db_dma_addr_h = upper_32_bits(eq->db_record_dma_addr); 232 224 233 225 return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL); 234 226 } ··· 235 229 static int erdma_ceq_init_one(struct erdma_dev *dev, u16 ceqn) 236 230 { 237 231 struct erdma_eq *eq = &dev->ceqs[ceqn].eq; 238 - u32 buf_size = ERDMA_DEFAULT_EQ_DEPTH << EQE_SHIFT; 239 232 int ret; 240 233 241 - eq->qbuf = 242 - dma_alloc_coherent(&dev->pdev->dev, WARPPED_BUFSIZE(buf_size), 243 - &eq->qbuf_dma_addr, GFP_KERNEL | __GFP_ZERO); 234 + eq->depth = ERDMA_DEFAULT_EQ_DEPTH; 235 + eq->qbuf = dma_alloc_coherent(&dev->pdev->dev, eq->depth << EQE_SHIFT, 236 + &eq->qbuf_dma_addr, 237 + GFP_KERNEL | __GFP_ZERO); 244 238 if (!eq->qbuf) 245 239 return -ENOMEM; 246 240 ··· 248 242 atomic64_set(&eq->event_num, 0); 249 243 atomic64_set(&eq->notify_num, 0); 250 244 251 - eq->depth = ERDMA_DEFAULT_EQ_DEPTH; 252 245 eq->db = dev->func_bar + ERDMA_REGS_CEQ_DB_BASE_REG + 253 246 (ceqn + 1) * ERDMA_DB_SIZE; 254 - eq->db_record = (u64 *)(eq->qbuf + buf_size); 247 + 248 + eq->db_record = dma_pool_zalloc(dev->db_pool, GFP_KERNEL, 249 + &eq->db_record_dma_addr); 250 + if (!eq->db_record) { 251 + dma_free_coherent(&dev->pdev->dev, eq->depth << EQE_SHIFT, 252 + eq->qbuf, eq->qbuf_dma_addr); 253 + return -ENOMEM; 254 + } 255 + 255 256 eq->ci = 0; 256 257 dev->ceqs[ceqn].dev = dev; 257 258 ··· 272 259 static void erdma_ceq_uninit_one(struct erdma_dev *dev, u16 ceqn) 273 260 { 274 261 struct erdma_eq *eq = &dev->ceqs[ceqn].eq; 275 - u32 buf_size = ERDMA_DEFAULT_EQ_DEPTH << EQE_SHIFT; 276 262 struct erdma_cmdq_destroy_eq_req req; 277 263 int err; 278 264 ··· 288 276 if (err) 289 277 return; 290 278 291 - dma_free_coherent(&dev->pdev->dev, WARPPED_BUFSIZE(buf_size), eq->qbuf, 279 + dma_free_coherent(&dev->pdev->dev, eq->depth << EQE_SHIFT, eq->qbuf, 292 280 eq->qbuf_dma_addr); 281 + dma_pool_free(dev->db_pool, eq->db_record, eq->db_record_dma_addr); 293 282 } 294 283 295 284 int erdma_ceqs_init(struct erdma_dev *dev)
+13 -2
drivers/infiniband/hw/erdma/erdma_main.c
··· 178 178 if (!dev->resp_pool) 179 179 return -ENOMEM; 180 180 181 + dev->db_pool = dma_pool_create("erdma_db_pool", &pdev->dev, 182 + ERDMA_DB_SIZE, ERDMA_DB_SIZE, 0); 183 + if (!dev->db_pool) { 184 + ret = -ENOMEM; 185 + goto destroy_resp_pool; 186 + } 187 + 181 188 ret = dma_set_mask_and_coherent(&pdev->dev, 182 189 DMA_BIT_MASK(ERDMA_PCI_WIDTH)); 183 190 if (ret) 184 - goto destroy_pool; 191 + goto destroy_db_pool; 185 192 186 193 dma_set_max_seg_size(&pdev->dev, UINT_MAX); 187 194 188 195 return 0; 189 196 190 - destroy_pool: 197 + destroy_db_pool: 198 + dma_pool_destroy(dev->db_pool); 199 + 200 + destroy_resp_pool: 191 201 dma_pool_destroy(dev->resp_pool); 192 202 193 203 return ret; ··· 205 195 206 196 static void erdma_device_uninit(struct erdma_dev *dev) 207 197 { 198 + dma_pool_destroy(dev->db_pool); 208 199 dma_pool_destroy(dev->resp_pool); 209 200 } 210 201
+49 -36
drivers/infiniband/hw/erdma/erdma_verbs.c
··· 76 76 77 77 req.rq_buf_addr = qp->kern_qp.rq_buf_dma_addr; 78 78 req.sq_buf_addr = qp->kern_qp.sq_buf_dma_addr; 79 - req.sq_db_info_dma_addr = qp->kern_qp.sq_buf_dma_addr + 80 - (qp->attrs.sq_size << SQEBB_SHIFT); 81 - req.rq_db_info_dma_addr = qp->kern_qp.rq_buf_dma_addr + 82 - (qp->attrs.rq_size << RQE_SHIFT); 79 + req.sq_db_info_dma_addr = qp->kern_qp.sq_db_info_dma_addr; 80 + req.rq_db_info_dma_addr = qp->kern_qp.rq_db_info_dma_addr; 83 81 } else { 84 82 user_qp = &qp->user_qp; 85 83 req.sq_cqn_mtt_cfg = FIELD_PREP( ··· 207 209 ERDMA_MR_MTT_0LEVEL); 208 210 209 211 req.first_page_offset = 0; 210 - req.cq_db_info_addr = 211 - cq->kern_cq.qbuf_dma_addr + (cq->depth << CQE_SHIFT); 212 + req.cq_db_info_addr = cq->kern_cq.db_record_dma_addr; 212 213 } else { 213 214 mem = &cq->user_cq.qbuf_mem; 214 215 req.cfg0 |= ··· 479 482 vfree(qp->kern_qp.rwr_tbl); 480 483 481 484 if (qp->kern_qp.sq_buf) 482 - dma_free_coherent( 483 - &dev->pdev->dev, 484 - WARPPED_BUFSIZE(qp->attrs.sq_size << SQEBB_SHIFT), 485 - qp->kern_qp.sq_buf, qp->kern_qp.sq_buf_dma_addr); 485 + dma_free_coherent(&dev->pdev->dev, 486 + qp->attrs.sq_size << SQEBB_SHIFT, 487 + qp->kern_qp.sq_buf, 488 + qp->kern_qp.sq_buf_dma_addr); 489 + 490 + if (qp->kern_qp.sq_db_info) 491 + dma_pool_free(dev->db_pool, qp->kern_qp.sq_db_info, 492 + qp->kern_qp.sq_db_info_dma_addr); 486 493 487 494 if (qp->kern_qp.rq_buf) 488 - dma_free_coherent( 489 - &dev->pdev->dev, 490 - WARPPED_BUFSIZE(qp->attrs.rq_size << RQE_SHIFT), 491 - qp->kern_qp.rq_buf, qp->kern_qp.rq_buf_dma_addr); 495 + dma_free_coherent(&dev->pdev->dev, 496 + qp->attrs.rq_size << RQE_SHIFT, 497 + qp->kern_qp.rq_buf, 498 + qp->kern_qp.rq_buf_dma_addr); 499 + 500 + if (qp->kern_qp.rq_db_info) 501 + dma_pool_free(dev->db_pool, qp->kern_qp.rq_db_info, 502 + qp->kern_qp.rq_db_info_dma_addr); 492 503 } 493 504 494 505 static int init_kernel_qp(struct erdma_dev *dev, struct erdma_qp *qp, ··· 521 516 if (!kqp->swr_tbl || !kqp->rwr_tbl) 522 517 goto err_out; 523 518 524 - size = (qp->attrs.sq_size << SQEBB_SHIFT) + ERDMA_EXTRA_BUFFER_SIZE; 519 + size = qp->attrs.sq_size << SQEBB_SHIFT; 525 520 kqp->sq_buf = dma_alloc_coherent(&dev->pdev->dev, size, 526 521 &kqp->sq_buf_dma_addr, GFP_KERNEL); 527 522 if (!kqp->sq_buf) 528 523 goto err_out; 529 524 530 - size = (qp->attrs.rq_size << RQE_SHIFT) + ERDMA_EXTRA_BUFFER_SIZE; 525 + kqp->sq_db_info = dma_pool_zalloc(dev->db_pool, GFP_KERNEL, 526 + &kqp->sq_db_info_dma_addr); 527 + if (!kqp->sq_db_info) 528 + goto err_out; 529 + 530 + size = qp->attrs.rq_size << RQE_SHIFT; 531 531 kqp->rq_buf = dma_alloc_coherent(&dev->pdev->dev, size, 532 532 &kqp->rq_buf_dma_addr, GFP_KERNEL); 533 533 if (!kqp->rq_buf) 534 534 goto err_out; 535 535 536 - kqp->sq_db_info = kqp->sq_buf + (qp->attrs.sq_size << SQEBB_SHIFT); 537 - kqp->rq_db_info = kqp->rq_buf + (qp->attrs.rq_size << RQE_SHIFT); 536 + kqp->rq_db_info = dma_pool_zalloc(dev->db_pool, GFP_KERNEL, 537 + &kqp->rq_db_info_dma_addr); 538 + if (!kqp->rq_db_info) 539 + goto err_out; 538 540 539 541 return 0; 540 542 ··· 1249 1237 return err; 1250 1238 1251 1239 if (rdma_is_kernel_res(&cq->ibcq.res)) { 1252 - dma_free_coherent(&dev->pdev->dev, 1253 - WARPPED_BUFSIZE(cq->depth << CQE_SHIFT), 1240 + dma_free_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT, 1254 1241 cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr); 1242 + dma_pool_free(dev->db_pool, cq->kern_cq.db_record, 1243 + cq->kern_cq.db_record_dma_addr); 1255 1244 } else { 1256 1245 erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page); 1257 1246 put_mtt_entries(dev, &cq->user_cq.qbuf_mem); ··· 1292 1279 wait_for_completion(&qp->safe_free); 1293 1280 1294 1281 if (rdma_is_kernel_res(&qp->ibqp.res)) { 1295 - vfree(qp->kern_qp.swr_tbl); 1296 - vfree(qp->kern_qp.rwr_tbl); 1297 - dma_free_coherent( 1298 - &dev->pdev->dev, 1299 - WARPPED_BUFSIZE(qp->attrs.rq_size << RQE_SHIFT), 1300 - qp->kern_qp.rq_buf, qp->kern_qp.rq_buf_dma_addr); 1301 - dma_free_coherent( 1302 - &dev->pdev->dev, 1303 - WARPPED_BUFSIZE(qp->attrs.sq_size << SQEBB_SHIFT), 1304 - qp->kern_qp.sq_buf, qp->kern_qp.sq_buf_dma_addr); 1282 + free_kernel_qp(qp); 1305 1283 } else { 1306 1284 put_mtt_entries(dev, &qp->user_qp.sq_mem); 1307 1285 put_mtt_entries(dev, &qp->user_qp.rq_mem); ··· 1604 1600 struct erdma_dev *dev = to_edev(cq->ibcq.device); 1605 1601 1606 1602 cq->kern_cq.qbuf = 1607 - dma_alloc_coherent(&dev->pdev->dev, 1608 - WARPPED_BUFSIZE(cq->depth << CQE_SHIFT), 1603 + dma_alloc_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT, 1609 1604 &cq->kern_cq.qbuf_dma_addr, GFP_KERNEL); 1610 1605 if (!cq->kern_cq.qbuf) 1611 1606 return -ENOMEM; 1612 1607 1613 - cq->kern_cq.db_record = 1614 - (u64 *)(cq->kern_cq.qbuf + (cq->depth << CQE_SHIFT)); 1608 + cq->kern_cq.db_record = dma_pool_zalloc( 1609 + dev->db_pool, GFP_KERNEL, &cq->kern_cq.db_record_dma_addr); 1610 + if (!cq->kern_cq.db_record) 1611 + goto err_out; 1612 + 1615 1613 spin_lock_init(&cq->kern_cq.lock); 1616 1614 /* use default cqdb addr */ 1617 1615 cq->kern_cq.db = dev->func_bar + ERDMA_BAR_CQDB_SPACE_OFFSET; 1618 1616 1619 1617 return 0; 1618 + 1619 + err_out: 1620 + dma_free_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT, 1621 + cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr); 1622 + 1623 + return -ENOMEM; 1620 1624 } 1621 1625 1622 1626 int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, ··· 1688 1676 erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page); 1689 1677 put_mtt_entries(dev, &cq->user_cq.qbuf_mem); 1690 1678 } else { 1691 - dma_free_coherent(&dev->pdev->dev, 1692 - WARPPED_BUFSIZE(depth << CQE_SHIFT), 1679 + dma_free_coherent(&dev->pdev->dev, depth << CQE_SHIFT, 1693 1680 cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr); 1681 + dma_pool_free(dev->db_pool, cq->kern_cq.db_record, 1682 + cq->kern_cq.db_record_dma_addr); 1694 1683 } 1695 1684 1696 1685 err_out_xa:
+4
drivers/infiniband/hw/erdma/erdma_verbs.h
··· 170 170 void *sq_db_info; 171 171 void *rq_db_info; 172 172 173 + dma_addr_t sq_db_info_dma_addr; 174 + dma_addr_t rq_db_info_dma_addr; 175 + 173 176 u8 sig_all; 174 177 }; 175 178 ··· 250 247 spinlock_t lock; 251 248 u8 __iomem *db; 252 249 u64 *db_record; 250 + dma_addr_t db_record_dma_addr; 253 251 }; 254 252 255 253 struct erdma_ucq_info {