Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2
3/* Authors: Cheng Xu <chengyou@linux.alibaba.com> */
4/* Kai Shen <kaishen@linux.alibaba.com> */
5/* Copyright (c) 2020-2022, Alibaba Group. */
6
7#include <linux/module.h>
8#include <net/addrconf.h>
9#include <rdma/erdma-abi.h>
10
11#include "erdma.h"
12#include "erdma_cm.h"
13#include "erdma_verbs.h"
14
15MODULE_AUTHOR("Cheng Xu <chengyou@linux.alibaba.com>");
16MODULE_DESCRIPTION("Alibaba elasticRDMA adapter driver");
17MODULE_LICENSE("Dual BSD/GPL");
18
19static int erdma_netdev_event(struct notifier_block *nb, unsigned long event,
20 void *arg)
21{
22 struct net_device *netdev = netdev_notifier_info_to_dev(arg);
23 struct erdma_dev *dev = container_of(nb, struct erdma_dev, netdev_nb);
24
25 if (dev->netdev == NULL || dev->netdev != netdev)
26 goto done;
27
28 switch (event) {
29 case NETDEV_CHANGEMTU:
30 if (dev->mtu != netdev->mtu) {
31 erdma_set_mtu(dev, netdev->mtu);
32 dev->mtu = netdev->mtu;
33 }
34 break;
35 case NETDEV_REGISTER:
36 case NETDEV_UNREGISTER:
37 case NETDEV_CHANGEADDR:
38 case NETDEV_GOING_DOWN:
39 case NETDEV_CHANGE:
40 default:
41 break;
42 }
43
44done:
45 return NOTIFY_OK;
46}
47
48static int erdma_enum_and_get_netdev(struct erdma_dev *dev)
49{
50 struct net_device *netdev;
51 int ret = -EPROBE_DEFER;
52
53 /* Already binded to a net_device, so we skip. */
54 if (dev->netdev)
55 return 0;
56
57 rtnl_lock();
58 for_each_netdev(&init_net, netdev) {
59 /*
60 * In erdma, the paired netdev and ibdev should have the same
61 * MAC address. erdma can get the value from its PCIe bar
62 * registers. Since erdma can not get the paired netdev
63 * reference directly, we do a traverse here to get the paired
64 * netdev.
65 */
66 if (ether_addr_equal_unaligned(netdev->perm_addr,
67 dev->attrs.peer_addr)) {
68 ret = ib_device_set_netdev(&dev->ibdev, netdev, 1);
69 if (ret) {
70 rtnl_unlock();
71 ibdev_warn(&dev->ibdev,
72 "failed (%d) to link netdev", ret);
73 return ret;
74 }
75
76 dev->netdev = netdev;
77 break;
78 }
79 }
80
81 rtnl_unlock();
82
83 return ret;
84}
85
86static int erdma_device_register(struct erdma_dev *dev)
87{
88 struct ib_device *ibdev = &dev->ibdev;
89 int ret;
90
91 ret = erdma_enum_and_get_netdev(dev);
92 if (ret)
93 return ret;
94
95 dev->mtu = dev->netdev->mtu;
96 addrconf_addr_eui48((u8 *)&ibdev->node_guid, dev->netdev->dev_addr);
97
98 ret = ib_register_device(ibdev, "erdma_%d", &dev->pdev->dev);
99 if (ret) {
100 dev_err(&dev->pdev->dev,
101 "ib_register_device failed: ret = %d\n", ret);
102 return ret;
103 }
104
105 dev->netdev_nb.notifier_call = erdma_netdev_event;
106 ret = register_netdevice_notifier(&dev->netdev_nb);
107 if (ret) {
108 ibdev_err(&dev->ibdev, "failed to register notifier.\n");
109 ib_unregister_device(ibdev);
110 }
111
112 return ret;
113}
114
115static irqreturn_t erdma_comm_irq_handler(int irq, void *data)
116{
117 struct erdma_dev *dev = data;
118
119 erdma_cmdq_completion_handler(&dev->cmdq);
120 erdma_aeq_event_handler(dev);
121
122 return IRQ_HANDLED;
123}
124
125static int erdma_request_vectors(struct erdma_dev *dev)
126{
127 int expect_irq_num = min(num_possible_cpus() + 1, ERDMA_NUM_MSIX_VEC);
128 int ret;
129
130 ret = pci_alloc_irq_vectors(dev->pdev, 1, expect_irq_num, PCI_IRQ_MSIX);
131 if (ret < 0) {
132 dev_err(&dev->pdev->dev, "request irq vectors failed(%d)\n",
133 ret);
134 return ret;
135 }
136 dev->attrs.irq_num = ret;
137
138 return 0;
139}
140
141static int erdma_comm_irq_init(struct erdma_dev *dev)
142{
143 snprintf(dev->comm_irq.name, ERDMA_IRQNAME_SIZE, "erdma-common@pci:%s",
144 pci_name(dev->pdev));
145 dev->comm_irq.msix_vector =
146 pci_irq_vector(dev->pdev, ERDMA_MSIX_VECTOR_CMDQ);
147
148 cpumask_set_cpu(cpumask_first(cpumask_of_pcibus(dev->pdev->bus)),
149 &dev->comm_irq.affinity_hint_mask);
150 irq_set_affinity_hint(dev->comm_irq.msix_vector,
151 &dev->comm_irq.affinity_hint_mask);
152
153 return request_irq(dev->comm_irq.msix_vector, erdma_comm_irq_handler, 0,
154 dev->comm_irq.name, dev);
155}
156
157static void erdma_comm_irq_uninit(struct erdma_dev *dev)
158{
159 irq_set_affinity_hint(dev->comm_irq.msix_vector, NULL);
160 free_irq(dev->comm_irq.msix_vector, dev);
161}
162
163static int erdma_device_init(struct erdma_dev *dev, struct pci_dev *pdev)
164{
165 int ret;
166
167 dev->proto = erdma_reg_read32(dev, ERDMA_REGS_DEV_PROTO_REG);
168
169 dev->resp_pool = dma_pool_create("erdma_resp_pool", &pdev->dev,
170 ERDMA_HW_RESP_SIZE, ERDMA_HW_RESP_SIZE,
171 0);
172 if (!dev->resp_pool)
173 return -ENOMEM;
174
175 dev->db_pool = dma_pool_create("erdma_db_pool", &pdev->dev,
176 ERDMA_DB_SIZE, ERDMA_DB_SIZE, 0);
177 if (!dev->db_pool) {
178 ret = -ENOMEM;
179 goto destroy_resp_pool;
180 }
181
182 ret = dma_set_mask_and_coherent(&pdev->dev,
183 DMA_BIT_MASK(ERDMA_PCI_WIDTH));
184 if (ret)
185 goto destroy_db_pool;
186
187 dma_set_max_seg_size(&pdev->dev, UINT_MAX);
188
189 return 0;
190
191destroy_db_pool:
192 dma_pool_destroy(dev->db_pool);
193
194destroy_resp_pool:
195 dma_pool_destroy(dev->resp_pool);
196
197 return ret;
198}
199
200static void erdma_device_uninit(struct erdma_dev *dev)
201{
202 dma_pool_destroy(dev->db_pool);
203 dma_pool_destroy(dev->resp_pool);
204}
205
206static void erdma_hw_reset(struct erdma_dev *dev)
207{
208 u32 ctrl = FIELD_PREP(ERDMA_REG_DEV_CTRL_RESET_MASK, 1);
209
210 erdma_reg_write32(dev, ERDMA_REGS_DEV_CTRL_REG, ctrl);
211}
212
213static int erdma_wait_hw_init_done(struct erdma_dev *dev)
214{
215 int i;
216
217 erdma_reg_write32(dev, ERDMA_REGS_DEV_CTRL_REG,
218 FIELD_PREP(ERDMA_REG_DEV_CTRL_INIT_MASK, 1));
219
220 for (i = 0; i < ERDMA_WAIT_DEV_DONE_CNT; i++) {
221 if (erdma_reg_read32_filed(dev, ERDMA_REGS_DEV_ST_REG,
222 ERDMA_REG_DEV_ST_INIT_DONE_MASK))
223 break;
224
225 msleep(ERDMA_REG_ACCESS_WAIT_MS);
226 }
227
228 if (i == ERDMA_WAIT_DEV_DONE_CNT) {
229 dev_err(&dev->pdev->dev, "wait init done failed.\n");
230 return -ETIMEDOUT;
231 }
232
233 return 0;
234}
235
236static const struct pci_device_id erdma_pci_tbl[] = {
237 { PCI_DEVICE(PCI_VENDOR_ID_ALIBABA, 0x107f) },
238 {}
239};
240
241static int erdma_probe_dev(struct pci_dev *pdev)
242{
243 struct erdma_dev *dev;
244 int bars, err;
245 u32 version;
246
247 err = pci_enable_device(pdev);
248 if (err) {
249 dev_err(&pdev->dev, "pci_enable_device failed(%d)\n", err);
250 return err;
251 }
252
253 pci_set_master(pdev);
254
255 dev = ib_alloc_device(erdma_dev, ibdev);
256 if (!dev) {
257 dev_err(&pdev->dev, "ib_alloc_device failed\n");
258 err = -ENOMEM;
259 goto err_disable_device;
260 }
261
262 pci_set_drvdata(pdev, dev);
263 dev->pdev = pdev;
264
265 bars = pci_select_bars(pdev, IORESOURCE_MEM);
266 err = pci_request_selected_regions(pdev, bars, DRV_MODULE_NAME);
267 if (bars != ERDMA_BAR_MASK || err) {
268 err = err ? err : -EINVAL;
269 goto err_ib_device_release;
270 }
271
272 dev->func_bar_addr = pci_resource_start(pdev, ERDMA_FUNC_BAR);
273 dev->func_bar_len = pci_resource_len(pdev, ERDMA_FUNC_BAR);
274
275 dev->func_bar =
276 devm_ioremap(&pdev->dev, dev->func_bar_addr, dev->func_bar_len);
277 if (!dev->func_bar) {
278 dev_err(&pdev->dev, "devm_ioremap failed.\n");
279 err = -EFAULT;
280 goto err_release_bars;
281 }
282
283 version = erdma_reg_read32(dev, ERDMA_REGS_VERSION_REG);
284 if (version == 0) {
285 /* we knows that it is a non-functional function. */
286 err = -ENODEV;
287 goto err_iounmap_func_bar;
288 }
289
290 err = erdma_device_init(dev, pdev);
291 if (err)
292 goto err_iounmap_func_bar;
293
294 err = erdma_request_vectors(dev);
295 if (err)
296 goto err_uninit_device;
297
298 err = erdma_comm_irq_init(dev);
299 if (err)
300 goto err_free_vectors;
301
302 err = erdma_aeq_init(dev);
303 if (err)
304 goto err_uninit_comm_irq;
305
306 err = erdma_cmdq_init(dev);
307 if (err)
308 goto err_uninit_aeq;
309
310 err = erdma_wait_hw_init_done(dev);
311 if (err)
312 goto err_uninit_cmdq;
313
314 err = erdma_ceqs_init(dev);
315 if (err)
316 goto err_reset_hw;
317
318 erdma_finish_cmdq_init(dev);
319
320 return 0;
321
322err_reset_hw:
323 erdma_hw_reset(dev);
324
325err_uninit_cmdq:
326 erdma_cmdq_destroy(dev);
327
328err_uninit_aeq:
329 erdma_eq_destroy(dev, &dev->aeq);
330
331err_uninit_comm_irq:
332 erdma_comm_irq_uninit(dev);
333
334err_free_vectors:
335 pci_free_irq_vectors(dev->pdev);
336
337err_uninit_device:
338 erdma_device_uninit(dev);
339
340err_iounmap_func_bar:
341 devm_iounmap(&pdev->dev, dev->func_bar);
342
343err_release_bars:
344 pci_release_selected_regions(pdev, bars);
345
346err_ib_device_release:
347 ib_dealloc_device(&dev->ibdev);
348
349err_disable_device:
350 pci_disable_device(pdev);
351
352 return err;
353}
354
355static void erdma_remove_dev(struct pci_dev *pdev)
356{
357 struct erdma_dev *dev = pci_get_drvdata(pdev);
358
359 erdma_ceqs_uninit(dev);
360 erdma_hw_reset(dev);
361 erdma_cmdq_destroy(dev);
362 erdma_eq_destroy(dev, &dev->aeq);
363 erdma_comm_irq_uninit(dev);
364 pci_free_irq_vectors(dev->pdev);
365 erdma_device_uninit(dev);
366
367 devm_iounmap(&pdev->dev, dev->func_bar);
368 pci_release_selected_regions(pdev, ERDMA_BAR_MASK);
369
370 ib_dealloc_device(&dev->ibdev);
371
372 pci_disable_device(pdev);
373}
374
375#define ERDMA_GET_CAP(name, cap) FIELD_GET(ERDMA_CMD_DEV_CAP_##name##_MASK, cap)
376
377static int erdma_dev_attrs_init(struct erdma_dev *dev)
378{
379 int err;
380 u64 req_hdr, cap0, cap1;
381
382 erdma_cmdq_build_reqhdr(&req_hdr, CMDQ_SUBMOD_RDMA,
383 CMDQ_OPCODE_QUERY_DEVICE);
384
385 err = erdma_post_cmd_wait(&dev->cmdq, &req_hdr, sizeof(req_hdr), &cap0,
386 &cap1, true);
387 if (err)
388 return err;
389
390 dev->attrs.max_cqe = 1 << ERDMA_GET_CAP(MAX_CQE, cap0);
391 dev->attrs.max_mr_size = 1ULL << ERDMA_GET_CAP(MAX_MR_SIZE, cap0);
392 dev->attrs.max_mw = 1 << ERDMA_GET_CAP(MAX_MW, cap1);
393 dev->attrs.max_recv_wr = 1 << ERDMA_GET_CAP(MAX_RECV_WR, cap0);
394 dev->attrs.max_gid = 1 << ERDMA_GET_CAP(MAX_GID, cap0);
395 dev->attrs.max_ah = 1 << ERDMA_GET_CAP(MAX_AH, cap0);
396 dev->attrs.local_dma_key = ERDMA_GET_CAP(DMA_LOCAL_KEY, cap1);
397 dev->attrs.cc = ERDMA_GET_CAP(DEFAULT_CC, cap1);
398 dev->attrs.max_qp = ERDMA_NQP_PER_QBLOCK * ERDMA_GET_CAP(QBLOCK, cap1);
399 dev->attrs.max_mr = dev->attrs.max_qp << 1;
400 dev->attrs.max_cq = dev->attrs.max_qp << 1;
401 dev->attrs.cap_flags = ERDMA_GET_CAP(FLAGS, cap0);
402
403 dev->attrs.max_send_wr = ERDMA_MAX_SEND_WR;
404 dev->attrs.max_ord = ERDMA_MAX_ORD;
405 dev->attrs.max_ird = ERDMA_MAX_IRD;
406 dev->attrs.max_send_sge = ERDMA_MAX_SEND_SGE;
407 dev->attrs.max_recv_sge = ERDMA_MAX_RECV_SGE;
408 dev->attrs.max_sge_rd = ERDMA_MAX_SGE_RD;
409 dev->attrs.max_pd = ERDMA_MAX_PD;
410
411 dev->res_cb[ERDMA_RES_TYPE_PD].max_cap = ERDMA_MAX_PD;
412 dev->res_cb[ERDMA_RES_TYPE_STAG_IDX].max_cap = dev->attrs.max_mr;
413 dev->res_cb[ERDMA_RES_TYPE_AH].max_cap = dev->attrs.max_ah;
414
415 erdma_cmdq_build_reqhdr(&req_hdr, CMDQ_SUBMOD_COMMON,
416 CMDQ_OPCODE_QUERY_FW_INFO);
417
418 err = erdma_post_cmd_wait(&dev->cmdq, &req_hdr, sizeof(req_hdr), &cap0,
419 &cap1, true);
420 if (!err)
421 dev->attrs.fw_version =
422 FIELD_GET(ERDMA_CMD_INFO0_FW_VER_MASK, cap0);
423
424 return err;
425}
426
427static int erdma_device_config(struct erdma_dev *dev)
428{
429 struct erdma_cmdq_config_device_req req = {};
430
431 if (!(dev->attrs.cap_flags & ERDMA_DEV_CAP_FLAGS_EXTEND_DB))
432 return 0;
433
434 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON,
435 CMDQ_OPCODE_CONF_DEVICE);
436
437 req.cfg = FIELD_PREP(ERDMA_CMD_CONFIG_DEVICE_PGSHIFT_MASK, PAGE_SHIFT) |
438 FIELD_PREP(ERDMA_CMD_CONFIG_DEVICE_PS_EN_MASK, 1);
439
440 return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
441 true);
442}
443
444static int erdma_res_cb_init(struct erdma_dev *dev)
445{
446 int i, j;
447
448 for (i = 0; i < ERDMA_RES_CNT; i++) {
449 dev->res_cb[i].next_alloc_idx = 1;
450 spin_lock_init(&dev->res_cb[i].lock);
451 dev->res_cb[i].bitmap =
452 bitmap_zalloc(dev->res_cb[i].max_cap, GFP_KERNEL);
453 if (!dev->res_cb[i].bitmap)
454 goto err;
455 }
456
457 return 0;
458
459err:
460 for (j = 0; j < i; j++)
461 bitmap_free(dev->res_cb[j].bitmap);
462
463 return -ENOMEM;
464}
465
466static void erdma_res_cb_free(struct erdma_dev *dev)
467{
468 int i;
469
470 for (i = 0; i < ERDMA_RES_CNT; i++)
471 bitmap_free(dev->res_cb[i].bitmap);
472}
473
474static const struct ib_device_ops erdma_device_ops_rocev2 = {
475 .get_link_layer = erdma_get_link_layer,
476 .add_gid = erdma_add_gid,
477 .del_gid = erdma_del_gid,
478 .query_pkey = erdma_query_pkey,
479 .create_ah = erdma_create_ah,
480 .destroy_ah = erdma_destroy_ah,
481 .query_ah = erdma_query_ah,
482
483 INIT_RDMA_OBJ_SIZE(ib_ah, erdma_ah, ibah),
484};
485
486static const struct ib_device_ops erdma_device_ops_iwarp = {
487 .iw_accept = erdma_accept,
488 .iw_add_ref = erdma_qp_get_ref,
489 .iw_connect = erdma_connect,
490 .iw_create_listen = erdma_create_listen,
491 .iw_destroy_listen = erdma_destroy_listen,
492 .iw_get_qp = erdma_get_ibqp,
493 .iw_reject = erdma_reject,
494 .iw_rem_ref = erdma_qp_put_ref,
495};
496
497static const struct ib_device_ops erdma_device_ops = {
498 .owner = THIS_MODULE,
499 .driver_id = RDMA_DRIVER_ERDMA,
500 .uverbs_abi_ver = ERDMA_ABI_VERSION,
501
502 .alloc_hw_port_stats = erdma_alloc_hw_port_stats,
503 .alloc_mr = erdma_ib_alloc_mr,
504 .alloc_pd = erdma_alloc_pd,
505 .alloc_ucontext = erdma_alloc_ucontext,
506 .create_cq = erdma_create_cq,
507 .create_qp = erdma_create_qp,
508 .dealloc_pd = erdma_dealloc_pd,
509 .dealloc_ucontext = erdma_dealloc_ucontext,
510 .dereg_mr = erdma_dereg_mr,
511 .destroy_cq = erdma_destroy_cq,
512 .destroy_qp = erdma_destroy_qp,
513 .disassociate_ucontext = erdma_disassociate_ucontext,
514 .get_dma_mr = erdma_get_dma_mr,
515 .get_hw_stats = erdma_get_hw_stats,
516 .get_port_immutable = erdma_get_port_immutable,
517 .map_mr_sg = erdma_map_mr_sg,
518 .mmap = erdma_mmap,
519 .mmap_free = erdma_mmap_free,
520 .post_recv = erdma_post_recv,
521 .post_send = erdma_post_send,
522 .poll_cq = erdma_poll_cq,
523 .query_device = erdma_query_device,
524 .query_gid = erdma_query_gid,
525 .query_port = erdma_query_port,
526 .query_qp = erdma_query_qp,
527 .req_notify_cq = erdma_req_notify_cq,
528 .reg_user_mr = erdma_reg_user_mr,
529 .modify_qp = erdma_modify_qp,
530
531 INIT_RDMA_OBJ_SIZE(ib_cq, erdma_cq, ibcq),
532 INIT_RDMA_OBJ_SIZE(ib_pd, erdma_pd, ibpd),
533 INIT_RDMA_OBJ_SIZE(ib_ucontext, erdma_ucontext, ibucontext),
534 INIT_RDMA_OBJ_SIZE(ib_qp, erdma_qp, ibqp),
535};
536
537static int erdma_ib_device_add(struct pci_dev *pdev)
538{
539 struct erdma_dev *dev = pci_get_drvdata(pdev);
540 struct ib_device *ibdev = &dev->ibdev;
541 u64 mac;
542 int ret;
543
544 ret = erdma_dev_attrs_init(dev);
545 if (ret)
546 return ret;
547
548 ret = erdma_device_config(dev);
549 if (ret)
550 return ret;
551
552 if (erdma_device_iwarp(dev)) {
553 ibdev->node_type = RDMA_NODE_RNIC;
554 ib_set_device_ops(ibdev, &erdma_device_ops_iwarp);
555 } else {
556 ibdev->node_type = RDMA_NODE_IB_CA;
557 ib_set_device_ops(ibdev, &erdma_device_ops_rocev2);
558 }
559
560 memcpy(ibdev->node_desc, ERDMA_NODE_DESC, sizeof(ERDMA_NODE_DESC));
561
562 /*
563 * Current model (one-to-one device association):
564 * One ERDMA device per net_device or, equivalently,
565 * per physical port.
566 */
567 ibdev->phys_port_cnt = 1;
568 ibdev->num_comp_vectors = dev->attrs.irq_num - 1;
569
570 ib_set_device_ops(ibdev, &erdma_device_ops);
571
572 INIT_LIST_HEAD(&dev->cep_list);
573
574 spin_lock_init(&dev->lock);
575 xa_init_flags(&dev->qp_xa, XA_FLAGS_ALLOC1);
576 xa_init_flags(&dev->cq_xa, XA_FLAGS_ALLOC1);
577 dev->next_alloc_cqn = 1;
578 dev->next_alloc_qpn = 1;
579
580 ret = erdma_res_cb_init(dev);
581 if (ret)
582 return ret;
583
584 atomic_set(&dev->num_ctx, 0);
585
586 mac = erdma_reg_read32(dev, ERDMA_REGS_NETDEV_MAC_L_REG);
587 mac |= (u64)erdma_reg_read32(dev, ERDMA_REGS_NETDEV_MAC_H_REG) << 32;
588
589 u64_to_ether_addr(mac, dev->attrs.peer_addr);
590
591 dev->reflush_wq = alloc_workqueue("erdma-reflush-wq", WQ_UNBOUND,
592 WQ_UNBOUND_MAX_ACTIVE);
593 if (!dev->reflush_wq) {
594 ret = -ENOMEM;
595 goto err_alloc_workqueue;
596 }
597
598 ret = erdma_device_register(dev);
599 if (ret)
600 goto err_register;
601
602 return 0;
603
604err_register:
605 destroy_workqueue(dev->reflush_wq);
606err_alloc_workqueue:
607 xa_destroy(&dev->qp_xa);
608 xa_destroy(&dev->cq_xa);
609
610 erdma_res_cb_free(dev);
611
612 return ret;
613}
614
615static void erdma_ib_device_remove(struct pci_dev *pdev)
616{
617 struct erdma_dev *dev = pci_get_drvdata(pdev);
618
619 unregister_netdevice_notifier(&dev->netdev_nb);
620 ib_unregister_device(&dev->ibdev);
621
622 destroy_workqueue(dev->reflush_wq);
623 erdma_res_cb_free(dev);
624 xa_destroy(&dev->qp_xa);
625 xa_destroy(&dev->cq_xa);
626}
627
628static int erdma_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
629{
630 int ret;
631
632 ret = erdma_probe_dev(pdev);
633 if (ret)
634 return ret;
635
636 ret = erdma_ib_device_add(pdev);
637 if (ret) {
638 erdma_remove_dev(pdev);
639 return ret;
640 }
641
642 return 0;
643}
644
645static void erdma_remove(struct pci_dev *pdev)
646{
647 erdma_ib_device_remove(pdev);
648 erdma_remove_dev(pdev);
649}
650
651static struct pci_driver erdma_pci_driver = {
652 .name = DRV_MODULE_NAME,
653 .id_table = erdma_pci_tbl,
654 .probe = erdma_probe,
655 .remove = erdma_remove
656};
657
658MODULE_DEVICE_TABLE(pci, erdma_pci_tbl);
659
660static __init int erdma_init_module(void)
661{
662 int ret;
663
664 ret = erdma_cm_init();
665 if (ret)
666 return ret;
667
668 ret = pci_register_driver(&erdma_pci_driver);
669 if (ret)
670 erdma_cm_exit();
671
672 return ret;
673}
674
675static void __exit erdma_exit_module(void)
676{
677 pci_unregister_driver(&erdma_pci_driver);
678
679 erdma_cm_exit();
680}
681
682module_init(erdma_init_module);
683module_exit(erdma_exit_module);