Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
PCI: endpoint: Add EP function driver to provide NTB functionality
Add a new endpoint function driver to provide NTB functionality using
multiple PCIe endpoint instances.
[arnd@arndb.de: Select configfs dependency]
[yebin10@huawei.com: Fix unused but set variables]
[geert+renesas@glider.be: Explain NTB in PCI_EPF_NTB help text]
Link: https://lore.kernel.org/r/20210201195809.7342-14-kishon@ti.com
Signed-off-by: Kishon Vijay Abraham I <kishon@ti.com>
Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Ye Bin <yebin10@huawei.com>
Signed-off-by: Geert Uytterhoeven <geert+renesas@glider.be>
Signed-off-by: Lorenzo Pieralisi <lorenzo.pieralisi@arm.com>
Signed-off-by: Bjorn Helgaas <bhelgaas@google.com>
authored by
Kishon Vijay Abraham I
and committed by
Bjorn Helgaas
8b821cf7
a62074a9
+2142
+13
drivers/pci/endpoint/functions/Kconfig
+13
drivers/pci/endpoint/functions/Kconfig
···
12
12
for PCI Endpoint.
13
13
14
14
If in doubt, say "N" to disable Endpoint test driver.
15
+
16
+
config PCI_EPF_NTB
17
+
tristate "PCI Endpoint NTB driver"
18
+
depends on PCI_ENDPOINT
19
+
select CONFIGFS_FS
20
+
help
21
+
Select this configuration option to enable the Non-Transparent
22
+
Bridge (NTB) driver for PCI Endpoint. NTB driver implements NTB
23
+
controller functionality using multiple PCIe endpoint instances.
24
+
It can support NTB endpoint function devices created using
25
+
device tree.
26
+
27
+
If in doubt, say "N" to disable Endpoint NTB driver.
+1
drivers/pci/endpoint/functions/Makefile
+1
drivers/pci/endpoint/functions/Makefile
+2128
drivers/pci/endpoint/functions/pci-epf-ntb.c
+2128
drivers/pci/endpoint/functions/pci-epf-ntb.c
···
1
+
// SPDX-License-Identifier: GPL-2.0
2
+
/**
3
+
* Endpoint Function Driver to implement Non-Transparent Bridge functionality
4
+
*
5
+
* Copyright (C) 2020 Texas Instruments
6
+
* Author: Kishon Vijay Abraham I <kishon@ti.com>
7
+
*/
8
+
9
+
/*
10
+
* The PCI NTB function driver configures the SoC with multiple PCIe Endpoint
11
+
* (EP) controller instances (see diagram below) in such a way that
12
+
* transactions from one EP controller are routed to the other EP controller.
13
+
* Once PCI NTB function driver configures the SoC with multiple EP instances,
14
+
* HOST1 and HOST2 can communicate with each other using SoC as a bridge.
15
+
*
16
+
* +-------------+ +-------------+
17
+
* | | | |
18
+
* | HOST1 | | HOST2 |
19
+
* | | | |
20
+
* +------^------+ +------^------+
21
+
* | |
22
+
* | |
23
+
* +---------|-------------------------------------------------|---------+
24
+
* | +------v------+ +------v------+ |
25
+
* | | | | | |
26
+
* | | EP | | EP | |
27
+
* | | CONTROLLER1 | | CONTROLLER2 | |
28
+
* | | <-----------------------------------> | |
29
+
* | | | | | |
30
+
* | | | | | |
31
+
* | | | SoC With Multiple EP Instances | | |
32
+
* | | | (Configured using NTB Function) | | |
33
+
* | +-------------+ +-------------+ |
34
+
* +---------------------------------------------------------------------+
35
+
*/
36
+
37
+
#include <linux/delay.h>
38
+
#include <linux/io.h>
39
+
#include <linux/module.h>
40
+
#include <linux/slab.h>
41
+
42
+
#include <linux/pci-epc.h>
43
+
#include <linux/pci-epf.h>
44
+
45
+
static struct workqueue_struct *kpcintb_workqueue;
46
+
47
+
#define COMMAND_CONFIGURE_DOORBELL 1
48
+
#define COMMAND_TEARDOWN_DOORBELL 2
49
+
#define COMMAND_CONFIGURE_MW 3
50
+
#define COMMAND_TEARDOWN_MW 4
51
+
#define COMMAND_LINK_UP 5
52
+
#define COMMAND_LINK_DOWN 6
53
+
54
+
#define COMMAND_STATUS_OK 1
55
+
#define COMMAND_STATUS_ERROR 2
56
+
57
+
#define LINK_STATUS_UP BIT(0)
58
+
59
+
#define SPAD_COUNT 64
60
+
#define DB_COUNT 4
61
+
#define NTB_MW_OFFSET 2
62
+
#define DB_COUNT_MASK GENMASK(15, 0)
63
+
#define MSIX_ENABLE BIT(16)
64
+
#define MAX_DB_COUNT 32
65
+
#define MAX_MW 4
66
+
67
+
enum epf_ntb_bar {
68
+
BAR_CONFIG,
69
+
BAR_PEER_SPAD,
70
+
BAR_DB_MW1,
71
+
BAR_MW2,
72
+
BAR_MW3,
73
+
BAR_MW4,
74
+
};
75
+
76
+
struct epf_ntb {
77
+
u32 num_mws;
78
+
u32 db_count;
79
+
u32 spad_count;
80
+
struct pci_epf *epf;
81
+
u64 mws_size[MAX_MW];
82
+
struct config_group group;
83
+
struct epf_ntb_epc *epc[2];
84
+
};
85
+
86
+
#define to_epf_ntb(epf_group) container_of((epf_group), struct epf_ntb, group)
87
+
88
+
struct epf_ntb_epc {
89
+
u8 func_no;
90
+
bool linkup;
91
+
bool is_msix;
92
+
int msix_bar;
93
+
u32 spad_size;
94
+
struct pci_epc *epc;
95
+
struct epf_ntb *epf_ntb;
96
+
void __iomem *mw_addr[6];
97
+
size_t msix_table_offset;
98
+
struct epf_ntb_ctrl *reg;
99
+
struct pci_epf_bar *epf_bar;
100
+
enum pci_barno epf_ntb_bar[6];
101
+
struct delayed_work cmd_handler;
102
+
enum pci_epc_interface_type type;
103
+
const struct pci_epc_features *epc_features;
104
+
};
105
+
106
+
struct epf_ntb_ctrl {
107
+
u32 command;
108
+
u32 argument;
109
+
u16 command_status;
110
+
u16 link_status;
111
+
u32 topology;
112
+
u64 addr;
113
+
u64 size;
114
+
u32 num_mws;
115
+
u32 mw1_offset;
116
+
u32 spad_offset;
117
+
u32 spad_count;
118
+
u32 db_entry_size;
119
+
u32 db_data[MAX_DB_COUNT];
120
+
u32 db_offset[MAX_DB_COUNT];
121
+
} __packed;
122
+
123
+
static struct pci_epf_header epf_ntb_header = {
124
+
.vendorid = PCI_ANY_ID,
125
+
.deviceid = PCI_ANY_ID,
126
+
.baseclass_code = PCI_BASE_CLASS_MEMORY,
127
+
.interrupt_pin = PCI_INTERRUPT_INTA,
128
+
};
129
+
130
+
/**
131
+
* epf_ntb_link_up() - Raise link_up interrupt to both the hosts
132
+
* @ntb: NTB device that facilitates communication between HOST1 and HOST2
133
+
* @link_up: true or false indicating Link is UP or Down
134
+
*
135
+
* Once NTB function in HOST1 and the NTB function in HOST2 invoke
136
+
* ntb_link_enable(), this NTB function driver will trigger a link event to
137
+
* the NTB client in both the hosts.
138
+
*/
139
+
static int epf_ntb_link_up(struct epf_ntb *ntb, bool link_up)
140
+
{
141
+
enum pci_epc_interface_type type;
142
+
enum pci_epc_irq_type irq_type;
143
+
struct epf_ntb_epc *ntb_epc;
144
+
struct epf_ntb_ctrl *ctrl;
145
+
struct pci_epc *epc;
146
+
bool is_msix;
147
+
u8 func_no;
148
+
int ret;
149
+
150
+
for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) {
151
+
ntb_epc = ntb->epc[type];
152
+
epc = ntb_epc->epc;
153
+
func_no = ntb_epc->func_no;
154
+
is_msix = ntb_epc->is_msix;
155
+
ctrl = ntb_epc->reg;
156
+
if (link_up)
157
+
ctrl->link_status |= LINK_STATUS_UP;
158
+
else
159
+
ctrl->link_status &= ~LINK_STATUS_UP;
160
+
irq_type = is_msix ? PCI_EPC_IRQ_MSIX : PCI_EPC_IRQ_MSI;
161
+
ret = pci_epc_raise_irq(epc, func_no, irq_type, 1);
162
+
if (ret) {
163
+
dev_err(&epc->dev,
164
+
"%s intf: Failed to raise Link Up IRQ\n",
165
+
pci_epc_interface_string(type));
166
+
return ret;
167
+
}
168
+
}
169
+
170
+
return 0;
171
+
}
172
+
173
+
/**
174
+
* epf_ntb_configure_mw() - Configure the Outbound Address Space for one host
175
+
* to access the memory window of other host
176
+
* @ntb: NTB device that facilitates communication between HOST1 and HOST2
177
+
* @type: PRIMARY interface or SECONDARY interface
178
+
* @mw: Index of the memory window (either 0, 1, 2 or 3)
179
+
*
180
+
* +-----------------+ +---->+----------------+-----------+-----------------+
181
+
* | BAR0 | | | Doorbell 1 +-----------> MSI|X ADDRESS 1 |
182
+
* +-----------------+ | +----------------+ +-----------------+
183
+
* | BAR1 | | | Doorbell 2 +---------+ | |
184
+
* +-----------------+----+ +----------------+ | | |
185
+
* | BAR2 | | Doorbell 3 +-------+ | +-----------------+
186
+
* +-----------------+----+ +----------------+ | +-> MSI|X ADDRESS 2 |
187
+
* | BAR3 | | | Doorbell 4 +-----+ | +-----------------+
188
+
* +-----------------+ | |----------------+ | | | |
189
+
* | BAR4 | | | | | | +-----------------+
190
+
* +-----------------+ | | MW1 +---+ | +-->+ MSI|X ADDRESS 3||
191
+
* | BAR5 | | | | | | +-----------------+
192
+
* +-----------------+ +---->-----------------+ | | | |
193
+
* EP CONTROLLER 1 | | | | +-----------------+
194
+
* | | | +---->+ MSI|X ADDRESS 4 |
195
+
* +----------------+ | +-----------------+
196
+
* (A) EP CONTROLLER 2 | | |
197
+
* (OB SPACE) | | |
198
+
* +-------> MW1 |
199
+
* | |
200
+
* | |
201
+
* (B) +-----------------+
202
+
* | |
203
+
* | |
204
+
* | |
205
+
* | |
206
+
* | |
207
+
* +-----------------+
208
+
* PCI Address Space
209
+
* (Managed by HOST2)
210
+
*
211
+
* This function performs stage (B) in the above diagram (see MW1) i.e., map OB
212
+
* address space of memory window to PCI address space.
213
+
*
214
+
* This operation requires 3 parameters
215
+
* 1) Address in the outbound address space
216
+
* 2) Address in the PCI Address space
217
+
* 3) Size of the address region to be mapped
218
+
*
219
+
* The address in the outbound address space (for MW1, MW2, MW3 and MW4) is
220
+
* stored in epf_bar corresponding to BAR_DB_MW1 for MW1 and BAR_MW2, BAR_MW3
221
+
* BAR_MW4 for rest of the BARs of epf_ntb_epc that is connected to HOST1. This
222
+
* is populated in epf_ntb_alloc_peer_mem() in this driver.
223
+
*
224
+
* The address and size of the PCI address region that has to be mapped would
225
+
* be provided by HOST2 in ctrl->addr and ctrl->size of epf_ntb_epc that is
226
+
* connected to HOST2.
227
+
*
228
+
* Please note Memory window1 (MW1) and Doorbell registers together will be
229
+
* mapped to a single BAR (BAR2) above for 32-bit BARs. The exact BAR that's
230
+
* used for Memory window (MW) can be obtained from epf_ntb_bar[BAR_DB_MW1],
231
+
* epf_ntb_bar[BAR_MW2], epf_ntb_bar[BAR_MW2], epf_ntb_bar[BAR_MW2].
232
+
*/
233
+
static int epf_ntb_configure_mw(struct epf_ntb *ntb,
234
+
enum pci_epc_interface_type type, u32 mw)
235
+
{
236
+
struct epf_ntb_epc *peer_ntb_epc, *ntb_epc;
237
+
struct pci_epf_bar *peer_epf_bar;
238
+
enum pci_barno peer_barno;
239
+
struct epf_ntb_ctrl *ctrl;
240
+
phys_addr_t phys_addr;
241
+
struct pci_epc *epc;
242
+
u64 addr, size;
243
+
int ret = 0;
244
+
u8 func_no;
245
+
246
+
ntb_epc = ntb->epc[type];
247
+
epc = ntb_epc->epc;
248
+
249
+
peer_ntb_epc = ntb->epc[!type];
250
+
peer_barno = peer_ntb_epc->epf_ntb_bar[mw + NTB_MW_OFFSET];
251
+
peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno];
252
+
253
+
phys_addr = peer_epf_bar->phys_addr;
254
+
ctrl = ntb_epc->reg;
255
+
addr = ctrl->addr;
256
+
size = ctrl->size;
257
+
if (mw + NTB_MW_OFFSET == BAR_DB_MW1)
258
+
phys_addr += ctrl->mw1_offset;
259
+
260
+
if (size > ntb->mws_size[mw]) {
261
+
dev_err(&epc->dev,
262
+
"%s intf: MW: %d Req Sz:%llxx > Supported Sz:%llx\n",
263
+
pci_epc_interface_string(type), mw, size,
264
+
ntb->mws_size[mw]);
265
+
ret = -EINVAL;
266
+
goto err_invalid_size;
267
+
}
268
+
269
+
func_no = ntb_epc->func_no;
270
+
271
+
ret = pci_epc_map_addr(epc, func_no, phys_addr, addr, size);
272
+
if (ret)
273
+
dev_err(&epc->dev,
274
+
"%s intf: Failed to map memory window %d address\n",
275
+
pci_epc_interface_string(type), mw);
276
+
277
+
err_invalid_size:
278
+
279
+
return ret;
280
+
}
281
+
282
+
/**
283
+
* epf_ntb_teardown_mw() - Teardown the configured OB ATU
284
+
* @ntb: NTB device that facilitates communication between HOST1 and HOST2
285
+
* @type: PRIMARY interface or SECONDARY interface
286
+
* @mw: Index of the memory window (either 0, 1, 2 or 3)
287
+
*
288
+
* Teardown the configured OB ATU configured in epf_ntb_configure_mw() using
289
+
* pci_epc_unmap_addr()
290
+
*/
291
+
static void epf_ntb_teardown_mw(struct epf_ntb *ntb,
292
+
enum pci_epc_interface_type type, u32 mw)
293
+
{
294
+
struct epf_ntb_epc *peer_ntb_epc, *ntb_epc;
295
+
struct pci_epf_bar *peer_epf_bar;
296
+
enum pci_barno peer_barno;
297
+
struct epf_ntb_ctrl *ctrl;
298
+
phys_addr_t phys_addr;
299
+
struct pci_epc *epc;
300
+
u8 func_no;
301
+
302
+
ntb_epc = ntb->epc[type];
303
+
epc = ntb_epc->epc;
304
+
305
+
peer_ntb_epc = ntb->epc[!type];
306
+
peer_barno = peer_ntb_epc->epf_ntb_bar[mw + NTB_MW_OFFSET];
307
+
peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno];
308
+
309
+
phys_addr = peer_epf_bar->phys_addr;
310
+
ctrl = ntb_epc->reg;
311
+
if (mw + NTB_MW_OFFSET == BAR_DB_MW1)
312
+
phys_addr += ctrl->mw1_offset;
313
+
func_no = ntb_epc->func_no;
314
+
315
+
pci_epc_unmap_addr(epc, func_no, phys_addr);
316
+
}
317
+
318
+
/**
319
+
* epf_ntb_configure_msi() - Map OB address space to MSI address
320
+
* @ntb: NTB device that facilitates communication between HOST1 and HOST2
321
+
* @type: PRIMARY interface or SECONDARY interface
322
+
* @db_count: Number of doorbell interrupts to map
323
+
*
324
+
*+-----------------+ +----->+----------------+-----------+-----------------+
325
+
*| BAR0 | | | Doorbell 1 +---+-------> MSI ADDRESS |
326
+
*+-----------------+ | +----------------+ | +-----------------+
327
+
*| BAR1 | | | Doorbell 2 +---+ | |
328
+
*+-----------------+----+ +----------------+ | | |
329
+
*| BAR2 | | Doorbell 3 +---+ | |
330
+
*+-----------------+----+ +----------------+ | | |
331
+
*| BAR3 | | | Doorbell 4 +---+ | |
332
+
*+-----------------+ | |----------------+ | |
333
+
*| BAR4 | | | | | |
334
+
*+-----------------+ | | MW1 | | |
335
+
*| BAR5 | | | | | |
336
+
*+-----------------+ +----->-----------------+ | |
337
+
* EP CONTROLLER 1 | | | |
338
+
* | | | |
339
+
* +----------------+ +-----------------+
340
+
* (A) EP CONTROLLER 2 | |
341
+
* (OB SPACE) | |
342
+
* | MW1 |
343
+
* | |
344
+
* | |
345
+
* (B) +-----------------+
346
+
* | |
347
+
* | |
348
+
* | |
349
+
* | |
350
+
* | |
351
+
* +-----------------+
352
+
* PCI Address Space
353
+
* (Managed by HOST2)
354
+
*
355
+
*
356
+
* This function performs stage (B) in the above diagram (see Doorbell 1,
357
+
* Doorbell 2, Doorbell 3, Doorbell 4) i.e map OB address space corresponding to
358
+
* doorbell to MSI address in PCI address space.
359
+
*
360
+
* This operation requires 3 parameters
361
+
* 1) Address reserved for doorbell in the outbound address space
362
+
* 2) MSI-X address in the PCIe Address space
363
+
* 3) Number of MSI-X interrupts that has to be configured
364
+
*
365
+
* The address in the outbound address space (for the Doorbell) is stored in
366
+
* epf_bar corresponding to BAR_DB_MW1 of epf_ntb_epc that is connected to
367
+
* HOST1. This is populated in epf_ntb_alloc_peer_mem() in this driver along
368
+
* with address for MW1.
369
+
*
370
+
* pci_epc_map_msi_irq() takes the MSI address from MSI capability register
371
+
* and maps the OB address (obtained in epf_ntb_alloc_peer_mem()) to the MSI
372
+
* address.
373
+
*
374
+
* epf_ntb_configure_msi() also stores the MSI data to raise each interrupt
375
+
* in db_data of the peer's control region. This helps the peer to raise
376
+
* doorbell of the other host by writing db_data to the BAR corresponding to
377
+
* BAR_DB_MW1.
378
+
*/
379
+
static int epf_ntb_configure_msi(struct epf_ntb *ntb,
380
+
enum pci_epc_interface_type type, u16 db_count)
381
+
{
382
+
struct epf_ntb_epc *peer_ntb_epc, *ntb_epc;
383
+
u32 db_entry_size, db_data, db_offset;
384
+
struct pci_epf_bar *peer_epf_bar;
385
+
struct epf_ntb_ctrl *peer_ctrl;
386
+
enum pci_barno peer_barno;
387
+
phys_addr_t phys_addr;
388
+
struct pci_epc *epc;
389
+
u8 func_no;
390
+
int ret, i;
391
+
392
+
ntb_epc = ntb->epc[type];
393
+
epc = ntb_epc->epc;
394
+
395
+
peer_ntb_epc = ntb->epc[!type];
396
+
peer_barno = peer_ntb_epc->epf_ntb_bar[BAR_DB_MW1];
397
+
peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno];
398
+
peer_ctrl = peer_ntb_epc->reg;
399
+
db_entry_size = peer_ctrl->db_entry_size;
400
+
401
+
phys_addr = peer_epf_bar->phys_addr;
402
+
func_no = ntb_epc->func_no;
403
+
404
+
ret = pci_epc_map_msi_irq(epc, func_no, phys_addr, db_count,
405
+
db_entry_size, &db_data, &db_offset);
406
+
if (ret) {
407
+
dev_err(&epc->dev, "%s intf: Failed to map MSI IRQ\n",
408
+
pci_epc_interface_string(type));
409
+
return ret;
410
+
}
411
+
412
+
for (i = 0; i < db_count; i++) {
413
+
peer_ctrl->db_data[i] = db_data | i;
414
+
peer_ctrl->db_offset[i] = db_offset;
415
+
}
416
+
417
+
return 0;
418
+
}
419
+
420
+
/**
421
+
* epf_ntb_configure_msix() - Map OB address space to MSI-X address
422
+
* @ntb: NTB device that facilitates communication between HOST1 and HOST2
423
+
* @type: PRIMARY interface or SECONDARY interface
424
+
* @db_count: Number of doorbell interrupts to map
425
+
*
426
+
*+-----------------+ +----->+----------------+-----------+-----------------+
427
+
*| BAR0 | | | Doorbell 1 +-----------> MSI-X ADDRESS 1 |
428
+
*+-----------------+ | +----------------+ +-----------------+
429
+
*| BAR1 | | | Doorbell 2 +---------+ | |
430
+
*+-----------------+----+ +----------------+ | | |
431
+
*| BAR2 | | Doorbell 3 +-------+ | +-----------------+
432
+
*+-----------------+----+ +----------------+ | +-> MSI-X ADDRESS 2 |
433
+
*| BAR3 | | | Doorbell 4 +-----+ | +-----------------+
434
+
*+-----------------+ | |----------------+ | | | |
435
+
*| BAR4 | | | | | | +-----------------+
436
+
*+-----------------+ | | MW1 + | +-->+ MSI-X ADDRESS 3||
437
+
*| BAR5 | | | | | +-----------------+
438
+
*+-----------------+ +----->-----------------+ | | |
439
+
* EP CONTROLLER 1 | | | +-----------------+
440
+
* | | +---->+ MSI-X ADDRESS 4 |
441
+
* +----------------+ +-----------------+
442
+
* (A) EP CONTROLLER 2 | |
443
+
* (OB SPACE) | |
444
+
* | MW1 |
445
+
* | |
446
+
* | |
447
+
* (B) +-----------------+
448
+
* | |
449
+
* | |
450
+
* | |
451
+
* | |
452
+
* | |
453
+
* +-----------------+
454
+
* PCI Address Space
455
+
* (Managed by HOST2)
456
+
*
457
+
* This function performs stage (B) in the above diagram (see Doorbell 1,
458
+
* Doorbell 2, Doorbell 3, Doorbell 4) i.e map OB address space corresponding to
459
+
* doorbell to MSI-X address in PCI address space.
460
+
*
461
+
* This operation requires 3 parameters
462
+
* 1) Address reserved for doorbell in the outbound address space
463
+
* 2) MSI-X address in the PCIe Address space
464
+
* 3) Number of MSI-X interrupts that has to be configured
465
+
*
466
+
* The address in the outbound address space (for the Doorbell) is stored in
467
+
* epf_bar corresponding to BAR_DB_MW1 of epf_ntb_epc that is connected to
468
+
* HOST1. This is populated in epf_ntb_alloc_peer_mem() in this driver along
469
+
* with address for MW1.
470
+
*
471
+
* The MSI-X address is in the MSI-X table of EP CONTROLLER 2 and
472
+
* the count of doorbell is in ctrl->argument of epf_ntb_epc that is connected
473
+
* to HOST2. MSI-X table is stored memory mapped to ntb_epc->msix_bar and the
474
+
* offset is in ntb_epc->msix_table_offset. From this epf_ntb_configure_msix()
475
+
* gets the MSI-X address and data.
476
+
*
477
+
* epf_ntb_configure_msix() also stores the MSI-X data to raise each interrupt
478
+
* in db_data of the peer's control region. This helps the peer to raise
479
+
* doorbell of the other host by writing db_data to the BAR corresponding to
480
+
* BAR_DB_MW1.
481
+
*/
482
+
static int epf_ntb_configure_msix(struct epf_ntb *ntb,
483
+
enum pci_epc_interface_type type,
484
+
u16 db_count)
485
+
{
486
+
const struct pci_epc_features *epc_features;
487
+
struct epf_ntb_epc *peer_ntb_epc, *ntb_epc;
488
+
struct pci_epf_bar *peer_epf_bar, *epf_bar;
489
+
struct pci_epf_msix_tbl *msix_tbl;
490
+
struct epf_ntb_ctrl *peer_ctrl;
491
+
u32 db_entry_size, msg_data;
492
+
enum pci_barno peer_barno;
493
+
phys_addr_t phys_addr;
494
+
struct pci_epc *epc;
495
+
size_t align;
496
+
u64 msg_addr;
497
+
u8 func_no;
498
+
int ret, i;
499
+
500
+
ntb_epc = ntb->epc[type];
501
+
epc = ntb_epc->epc;
502
+
503
+
epf_bar = &ntb_epc->epf_bar[ntb_epc->msix_bar];
504
+
msix_tbl = epf_bar->addr + ntb_epc->msix_table_offset;
505
+
506
+
peer_ntb_epc = ntb->epc[!type];
507
+
peer_barno = peer_ntb_epc->epf_ntb_bar[BAR_DB_MW1];
508
+
peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno];
509
+
phys_addr = peer_epf_bar->phys_addr;
510
+
peer_ctrl = peer_ntb_epc->reg;
511
+
epc_features = ntb_epc->epc_features;
512
+
align = epc_features->align;
513
+
514
+
func_no = ntb_epc->func_no;
515
+
db_entry_size = peer_ctrl->db_entry_size;
516
+
517
+
for (i = 0; i < db_count; i++) {
518
+
msg_addr = ALIGN_DOWN(msix_tbl[i].msg_addr, align);
519
+
msg_data = msix_tbl[i].msg_data;
520
+
ret = pci_epc_map_addr(epc, func_no, phys_addr, msg_addr,
521
+
db_entry_size);
522
+
if (ret) {
523
+
dev_err(&epc->dev,
524
+
"%s intf: Failed to configure MSI-X IRQ\n",
525
+
pci_epc_interface_string(type));
526
+
return ret;
527
+
}
528
+
phys_addr = phys_addr + db_entry_size;
529
+
peer_ctrl->db_data[i] = msg_data;
530
+
peer_ctrl->db_offset[i] = msix_tbl[i].msg_addr & (align - 1);
531
+
}
532
+
ntb_epc->is_msix = true;
533
+
534
+
return 0;
535
+
}
536
+
537
+
/**
538
+
* epf_ntb_configure_db() - Configure the Outbound Address Space for one host
539
+
* to ring the doorbell of other host
540
+
* @ntb: NTB device that facilitates communication between HOST1 and HOST2
541
+
* @type: PRIMARY interface or SECONDARY interface
542
+
* @db_count: Count of the number of doorbells that has to be configured
543
+
* @msix: Indicates whether MSI-X or MSI should be used
544
+
*
545
+
* Invokes epf_ntb_configure_msix() or epf_ntb_configure_msi() required for
546
+
* one HOST to ring the doorbell of other HOST.
547
+
*/
548
+
static int epf_ntb_configure_db(struct epf_ntb *ntb,
549
+
enum pci_epc_interface_type type,
550
+
u16 db_count, bool msix)
551
+
{
552
+
struct epf_ntb_epc *ntb_epc;
553
+
struct pci_epc *epc;
554
+
int ret;
555
+
556
+
if (db_count > MAX_DB_COUNT)
557
+
return -EINVAL;
558
+
559
+
ntb_epc = ntb->epc[type];
560
+
epc = ntb_epc->epc;
561
+
562
+
if (msix)
563
+
ret = epf_ntb_configure_msix(ntb, type, db_count);
564
+
else
565
+
ret = epf_ntb_configure_msi(ntb, type, db_count);
566
+
567
+
if (ret)
568
+
dev_err(&epc->dev, "%s intf: Failed to configure DB\n",
569
+
pci_epc_interface_string(type));
570
+
571
+
return ret;
572
+
}
573
+
574
+
/**
575
+
* epf_ntb_teardown_db() - Unmap address in OB address space to MSI/MSI-X
576
+
* address
577
+
* @ntb: NTB device that facilitates communication between HOST1 and HOST2
578
+
* @type: PRIMARY interface or SECONDARY interface
579
+
*
580
+
* Invoke pci_epc_unmap_addr() to unmap OB address to MSI/MSI-X address.
581
+
*/
582
+
static void
583
+
epf_ntb_teardown_db(struct epf_ntb *ntb, enum pci_epc_interface_type type)
584
+
{
585
+
struct epf_ntb_epc *peer_ntb_epc, *ntb_epc;
586
+
struct pci_epf_bar *peer_epf_bar;
587
+
enum pci_barno peer_barno;
588
+
phys_addr_t phys_addr;
589
+
struct pci_epc *epc;
590
+
u8 func_no;
591
+
592
+
ntb_epc = ntb->epc[type];
593
+
epc = ntb_epc->epc;
594
+
595
+
peer_ntb_epc = ntb->epc[!type];
596
+
peer_barno = peer_ntb_epc->epf_ntb_bar[BAR_DB_MW1];
597
+
peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno];
598
+
phys_addr = peer_epf_bar->phys_addr;
599
+
func_no = ntb_epc->func_no;
600
+
601
+
pci_epc_unmap_addr(epc, func_no, phys_addr);
602
+
}
603
+
604
+
/**
605
+
* epf_ntb_cmd_handler() - Handle commands provided by the NTB Host
606
+
* @work: work_struct for the two epf_ntb_epc (PRIMARY and SECONDARY)
607
+
*
608
+
* Workqueue function that gets invoked for the two epf_ntb_epc
609
+
* periodically (once every 5ms) to see if it has received any commands
610
+
* from NTB host. The host can send commands to configure doorbell or
611
+
* configure memory window or to update link status.
612
+
*/
613
+
static void epf_ntb_cmd_handler(struct work_struct *work)
614
+
{
615
+
enum pci_epc_interface_type type;
616
+
struct epf_ntb_epc *ntb_epc;
617
+
struct epf_ntb_ctrl *ctrl;
618
+
u32 command, argument;
619
+
struct epf_ntb *ntb;
620
+
struct device *dev;
621
+
u16 db_count;
622
+
bool is_msix;
623
+
int ret;
624
+
625
+
ntb_epc = container_of(work, struct epf_ntb_epc, cmd_handler.work);
626
+
ctrl = ntb_epc->reg;
627
+
command = ctrl->command;
628
+
if (!command)
629
+
goto reset_handler;
630
+
argument = ctrl->argument;
631
+
632
+
ctrl->command = 0;
633
+
ctrl->argument = 0;
634
+
635
+
ctrl = ntb_epc->reg;
636
+
type = ntb_epc->type;
637
+
ntb = ntb_epc->epf_ntb;
638
+
dev = &ntb->epf->dev;
639
+
640
+
switch (command) {
641
+
case COMMAND_CONFIGURE_DOORBELL:
642
+
db_count = argument & DB_COUNT_MASK;
643
+
is_msix = argument & MSIX_ENABLE;
644
+
ret = epf_ntb_configure_db(ntb, type, db_count, is_msix);
645
+
if (ret < 0)
646
+
ctrl->command_status = COMMAND_STATUS_ERROR;
647
+
else
648
+
ctrl->command_status = COMMAND_STATUS_OK;
649
+
break;
650
+
case COMMAND_TEARDOWN_DOORBELL:
651
+
epf_ntb_teardown_db(ntb, type);
652
+
ctrl->command_status = COMMAND_STATUS_OK;
653
+
break;
654
+
case COMMAND_CONFIGURE_MW:
655
+
ret = epf_ntb_configure_mw(ntb, type, argument);
656
+
if (ret < 0)
657
+
ctrl->command_status = COMMAND_STATUS_ERROR;
658
+
else
659
+
ctrl->command_status = COMMAND_STATUS_OK;
660
+
break;
661
+
case COMMAND_TEARDOWN_MW:
662
+
epf_ntb_teardown_mw(ntb, type, argument);
663
+
ctrl->command_status = COMMAND_STATUS_OK;
664
+
break;
665
+
case COMMAND_LINK_UP:
666
+
ntb_epc->linkup = true;
667
+
if (ntb->epc[PRIMARY_INTERFACE]->linkup &&
668
+
ntb->epc[SECONDARY_INTERFACE]->linkup) {
669
+
ret = epf_ntb_link_up(ntb, true);
670
+
if (ret < 0)
671
+
ctrl->command_status = COMMAND_STATUS_ERROR;
672
+
else
673
+
ctrl->command_status = COMMAND_STATUS_OK;
674
+
goto reset_handler;
675
+
}
676
+
ctrl->command_status = COMMAND_STATUS_OK;
677
+
break;
678
+
case COMMAND_LINK_DOWN:
679
+
ntb_epc->linkup = false;
680
+
ret = epf_ntb_link_up(ntb, false);
681
+
if (ret < 0)
682
+
ctrl->command_status = COMMAND_STATUS_ERROR;
683
+
else
684
+
ctrl->command_status = COMMAND_STATUS_OK;
685
+
break;
686
+
default:
687
+
dev_err(dev, "%s intf UNKNOWN command: %d\n",
688
+
pci_epc_interface_string(type), command);
689
+
break;
690
+
}
691
+
692
+
reset_handler:
693
+
queue_delayed_work(kpcintb_workqueue, &ntb_epc->cmd_handler,
694
+
msecs_to_jiffies(5));
695
+
}
696
+
697
+
/**
698
+
* epf_ntb_peer_spad_bar_clear() - Clear Peer Scratchpad BAR
699
+
* @ntb: NTB device that facilitates communication between HOST1 and HOST2
700
+
*
701
+
*+-----------------+------->+------------------+ +-----------------+
702
+
*| BAR0 | | CONFIG REGION | | BAR0 |
703
+
*+-----------------+----+ +------------------+<-------+-----------------+
704
+
*| BAR1 | | |SCRATCHPAD REGION | | BAR1 |
705
+
*+-----------------+ +-->+------------------+<-------+-----------------+
706
+
*| BAR2 | Local Memory | BAR2 |
707
+
*+-----------------+ +-----------------+
708
+
*| BAR3 | | BAR3 |
709
+
*+-----------------+ +-----------------+
710
+
*| BAR4 | | BAR4 |
711
+
*+-----------------+ +-----------------+
712
+
*| BAR5 | | BAR5 |
713
+
*+-----------------+ +-----------------+
714
+
* EP CONTROLLER 1 EP CONTROLLER 2
715
+
*
716
+
* Clear BAR1 of EP CONTROLLER 2 which contains the HOST2's peer scratchpad
717
+
* region. While BAR1 is the default peer scratchpad BAR, an NTB could have
718
+
* other BARs for peer scratchpad (because of 64-bit BARs or reserved BARs).
719
+
* This function can get the exact BAR used for peer scratchpad from
720
+
* epf_ntb_bar[BAR_PEER_SPAD].
721
+
*
722
+
* Since HOST2's peer scratchpad is also HOST1's self scratchpad, this function
723
+
* gets the address of peer scratchpad from
724
+
* peer_ntb_epc->epf_ntb_bar[BAR_CONFIG].
725
+
*/
726
+
static void epf_ntb_peer_spad_bar_clear(struct epf_ntb_epc *ntb_epc)
727
+
{
728
+
struct pci_epf_bar *epf_bar;
729
+
enum pci_barno barno;
730
+
struct pci_epc *epc;
731
+
u8 func_no;
732
+
733
+
epc = ntb_epc->epc;
734
+
func_no = ntb_epc->func_no;
735
+
barno = ntb_epc->epf_ntb_bar[BAR_PEER_SPAD];
736
+
epf_bar = &ntb_epc->epf_bar[barno];
737
+
pci_epc_clear_bar(epc, func_no, epf_bar);
738
+
}
739
+
740
+
/**
741
+
* epf_ntb_peer_spad_bar_set() - Set peer scratchpad BAR
742
+
* @ntb: NTB device that facilitates communication between HOST1 and HOST2
743
+
*
744
+
*+-----------------+------->+------------------+ +-----------------+
745
+
*| BAR0 | | CONFIG REGION | | BAR0 |
746
+
*+-----------------+----+ +------------------+<-------+-----------------+
747
+
*| BAR1 | | |SCRATCHPAD REGION | | BAR1 |
748
+
*+-----------------+ +-->+------------------+<-------+-----------------+
749
+
*| BAR2 | Local Memory | BAR2 |
750
+
*+-----------------+ +-----------------+
751
+
*| BAR3 | | BAR3 |
752
+
*+-----------------+ +-----------------+
753
+
*| BAR4 | | BAR4 |
754
+
*+-----------------+ +-----------------+
755
+
*| BAR5 | | BAR5 |
756
+
*+-----------------+ +-----------------+
757
+
* EP CONTROLLER 1 EP CONTROLLER 2
758
+
*
759
+
* Set BAR1 of EP CONTROLLER 2 which contains the HOST2's peer scratchpad
760
+
* region. While BAR1 is the default peer scratchpad BAR, an NTB could have
761
+
* other BARs for peer scratchpad (because of 64-bit BARs or reserved BARs).
762
+
* This function can get the exact BAR used for peer scratchpad from
763
+
* epf_ntb_bar[BAR_PEER_SPAD].
764
+
*
765
+
* Since HOST2's peer scratchpad is also HOST1's self scratchpad, this function
766
+
* gets the address of peer scratchpad from
767
+
* peer_ntb_epc->epf_ntb_bar[BAR_CONFIG].
768
+
*/
769
+
static int epf_ntb_peer_spad_bar_set(struct epf_ntb *ntb,
770
+
enum pci_epc_interface_type type)
771
+
{
772
+
struct epf_ntb_epc *peer_ntb_epc, *ntb_epc;
773
+
struct pci_epf_bar *peer_epf_bar, *epf_bar;
774
+
enum pci_barno peer_barno, barno;
775
+
u32 peer_spad_offset;
776
+
struct pci_epc *epc;
777
+
struct device *dev;
778
+
u8 func_no;
779
+
int ret;
780
+
781
+
dev = &ntb->epf->dev;
782
+
783
+
peer_ntb_epc = ntb->epc[!type];
784
+
peer_barno = peer_ntb_epc->epf_ntb_bar[BAR_CONFIG];
785
+
peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno];
786
+
787
+
ntb_epc = ntb->epc[type];
788
+
barno = ntb_epc->epf_ntb_bar[BAR_PEER_SPAD];
789
+
epf_bar = &ntb_epc->epf_bar[barno];
790
+
func_no = ntb_epc->func_no;
791
+
epc = ntb_epc->epc;
792
+
793
+
peer_spad_offset = peer_ntb_epc->reg->spad_offset;
794
+
epf_bar->phys_addr = peer_epf_bar->phys_addr + peer_spad_offset;
795
+
epf_bar->size = peer_ntb_epc->spad_size;
796
+
epf_bar->barno = barno;
797
+
epf_bar->flags = PCI_BASE_ADDRESS_MEM_TYPE_32;
798
+
799
+
ret = pci_epc_set_bar(epc, func_no, epf_bar);
800
+
if (ret) {
801
+
dev_err(dev, "%s intf: peer SPAD BAR set failed\n",
802
+
pci_epc_interface_string(type));
803
+
return ret;
804
+
}
805
+
806
+
return 0;
807
+
}
808
+
809
+
/**
810
+
* epf_ntb_config_sspad_bar_clear() - Clear Config + Self scratchpad BAR
811
+
* @ntb: NTB device that facilitates communication between HOST1 and HOST2
812
+
*
813
+
* +-----------------+------->+------------------+ +-----------------+
814
+
* | BAR0 | | CONFIG REGION | | BAR0 |
815
+
* +-----------------+----+ +------------------+<-------+-----------------+
816
+
* | BAR1 | | |SCRATCHPAD REGION | | BAR1 |
817
+
* +-----------------+ +-->+------------------+<-------+-----------------+
818
+
* | BAR2 | Local Memory | BAR2 |
819
+
* +-----------------+ +-----------------+
820
+
* | BAR3 | | BAR3 |
821
+
* +-----------------+ +-----------------+
822
+
* | BAR4 | | BAR4 |
823
+
* +-----------------+ +-----------------+
824
+
* | BAR5 | | BAR5 |
825
+
* +-----------------+ +-----------------+
826
+
* EP CONTROLLER 1 EP CONTROLLER 2
827
+
*
828
+
* Clear BAR0 of EP CONTROLLER 1 which contains the HOST1's config and
829
+
* self scratchpad region (removes inbound ATU configuration). While BAR0 is
830
+
* the default self scratchpad BAR, an NTB could have other BARs for self
831
+
* scratchpad (because of reserved BARs). This function can get the exact BAR
832
+
* used for self scratchpad from epf_ntb_bar[BAR_CONFIG].
833
+
*
834
+
* Please note the self scratchpad region and config region is combined to
835
+
* a single region and mapped using the same BAR. Also note HOST2's peer
836
+
* scratchpad is HOST1's self scratchpad.
837
+
*/
838
+
static void epf_ntb_config_sspad_bar_clear(struct epf_ntb_epc *ntb_epc)
839
+
{
840
+
struct pci_epf_bar *epf_bar;
841
+
enum pci_barno barno;
842
+
struct pci_epc *epc;
843
+
u8 func_no;
844
+
845
+
epc = ntb_epc->epc;
846
+
func_no = ntb_epc->func_no;
847
+
barno = ntb_epc->epf_ntb_bar[BAR_CONFIG];
848
+
epf_bar = &ntb_epc->epf_bar[barno];
849
+
pci_epc_clear_bar(epc, func_no, epf_bar);
850
+
}
851
+
852
+
/**
853
+
* epf_ntb_config_sspad_bar_set() - Set Config + Self scratchpad BAR
854
+
* @ntb: NTB device that facilitates communication between HOST1 and HOST2
855
+
*
856
+
* +-----------------+------->+------------------+ +-----------------+
857
+
* | BAR0 | | CONFIG REGION | | BAR0 |
858
+
* +-----------------+----+ +------------------+<-------+-----------------+
859
+
* | BAR1 | | |SCRATCHPAD REGION | | BAR1 |
860
+
* +-----------------+ +-->+------------------+<-------+-----------------+
861
+
* | BAR2 | Local Memory | BAR2 |
862
+
* +-----------------+ +-----------------+
863
+
* | BAR3 | | BAR3 |
864
+
* +-----------------+ +-----------------+
865
+
* | BAR4 | | BAR4 |
866
+
* +-----------------+ +-----------------+
867
+
* | BAR5 | | BAR5 |
868
+
* +-----------------+ +-----------------+
869
+
* EP CONTROLLER 1 EP CONTROLLER 2
870
+
*
871
+
* Map BAR0 of EP CONTROLLER 1 which contains the HOST1's config and
872
+
* self scratchpad region. While BAR0 is the default self scratchpad BAR, an
873
+
* NTB could have other BARs for self scratchpad (because of reserved BARs).
874
+
* This function can get the exact BAR used for self scratchpad from
875
+
* epf_ntb_bar[BAR_CONFIG].
876
+
*
877
+
* Please note the self scratchpad region and config region is combined to
878
+
* a single region and mapped using the same BAR. Also note HOST2's peer
879
+
* scratchpad is HOST1's self scratchpad.
880
+
*/
881
+
static int epf_ntb_config_sspad_bar_set(struct epf_ntb_epc *ntb_epc)
882
+
{
883
+
struct pci_epf_bar *epf_bar;
884
+
enum pci_barno barno;
885
+
struct epf_ntb *ntb;
886
+
struct pci_epc *epc;
887
+
struct device *dev;
888
+
u8 func_no;
889
+
int ret;
890
+
891
+
ntb = ntb_epc->epf_ntb;
892
+
dev = &ntb->epf->dev;
893
+
894
+
epc = ntb_epc->epc;
895
+
func_no = ntb_epc->func_no;
896
+
barno = ntb_epc->epf_ntb_bar[BAR_CONFIG];
897
+
epf_bar = &ntb_epc->epf_bar[barno];
898
+
899
+
ret = pci_epc_set_bar(epc, func_no, epf_bar);
900
+
if (ret) {
901
+
dev_err(dev, "%s inft: Config/Status/SPAD BAR set failed\n",
902
+
pci_epc_interface_string(ntb_epc->type));
903
+
return ret;
904
+
}
905
+
906
+
return 0;
907
+
}
908
+
909
+
/**
910
+
* epf_ntb_config_spad_bar_free() - Free the physical memory associated with
911
+
* config + scratchpad region
912
+
* @ntb: NTB device that facilitates communication between HOST1 and HOST2
913
+
*
914
+
* +-----------------+------->+------------------+ +-----------------+
915
+
* | BAR0 | | CONFIG REGION | | BAR0 |
916
+
* +-----------------+----+ +------------------+<-------+-----------------+
917
+
* | BAR1 | | |SCRATCHPAD REGION | | BAR1 |
918
+
* +-----------------+ +-->+------------------+<-------+-----------------+
919
+
* | BAR2 | Local Memory | BAR2 |
920
+
* +-----------------+ +-----------------+
921
+
* | BAR3 | | BAR3 |
922
+
* +-----------------+ +-----------------+
923
+
* | BAR4 | | BAR4 |
924
+
* +-----------------+ +-----------------+
925
+
* | BAR5 | | BAR5 |
926
+
* +-----------------+ +-----------------+
927
+
* EP CONTROLLER 1 EP CONTROLLER 2
928
+
*
929
+
* Free the Local Memory mentioned in the above diagram. After invoking this
930
+
* function, any of config + self scratchpad region of HOST1 or peer scratchpad
931
+
* region of HOST2 should not be accessed.
932
+
*/
933
+
static void epf_ntb_config_spad_bar_free(struct epf_ntb *ntb)
934
+
{
935
+
enum pci_epc_interface_type type;
936
+
struct epf_ntb_epc *ntb_epc;
937
+
enum pci_barno barno;
938
+
struct pci_epf *epf;
939
+
940
+
epf = ntb->epf;
941
+
for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) {
942
+
ntb_epc = ntb->epc[type];
943
+
barno = ntb_epc->epf_ntb_bar[BAR_CONFIG];
944
+
if (ntb_epc->reg)
945
+
pci_epf_free_space(epf, ntb_epc->reg, barno, type);
946
+
}
947
+
}
948
+
949
+
/**
950
+
* epf_ntb_config_spad_bar_alloc() - Allocate memory for config + scratchpad
951
+
* region
952
+
* @ntb: NTB device that facilitates communication between HOST1 and HOST2
953
+
* @type: PRIMARY interface or SECONDARY interface
954
+
*
955
+
* +-----------------+------->+------------------+ +-----------------+
956
+
* | BAR0 | | CONFIG REGION | | BAR0 |
957
+
* +-----------------+----+ +------------------+<-------+-----------------+
958
+
* | BAR1 | | |SCRATCHPAD REGION | | BAR1 |
959
+
* +-----------------+ +-->+------------------+<-------+-----------------+
960
+
* | BAR2 | Local Memory | BAR2 |
961
+
* +-----------------+ +-----------------+
962
+
* | BAR3 | | BAR3 |
963
+
* +-----------------+ +-----------------+
964
+
* | BAR4 | | BAR4 |
965
+
* +-----------------+ +-----------------+
966
+
* | BAR5 | | BAR5 |
967
+
* +-----------------+ +-----------------+
968
+
* EP CONTROLLER 1 EP CONTROLLER 2
969
+
*
970
+
* Allocate the Local Memory mentioned in the above diagram. The size of
971
+
* CONFIG REGION is sizeof(struct epf_ntb_ctrl) and size of SCRATCHPAD REGION
972
+
* is obtained from "spad-count" configfs entry.
973
+
*
974
+
* The size of both config region and scratchpad region has to be aligned,
975
+
* since the scratchpad region will also be mapped as PEER SCRATCHPAD of
976
+
* other host using a separate BAR.
977
+
*/
978
+
static int epf_ntb_config_spad_bar_alloc(struct epf_ntb *ntb,
979
+
enum pci_epc_interface_type type)
980
+
{
981
+
const struct pci_epc_features *peer_epc_features, *epc_features;
982
+
struct epf_ntb_epc *peer_ntb_epc, *ntb_epc;
983
+
size_t msix_table_size, pba_size, align;
984
+
enum pci_barno peer_barno, barno;
985
+
struct epf_ntb_ctrl *ctrl;
986
+
u32 spad_size, ctrl_size;
987
+
u64 size, peer_size;
988
+
struct pci_epf *epf;
989
+
struct device *dev;
990
+
bool msix_capable;
991
+
u32 spad_count;
992
+
void *base;
993
+
994
+
epf = ntb->epf;
995
+
dev = &epf->dev;
996
+
ntb_epc = ntb->epc[type];
997
+
998
+
epc_features = ntb_epc->epc_features;
999
+
barno = ntb_epc->epf_ntb_bar[BAR_CONFIG];
1000
+
size = epc_features->bar_fixed_size[barno];
1001
+
align = epc_features->align;
1002
+
1003
+
peer_ntb_epc = ntb->epc[!type];
1004
+
peer_epc_features = peer_ntb_epc->epc_features;
1005
+
peer_barno = ntb_epc->epf_ntb_bar[BAR_PEER_SPAD];
1006
+
peer_size = peer_epc_features->bar_fixed_size[peer_barno];
1007
+
1008
+
/* Check if epc_features is populated incorrectly */
1009
+
if ((!IS_ALIGNED(size, align)))
1010
+
return -EINVAL;
1011
+
1012
+
spad_count = ntb->spad_count;
1013
+
1014
+
ctrl_size = sizeof(struct epf_ntb_ctrl);
1015
+
spad_size = spad_count * 4;
1016
+
1017
+
msix_capable = epc_features->msix_capable;
1018
+
if (msix_capable) {
1019
+
msix_table_size = PCI_MSIX_ENTRY_SIZE * ntb->db_count;
1020
+
ctrl_size = ALIGN(ctrl_size, 8);
1021
+
ntb_epc->msix_table_offset = ctrl_size;
1022
+
ntb_epc->msix_bar = barno;
1023
+
/* Align to QWORD or 8 Bytes */
1024
+
pba_size = ALIGN(DIV_ROUND_UP(ntb->db_count, 8), 8);
1025
+
ctrl_size = ctrl_size + msix_table_size + pba_size;
1026
+
}
1027
+
1028
+
if (!align) {
1029
+
ctrl_size = roundup_pow_of_two(ctrl_size);
1030
+
spad_size = roundup_pow_of_two(spad_size);
1031
+
} else {
1032
+
ctrl_size = ALIGN(ctrl_size, align);
1033
+
spad_size = ALIGN(spad_size, align);
1034
+
}
1035
+
1036
+
if (peer_size) {
1037
+
if (peer_size < spad_size)
1038
+
spad_count = peer_size / 4;
1039
+
spad_size = peer_size;
1040
+
}
1041
+
1042
+
/*
1043
+
* In order to make sure SPAD offset is aligned to its size,
1044
+
* expand control region size to the size of SPAD if SPAD size
1045
+
* is greater than control region size.
1046
+
*/
1047
+
if (spad_size > ctrl_size)
1048
+
ctrl_size = spad_size;
1049
+
1050
+
if (!size)
1051
+
size = ctrl_size + spad_size;
1052
+
else if (size < ctrl_size + spad_size)
1053
+
return -EINVAL;
1054
+
1055
+
base = pci_epf_alloc_space(epf, size, barno, align, type);
1056
+
if (!base) {
1057
+
dev_err(dev, "%s intf: Config/Status/SPAD alloc region fail\n",
1058
+
pci_epc_interface_string(type));
1059
+
return -ENOMEM;
1060
+
}
1061
+
1062
+
ntb_epc->reg = base;
1063
+
1064
+
ctrl = ntb_epc->reg;
1065
+
ctrl->spad_offset = ctrl_size;
1066
+
ctrl->spad_count = spad_count;
1067
+
ctrl->num_mws = ntb->num_mws;
1068
+
ctrl->db_entry_size = align ? align : 4;
1069
+
ntb_epc->spad_size = spad_size;
1070
+
1071
+
return 0;
1072
+
}
1073
+
1074
+
/**
1075
+
* epf_ntb_config_spad_bar_alloc_interface() - Allocate memory for config +
1076
+
* scratchpad region for each of PRIMARY and SECONDARY interface
1077
+
* @ntb: NTB device that facilitates communication between HOST1 and HOST2
1078
+
*
1079
+
* Wrapper for epf_ntb_config_spad_bar_alloc() which allocates memory for
1080
+
* config + scratchpad region for a specific interface
1081
+
*/
1082
+
static int epf_ntb_config_spad_bar_alloc_interface(struct epf_ntb *ntb)
1083
+
{
1084
+
enum pci_epc_interface_type type;
1085
+
struct device *dev;
1086
+
int ret;
1087
+
1088
+
dev = &ntb->epf->dev;
1089
+
1090
+
for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) {
1091
+
ret = epf_ntb_config_spad_bar_alloc(ntb, type);
1092
+
if (ret) {
1093
+
dev_err(dev, "%s intf: Config/SPAD BAR alloc failed\n",
1094
+
pci_epc_interface_string(type));
1095
+
return ret;
1096
+
}
1097
+
}
1098
+
1099
+
return 0;
1100
+
}
1101
+
1102
+
/**
1103
+
* epf_ntb_free_peer_mem() - Free memory allocated in peers outbound address
1104
+
* space
1105
+
* @ntb_epc: EPC associated with one of the HOST which holds peers outbound
1106
+
* address regions
1107
+
*
1108
+
* +-----------------+ +---->+----------------+-----------+-----------------+
1109
+
* | BAR0 | | | Doorbell 1 +-----------> MSI|X ADDRESS 1 |
1110
+
* +-----------------+ | +----------------+ +-----------------+
1111
+
* | BAR1 | | | Doorbell 2 +---------+ | |
1112
+
* +-----------------+----+ +----------------+ | | |
1113
+
* | BAR2 | | Doorbell 3 +-------+ | +-----------------+
1114
+
* +-----------------+----+ +----------------+ | +-> MSI|X ADDRESS 2 |
1115
+
* | BAR3 | | | Doorbell 4 +-----+ | +-----------------+
1116
+
* +-----------------+ | |----------------+ | | | |
1117
+
* | BAR4 | | | | | | +-----------------+
1118
+
* +-----------------+ | | MW1 +---+ | +-->+ MSI|X ADDRESS 3||
1119
+
* | BAR5 | | | | | | +-----------------+
1120
+
* +-----------------+ +---->-----------------+ | | | |
1121
+
* EP CONTROLLER 1 | | | | +-----------------+
1122
+
* | | | +---->+ MSI|X ADDRESS 4 |
1123
+
* +----------------+ | +-----------------+
1124
+
* (A) EP CONTROLLER 2 | | |
1125
+
* (OB SPACE) | | |
1126
+
* +-------> MW1 |
1127
+
* | |
1128
+
* | |
1129
+
* (B) +-----------------+
1130
+
* | |
1131
+
* | |
1132
+
* | |
1133
+
* | |
1134
+
* | |
1135
+
* +-----------------+
1136
+
* PCI Address Space
1137
+
* (Managed by HOST2)
1138
+
*
1139
+
* Free memory allocated in EP CONTROLLER 2 (OB SPACE) in the above diagram.
1140
+
* It'll free Doorbell 1, Doorbell 2, Doorbell 3, Doorbell 4, MW1 (and MW2, MW3,
1141
+
* MW4).
1142
+
*/
1143
+
static void epf_ntb_free_peer_mem(struct epf_ntb_epc *ntb_epc)
1144
+
{
1145
+
struct pci_epf_bar *epf_bar;
1146
+
void __iomem *mw_addr;
1147
+
phys_addr_t phys_addr;
1148
+
enum epf_ntb_bar bar;
1149
+
enum pci_barno barno;
1150
+
struct pci_epc *epc;
1151
+
size_t size;
1152
+
1153
+
epc = ntb_epc->epc;
1154
+
1155
+
for (bar = BAR_DB_MW1; bar < BAR_MW4; bar++) {
1156
+
barno = ntb_epc->epf_ntb_bar[bar];
1157
+
mw_addr = ntb_epc->mw_addr[barno];
1158
+
epf_bar = &ntb_epc->epf_bar[barno];
1159
+
phys_addr = epf_bar->phys_addr;
1160
+
size = epf_bar->size;
1161
+
if (mw_addr) {
1162
+
pci_epc_mem_free_addr(epc, phys_addr, mw_addr, size);
1163
+
ntb_epc->mw_addr[barno] = NULL;
1164
+
}
1165
+
}
1166
+
}
1167
+
1168
+
/**
1169
+
* epf_ntb_db_mw_bar_clear() - Clear doorbell and memory BAR
1170
+
* @ntb_epc: EPC associated with one of the HOST which holds peer's outbound
1171
+
* address
1172
+
*
1173
+
* +-----------------+ +---->+----------------+-----------+-----------------+
1174
+
* | BAR0 | | | Doorbell 1 +-----------> MSI|X ADDRESS 1 |
1175
+
* +-----------------+ | +----------------+ +-----------------+
1176
+
* | BAR1 | | | Doorbell 2 +---------+ | |
1177
+
* +-----------------+----+ +----------------+ | | |
1178
+
* | BAR2 | | Doorbell 3 +-------+ | +-----------------+
1179
+
* +-----------------+----+ +----------------+ | +-> MSI|X ADDRESS 2 |
1180
+
* | BAR3 | | | Doorbell 4 +-----+ | +-----------------+
1181
+
* +-----------------+ | |----------------+ | | | |
1182
+
* | BAR4 | | | | | | +-----------------+
1183
+
* +-----------------+ | | MW1 +---+ | +-->+ MSI|X ADDRESS 3||
1184
+
* | BAR5 | | | | | | +-----------------+
1185
+
* +-----------------+ +---->-----------------+ | | | |
1186
+
* EP CONTROLLER 1 | | | | +-----------------+
1187
+
* | | | +---->+ MSI|X ADDRESS 4 |
1188
+
* +----------------+ | +-----------------+
1189
+
* (A) EP CONTROLLER 2 | | |
1190
+
* (OB SPACE) | | |
1191
+
* +-------> MW1 |
1192
+
* | |
1193
+
* | |
1194
+
* (B) +-----------------+
1195
+
* | |
1196
+
* | |
1197
+
* | |
1198
+
* | |
1199
+
* | |
1200
+
* +-----------------+
1201
+
* PCI Address Space
1202
+
* (Managed by HOST2)
1203
+
*
1204
+
* Clear doorbell and memory BARs (remove inbound ATU configuration). In the above
1205
+
* diagram it clears BAR2 TO BAR5 of EP CONTROLLER 1 (Doorbell BAR, MW1 BAR, MW2
1206
+
* BAR, MW3 BAR and MW4 BAR).
1207
+
*/
1208
+
static void epf_ntb_db_mw_bar_clear(struct epf_ntb_epc *ntb_epc)
1209
+
{
1210
+
struct pci_epf_bar *epf_bar;
1211
+
enum epf_ntb_bar bar;
1212
+
enum pci_barno barno;
1213
+
struct pci_epc *epc;
1214
+
u8 func_no;
1215
+
1216
+
epc = ntb_epc->epc;
1217
+
1218
+
func_no = ntb_epc->func_no;
1219
+
1220
+
for (bar = BAR_DB_MW1; bar < BAR_MW4; bar++) {
1221
+
barno = ntb_epc->epf_ntb_bar[bar];
1222
+
epf_bar = &ntb_epc->epf_bar[barno];
1223
+
pci_epc_clear_bar(epc, func_no, epf_bar);
1224
+
}
1225
+
}
1226
+
1227
+
/**
1228
+
* epf_ntb_db_mw_bar_cleanup() - Clear doorbell/memory BAR and free memory
1229
+
* allocated in peers outbound address space
1230
+
* @ntb: NTB device that facilitates communication between HOST1 and HOST2
1231
+
* @type: PRIMARY interface or SECONDARY interface
1232
+
*
1233
+
* Wrapper for epf_ntb_db_mw_bar_clear() to clear HOST1's BAR and
1234
+
* epf_ntb_free_peer_mem() which frees up HOST2 outbound memory.
1235
+
*/
1236
+
static void epf_ntb_db_mw_bar_cleanup(struct epf_ntb *ntb,
1237
+
enum pci_epc_interface_type type)
1238
+
{
1239
+
struct epf_ntb_epc *peer_ntb_epc, *ntb_epc;
1240
+
1241
+
ntb_epc = ntb->epc[type];
1242
+
peer_ntb_epc = ntb->epc[!type];
1243
+
1244
+
epf_ntb_db_mw_bar_clear(ntb_epc);
1245
+
epf_ntb_free_peer_mem(peer_ntb_epc);
1246
+
}
1247
+
1248
+
/**
1249
+
* epf_ntb_configure_interrupt() - Configure MSI/MSI-X capaiblity
1250
+
* @ntb: NTB device that facilitates communication between HOST1 and HOST2
1251
+
* @type: PRIMARY interface or SECONDARY interface
1252
+
*
1253
+
* Configure MSI/MSI-X capability for each interface with number of
1254
+
* interrupts equal to "db_count" configfs entry.
1255
+
*/
1256
+
static int epf_ntb_configure_interrupt(struct epf_ntb *ntb,
1257
+
enum pci_epc_interface_type type)
1258
+
{
1259
+
const struct pci_epc_features *epc_features;
1260
+
bool msix_capable, msi_capable;
1261
+
struct epf_ntb_epc *ntb_epc;
1262
+
struct pci_epc *epc;
1263
+
struct device *dev;
1264
+
u32 db_count;
1265
+
u8 func_no;
1266
+
int ret;
1267
+
1268
+
ntb_epc = ntb->epc[type];
1269
+
dev = &ntb->epf->dev;
1270
+
1271
+
epc_features = ntb_epc->epc_features;
1272
+
msix_capable = epc_features->msix_capable;
1273
+
msi_capable = epc_features->msi_capable;
1274
+
1275
+
if (!(msix_capable || msi_capable)) {
1276
+
dev_err(dev, "MSI or MSI-X is required for doorbell\n");
1277
+
return -EINVAL;
1278
+
}
1279
+
1280
+
func_no = ntb_epc->func_no;
1281
+
1282
+
db_count = ntb->db_count;
1283
+
if (db_count > MAX_DB_COUNT) {
1284
+
dev_err(dev, "DB count cannot be more than %d\n", MAX_DB_COUNT);
1285
+
return -EINVAL;
1286
+
}
1287
+
1288
+
ntb->db_count = db_count;
1289
+
epc = ntb_epc->epc;
1290
+
1291
+
if (msi_capable) {
1292
+
ret = pci_epc_set_msi(epc, func_no, db_count);
1293
+
if (ret) {
1294
+
dev_err(dev, "%s intf: MSI configuration failed\n",
1295
+
pci_epc_interface_string(type));
1296
+
return ret;
1297
+
}
1298
+
}
1299
+
1300
+
if (msix_capable) {
1301
+
ret = pci_epc_set_msix(epc, func_no, db_count,
1302
+
ntb_epc->msix_bar,
1303
+
ntb_epc->msix_table_offset);
1304
+
if (ret) {
1305
+
dev_err(dev, "MSI configuration failed\n");
1306
+
return ret;
1307
+
}
1308
+
}
1309
+
1310
+
return 0;
1311
+
}
1312
+
1313
+
/**
1314
+
* epf_ntb_alloc_peer_mem() - Allocate memory in peer's outbound address space
1315
+
* @ntb_epc: EPC associated with one of the HOST whose BAR holds peer's outbound
1316
+
* address
1317
+
* @bar: BAR of @ntb_epc in for which memory has to be allocated (could be
1318
+
* BAR_DB_MW1, BAR_MW2, BAR_MW3, BAR_MW4)
1319
+
* @peer_ntb_epc: EPC associated with HOST whose outbound address space is
1320
+
* used by @ntb_epc
1321
+
* @size: Size of the address region that has to be allocated in peers OB SPACE
1322
+
*
1323
+
*
1324
+
* +-----------------+ +---->+----------------+-----------+-----------------+
1325
+
* | BAR0 | | | Doorbell 1 +-----------> MSI|X ADDRESS 1 |
1326
+
* +-----------------+ | +----------------+ +-----------------+
1327
+
* | BAR1 | | | Doorbell 2 +---------+ | |
1328
+
* +-----------------+----+ +----------------+ | | |
1329
+
* | BAR2 | | Doorbell 3 +-------+ | +-----------------+
1330
+
* +-----------------+----+ +----------------+ | +-> MSI|X ADDRESS 2 |
1331
+
* | BAR3 | | | Doorbell 4 +-----+ | +-----------------+
1332
+
* +-----------------+ | |----------------+ | | | |
1333
+
* | BAR4 | | | | | | +-----------------+
1334
+
* +-----------------+ | | MW1 +---+ | +-->+ MSI|X ADDRESS 3||
1335
+
* | BAR5 | | | | | | +-----------------+
1336
+
* +-----------------+ +---->-----------------+ | | | |
1337
+
* EP CONTROLLER 1 | | | | +-----------------+
1338
+
* | | | +---->+ MSI|X ADDRESS 4 |
1339
+
* +----------------+ | +-----------------+
1340
+
* (A) EP CONTROLLER 2 | | |
1341
+
* (OB SPACE) | | |
1342
+
* +-------> MW1 |
1343
+
* | |
1344
+
* | |
1345
+
* (B) +-----------------+
1346
+
* | |
1347
+
* | |
1348
+
* | |
1349
+
* | |
1350
+
* | |
1351
+
* +-----------------+
1352
+
* PCI Address Space
1353
+
* (Managed by HOST2)
1354
+
*
1355
+
* Allocate memory in OB space of EP CONTROLLER 2 in the above diagram. Allocate
1356
+
* for Doorbell 1, Doorbell 2, Doorbell 3, Doorbell 4, MW1 (and MW2, MW3, MW4).
1357
+
*/
1358
+
static int epf_ntb_alloc_peer_mem(struct device *dev,
1359
+
struct epf_ntb_epc *ntb_epc,
1360
+
enum epf_ntb_bar bar,
1361
+
struct epf_ntb_epc *peer_ntb_epc,
1362
+
size_t size)
1363
+
{
1364
+
const struct pci_epc_features *epc_features;
1365
+
struct pci_epf_bar *epf_bar;
1366
+
struct pci_epc *peer_epc;
1367
+
phys_addr_t phys_addr;
1368
+
void __iomem *mw_addr;
1369
+
enum pci_barno barno;
1370
+
size_t align;
1371
+
1372
+
epc_features = ntb_epc->epc_features;
1373
+
align = epc_features->align;
1374
+
1375
+
if (size < 128)
1376
+
size = 128;
1377
+
1378
+
if (align)
1379
+
size = ALIGN(size, align);
1380
+
else
1381
+
size = roundup_pow_of_two(size);
1382
+
1383
+
peer_epc = peer_ntb_epc->epc;
1384
+
mw_addr = pci_epc_mem_alloc_addr(peer_epc, &phys_addr, size);
1385
+
if (!mw_addr) {
1386
+
dev_err(dev, "%s intf: Failed to allocate OB address\n",
1387
+
pci_epc_interface_string(peer_ntb_epc->type));
1388
+
return -ENOMEM;
1389
+
}
1390
+
1391
+
barno = ntb_epc->epf_ntb_bar[bar];
1392
+
epf_bar = &ntb_epc->epf_bar[barno];
1393
+
ntb_epc->mw_addr[barno] = mw_addr;
1394
+
1395
+
epf_bar->phys_addr = phys_addr;
1396
+
epf_bar->size = size;
1397
+
epf_bar->barno = barno;
1398
+
epf_bar->flags = PCI_BASE_ADDRESS_MEM_TYPE_32;
1399
+
1400
+
return 0;
1401
+
}
1402
+
1403
+
/**
1404
+
* epf_ntb_db_mw_bar_init() - Configure Doorbell and Memory window BARs
1405
+
* @ntb: NTB device that facilitates communication between HOST1 and HOST2
1406
+
* @type: PRIMARY interface or SECONDARY interface
1407
+
*
1408
+
* Wrapper for epf_ntb_alloc_peer_mem() and pci_epc_set_bar() that allocates
1409
+
* memory in OB address space of HOST2 and configures BAR of HOST1
1410
+
*/
1411
+
static int epf_ntb_db_mw_bar_init(struct epf_ntb *ntb,
1412
+
enum pci_epc_interface_type type)
1413
+
{
1414
+
const struct pci_epc_features *epc_features;
1415
+
struct epf_ntb_epc *peer_ntb_epc, *ntb_epc;
1416
+
struct pci_epf_bar *epf_bar;
1417
+
struct epf_ntb_ctrl *ctrl;
1418
+
u32 num_mws, db_count;
1419
+
enum epf_ntb_bar bar;
1420
+
enum pci_barno barno;
1421
+
struct pci_epc *epc;
1422
+
struct device *dev;
1423
+
size_t align;
1424
+
int ret, i;
1425
+
u8 func_no;
1426
+
u64 size;
1427
+
1428
+
ntb_epc = ntb->epc[type];
1429
+
peer_ntb_epc = ntb->epc[!type];
1430
+
1431
+
dev = &ntb->epf->dev;
1432
+
epc_features = ntb_epc->epc_features;
1433
+
align = epc_features->align;
1434
+
func_no = ntb_epc->func_no;
1435
+
epc = ntb_epc->epc;
1436
+
num_mws = ntb->num_mws;
1437
+
db_count = ntb->db_count;
1438
+
1439
+
for (bar = BAR_DB_MW1, i = 0; i < num_mws; bar++, i++) {
1440
+
if (bar == BAR_DB_MW1) {
1441
+
align = align ? align : 4;
1442
+
size = db_count * align;
1443
+
size = ALIGN(size, ntb->mws_size[i]);
1444
+
ctrl = ntb_epc->reg;
1445
+
ctrl->mw1_offset = size;
1446
+
size += ntb->mws_size[i];
1447
+
} else {
1448
+
size = ntb->mws_size[i];
1449
+
}
1450
+
1451
+
ret = epf_ntb_alloc_peer_mem(dev, ntb_epc, bar,
1452
+
peer_ntb_epc, size);
1453
+
if (ret) {
1454
+
dev_err(dev, "%s intf: DoorBell mem alloc failed\n",
1455
+
pci_epc_interface_string(type));
1456
+
goto err_alloc_peer_mem;
1457
+
}
1458
+
1459
+
barno = ntb_epc->epf_ntb_bar[bar];
1460
+
epf_bar = &ntb_epc->epf_bar[barno];
1461
+
1462
+
ret = pci_epc_set_bar(epc, func_no, epf_bar);
1463
+
if (ret) {
1464
+
dev_err(dev, "%s intf: DoorBell BAR set failed\n",
1465
+
pci_epc_interface_string(type));
1466
+
goto err_alloc_peer_mem;
1467
+
}
1468
+
}
1469
+
1470
+
return 0;
1471
+
1472
+
err_alloc_peer_mem:
1473
+
epf_ntb_db_mw_bar_cleanup(ntb, type);
1474
+
1475
+
return ret;
1476
+
}
1477
+
1478
+
/**
1479
+
* epf_ntb_epc_destroy_interface() - Cleanup NTB EPC interface
1480
+
* @ntb: NTB device that facilitates communication between HOST1 and HOST2
1481
+
* @type: PRIMARY interface or SECONDARY interface
1482
+
*
1483
+
* Unbind NTB function device from EPC and relinquish reference to pci_epc
1484
+
* for each of the interface.
1485
+
*/
1486
+
static void epf_ntb_epc_destroy_interface(struct epf_ntb *ntb,
1487
+
enum pci_epc_interface_type type)
1488
+
{
1489
+
struct epf_ntb_epc *ntb_epc;
1490
+
struct pci_epc *epc;
1491
+
struct pci_epf *epf;
1492
+
1493
+
if (type < 0)
1494
+
return;
1495
+
1496
+
epf = ntb->epf;
1497
+
ntb_epc = ntb->epc[type];
1498
+
if (!ntb_epc)
1499
+
return;
1500
+
epc = ntb_epc->epc;
1501
+
pci_epc_remove_epf(epc, epf, type);
1502
+
pci_epc_put(epc);
1503
+
}
1504
+
1505
+
/**
1506
+
* epf_ntb_epc_destroy() - Cleanup NTB EPC interface
1507
+
* @ntb: NTB device that facilitates communication between HOST1 and HOST2
1508
+
*
1509
+
* Wrapper for epf_ntb_epc_destroy_interface() to cleanup all the NTB interfaces
1510
+
*/
1511
+
static void epf_ntb_epc_destroy(struct epf_ntb *ntb)
1512
+
{
1513
+
enum pci_epc_interface_type type;
1514
+
1515
+
for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++)
1516
+
epf_ntb_epc_destroy_interface(ntb, type);
1517
+
}
1518
+
1519
+
/**
1520
+
* epf_ntb_epc_create_interface() - Create and initialize NTB EPC interface
1521
+
* @ntb: NTB device that facilitates communication between HOST1 and HOST2
1522
+
* @epc: struct pci_epc to which a particular NTB interface should be associated
1523
+
* @type: PRIMARY interface or SECONDARY interface
1524
+
*
1525
+
* Allocate memory for NTB EPC interface and initialize it.
1526
+
*/
1527
+
static int epf_ntb_epc_create_interface(struct epf_ntb *ntb,
1528
+
struct pci_epc *epc,
1529
+
enum pci_epc_interface_type type)
1530
+
{
1531
+
const struct pci_epc_features *epc_features;
1532
+
struct pci_epf_bar *epf_bar;
1533
+
struct epf_ntb_epc *ntb_epc;
1534
+
struct pci_epf *epf;
1535
+
struct device *dev;
1536
+
u8 func_no;
1537
+
1538
+
dev = &ntb->epf->dev;
1539
+
1540
+
ntb_epc = devm_kzalloc(dev, sizeof(*ntb_epc), GFP_KERNEL);
1541
+
if (!ntb_epc)
1542
+
return -ENOMEM;
1543
+
1544
+
epf = ntb->epf;
1545
+
if (type == PRIMARY_INTERFACE) {
1546
+
func_no = epf->func_no;
1547
+
epf_bar = epf->bar;
1548
+
} else {
1549
+
func_no = epf->sec_epc_func_no;
1550
+
epf_bar = epf->sec_epc_bar;
1551
+
}
1552
+
1553
+
ntb_epc->linkup = false;
1554
+
ntb_epc->epc = epc;
1555
+
ntb_epc->func_no = func_no;
1556
+
ntb_epc->type = type;
1557
+
ntb_epc->epf_bar = epf_bar;
1558
+
ntb_epc->epf_ntb = ntb;
1559
+
1560
+
epc_features = pci_epc_get_features(epc, func_no);
1561
+
if (!epc_features)
1562
+
return -EINVAL;
1563
+
ntb_epc->epc_features = epc_features;
1564
+
1565
+
ntb->epc[type] = ntb_epc;
1566
+
1567
+
return 0;
1568
+
}
1569
+
1570
+
/**
1571
+
* epf_ntb_epc_create() - Create and initialize NTB EPC interface
1572
+
* @ntb: NTB device that facilitates communication between HOST1 and HOST2
1573
+
*
1574
+
* Get a reference to EPC device and bind NTB function device to that EPC
1575
+
* for each of the interface. It is also a wrapper to
1576
+
* epf_ntb_epc_create_interface() to allocate memory for NTB EPC interface
1577
+
* and initialize it
1578
+
*/
1579
+
static int epf_ntb_epc_create(struct epf_ntb *ntb)
1580
+
{
1581
+
struct pci_epf *epf;
1582
+
struct device *dev;
1583
+
int ret;
1584
+
1585
+
epf = ntb->epf;
1586
+
dev = &epf->dev;
1587
+
1588
+
ret = epf_ntb_epc_create_interface(ntb, epf->epc, PRIMARY_INTERFACE);
1589
+
if (ret) {
1590
+
dev_err(dev, "PRIMARY intf: Fail to create NTB EPC\n");
1591
+
return ret;
1592
+
}
1593
+
1594
+
ret = epf_ntb_epc_create_interface(ntb, epf->sec_epc,
1595
+
SECONDARY_INTERFACE);
1596
+
if (ret) {
1597
+
dev_err(dev, "SECONDARY intf: Fail to create NTB EPC\n");
1598
+
goto err_epc_create;
1599
+
}
1600
+
1601
+
return 0;
1602
+
1603
+
err_epc_create:
1604
+
epf_ntb_epc_destroy_interface(ntb, PRIMARY_INTERFACE);
1605
+
1606
+
return ret;
1607
+
}
1608
+
1609
+
/**
1610
+
* epf_ntb_init_epc_bar_interface() - Identify BARs to be used for each of
1611
+
* the NTB constructs (scratchpad region, doorbell, memorywindow)
1612
+
* @ntb: NTB device that facilitates communication between HOST1 and HOST2
1613
+
* @type: PRIMARY interface or SECONDARY interface
1614
+
*
1615
+
* Identify the free BARs to be used for each of BAR_CONFIG, BAR_PEER_SPAD,
1616
+
* BAR_DB_MW1, BAR_MW2, BAR_MW3 and BAR_MW4.
1617
+
*/
1618
+
static int epf_ntb_init_epc_bar_interface(struct epf_ntb *ntb,
1619
+
enum pci_epc_interface_type type)
1620
+
{
1621
+
const struct pci_epc_features *epc_features;
1622
+
struct epf_ntb_epc *ntb_epc;
1623
+
enum pci_barno barno;
1624
+
enum epf_ntb_bar bar;
1625
+
struct device *dev;
1626
+
u32 num_mws;
1627
+
int i;
1628
+
1629
+
barno = BAR_0;
1630
+
ntb_epc = ntb->epc[type];
1631
+
num_mws = ntb->num_mws;
1632
+
dev = &ntb->epf->dev;
1633
+
epc_features = ntb_epc->epc_features;
1634
+
1635
+
/* These are required BARs which are mandatory for NTB functionality */
1636
+
for (bar = BAR_CONFIG; bar <= BAR_DB_MW1; bar++, barno++) {
1637
+
barno = pci_epc_get_next_free_bar(epc_features, barno);
1638
+
if (barno < 0) {
1639
+
dev_err(dev, "%s intf: Fail to get NTB function BAR\n",
1640
+
pci_epc_interface_string(type));
1641
+
return barno;
1642
+
}
1643
+
ntb_epc->epf_ntb_bar[bar] = barno;
1644
+
}
1645
+
1646
+
/* These are optional BARs which don't impact NTB functionality */
1647
+
for (bar = BAR_MW2, i = 1; i < num_mws; bar++, barno++, i++) {
1648
+
barno = pci_epc_get_next_free_bar(epc_features, barno);
1649
+
if (barno < 0) {
1650
+
ntb->num_mws = i;
1651
+
dev_dbg(dev, "BAR not available for > MW%d\n", i + 1);
1652
+
}
1653
+
ntb_epc->epf_ntb_bar[bar] = barno;
1654
+
}
1655
+
1656
+
return 0;
1657
+
}
1658
+
1659
+
/**
1660
+
* epf_ntb_init_epc_bar() - Identify BARs to be used for each of the NTB
1661
+
* constructs (scratchpad region, doorbell, memorywindow)
1662
+
* @ntb: NTB device that facilitates communication between HOST1 and HOST2
1663
+
* @type: PRIMARY interface or SECONDARY interface
1664
+
*
1665
+
* Wrapper to epf_ntb_init_epc_bar_interface() to identify the free BARs
1666
+
* to be used for each of BAR_CONFIG, BAR_PEER_SPAD, BAR_DB_MW1, BAR_MW2,
1667
+
* BAR_MW3 and BAR_MW4 for all the interfaces.
1668
+
*/
1669
+
static int epf_ntb_init_epc_bar(struct epf_ntb *ntb)
1670
+
{
1671
+
enum pci_epc_interface_type type;
1672
+
struct device *dev;
1673
+
int ret;
1674
+
1675
+
dev = &ntb->epf->dev;
1676
+
for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) {
1677
+
ret = epf_ntb_init_epc_bar_interface(ntb, type);
1678
+
if (ret) {
1679
+
dev_err(dev, "Fail to init EPC bar for %s interface\n",
1680
+
pci_epc_interface_string(type));
1681
+
return ret;
1682
+
}
1683
+
}
1684
+
1685
+
return 0;
1686
+
}
1687
+
1688
+
/**
1689
+
* epf_ntb_epc_init_interface() - Initialize NTB interface
1690
+
* @ntb: NTB device that facilitates communication between HOST1 and HOST2
1691
+
* @type: PRIMARY interface or SECONDARY interface
1692
+
*
1693
+
* Wrapper to initialize a particular EPC interface and start the workqueue
1694
+
* to check for commands from host. This function will write to the
1695
+
* EP controller HW for configuring it.
1696
+
*/
1697
+
static int epf_ntb_epc_init_interface(struct epf_ntb *ntb,
1698
+
enum pci_epc_interface_type type)
1699
+
{
1700
+
struct epf_ntb_epc *ntb_epc;
1701
+
struct pci_epc *epc;
1702
+
struct pci_epf *epf;
1703
+
struct device *dev;
1704
+
u8 func_no;
1705
+
int ret;
1706
+
1707
+
ntb_epc = ntb->epc[type];
1708
+
epf = ntb->epf;
1709
+
dev = &epf->dev;
1710
+
epc = ntb_epc->epc;
1711
+
func_no = ntb_epc->func_no;
1712
+
1713
+
ret = epf_ntb_config_sspad_bar_set(ntb->epc[type]);
1714
+
if (ret) {
1715
+
dev_err(dev, "%s intf: Config/self SPAD BAR init failed\n",
1716
+
pci_epc_interface_string(type));
1717
+
return ret;
1718
+
}
1719
+
1720
+
ret = epf_ntb_peer_spad_bar_set(ntb, type);
1721
+
if (ret) {
1722
+
dev_err(dev, "%s intf: Peer SPAD BAR init failed\n",
1723
+
pci_epc_interface_string(type));
1724
+
goto err_peer_spad_bar_init;
1725
+
}
1726
+
1727
+
ret = epf_ntb_configure_interrupt(ntb, type);
1728
+
if (ret) {
1729
+
dev_err(dev, "%s intf: Interrupt configuration failed\n",
1730
+
pci_epc_interface_string(type));
1731
+
goto err_peer_spad_bar_init;
1732
+
}
1733
+
1734
+
ret = epf_ntb_db_mw_bar_init(ntb, type);
1735
+
if (ret) {
1736
+
dev_err(dev, "%s intf: DB/MW BAR init failed\n",
1737
+
pci_epc_interface_string(type));
1738
+
goto err_db_mw_bar_init;
1739
+
}
1740
+
1741
+
ret = pci_epc_write_header(epc, func_no, epf->header);
1742
+
if (ret) {
1743
+
dev_err(dev, "%s intf: Configuration header write failed\n",
1744
+
pci_epc_interface_string(type));
1745
+
goto err_write_header;
1746
+
}
1747
+
1748
+
INIT_DELAYED_WORK(&ntb->epc[type]->cmd_handler, epf_ntb_cmd_handler);
1749
+
queue_work(kpcintb_workqueue, &ntb->epc[type]->cmd_handler.work);
1750
+
1751
+
return 0;
1752
+
1753
+
err_write_header:
1754
+
epf_ntb_db_mw_bar_cleanup(ntb, type);
1755
+
1756
+
err_db_mw_bar_init:
1757
+
epf_ntb_peer_spad_bar_clear(ntb->epc[type]);
1758
+
1759
+
err_peer_spad_bar_init:
1760
+
epf_ntb_config_sspad_bar_clear(ntb->epc[type]);
1761
+
1762
+
return ret;
1763
+
}
1764
+
1765
+
/**
1766
+
* epf_ntb_epc_cleanup_interface() - Cleanup NTB interface
1767
+
* @ntb: NTB device that facilitates communication between HOST1 and HOST2
1768
+
* @type: PRIMARY interface or SECONDARY interface
1769
+
*
1770
+
* Wrapper to cleanup a particular NTB interface.
1771
+
*/
1772
+
static void epf_ntb_epc_cleanup_interface(struct epf_ntb *ntb,
1773
+
enum pci_epc_interface_type type)
1774
+
{
1775
+
struct epf_ntb_epc *ntb_epc;
1776
+
1777
+
if (type < 0)
1778
+
return;
1779
+
1780
+
ntb_epc = ntb->epc[type];
1781
+
cancel_delayed_work(&ntb_epc->cmd_handler);
1782
+
epf_ntb_db_mw_bar_cleanup(ntb, type);
1783
+
epf_ntb_peer_spad_bar_clear(ntb_epc);
1784
+
epf_ntb_config_sspad_bar_clear(ntb_epc);
1785
+
}
1786
+
1787
+
/**
1788
+
* epf_ntb_epc_cleanup() - Cleanup all NTB interfaces
1789
+
* @ntb: NTB device that facilitates communication between HOST1 and HOST2
1790
+
*
1791
+
* Wrapper to cleanup all NTB interfaces.
1792
+
*/
1793
+
static void epf_ntb_epc_cleanup(struct epf_ntb *ntb)
1794
+
{
1795
+
enum pci_epc_interface_type type;
1796
+
1797
+
for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++)
1798
+
epf_ntb_epc_cleanup_interface(ntb, type);
1799
+
}
1800
+
1801
+
/**
1802
+
* epf_ntb_epc_init() - Initialize all NTB interfaces
1803
+
* @ntb: NTB device that facilitates communication between HOST1 and HOST2
1804
+
*
1805
+
* Wrapper to initialize all NTB interface and start the workqueue
1806
+
* to check for commands from host.
1807
+
*/
1808
+
static int epf_ntb_epc_init(struct epf_ntb *ntb)
1809
+
{
1810
+
enum pci_epc_interface_type type;
1811
+
struct device *dev;
1812
+
int ret;
1813
+
1814
+
dev = &ntb->epf->dev;
1815
+
1816
+
for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) {
1817
+
ret = epf_ntb_epc_init_interface(ntb, type);
1818
+
if (ret) {
1819
+
dev_err(dev, "%s intf: Failed to initialize\n",
1820
+
pci_epc_interface_string(type));
1821
+
goto err_init_type;
1822
+
}
1823
+
}
1824
+
1825
+
return 0;
1826
+
1827
+
err_init_type:
1828
+
epf_ntb_epc_cleanup_interface(ntb, type - 1);
1829
+
1830
+
return ret;
1831
+
}
1832
+
1833
+
/**
1834
+
* epf_ntb_bind() - Initialize endpoint controller to provide NTB functionality
1835
+
* @epf: NTB endpoint function device
1836
+
*
1837
+
* Initialize both the endpoint controllers associated with NTB function device.
1838
+
* Invoked when a primary interface or secondary interface is bound to EPC
1839
+
* device. This function will succeed only when EPC is bound to both the
1840
+
* interfaces.
1841
+
*/
1842
+
static int epf_ntb_bind(struct pci_epf *epf)
1843
+
{
1844
+
struct epf_ntb *ntb = epf_get_drvdata(epf);
1845
+
struct device *dev = &epf->dev;
1846
+
int ret;
1847
+
1848
+
if (!epf->epc) {
1849
+
dev_dbg(dev, "PRIMARY EPC interface not yet bound\n");
1850
+
return 0;
1851
+
}
1852
+
1853
+
if (!epf->sec_epc) {
1854
+
dev_dbg(dev, "SECONDARY EPC interface not yet bound\n");
1855
+
return 0;
1856
+
}
1857
+
1858
+
ret = epf_ntb_epc_create(ntb);
1859
+
if (ret) {
1860
+
dev_err(dev, "Failed to create NTB EPC\n");
1861
+
return ret;
1862
+
}
1863
+
1864
+
ret = epf_ntb_init_epc_bar(ntb);
1865
+
if (ret) {
1866
+
dev_err(dev, "Failed to create NTB EPC\n");
1867
+
goto err_bar_init;
1868
+
}
1869
+
1870
+
ret = epf_ntb_config_spad_bar_alloc_interface(ntb);
1871
+
if (ret) {
1872
+
dev_err(dev, "Failed to allocate BAR memory\n");
1873
+
goto err_bar_alloc;
1874
+
}
1875
+
1876
+
ret = epf_ntb_epc_init(ntb);
1877
+
if (ret) {
1878
+
dev_err(dev, "Failed to initialize EPC\n");
1879
+
goto err_bar_alloc;
1880
+
}
1881
+
1882
+
epf_set_drvdata(epf, ntb);
1883
+
1884
+
return 0;
1885
+
1886
+
err_bar_alloc:
1887
+
epf_ntb_config_spad_bar_free(ntb);
1888
+
1889
+
err_bar_init:
1890
+
epf_ntb_epc_destroy(ntb);
1891
+
1892
+
return ret;
1893
+
}
1894
+
1895
+
/**
1896
+
* epf_ntb_unbind() - Cleanup the initialization from epf_ntb_bind()
1897
+
* @epf: NTB endpoint function device
1898
+
*
1899
+
* Cleanup the initialization from epf_ntb_bind()
1900
+
*/
1901
+
static void epf_ntb_unbind(struct pci_epf *epf)
1902
+
{
1903
+
struct epf_ntb *ntb = epf_get_drvdata(epf);
1904
+
1905
+
epf_ntb_epc_cleanup(ntb);
1906
+
epf_ntb_config_spad_bar_free(ntb);
1907
+
epf_ntb_epc_destroy(ntb);
1908
+
}
1909
+
1910
+
#define EPF_NTB_R(_name) \
1911
+
static ssize_t epf_ntb_##_name##_show(struct config_item *item, \
1912
+
char *page) \
1913
+
{ \
1914
+
struct config_group *group = to_config_group(item); \
1915
+
struct epf_ntb *ntb = to_epf_ntb(group); \
1916
+
\
1917
+
return sprintf(page, "%d\n", ntb->_name); \
1918
+
}
1919
+
1920
+
#define EPF_NTB_W(_name) \
1921
+
static ssize_t epf_ntb_##_name##_store(struct config_item *item, \
1922
+
const char *page, size_t len) \
1923
+
{ \
1924
+
struct config_group *group = to_config_group(item); \
1925
+
struct epf_ntb *ntb = to_epf_ntb(group); \
1926
+
u32 val; \
1927
+
int ret; \
1928
+
\
1929
+
ret = kstrtou32(page, 0, &val); \
1930
+
if (ret) \
1931
+
return ret; \
1932
+
\
1933
+
ntb->_name = val; \
1934
+
\
1935
+
return len; \
1936
+
}
1937
+
1938
+
#define EPF_NTB_MW_R(_name) \
1939
+
static ssize_t epf_ntb_##_name##_show(struct config_item *item, \
1940
+
char *page) \
1941
+
{ \
1942
+
struct config_group *group = to_config_group(item); \
1943
+
struct epf_ntb *ntb = to_epf_ntb(group); \
1944
+
int win_no; \
1945
+
\
1946
+
sscanf(#_name, "mw%d", &win_no); \
1947
+
\
1948
+
return sprintf(page, "%lld\n", ntb->mws_size[win_no - 1]); \
1949
+
}
1950
+
1951
+
#define EPF_NTB_MW_W(_name) \
1952
+
static ssize_t epf_ntb_##_name##_store(struct config_item *item, \
1953
+
const char *page, size_t len) \
1954
+
{ \
1955
+
struct config_group *group = to_config_group(item); \
1956
+
struct epf_ntb *ntb = to_epf_ntb(group); \
1957
+
struct device *dev = &ntb->epf->dev; \
1958
+
int win_no; \
1959
+
u64 val; \
1960
+
int ret; \
1961
+
\
1962
+
ret = kstrtou64(page, 0, &val); \
1963
+
if (ret) \
1964
+
return ret; \
1965
+
\
1966
+
if (sscanf(#_name, "mw%d", &win_no) != 1) \
1967
+
return -EINVAL; \
1968
+
\
1969
+
if (ntb->num_mws < win_no) { \
1970
+
dev_err(dev, "Invalid num_nws: %d value\n", ntb->num_mws); \
1971
+
return -EINVAL; \
1972
+
} \
1973
+
\
1974
+
ntb->mws_size[win_no - 1] = val; \
1975
+
\
1976
+
return len; \
1977
+
}
1978
+
1979
+
static ssize_t epf_ntb_num_mws_store(struct config_item *item,
1980
+
const char *page, size_t len)
1981
+
{
1982
+
struct config_group *group = to_config_group(item);
1983
+
struct epf_ntb *ntb = to_epf_ntb(group);
1984
+
u32 val;
1985
+
int ret;
1986
+
1987
+
ret = kstrtou32(page, 0, &val);
1988
+
if (ret)
1989
+
return ret;
1990
+
1991
+
if (val > MAX_MW)
1992
+
return -EINVAL;
1993
+
1994
+
ntb->num_mws = val;
1995
+
1996
+
return len;
1997
+
}
1998
+
1999
+
EPF_NTB_R(spad_count)
2000
+
EPF_NTB_W(spad_count)
2001
+
EPF_NTB_R(db_count)
2002
+
EPF_NTB_W(db_count)
2003
+
EPF_NTB_R(num_mws)
2004
+
EPF_NTB_MW_R(mw1)
2005
+
EPF_NTB_MW_W(mw1)
2006
+
EPF_NTB_MW_R(mw2)
2007
+
EPF_NTB_MW_W(mw2)
2008
+
EPF_NTB_MW_R(mw3)
2009
+
EPF_NTB_MW_W(mw3)
2010
+
EPF_NTB_MW_R(mw4)
2011
+
EPF_NTB_MW_W(mw4)
2012
+
2013
+
CONFIGFS_ATTR(epf_ntb_, spad_count);
2014
+
CONFIGFS_ATTR(epf_ntb_, db_count);
2015
+
CONFIGFS_ATTR(epf_ntb_, num_mws);
2016
+
CONFIGFS_ATTR(epf_ntb_, mw1);
2017
+
CONFIGFS_ATTR(epf_ntb_, mw2);
2018
+
CONFIGFS_ATTR(epf_ntb_, mw3);
2019
+
CONFIGFS_ATTR(epf_ntb_, mw4);
2020
+
2021
+
static struct configfs_attribute *epf_ntb_attrs[] = {
2022
+
&epf_ntb_attr_spad_count,
2023
+
&epf_ntb_attr_db_count,
2024
+
&epf_ntb_attr_num_mws,
2025
+
&epf_ntb_attr_mw1,
2026
+
&epf_ntb_attr_mw2,
2027
+
&epf_ntb_attr_mw3,
2028
+
&epf_ntb_attr_mw4,
2029
+
NULL,
2030
+
};
2031
+
2032
+
static const struct config_item_type ntb_group_type = {
2033
+
.ct_attrs = epf_ntb_attrs,
2034
+
.ct_owner = THIS_MODULE,
2035
+
};
2036
+
2037
+
/**
2038
+
* epf_ntb_add_cfs() - Add configfs directory specific to NTB
2039
+
* @epf: NTB endpoint function device
2040
+
*
2041
+
* Add configfs directory specific to NTB. This directory will hold
2042
+
* NTB specific properties like db_count, spad_count, num_mws etc.,
2043
+
*/
2044
+
static struct config_group *epf_ntb_add_cfs(struct pci_epf *epf,
2045
+
struct config_group *group)
2046
+
{
2047
+
struct epf_ntb *ntb = epf_get_drvdata(epf);
2048
+
struct config_group *ntb_group = &ntb->group;
2049
+
struct device *dev = &epf->dev;
2050
+
2051
+
config_group_init_type_name(ntb_group, dev_name(dev), &ntb_group_type);
2052
+
2053
+
return ntb_group;
2054
+
}
2055
+
2056
+
/**
2057
+
* epf_ntb_probe() - Probe NTB function driver
2058
+
* @epf: NTB endpoint function device
2059
+
*
2060
+
* Probe NTB function driver when endpoint function bus detects a NTB
2061
+
* endpoint function.
2062
+
*/
2063
+
static int epf_ntb_probe(struct pci_epf *epf)
2064
+
{
2065
+
struct epf_ntb *ntb;
2066
+
struct device *dev;
2067
+
2068
+
dev = &epf->dev;
2069
+
2070
+
ntb = devm_kzalloc(dev, sizeof(*ntb), GFP_KERNEL);
2071
+
if (!ntb)
2072
+
return -ENOMEM;
2073
+
2074
+
epf->header = &epf_ntb_header;
2075
+
ntb->epf = epf;
2076
+
epf_set_drvdata(epf, ntb);
2077
+
2078
+
return 0;
2079
+
}
2080
+
2081
+
static struct pci_epf_ops epf_ntb_ops = {
2082
+
.bind = epf_ntb_bind,
2083
+
.unbind = epf_ntb_unbind,
2084
+
.add_cfs = epf_ntb_add_cfs,
2085
+
};
2086
+
2087
+
static const struct pci_epf_device_id epf_ntb_ids[] = {
2088
+
{
2089
+
.name = "pci_epf_ntb",
2090
+
},
2091
+
{},
2092
+
};
2093
+
2094
+
static struct pci_epf_driver epf_ntb_driver = {
2095
+
.driver.name = "pci_epf_ntb",
2096
+
.probe = epf_ntb_probe,
2097
+
.id_table = epf_ntb_ids,
2098
+
.ops = &epf_ntb_ops,
2099
+
.owner = THIS_MODULE,
2100
+
};
2101
+
2102
+
static int __init epf_ntb_init(void)
2103
+
{
2104
+
int ret;
2105
+
2106
+
kpcintb_workqueue = alloc_workqueue("kpcintb", WQ_MEM_RECLAIM |
2107
+
WQ_HIGHPRI, 0);
2108
+
ret = pci_epf_register_driver(&epf_ntb_driver);
2109
+
if (ret) {
2110
+
destroy_workqueue(kpcintb_workqueue);
2111
+
pr_err("Failed to register pci epf ntb driver --> %d\n", ret);
2112
+
return ret;
2113
+
}
2114
+
2115
+
return 0;
2116
+
}
2117
+
module_init(epf_ntb_init);
2118
+
2119
+
static void __exit epf_ntb_exit(void)
2120
+
{
2121
+
pci_epf_unregister_driver(&epf_ntb_driver);
2122
+
destroy_workqueue(kpcintb_workqueue);
2123
+
}
2124
+
module_exit(epf_ntb_exit);
2125
+
2126
+
MODULE_DESCRIPTION("PCI EPF NTB DRIVER");
2127
+
MODULE_AUTHOR("Kishon Vijay Abraham I <kishon@ti.com>");
2128
+
MODULE_LICENSE("GPL v2");