Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
dibs: Move data path to dibs layer
Use struct dibs_dmb instead of struct smc_dmb and move the corresponding
client tables to dibs_dev. Leave driver specific implementation details
like sba in the device drivers.
Register and unregister dmbs via dibs_dev_ops. A dmb is dedicated to a
single client, but a dibs device can have dmbs for more than one client.
Trigger dibs clients via dibs_client_ops->handle_irq(), when data is
received into a dmb. For dibs_loopback replace scheduling an smcd receive
tasklet with calling dibs_client_ops->handle_irq().
For loopback devices attach_dmb(), detach_dmb() and move_data() need to
access the dmb tables, so move those to dibs_dev_ops in this patch as well.
Remove remaining definitions of smc_loopback as they are no longer
required, now that everything is in dibs_loopback.
Note that struct ism_client and struct ism_dev are still required in smc
until a follow-on patch moves event handling to dibs. (Loopback does not
use events).
Signed-off-by: Alexandra Winter <wintera@linux.ibm.com>
Link: https://patch.msgid.link/20250918110500.1731261-14-wintera@linux.ibm.com
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
authored by
Alexandra Winter
and committed by
Paolo Abeni
cc21191b
719c3b67
+592
-501
+257
drivers/dibs/dibs_loopback.c
+257
drivers/dibs/dibs_loopback.c
···
9
9
*
10
10
*/
11
11
12
+
#include <linux/bitops.h>
13
+
#include <linux/device.h>
12
14
#include <linux/dibs.h>
13
15
#include <linux/slab.h>
16
+
#include <linux/spinlock.h>
14
17
#include <linux/types.h>
15
18
16
19
#include "dibs_loopback.h"
20
+
21
+
#define DIBS_LO_SUPPORT_NOCOPY 0x1
22
+
#define DIBS_DMA_ADDR_INVALID (~(dma_addr_t)0)
17
23
18
24
static const char dibs_lo_dev_name[] = "lo";
19
25
/* global loopback device */
···
39
33
return 0;
40
34
}
41
35
36
+
static int dibs_lo_max_dmbs(void)
37
+
{
38
+
return DIBS_LO_MAX_DMBS;
39
+
}
40
+
41
+
static int dibs_lo_register_dmb(struct dibs_dev *dibs, struct dibs_dmb *dmb,
42
+
struct dibs_client *client)
43
+
{
44
+
struct dibs_lo_dmb_node *dmb_node, *tmp_node;
45
+
struct dibs_lo_dev *ldev;
46
+
unsigned long flags;
47
+
int sba_idx, rc;
48
+
49
+
ldev = dibs->drv_priv;
50
+
sba_idx = dmb->idx;
51
+
/* check space for new dmb */
52
+
for_each_clear_bit(sba_idx, ldev->sba_idx_mask, DIBS_LO_MAX_DMBS) {
53
+
if (!test_and_set_bit(sba_idx, ldev->sba_idx_mask))
54
+
break;
55
+
}
56
+
if (sba_idx == DIBS_LO_MAX_DMBS)
57
+
return -ENOSPC;
58
+
59
+
dmb_node = kzalloc(sizeof(*dmb_node), GFP_KERNEL);
60
+
if (!dmb_node) {
61
+
rc = -ENOMEM;
62
+
goto err_bit;
63
+
}
64
+
65
+
dmb_node->sba_idx = sba_idx;
66
+
dmb_node->len = dmb->dmb_len;
67
+
dmb_node->cpu_addr = kzalloc(dmb_node->len, GFP_KERNEL |
68
+
__GFP_NOWARN | __GFP_NORETRY |
69
+
__GFP_NOMEMALLOC);
70
+
if (!dmb_node->cpu_addr) {
71
+
rc = -ENOMEM;
72
+
goto err_node;
73
+
}
74
+
dmb_node->dma_addr = DIBS_DMA_ADDR_INVALID;
75
+
refcount_set(&dmb_node->refcnt, 1);
76
+
77
+
again:
78
+
/* add new dmb into hash table */
79
+
get_random_bytes(&dmb_node->token, sizeof(dmb_node->token));
80
+
write_lock_bh(&ldev->dmb_ht_lock);
81
+
hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb_node->token) {
82
+
if (tmp_node->token == dmb_node->token) {
83
+
write_unlock_bh(&ldev->dmb_ht_lock);
84
+
goto again;
85
+
}
86
+
}
87
+
hash_add(ldev->dmb_ht, &dmb_node->list, dmb_node->token);
88
+
write_unlock_bh(&ldev->dmb_ht_lock);
89
+
atomic_inc(&ldev->dmb_cnt);
90
+
91
+
dmb->idx = dmb_node->sba_idx;
92
+
dmb->dmb_tok = dmb_node->token;
93
+
dmb->cpu_addr = dmb_node->cpu_addr;
94
+
dmb->dma_addr = dmb_node->dma_addr;
95
+
dmb->dmb_len = dmb_node->len;
96
+
97
+
spin_lock_irqsave(&dibs->lock, flags);
98
+
dibs->dmb_clientid_arr[sba_idx] = client->id;
99
+
spin_unlock_irqrestore(&dibs->lock, flags);
100
+
101
+
return 0;
102
+
103
+
err_node:
104
+
kfree(dmb_node);
105
+
err_bit:
106
+
clear_bit(sba_idx, ldev->sba_idx_mask);
107
+
return rc;
108
+
}
109
+
110
+
static void __dibs_lo_unregister_dmb(struct dibs_lo_dev *ldev,
111
+
struct dibs_lo_dmb_node *dmb_node)
112
+
{
113
+
/* remove dmb from hash table */
114
+
write_lock_bh(&ldev->dmb_ht_lock);
115
+
hash_del(&dmb_node->list);
116
+
write_unlock_bh(&ldev->dmb_ht_lock);
117
+
118
+
clear_bit(dmb_node->sba_idx, ldev->sba_idx_mask);
119
+
kfree(dmb_node->cpu_addr);
120
+
kfree(dmb_node);
121
+
122
+
if (atomic_dec_and_test(&ldev->dmb_cnt))
123
+
wake_up(&ldev->ldev_release);
124
+
}
125
+
126
+
static int dibs_lo_unregister_dmb(struct dibs_dev *dibs, struct dibs_dmb *dmb)
127
+
{
128
+
struct dibs_lo_dmb_node *dmb_node = NULL, *tmp_node;
129
+
struct dibs_lo_dev *ldev;
130
+
unsigned long flags;
131
+
132
+
ldev = dibs->drv_priv;
133
+
134
+
/* find dmb from hash table */
135
+
read_lock_bh(&ldev->dmb_ht_lock);
136
+
hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb->dmb_tok) {
137
+
if (tmp_node->token == dmb->dmb_tok) {
138
+
dmb_node = tmp_node;
139
+
break;
140
+
}
141
+
}
142
+
read_unlock_bh(&ldev->dmb_ht_lock);
143
+
if (!dmb_node)
144
+
return -EINVAL;
145
+
146
+
if (refcount_dec_and_test(&dmb_node->refcnt)) {
147
+
spin_lock_irqsave(&dibs->lock, flags);
148
+
dibs->dmb_clientid_arr[dmb_node->sba_idx] = NO_DIBS_CLIENT;
149
+
spin_unlock_irqrestore(&dibs->lock, flags);
150
+
151
+
__dibs_lo_unregister_dmb(ldev, dmb_node);
152
+
}
153
+
return 0;
154
+
}
155
+
156
+
static int dibs_lo_support_dmb_nocopy(struct dibs_dev *dibs)
157
+
{
158
+
return DIBS_LO_SUPPORT_NOCOPY;
159
+
}
160
+
161
+
static int dibs_lo_attach_dmb(struct dibs_dev *dibs, struct dibs_dmb *dmb)
162
+
{
163
+
struct dibs_lo_dmb_node *dmb_node = NULL, *tmp_node;
164
+
struct dibs_lo_dev *ldev;
165
+
166
+
ldev = dibs->drv_priv;
167
+
168
+
/* find dmb_node according to dmb->dmb_tok */
169
+
read_lock_bh(&ldev->dmb_ht_lock);
170
+
hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb->dmb_tok) {
171
+
if (tmp_node->token == dmb->dmb_tok) {
172
+
dmb_node = tmp_node;
173
+
break;
174
+
}
175
+
}
176
+
if (!dmb_node) {
177
+
read_unlock_bh(&ldev->dmb_ht_lock);
178
+
return -EINVAL;
179
+
}
180
+
read_unlock_bh(&ldev->dmb_ht_lock);
181
+
182
+
if (!refcount_inc_not_zero(&dmb_node->refcnt))
183
+
/* the dmb is being unregistered, but has
184
+
* not been removed from the hash table.
185
+
*/
186
+
return -EINVAL;
187
+
188
+
/* provide dmb information */
189
+
dmb->idx = dmb_node->sba_idx;
190
+
dmb->dmb_tok = dmb_node->token;
191
+
dmb->cpu_addr = dmb_node->cpu_addr;
192
+
dmb->dma_addr = dmb_node->dma_addr;
193
+
dmb->dmb_len = dmb_node->len;
194
+
return 0;
195
+
}
196
+
197
+
static int dibs_lo_detach_dmb(struct dibs_dev *dibs, u64 token)
198
+
{
199
+
struct dibs_lo_dmb_node *dmb_node = NULL, *tmp_node;
200
+
struct dibs_lo_dev *ldev;
201
+
202
+
ldev = dibs->drv_priv;
203
+
204
+
/* find dmb_node according to dmb->dmb_tok */
205
+
read_lock_bh(&ldev->dmb_ht_lock);
206
+
hash_for_each_possible(ldev->dmb_ht, tmp_node, list, token) {
207
+
if (tmp_node->token == token) {
208
+
dmb_node = tmp_node;
209
+
break;
210
+
}
211
+
}
212
+
if (!dmb_node) {
213
+
read_unlock_bh(&ldev->dmb_ht_lock);
214
+
return -EINVAL;
215
+
}
216
+
read_unlock_bh(&ldev->dmb_ht_lock);
217
+
218
+
if (refcount_dec_and_test(&dmb_node->refcnt))
219
+
__dibs_lo_unregister_dmb(ldev, dmb_node);
220
+
return 0;
221
+
}
222
+
223
+
static int dibs_lo_move_data(struct dibs_dev *dibs, u64 dmb_tok,
224
+
unsigned int idx, bool sf, unsigned int offset,
225
+
void *data, unsigned int size)
226
+
{
227
+
struct dibs_lo_dmb_node *rmb_node = NULL, *tmp_node;
228
+
struct dibs_lo_dev *ldev;
229
+
u16 s_mask;
230
+
u8 client_id;
231
+
u32 sba_idx;
232
+
233
+
ldev = dibs->drv_priv;
234
+
235
+
read_lock_bh(&ldev->dmb_ht_lock);
236
+
hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb_tok) {
237
+
if (tmp_node->token == dmb_tok) {
238
+
rmb_node = tmp_node;
239
+
break;
240
+
}
241
+
}
242
+
if (!rmb_node) {
243
+
read_unlock_bh(&ldev->dmb_ht_lock);
244
+
return -EINVAL;
245
+
}
246
+
memcpy((char *)rmb_node->cpu_addr + offset, data, size);
247
+
sba_idx = rmb_node->sba_idx;
248
+
read_unlock_bh(&ldev->dmb_ht_lock);
249
+
250
+
if (!sf)
251
+
return 0;
252
+
253
+
spin_lock(&dibs->lock);
254
+
client_id = dibs->dmb_clientid_arr[sba_idx];
255
+
s_mask = ror16(0x1000, idx);
256
+
if (likely(client_id != NO_DIBS_CLIENT && dibs->subs[client_id]))
257
+
dibs->subs[client_id]->ops->handle_irq(dibs, sba_idx, s_mask);
258
+
spin_unlock(&dibs->lock);
259
+
260
+
return 0;
261
+
}
262
+
42
263
static const struct dibs_dev_ops dibs_lo_ops = {
43
264
.get_fabric_id = dibs_lo_get_fabric_id,
44
265
.query_remote_gid = dibs_lo_query_rgid,
266
+
.max_dmbs = dibs_lo_max_dmbs,
267
+
.register_dmb = dibs_lo_register_dmb,
268
+
.unregister_dmb = dibs_lo_unregister_dmb,
269
+
.move_data = dibs_lo_move_data,
270
+
.support_mmapped_rdmb = dibs_lo_support_dmb_nocopy,
271
+
.attach_dmb = dibs_lo_attach_dmb,
272
+
.detach_dmb = dibs_lo_detach_dmb,
45
273
};
274
+
275
+
static void dibs_lo_dev_init(struct dibs_lo_dev *ldev)
276
+
{
277
+
rwlock_init(&ldev->dmb_ht_lock);
278
+
hash_init(ldev->dmb_ht);
279
+
atomic_set(&ldev->dmb_cnt, 0);
280
+
init_waitqueue_head(&ldev->ldev_release);
281
+
}
282
+
283
+
static void dibs_lo_dev_exit(struct dibs_lo_dev *ldev)
284
+
{
285
+
if (atomic_read(&ldev->dmb_cnt))
286
+
wait_event(ldev->ldev_release, !atomic_read(&ldev->dmb_cnt));
287
+
}
46
288
47
289
static int dibs_lo_dev_probe(void)
48
290
{
···
310
56
311
57
ldev->dibs = dibs;
312
58
dibs->drv_priv = ldev;
59
+
dibs_lo_dev_init(ldev);
313
60
uuid_gen(&dibs->gid);
314
61
dibs->ops = &dibs_lo_ops;
315
62
···
324
69
return 0;
325
70
326
71
err_reg:
72
+
kfree(dibs->dmb_clientid_arr);
327
73
/* pairs with dibs_dev_alloc() */
328
74
put_device(&dibs->dev);
329
75
kfree(ldev);
···
338
82
return;
339
83
340
84
dibs_dev_del(lo_dev->dibs);
85
+
dibs_lo_dev_exit(lo_dev);
341
86
/* pairs with dibs_dev_alloc() */
342
87
put_device(&lo_dev->dibs->dev);
343
88
kfree(lo_dev);
+19
drivers/dibs/dibs_loopback.h
+19
drivers/dibs/dibs_loopback.h
···
13
13
#define _DIBS_LOOPBACK_H
14
14
15
15
#include <linux/dibs.h>
16
+
#include <linux/hashtable.h>
17
+
#include <linux/spinlock.h>
16
18
#include <linux/types.h>
17
19
#include <linux/wait.h>
18
20
19
21
#if IS_ENABLED(CONFIG_DIBS_LO)
22
+
#define DIBS_LO_DMBS_HASH_BITS 12
23
+
#define DIBS_LO_MAX_DMBS 5000
24
+
25
+
struct dibs_lo_dmb_node {
26
+
struct hlist_node list;
27
+
u64 token;
28
+
u32 len;
29
+
u32 sba_idx;
30
+
void *cpu_addr;
31
+
dma_addr_t dma_addr;
32
+
refcount_t refcnt;
33
+
};
20
34
21
35
struct dibs_lo_dev {
22
36
struct dibs_dev *dibs;
37
+
atomic_t dmb_cnt;
38
+
rwlock_t dmb_ht_lock;
39
+
DECLARE_BITMAP(sba_idx_mask, DIBS_LO_MAX_DMBS);
40
+
DECLARE_HASHTABLE(dmb_ht, DIBS_LO_DMBS_HASH_BITS);
41
+
wait_queue_head_t ldev_release;
23
42
};
24
43
25
44
int dibs_loopback_init(void);
+54
-2
drivers/dibs/dibs_main.c
+54
-2
drivers/dibs/dibs_main.c
···
36
36
.mutex = __MUTEX_INITIALIZER(dibs_dev_list.mutex),
37
37
};
38
38
39
+
static void dibs_setup_forwarding(struct dibs_client *client,
40
+
struct dibs_dev *dibs)
41
+
{
42
+
unsigned long flags;
43
+
44
+
spin_lock_irqsave(&dibs->lock, flags);
45
+
dibs->subs[client->id] = client;
46
+
spin_unlock_irqrestore(&dibs->lock, flags);
47
+
}
48
+
39
49
int dibs_register_client(struct dibs_client *client)
40
50
{
41
51
struct dibs_dev *dibs;
···
70
60
list_for_each_entry(dibs, &dibs_dev_list.list, list) {
71
61
dibs->priv[i] = NULL;
72
62
client->ops->add_dev(dibs);
63
+
dibs_setup_forwarding(client, dibs);
73
64
}
74
65
}
75
66
mutex_unlock(&dibs_dev_list.mutex);
···
82
71
int dibs_unregister_client(struct dibs_client *client)
83
72
{
84
73
struct dibs_dev *dibs;
74
+
unsigned long flags;
75
+
int max_dmbs;
85
76
int rc = 0;
86
77
87
78
mutex_lock(&dibs_dev_list.mutex);
88
79
list_for_each_entry(dibs, &dibs_dev_list.list, list) {
80
+
spin_lock_irqsave(&dibs->lock, flags);
81
+
max_dmbs = dibs->ops->max_dmbs();
82
+
for (int i = 0; i < max_dmbs; ++i) {
83
+
if (dibs->dmb_clientid_arr[i] == client->id) {
84
+
WARN(1, "%s: attempt to unregister '%s' with registered dmb(s)\n",
85
+
__func__, client->name);
86
+
rc = -EBUSY;
87
+
goto err_reg_dmb;
88
+
}
89
+
}
90
+
/* Stop forwarding IRQs */
91
+
dibs->subs[client->id] = NULL;
92
+
spin_unlock_irqrestore(&dibs->lock, flags);
89
93
clients[client->id]->ops->del_dev(dibs);
90
94
dibs->priv[client->id] = NULL;
91
95
}
···
111
85
max_client--;
112
86
mutex_unlock(&clients_lock);
113
87
88
+
mutex_unlock(&dibs_dev_list.mutex);
89
+
return rc;
90
+
91
+
err_reg_dmb:
92
+
spin_unlock_irqrestore(&dibs->lock, flags);
114
93
mutex_unlock(&dibs_dev_list.mutex);
115
94
return rc;
116
95
}
···
181
150
182
151
int dibs_dev_add(struct dibs_dev *dibs)
183
152
{
153
+
int max_dmbs;
184
154
int i, ret;
155
+
156
+
max_dmbs = dibs->ops->max_dmbs();
157
+
spin_lock_init(&dibs->lock);
158
+
dibs->dmb_clientid_arr = kzalloc(max_dmbs, GFP_KERNEL);
159
+
if (!dibs->dmb_clientid_arr)
160
+
return -ENOMEM;
161
+
memset(dibs->dmb_clientid_arr, NO_DIBS_CLIENT, max_dmbs);
185
162
186
163
ret = device_add(&dibs->dev);
187
164
if (ret)
188
-
return ret;
165
+
goto free_client_arr;
189
166
190
167
ret = sysfs_create_group(&dibs->dev.kobj, &dibs_dev_attr_group);
191
168
if (ret) {
···
203
164
mutex_lock(&dibs_dev_list.mutex);
204
165
mutex_lock(&clients_lock);
205
166
for (i = 0; i < max_client; ++i) {
206
-
if (clients[i])
167
+
if (clients[i]) {
207
168
clients[i]->ops->add_dev(dibs);
169
+
dibs_setup_forwarding(clients[i], dibs);
170
+
}
208
171
}
209
172
mutex_unlock(&clients_lock);
210
173
list_add(&dibs->list, &dibs_dev_list.list);
···
216
175
217
176
err_device_del:
218
177
device_del(&dibs->dev);
178
+
free_client_arr:
179
+
kfree(dibs->dmb_clientid_arr);
219
180
return ret;
220
181
221
182
}
···
225
182
226
183
void dibs_dev_del(struct dibs_dev *dibs)
227
184
{
185
+
unsigned long flags;
228
186
int i;
187
+
188
+
sysfs_remove_group(&dibs->dev.kobj, &dibs_dev_attr_group);
189
+
190
+
spin_lock_irqsave(&dibs->lock, flags);
191
+
for (i = 0; i < MAX_DIBS_CLIENTS; ++i)
192
+
dibs->subs[i] = NULL;
193
+
spin_unlock_irqrestore(&dibs->lock, flags);
229
194
230
195
mutex_lock(&dibs_dev_list.mutex);
231
196
mutex_lock(&clients_lock);
···
246
195
mutex_unlock(&dibs_dev_list.mutex);
247
196
248
197
device_del(&dibs->dev);
198
+
kfree(dibs->dmb_clientid_arr);
249
199
}
250
200
EXPORT_SYMBOL_GPL(dibs_dev_del);
251
201
+49
-72
drivers/s390/net/ism_drv.c
+49
-72
drivers/s390/net/ism_drv.c
···
98
98
spin_lock_irqsave(&ism->lock, flags);
99
99
/* Stop forwarding IRQs and events */
100
100
ism->subs[client->id] = NULL;
101
-
for (int i = 0; i < ISM_NR_DMBS; ++i) {
102
-
if (ism->sba_client_arr[i] == client->id) {
103
-
WARN(1, "%s: attempt to unregister '%s' with registered dmb(s)\n",
104
-
__func__, client->name);
105
-
rc = -EBUSY;
106
-
goto err_reg_dmb;
107
-
}
108
-
}
109
101
spin_unlock_irqrestore(&ism->lock, flags);
110
102
}
111
103
mutex_unlock(&ism_dev_list.mutex);
···
107
115
if (client->id + 1 == max_client)
108
116
max_client--;
109
117
mutex_unlock(&clients_lock);
110
-
return rc;
111
-
112
-
err_reg_dmb:
113
-
spin_unlock_irqrestore(&ism->lock, flags);
114
-
mutex_unlock(&ism_dev_list.mutex);
115
118
return rc;
116
119
}
117
120
EXPORT_SYMBOL_GPL(ism_unregister_client);
···
295
308
return ism_cmd(ism, &cmd);
296
309
}
297
310
298
-
static void ism_free_dmb(struct ism_dev *ism, struct ism_dmb *dmb)
311
+
static int ism_max_dmbs(void)
299
312
{
300
-
clear_bit(dmb->sba_idx, ism->sba_bitmap);
313
+
return ISM_NR_DMBS;
314
+
}
315
+
316
+
static void ism_free_dmb(struct ism_dev *ism, struct dibs_dmb *dmb)
317
+
{
318
+
clear_bit(dmb->idx, ism->sba_bitmap);
301
319
dma_unmap_page(&ism->pdev->dev, dmb->dma_addr, dmb->dmb_len,
302
320
DMA_FROM_DEVICE);
303
321
folio_put(virt_to_folio(dmb->cpu_addr));
304
322
}
305
323
306
-
static int ism_alloc_dmb(struct ism_dev *ism, struct ism_dmb *dmb)
324
+
static int ism_alloc_dmb(struct ism_dev *ism, struct dibs_dmb *dmb)
307
325
{
308
326
struct folio *folio;
309
327
unsigned long bit;
···
317
325
if (PAGE_ALIGN(dmb->dmb_len) > dma_get_max_seg_size(&ism->pdev->dev))
318
326
return -EINVAL;
319
327
320
-
if (!dmb->sba_idx) {
328
+
if (!dmb->idx) {
321
329
bit = find_next_zero_bit(ism->sba_bitmap, ISM_NR_DMBS,
322
330
ISM_DMB_BIT_OFFSET);
323
331
if (bit == ISM_NR_DMBS)
324
332
return -ENOSPC;
325
333
326
-
dmb->sba_idx = bit;
334
+
dmb->idx = bit;
327
335
}
328
-
if (dmb->sba_idx < ISM_DMB_BIT_OFFSET ||
329
-
test_and_set_bit(dmb->sba_idx, ism->sba_bitmap))
336
+
if (dmb->idx < ISM_DMB_BIT_OFFSET ||
337
+
test_and_set_bit(dmb->idx, ism->sba_bitmap))
330
338
return -EINVAL;
331
339
332
340
folio = folio_alloc(GFP_KERNEL | __GFP_NOWARN | __GFP_NOMEMALLOC |
···
351
359
out_free:
352
360
kfree(dmb->cpu_addr);
353
361
out_bit:
354
-
clear_bit(dmb->sba_idx, ism->sba_bitmap);
362
+
clear_bit(dmb->idx, ism->sba_bitmap);
355
363
return rc;
356
364
}
357
365
358
-
int ism_register_dmb(struct ism_dev *ism, struct ism_dmb *dmb,
359
-
struct ism_client *client)
366
+
static int ism_register_dmb(struct dibs_dev *dibs, struct dibs_dmb *dmb,
367
+
struct dibs_client *client)
360
368
{
369
+
struct ism_dev *ism = dibs->drv_priv;
361
370
union ism_reg_dmb cmd;
362
371
unsigned long flags;
363
372
int ret;
···
373
380
374
381
cmd.request.dmb = dmb->dma_addr;
375
382
cmd.request.dmb_len = dmb->dmb_len;
376
-
cmd.request.sba_idx = dmb->sba_idx;
383
+
cmd.request.sba_idx = dmb->idx;
377
384
cmd.request.vlan_valid = dmb->vlan_valid;
378
385
cmd.request.vlan_id = dmb->vlan_id;
379
-
cmd.request.rgid = dmb->rgid;
386
+
memcpy(&cmd.request.rgid, &dmb->rgid, sizeof(u64));
380
387
381
388
ret = ism_cmd(ism, &cmd);
382
389
if (ret) {
···
384
391
goto out;
385
392
}
386
393
dmb->dmb_tok = cmd.response.dmb_tok;
387
-
spin_lock_irqsave(&ism->lock, flags);
388
-
ism->sba_client_arr[dmb->sba_idx - ISM_DMB_BIT_OFFSET] = client->id;
389
-
spin_unlock_irqrestore(&ism->lock, flags);
394
+
spin_lock_irqsave(&dibs->lock, flags);
395
+
dibs->dmb_clientid_arr[dmb->idx - ISM_DMB_BIT_OFFSET] = client->id;
396
+
spin_unlock_irqrestore(&dibs->lock, flags);
390
397
out:
391
398
return ret;
392
399
}
393
-
EXPORT_SYMBOL_GPL(ism_register_dmb);
394
400
395
-
int ism_unregister_dmb(struct ism_dev *ism, struct ism_dmb *dmb)
401
+
static int ism_unregister_dmb(struct dibs_dev *dibs, struct dibs_dmb *dmb)
396
402
{
403
+
struct ism_dev *ism = dibs->drv_priv;
397
404
union ism_unreg_dmb cmd;
398
405
unsigned long flags;
399
406
int ret;
···
404
411
405
412
cmd.request.dmb_tok = dmb->dmb_tok;
406
413
407
-
spin_lock_irqsave(&ism->lock, flags);
408
-
ism->sba_client_arr[dmb->sba_idx - ISM_DMB_BIT_OFFSET] = NO_CLIENT;
409
-
spin_unlock_irqrestore(&ism->lock, flags);
414
+
spin_lock_irqsave(&dibs->lock, flags);
415
+
dibs->dmb_clientid_arr[dmb->idx - ISM_DMB_BIT_OFFSET] = NO_DIBS_CLIENT;
416
+
spin_unlock_irqrestore(&dibs->lock, flags);
410
417
411
418
ret = ism_cmd(ism, &cmd);
412
419
if (ret && ret != ISM_ERROR)
···
416
423
out:
417
424
return ret;
418
425
}
419
-
EXPORT_SYMBOL_GPL(ism_unregister_dmb);
420
426
421
427
static int ism_add_vlan_id(struct dibs_dev *dibs, u64 vlan_id)
422
428
{
···
451
459
return min(boundary - (start & (boundary - 1)), len);
452
460
}
453
461
454
-
int ism_move(struct ism_dev *ism, u64 dmb_tok, unsigned int idx, bool sf,
455
-
unsigned int offset, void *data, unsigned int size)
462
+
static int ism_move(struct dibs_dev *dibs, u64 dmb_tok, unsigned int idx,
463
+
bool sf, unsigned int offset, void *data,
464
+
unsigned int size)
456
465
{
466
+
struct ism_dev *ism = dibs->drv_priv;
457
467
unsigned int bytes;
458
468
u64 dmb_req;
459
469
int ret;
···
476
482
477
483
return 0;
478
484
}
479
-
EXPORT_SYMBOL_GPL(ism_move);
480
485
481
486
static u16 ism_get_chid(struct dibs_dev *dibs)
482
487
{
···
511
518
{
512
519
struct ism_dev *ism = data;
513
520
unsigned long bit, end;
521
+
struct dibs_dev *dibs;
514
522
unsigned long *bv;
515
523
u16 dmbemask;
516
524
u8 client_id;
517
525
526
+
dibs = ism->dibs;
527
+
518
528
bv = (void *) &ism->sba->dmb_bits[ISM_DMB_WORD_OFFSET];
519
529
end = sizeof(ism->sba->dmb_bits) * BITS_PER_BYTE - ISM_DMB_BIT_OFFSET;
520
530
521
-
spin_lock(&ism->lock);
531
+
spin_lock(&dibs->lock);
522
532
ism->sba->s = 0;
523
533
barrier();
524
534
for (bit = 0;;) {
···
533
537
dmbemask = ism->sba->dmbe_mask[bit + ISM_DMB_BIT_OFFSET];
534
538
ism->sba->dmbe_mask[bit + ISM_DMB_BIT_OFFSET] = 0;
535
539
barrier();
536
-
client_id = ism->sba_client_arr[bit];
537
-
if (unlikely(client_id == NO_CLIENT || !ism->subs[client_id]))
540
+
client_id = dibs->dmb_clientid_arr[bit];
541
+
if (unlikely(client_id == NO_DIBS_CLIENT ||
542
+
!dibs->subs[client_id]))
538
543
continue;
539
-
ism->subs[client_id]->handle_irq(ism, bit + ISM_DMB_BIT_OFFSET, dmbemask);
544
+
dibs->subs[client_id]->ops->handle_irq(dibs,
545
+
bit + ISM_DMB_BIT_OFFSET,
546
+
dmbemask);
540
547
}
541
548
542
549
if (ism->sba->e) {
···
547
548
barrier();
548
549
ism_handle_event(ism);
549
550
}
550
-
spin_unlock(&ism->lock);
551
+
spin_unlock(&dibs->lock);
551
552
return IRQ_HANDLED;
552
553
}
553
554
554
555
static const struct dibs_dev_ops ism_ops = {
555
556
.get_fabric_id = ism_get_chid,
556
557
.query_remote_gid = ism_query_rgid,
558
+
.max_dmbs = ism_max_dmbs,
559
+
.register_dmb = ism_register_dmb,
560
+
.unregister_dmb = ism_unregister_dmb,
561
+
.move_data = ism_move,
557
562
.add_vlan_id = ism_add_vlan_id,
558
563
.del_vlan_id = ism_del_vlan_id,
559
564
};
···
571
568
if (ret <= 0)
572
569
goto out;
573
570
574
-
ism->sba_client_arr = kzalloc(ISM_NR_DMBS, GFP_KERNEL);
575
-
if (!ism->sba_client_arr)
576
-
goto free_vectors;
577
-
memset(ism->sba_client_arr, NO_CLIENT, ISM_NR_DMBS);
578
-
579
571
ret = request_irq(pci_irq_vector(pdev, 0), ism_handle_irq, 0,
580
572
pci_name(pdev), ism);
581
573
if (ret)
582
-
goto free_client_arr;
574
+
goto free_vectors;
583
575
584
576
ret = register_sba(ism);
585
577
if (ret)
···
603
605
unregister_sba(ism);
604
606
free_irq:
605
607
free_irq(pci_irq_vector(pdev, 0), ism);
606
-
free_client_arr:
607
-
kfree(ism->sba_client_arr);
608
608
free_vectors:
609
609
pci_free_irq_vectors(pdev);
610
610
out:
···
625
629
unregister_ieq(ism);
626
630
unregister_sba(ism);
627
631
free_irq(pci_irq_vector(pdev, 0), ism);
628
-
kfree(ism->sba_client_arr);
629
632
pci_free_irq_vectors(pdev);
630
633
list_del_init(&ism->list);
631
634
mutex_unlock(&ism_dev_list.mutex);
···
672
677
dibs->drv_priv = ism;
673
678
dibs->ops = &ism_ops;
674
679
680
+
/* enable ism device, but any interrupts and events will be ignored
681
+
* before dibs_dev_add() adds it to any clients.
682
+
*/
675
683
ret = ism_dev_init(ism);
676
684
if (ret)
677
685
goto err_dibs;
···
764
766
/*************************** SMC-D Implementation *****************************/
765
767
766
768
#if IS_ENABLED(CONFIG_SMC)
767
-
static int smcd_register_dmb(struct smcd_dev *smcd, struct smcd_dmb *dmb,
768
-
void *client)
769
-
{
770
-
return ism_register_dmb(smcd->priv, (struct ism_dmb *)dmb, client);
771
-
}
772
-
773
-
static int smcd_unregister_dmb(struct smcd_dev *smcd, struct smcd_dmb *dmb)
774
-
{
775
-
return ism_unregister_dmb(smcd->priv, (struct ism_dmb *)dmb);
776
-
}
777
-
778
769
static int ism_signal_ieq(struct ism_dev *ism, u64 rgid, u32 trigger_irq,
779
770
u32 event_code, u64 info)
780
771
{
···
788
801
trigger_irq, event_code, info);
789
802
}
790
803
791
-
static int smcd_move(struct smcd_dev *smcd, u64 dmb_tok, unsigned int idx,
792
-
bool sf, unsigned int offset, void *data,
793
-
unsigned int size)
794
-
{
795
-
return ism_move(smcd->priv, dmb_tok, idx, sf, offset, data, size);
796
-
}
797
-
798
804
static const struct smcd_ops ism_smcd_ops = {
799
-
.register_dmb = smcd_register_dmb,
800
-
.unregister_dmb = smcd_unregister_dmb,
801
805
.signal_event = smcd_signal_ieq,
802
-
.move_data = smcd_move,
803
806
};
804
807
805
808
const struct smcd_ops *ism_get_smcd_ops(void)
+177
include/linux/dibs.h
+177
include/linux/dibs.h
···
36
36
* clients.
37
37
*/
38
38
39
+
/* DMB - Direct Memory Buffer
40
+
* --------------------------
41
+
* A dibs client provides a dmb as input buffer for a local receiving
42
+
* dibs device for exactly one (remote) sending dibs device. Only this
43
+
* sending device can send data into this dmb using move_data(). Sender
44
+
* and receiver can be the same device. A dmb belongs to exactly one client.
45
+
*/
46
+
struct dibs_dmb {
47
+
/* tok - Token for this dmb
48
+
* Used by remote and local devices and clients to address this dmb.
49
+
* Provided by dibs fabric. Unique per dibs fabric.
50
+
*/
51
+
u64 dmb_tok;
52
+
/* rgid - GID of designated remote sending device */
53
+
uuid_t rgid;
54
+
/* cpu_addr - buffer address */
55
+
void *cpu_addr;
56
+
/* len - buffer length */
57
+
u32 dmb_len;
58
+
/* idx - Index of this DMB on this receiving device */
59
+
u32 idx;
60
+
/* VLAN support (deprecated)
61
+
* In order to write into a vlan-tagged dmb, the remote device needs
62
+
* to belong to the this vlan
63
+
*/
64
+
u32 vlan_valid;
65
+
u32 vlan_id;
66
+
/* optional, used by device driver */
67
+
dma_addr_t dma_addr;
68
+
};
69
+
39
70
struct dibs_dev;
40
71
41
72
/* DIBS client
42
73
* -----------
43
74
*/
44
75
#define MAX_DIBS_CLIENTS 8
76
+
#define NO_DIBS_CLIENT 0xff
45
77
/* All dibs clients have access to all dibs devices.
46
78
* A dibs client provides the following functions to be called by dibs layer or
47
79
* dibs device drivers:
···
101
69
* The device is no longer usable by this client after this call.
102
70
*/
103
71
void (*del_dev)(struct dibs_dev *dev);
72
+
/**
73
+
* handle_irq() - Handle signaling for a DMB
74
+
* @dev: device that owns the dmb
75
+
* @idx: Index of the dmb that got signalled
76
+
* @dmbemask: signaling mask of the dmb
77
+
*
78
+
* Handle signaling for a dmb that was registered by this client
79
+
* for this device.
80
+
* The dibs device can coalesce multiple signaling triggers into a
81
+
* single call of handle_irq(). dmbemask can be used to indicate
82
+
* different kinds of triggers.
83
+
*
84
+
* Context: Called in IRQ context by dibs device driver
85
+
*/
86
+
void (*handle_irq)(struct dibs_dev *dev, unsigned int idx,
87
+
u16 dmbemask);
104
88
};
105
89
106
90
struct dibs_client {
···
196
148
int (*query_remote_gid)(struct dibs_dev *dev, const uuid_t *rgid,
197
149
u32 vid_valid, u32 vid);
198
150
/**
151
+
* max_dmbs()
152
+
* Return: Max number of DMBs that can be registered for this kind of
153
+
* dibs_dev
154
+
*/
155
+
int (*max_dmbs)(void);
156
+
/**
157
+
* register_dmb() - allocate and register a dmb
158
+
* @dev: dibs device
159
+
* @dmb: dmb struct to be registered
160
+
* @client: dibs client
161
+
* @vid: VLAN id; deprecated, ignored if device does not support vlan
162
+
*
163
+
* The following fields of dmb must provide valid input:
164
+
* @rgid: gid of remote user device
165
+
* @dmb_len: buffer length
166
+
* @idx: Optionally:requested idx (if non-zero)
167
+
* @vlan_valid: if zero, vlan_id will be ignored;
168
+
* deprecated, ignored if device does not support vlan
169
+
* @vlan_id: deprecated, ignored if device does not support vlan
170
+
* Upon return in addition the following fields will be valid:
171
+
* @dmb_tok: for usage by remote and local devices and clients
172
+
* @cpu_addr: allocated buffer
173
+
* @idx: dmb index, unique per dibs device
174
+
* @dma_addr: to be used by device driver,if applicable
175
+
*
176
+
* Allocate a dmb buffer and register it with this device and for this
177
+
* client.
178
+
* Return: zero on success
179
+
*/
180
+
int (*register_dmb)(struct dibs_dev *dev, struct dibs_dmb *dmb,
181
+
struct dibs_client *client);
182
+
/**
183
+
* unregister_dmb() - unregister and free a dmb
184
+
* @dev: dibs device
185
+
* @dmb: dmb struct to be unregistered
186
+
* The following fields of dmb must provide valid input:
187
+
* @dmb_tok
188
+
* @cpu_addr
189
+
* @idx
190
+
*
191
+
* Free dmb.cpu_addr and unregister the dmb from this device.
192
+
* Return: zero on success
193
+
*/
194
+
int (*unregister_dmb)(struct dibs_dev *dev, struct dibs_dmb *dmb);
195
+
/**
196
+
* move_data() - write into a remote dmb
197
+
* @dev: Local sending dibs device
198
+
* @dmb_tok: Token of the remote dmb
199
+
* @idx: signaling index in dmbemask
200
+
* @sf: signaling flag;
201
+
* if true, idx will be turned on at target dmbemask mask
202
+
* and target device will be signaled.
203
+
* @offset: offset within target dmb
204
+
* @data: pointer to data to be sent
205
+
* @size: length of data to be sent, can be zero.
206
+
*
207
+
* Use dev to write data of size at offset into a remote dmb
208
+
* identified by dmb_tok. Data is moved synchronously, *data can
209
+
* be freed when this function returns.
210
+
*
211
+
* If signaling flag (sf) is true, bit number idx bit will be turned
212
+
* on in the dmbemask mask when handle_irq() is called at the remote
213
+
* dibs client that owns the target dmb. The target device may chose
214
+
* to coalesce the signaling triggers of multiple move_data() calls
215
+
* to the same target dmb into a single handle_irq() call.
216
+
* Return: zero on success
217
+
*/
218
+
int (*move_data)(struct dibs_dev *dev, u64 dmb_tok, unsigned int idx,
219
+
bool sf, unsigned int offset, void *data,
220
+
unsigned int size);
221
+
/**
199
222
* add_vlan_id() - add dibs device to vlan (optional, deprecated)
200
223
* @dev: dibs device
201
224
* @vlan_id: vlan id
···
285
166
* Return: zero on success
286
167
*/
287
168
int (*del_vlan_id)(struct dibs_dev *dev, u64 vlan_id);
169
+
/**
170
+
* support_mmapped_rdmb() - can this device provide memory mapped
171
+
* remote dmbs? (optional)
172
+
* @dev: dibs device
173
+
*
174
+
* A dibs device can provide a kernel address + length, that represent
175
+
* a remote target dmb (like MMIO). Alternatively to calling
176
+
* move_data(), a dibs client can write into such a ghost-send-buffer
177
+
* (= to this kernel address) and the data will automatically
178
+
* immediately appear in the target dmb, even without calling
179
+
* move_data().
180
+
*
181
+
* Either all 3 function pointers for support_dmb_nocopy(),
182
+
* attach_dmb() and detach_dmb() are defined, or all of them must
183
+
* be NULL.
184
+
*
185
+
* Return: non-zero, if memory mapped remote dmbs are supported.
186
+
*/
187
+
int (*support_mmapped_rdmb)(struct dibs_dev *dev);
188
+
/**
189
+
* attach_dmb() - attach local memory to a remote dmb
190
+
* @dev: Local sending ism device
191
+
* @dmb: all other parameters are passed in the form of a
192
+
* dmb struct
193
+
* TODO: (THIS IS CONFUSING, should be changed)
194
+
* dmb_tok: (in) Token of the remote dmb, we want to attach to
195
+
* cpu_addr: (out) MMIO address
196
+
* dma_addr: (out) MMIO address (if applicable, invalid otherwise)
197
+
* dmb_len: (out) length of local MMIO region,
198
+
* equal to length of remote DMB.
199
+
* sba_idx: (out) index of remote dmb (NOT HELPFUL, should be removed)
200
+
*
201
+
* Provides a memory address to the sender that can be used to
202
+
* directly write into the remote dmb.
203
+
* Memory is available until detach_dmb is called
204
+
*
205
+
* Return: Zero upon success, Error code otherwise
206
+
*/
207
+
int (*attach_dmb)(struct dibs_dev *dev, struct dibs_dmb *dmb);
208
+
/**
209
+
* detach_dmb() - Detach the ghost buffer from a remote dmb
210
+
* @dev: ism device
211
+
* @token: dmb token of the remote dmb
212
+
*
213
+
* No need to free cpu_addr.
214
+
*
215
+
* Return: Zero upon success, Error code otherwise
216
+
*/
217
+
int (*detach_dmb)(struct dibs_dev *dev, u64 token);
288
218
};
289
219
290
220
struct dibs_dev {
···
347
179
348
180
/* priv pointer per client; for client usage only */
349
181
void *priv[MAX_DIBS_CLIENTS];
182
+
183
+
/* get this lock before accessing any of the fields below */
184
+
spinlock_t lock;
185
+
/* array of client ids indexed by dmb idx;
186
+
* can be used as indices into priv and subs arrays
187
+
*/
188
+
u8 *dmb_clientid_arr;
189
+
/* Sparse array of all ISM clients */
190
+
struct dibs_client *subs[MAX_DIBS_CLIENTS];
350
191
};
351
192
352
193
static inline void dibs_set_priv(struct dibs_dev *dev,
-23
include/linux/ism.h
-23
include/linux/ism.h
···
11
11
12
12
#include <linux/workqueue.h>
13
13
14
-
struct ism_dmb {
15
-
u64 dmb_tok;
16
-
u64 rgid;
17
-
u32 dmb_len;
18
-
u32 sba_idx;
19
-
u32 vlan_valid;
20
-
u32 vlan_id;
21
-
void *cpu_addr;
22
-
dma_addr_t dma_addr;
23
-
};
24
-
25
14
/* Unless we gain unexpected popularity, this limit should hold for a while */
26
15
#define MAX_CLIENTS 8
27
16
#define ISM_NR_DMBS 1920
···
25
36
struct ism_sba *sba;
26
37
dma_addr_t sba_dma_addr;
27
38
DECLARE_BITMAP(sba_bitmap, ISM_NR_DMBS);
28
-
u8 *sba_client_arr; /* entries are indices into 'clients' array */
29
39
void *priv[MAX_CLIENTS];
30
40
31
41
struct ism_eq *ieq;
···
46
58
struct ism_client {
47
59
const char *name;
48
60
void (*handle_event)(struct ism_dev *dev, struct ism_event *event);
49
-
/* Parameter dmbemask contains a bit vector with updated DMBEs, if sent
50
-
* via ism_move_data(). Callback function must handle all active bits
51
-
* indicated by dmbemask.
52
-
*/
53
-
void (*handle_irq)(struct ism_dev *dev, unsigned int bit, u16 dmbemask);
54
61
/* Private area - don't touch! */
55
62
u8 id;
56
63
};
···
61
78
void *priv) {
62
79
dev->priv[client->id] = priv;
63
80
}
64
-
65
-
int ism_register_dmb(struct ism_dev *dev, struct ism_dmb *dmb,
66
-
struct ism_client *client);
67
-
int ism_unregister_dmb(struct ism_dev *dev, struct ism_dmb *dmb);
68
-
int ism_move(struct ism_dev *dev, u64 dmb_tok, unsigned int idx, bool sf,
69
-
unsigned int offset, void *data, unsigned int size);
70
81
71
82
const struct smcd_ops *ism_get_smcd_ops(void);
72
83
-22
include/net/smc.h
-22
include/net/smc.h
···
28
28
};
29
29
30
30
/* SMCD/ISM device driver interface */
31
-
struct smcd_dmb {
32
-
u64 dmb_tok;
33
-
u64 rgid;
34
-
u32 dmb_len;
35
-
u32 sba_idx;
36
-
u32 vlan_valid;
37
-
u32 vlan_id;
38
-
void *cpu_addr;
39
-
dma_addr_t dma_addr;
40
-
};
41
-
42
31
#define ISM_EVENT_DMB 0
43
32
#define ISM_EVENT_GID 1
44
33
#define ISM_EVENT_SWR 2
···
42
53
};
43
54
44
55
struct smcd_ops {
45
-
int (*register_dmb)(struct smcd_dev *dev, struct smcd_dmb *dmb,
46
-
void *client);
47
-
int (*unregister_dmb)(struct smcd_dev *dev, struct smcd_dmb *dmb);
48
-
int (*move_data)(struct smcd_dev *dev, u64 dmb_tok, unsigned int idx,
49
-
bool sf, unsigned int offset, void *data,
50
-
unsigned int size);
51
-
52
56
/* optional operations */
53
57
int (*signal_event)(struct smcd_dev *dev, struct smcd_gid *rgid,
54
58
u32 trigger_irq, u32 event_code, u64 info);
55
-
int (*support_dmb_nocopy)(struct smcd_dev *dev);
56
-
int (*attach_dmb)(struct smcd_dev *dev, struct smcd_dmb *dmb);
57
-
int (*detach_dmb)(struct smcd_dev *dev, u64 token);
58
59
};
59
60
60
61
struct smcd_dev {
61
62
const struct smcd_ops *ops;
62
63
void *priv;
63
-
void *client;
64
64
struct dibs_dev *dibs;
65
65
struct list_head list;
66
66
spinlock_t lock;
-1
net/smc/Makefile
-1
net/smc/Makefile
+33
-39
net/smc/smc_ism.c
+33
-39
net/smc/smc_ism.c
···
15
15
#include "smc.h"
16
16
#include "smc_core.h"
17
17
#include "smc_ism.h"
18
-
#include "smc_loopback.h"
19
18
#include "smc_pnet.h"
20
19
#include "smc_netlink.h"
21
20
#include "linux/ism.h"
···
32
33
static void smcd_unregister_dev(struct dibs_dev *dibs);
33
34
#if IS_ENABLED(CONFIG_ISM)
34
35
static void smcd_handle_event(struct ism_dev *ism, struct ism_event *event);
35
-
static void smcd_handle_irq(struct ism_dev *ism, unsigned int dmbno,
36
-
u16 dmbemask);
37
36
38
37
static struct ism_client smc_ism_client = {
39
38
.name = "SMC-D",
40
39
.handle_event = smcd_handle_event,
41
-
.handle_irq = smcd_handle_irq,
42
40
};
43
41
#endif
42
+
static void smcd_handle_irq(struct dibs_dev *dibs, unsigned int dmbno,
43
+
u16 dmbemask);
44
+
44
45
static struct dibs_client_ops smc_client_ops = {
45
46
.add_dev = smcd_register_dev,
46
47
.del_dev = smcd_unregister_dev,
48
+
.handle_irq = smcd_handle_irq,
47
49
};
48
50
49
51
static struct dibs_client smc_dibs_client = {
···
221
221
void smc_ism_unregister_dmb(struct smcd_dev *smcd,
222
222
struct smc_buf_desc *dmb_desc)
223
223
{
224
-
struct smcd_dmb dmb;
224
+
struct dibs_dmb dmb;
225
225
226
226
if (!dmb_desc->dma_addr)
227
227
return;
228
228
229
229
memset(&dmb, 0, sizeof(dmb));
230
230
dmb.dmb_tok = dmb_desc->token;
231
-
dmb.sba_idx = dmb_desc->sba_idx;
231
+
dmb.idx = dmb_desc->sba_idx;
232
232
dmb.cpu_addr = dmb_desc->cpu_addr;
233
233
dmb.dma_addr = dmb_desc->dma_addr;
234
234
dmb.dmb_len = dmb_desc->len;
235
-
smcd->ops->unregister_dmb(smcd, &dmb);
235
+
236
+
smcd->dibs->ops->unregister_dmb(smcd->dibs, &dmb);
236
237
237
238
return;
238
239
}
···
241
240
int smc_ism_register_dmb(struct smc_link_group *lgr, int dmb_len,
242
241
struct smc_buf_desc *dmb_desc)
243
242
{
244
-
struct smcd_dmb dmb;
243
+
struct dibs_dev *dibs;
244
+
struct dibs_dmb dmb;
245
245
int rc;
246
246
247
247
memset(&dmb, 0, sizeof(dmb));
248
248
dmb.dmb_len = dmb_len;
249
-
dmb.sba_idx = dmb_desc->sba_idx;
249
+
dmb.idx = dmb_desc->sba_idx;
250
250
dmb.vlan_id = lgr->vlan_id;
251
-
dmb.rgid = lgr->peer_gid.gid;
252
-
rc = lgr->smcd->ops->register_dmb(lgr->smcd, &dmb, lgr->smcd->client);
251
+
copy_to_dibsgid(&dmb.rgid, &lgr->peer_gid);
252
+
253
+
dibs = lgr->smcd->dibs;
254
+
rc = dibs->ops->register_dmb(dibs, &dmb, &smc_dibs_client);
253
255
if (!rc) {
254
-
dmb_desc->sba_idx = dmb.sba_idx;
256
+
dmb_desc->sba_idx = dmb.idx;
255
257
dmb_desc->token = dmb.dmb_tok;
256
258
dmb_desc->cpu_addr = dmb.cpu_addr;
257
259
dmb_desc->dma_addr = dmb.dma_addr;
···
269
265
* merging sndbuf with peer DMB to avoid
270
266
* data copies between them.
271
267
*/
272
-
return (smcd->ops->support_dmb_nocopy &&
273
-
smcd->ops->support_dmb_nocopy(smcd));
268
+
return (smcd->dibs->ops->support_mmapped_rdmb &&
269
+
smcd->dibs->ops->support_mmapped_rdmb(smcd->dibs));
274
270
}
275
271
276
272
int smc_ism_attach_dmb(struct smcd_dev *dev, u64 token,
277
273
struct smc_buf_desc *dmb_desc)
278
274
{
279
-
struct smcd_dmb dmb;
275
+
struct dibs_dmb dmb;
280
276
int rc = 0;
281
277
282
-
if (!dev->ops->attach_dmb)
278
+
if (!dev->dibs->ops->attach_dmb)
283
279
return -EINVAL;
284
280
285
281
memset(&dmb, 0, sizeof(dmb));
286
282
dmb.dmb_tok = token;
287
-
rc = dev->ops->attach_dmb(dev, &dmb);
283
+
rc = dev->dibs->ops->attach_dmb(dev->dibs, &dmb);
288
284
if (!rc) {
289
-
dmb_desc->sba_idx = dmb.sba_idx;
285
+
dmb_desc->sba_idx = dmb.idx;
290
286
dmb_desc->token = dmb.dmb_tok;
291
287
dmb_desc->cpu_addr = dmb.cpu_addr;
292
288
dmb_desc->dma_addr = dmb.dma_addr;
···
298
294
299
295
int smc_ism_detach_dmb(struct smcd_dev *dev, u64 token)
300
296
{
301
-
if (!dev->ops->detach_dmb)
297
+
if (!dev->dibs->ops->detach_dmb)
302
298
return -EINVAL;
303
299
304
-
return dev->ops->detach_dmb(dev, token);
300
+
return dev->dibs->ops->detach_dmb(dev->dibs, token);
305
301
}
306
302
307
303
static int smc_nl_handle_smcd_dev(struct smcd_dev *smcd,
···
507
503
{
508
504
struct smcd_dev *smcd, *fentry;
509
505
const struct smcd_ops *ops;
510
-
struct smc_lo_dev *smc_lo;
511
506
struct ism_dev *ism;
507
+
int max_dmbs;
508
+
509
+
max_dmbs = dibs->ops->max_dmbs();
512
510
513
511
if (smc_ism_is_loopback(dibs)) {
514
-
if (smc_loopback_init(&smc_lo))
515
-
return;
516
-
}
517
-
518
-
if (smc_ism_is_loopback(dibs)) {
519
-
ops = smc_lo_get_smcd_ops();
520
-
smcd = smcd_alloc_dev(dev_name(&dibs->dev), ops,
521
-
SMC_LO_MAX_DMBS);
512
+
ops = NULL;
522
513
} else {
523
514
ism = dibs->drv_priv;
524
515
#if IS_ENABLED(CONFIG_ISM)
525
516
ops = ism_get_smcd_ops();
526
517
#endif
527
-
smcd = smcd_alloc_dev(dev_name(&dibs->dev), ops,
528
-
ISM_NR_DMBS);
529
518
}
519
+
smcd = smcd_alloc_dev(dev_name(&dibs->dev), ops, max_dmbs);
530
520
if (!smcd)
531
521
return;
532
522
···
528
530
dibs_set_priv(dibs, &smc_dibs_client, smcd);
529
531
530
532
if (smc_ism_is_loopback(dibs)) {
531
-
smcd->priv = smc_lo;
532
-
smc_lo->smcd = smcd;
533
+
smcd->priv = NULL;
533
534
} else {
534
535
smcd->priv = ism;
535
536
#if IS_ENABLED(CONFIG_ISM)
536
537
ism_set_priv(ism, &smc_ism_client, smcd);
537
-
smcd->client = &smc_ism_client;
538
538
#endif
539
539
}
540
540
···
586
590
list_del_init(&smcd->list);
587
591
mutex_unlock(&smcd_dev_list.mutex);
588
592
destroy_workqueue(smcd->event_wq);
589
-
if (smc_ism_is_loopback(dibs))
590
-
smc_loopback_exit();
591
593
kfree(smcd->conn);
592
594
kfree(smcd);
593
595
}
···
618
624
wrk->event = *event;
619
625
queue_work(smcd->event_wq, &wrk->work);
620
626
}
627
+
#endif
621
628
622
629
/* SMCD Device interrupt handler. Called from ISM device interrupt handler.
623
630
* Parameters are the ism device pointer, DMB number, and the DMBE bitmask.
···
627
632
* Context:
628
633
* - Function called in IRQ context from ISM device driver IRQ handler.
629
634
*/
630
-
static void smcd_handle_irq(struct ism_dev *ism, unsigned int dmbno,
635
+
static void smcd_handle_irq(struct dibs_dev *dibs, unsigned int dmbno,
631
636
u16 dmbemask)
632
637
{
633
-
struct smcd_dev *smcd = ism_get_priv(ism, &smc_ism_client);
638
+
struct smcd_dev *smcd = dibs_get_priv(dibs, &smc_dibs_client);
634
639
struct smc_connection *conn = NULL;
635
640
unsigned long flags;
636
641
···
640
645
tasklet_schedule(&conn->rx_tsklet);
641
646
spin_unlock_irqrestore(&smcd->lock, flags);
642
647
}
643
-
#endif
644
648
645
649
int smc_ism_signal_shutdown(struct smc_link_group *lgr)
646
650
{
+3
-1
net/smc/smc_ism.h
+3
-1
net/smc/smc_ism.h
-294
net/smc/smc_loopback.c
-294
net/smc/smc_loopback.c
···
1
-
// SPDX-License-Identifier: GPL-2.0
2
-
/*
3
-
* Shared Memory Communications Direct over loopback-ism device.
4
-
*
5
-
* Functions for loopback-ism device.
6
-
*
7
-
* Copyright (c) 2024, Alibaba Inc.
8
-
*
9
-
* Author: Wen Gu <guwen@linux.alibaba.com>
10
-
* Tony Lu <tonylu@linux.alibaba.com>
11
-
*
12
-
*/
13
-
14
-
#include <linux/device.h>
15
-
#include <linux/types.h>
16
-
#include <linux/dibs.h>
17
-
#include <net/smc.h>
18
-
19
-
#include "smc_cdc.h"
20
-
#include "smc_ism.h"
21
-
#include "smc_loopback.h"
22
-
23
-
#define SMC_LO_SUPPORT_NOCOPY 0x1
24
-
#define SMC_DMA_ADDR_INVALID (~(dma_addr_t)0)
25
-
26
-
static struct smc_lo_dev *lo_dev;
27
-
28
-
static int smc_lo_register_dmb(struct smcd_dev *smcd, struct smcd_dmb *dmb,
29
-
void *client_priv)
30
-
{
31
-
struct smc_lo_dmb_node *dmb_node, *tmp_node;
32
-
struct smc_lo_dev *ldev = smcd->priv;
33
-
int sba_idx, rc;
34
-
35
-
/* check space for new dmb */
36
-
for_each_clear_bit(sba_idx, ldev->sba_idx_mask, SMC_LO_MAX_DMBS) {
37
-
if (!test_and_set_bit(sba_idx, ldev->sba_idx_mask))
38
-
break;
39
-
}
40
-
if (sba_idx == SMC_LO_MAX_DMBS)
41
-
return -ENOSPC;
42
-
43
-
dmb_node = kzalloc(sizeof(*dmb_node), GFP_KERNEL);
44
-
if (!dmb_node) {
45
-
rc = -ENOMEM;
46
-
goto err_bit;
47
-
}
48
-
49
-
dmb_node->sba_idx = sba_idx;
50
-
dmb_node->len = dmb->dmb_len;
51
-
dmb_node->cpu_addr = kzalloc(dmb_node->len, GFP_KERNEL |
52
-
__GFP_NOWARN | __GFP_NORETRY |
53
-
__GFP_NOMEMALLOC);
54
-
if (!dmb_node->cpu_addr) {
55
-
rc = -ENOMEM;
56
-
goto err_node;
57
-
}
58
-
dmb_node->dma_addr = SMC_DMA_ADDR_INVALID;
59
-
refcount_set(&dmb_node->refcnt, 1);
60
-
61
-
again:
62
-
/* add new dmb into hash table */
63
-
get_random_bytes(&dmb_node->token, sizeof(dmb_node->token));
64
-
write_lock_bh(&ldev->dmb_ht_lock);
65
-
hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb_node->token) {
66
-
if (tmp_node->token == dmb_node->token) {
67
-
write_unlock_bh(&ldev->dmb_ht_lock);
68
-
goto again;
69
-
}
70
-
}
71
-
hash_add(ldev->dmb_ht, &dmb_node->list, dmb_node->token);
72
-
write_unlock_bh(&ldev->dmb_ht_lock);
73
-
atomic_inc(&ldev->dmb_cnt);
74
-
75
-
dmb->sba_idx = dmb_node->sba_idx;
76
-
dmb->dmb_tok = dmb_node->token;
77
-
dmb->cpu_addr = dmb_node->cpu_addr;
78
-
dmb->dma_addr = dmb_node->dma_addr;
79
-
dmb->dmb_len = dmb_node->len;
80
-
81
-
return 0;
82
-
83
-
err_node:
84
-
kfree(dmb_node);
85
-
err_bit:
86
-
clear_bit(sba_idx, ldev->sba_idx_mask);
87
-
return rc;
88
-
}
89
-
90
-
static void __smc_lo_unregister_dmb(struct smc_lo_dev *ldev,
91
-
struct smc_lo_dmb_node *dmb_node)
92
-
{
93
-
/* remove dmb from hash table */
94
-
write_lock_bh(&ldev->dmb_ht_lock);
95
-
hash_del(&dmb_node->list);
96
-
write_unlock_bh(&ldev->dmb_ht_lock);
97
-
98
-
clear_bit(dmb_node->sba_idx, ldev->sba_idx_mask);
99
-
kvfree(dmb_node->cpu_addr);
100
-
kfree(dmb_node);
101
-
102
-
if (atomic_dec_and_test(&ldev->dmb_cnt))
103
-
wake_up(&ldev->ldev_release);
104
-
}
105
-
106
-
static int smc_lo_unregister_dmb(struct smcd_dev *smcd, struct smcd_dmb *dmb)
107
-
{
108
-
struct smc_lo_dmb_node *dmb_node = NULL, *tmp_node;
109
-
struct smc_lo_dev *ldev = smcd->priv;
110
-
111
-
/* find dmb from hash table */
112
-
read_lock_bh(&ldev->dmb_ht_lock);
113
-
hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb->dmb_tok) {
114
-
if (tmp_node->token == dmb->dmb_tok) {
115
-
dmb_node = tmp_node;
116
-
break;
117
-
}
118
-
}
119
-
if (!dmb_node) {
120
-
read_unlock_bh(&ldev->dmb_ht_lock);
121
-
return -EINVAL;
122
-
}
123
-
read_unlock_bh(&ldev->dmb_ht_lock);
124
-
125
-
if (refcount_dec_and_test(&dmb_node->refcnt))
126
-
__smc_lo_unregister_dmb(ldev, dmb_node);
127
-
return 0;
128
-
}
129
-
130
-
static int smc_lo_support_dmb_nocopy(struct smcd_dev *smcd)
131
-
{
132
-
return SMC_LO_SUPPORT_NOCOPY;
133
-
}
134
-
135
-
static int smc_lo_attach_dmb(struct smcd_dev *smcd, struct smcd_dmb *dmb)
136
-
{
137
-
struct smc_lo_dmb_node *dmb_node = NULL, *tmp_node;
138
-
struct smc_lo_dev *ldev = smcd->priv;
139
-
140
-
/* find dmb_node according to dmb->dmb_tok */
141
-
read_lock_bh(&ldev->dmb_ht_lock);
142
-
hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb->dmb_tok) {
143
-
if (tmp_node->token == dmb->dmb_tok) {
144
-
dmb_node = tmp_node;
145
-
break;
146
-
}
147
-
}
148
-
if (!dmb_node) {
149
-
read_unlock_bh(&ldev->dmb_ht_lock);
150
-
return -EINVAL;
151
-
}
152
-
read_unlock_bh(&ldev->dmb_ht_lock);
153
-
154
-
if (!refcount_inc_not_zero(&dmb_node->refcnt))
155
-
/* the dmb is being unregistered, but has
156
-
* not been removed from the hash table.
157
-
*/
158
-
return -EINVAL;
159
-
160
-
/* provide dmb information */
161
-
dmb->sba_idx = dmb_node->sba_idx;
162
-
dmb->dmb_tok = dmb_node->token;
163
-
dmb->cpu_addr = dmb_node->cpu_addr;
164
-
dmb->dma_addr = dmb_node->dma_addr;
165
-
dmb->dmb_len = dmb_node->len;
166
-
return 0;
167
-
}
168
-
169
-
static int smc_lo_detach_dmb(struct smcd_dev *smcd, u64 token)
170
-
{
171
-
struct smc_lo_dmb_node *dmb_node = NULL, *tmp_node;
172
-
struct smc_lo_dev *ldev = smcd->priv;
173
-
174
-
/* find dmb_node according to dmb->dmb_tok */
175
-
read_lock_bh(&ldev->dmb_ht_lock);
176
-
hash_for_each_possible(ldev->dmb_ht, tmp_node, list, token) {
177
-
if (tmp_node->token == token) {
178
-
dmb_node = tmp_node;
179
-
break;
180
-
}
181
-
}
182
-
if (!dmb_node) {
183
-
read_unlock_bh(&ldev->dmb_ht_lock);
184
-
return -EINVAL;
185
-
}
186
-
read_unlock_bh(&ldev->dmb_ht_lock);
187
-
188
-
if (refcount_dec_and_test(&dmb_node->refcnt))
189
-
__smc_lo_unregister_dmb(ldev, dmb_node);
190
-
return 0;
191
-
}
192
-
193
-
static int smc_lo_move_data(struct smcd_dev *smcd, u64 dmb_tok,
194
-
unsigned int idx, bool sf, unsigned int offset,
195
-
void *data, unsigned int size)
196
-
{
197
-
struct smc_lo_dmb_node *rmb_node = NULL, *tmp_node;
198
-
struct smc_lo_dev *ldev = smcd->priv;
199
-
struct smc_connection *conn;
200
-
201
-
read_lock_bh(&ldev->dmb_ht_lock);
202
-
hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb_tok) {
203
-
if (tmp_node->token == dmb_tok) {
204
-
rmb_node = tmp_node;
205
-
break;
206
-
}
207
-
}
208
-
if (!rmb_node) {
209
-
read_unlock_bh(&ldev->dmb_ht_lock);
210
-
return -EINVAL;
211
-
}
212
-
memcpy((char *)rmb_node->cpu_addr + offset, data, size);
213
-
read_unlock_bh(&ldev->dmb_ht_lock);
214
-
215
-
if (!sf)
216
-
return 0;
217
-
218
-
conn = smcd->conn[rmb_node->sba_idx];
219
-
if (!conn || conn->killed)
220
-
return -EPIPE;
221
-
tasklet_schedule(&conn->rx_tsklet);
222
-
return 0;
223
-
}
224
-
225
-
static const struct smcd_ops lo_ops = {
226
-
.register_dmb = smc_lo_register_dmb,
227
-
.unregister_dmb = smc_lo_unregister_dmb,
228
-
.support_dmb_nocopy = smc_lo_support_dmb_nocopy,
229
-
.attach_dmb = smc_lo_attach_dmb,
230
-
.detach_dmb = smc_lo_detach_dmb,
231
-
.signal_event = NULL,
232
-
.move_data = smc_lo_move_data,
233
-
};
234
-
235
-
const struct smcd_ops *smc_lo_get_smcd_ops(void)
236
-
{
237
-
return &lo_ops;
238
-
}
239
-
240
-
static void smc_lo_dev_init(struct smc_lo_dev *ldev)
241
-
{
242
-
rwlock_init(&ldev->dmb_ht_lock);
243
-
hash_init(ldev->dmb_ht);
244
-
atomic_set(&ldev->dmb_cnt, 0);
245
-
init_waitqueue_head(&ldev->ldev_release);
246
-
247
-
return;
248
-
}
249
-
250
-
static void smc_lo_dev_exit(struct smc_lo_dev *ldev)
251
-
{
252
-
if (atomic_read(&ldev->dmb_cnt))
253
-
wait_event(ldev->ldev_release, !atomic_read(&ldev->dmb_cnt));
254
-
}
255
-
256
-
static int smc_lo_dev_probe(void)
257
-
{
258
-
struct smc_lo_dev *ldev;
259
-
260
-
ldev = kzalloc(sizeof(*ldev), GFP_KERNEL);
261
-
if (!ldev)
262
-
return -ENOMEM;
263
-
264
-
smc_lo_dev_init(ldev);
265
-
266
-
lo_dev = ldev; /* global loopback device */
267
-
268
-
return 0;
269
-
}
270
-
271
-
static void smc_lo_dev_remove(void)
272
-
{
273
-
if (!lo_dev)
274
-
return;
275
-
276
-
smc_lo_dev_exit(lo_dev);
277
-
kfree(lo_dev);
278
-
lo_dev = NULL;
279
-
}
280
-
281
-
int smc_loopback_init(struct smc_lo_dev **smc_lb)
282
-
{
283
-
int ret;
284
-
285
-
ret = smc_lo_dev_probe();
286
-
if (!ret)
287
-
*smc_lb = lo_dev;
288
-
return ret;
289
-
}
290
-
291
-
void smc_loopback_exit(void)
292
-
{
293
-
smc_lo_dev_remove();
294
-
}
-47
net/smc/smc_loopback.h
-47
net/smc/smc_loopback.h
···
1
-
/* SPDX-License-Identifier: GPL-2.0 */
2
-
/*
3
-
* Shared Memory Communications Direct over loopback-ism device.
4
-
*
5
-
* SMC-D loopback-ism device structure definitions.
6
-
*
7
-
* Copyright (c) 2024, Alibaba Inc.
8
-
*
9
-
* Author: Wen Gu <guwen@linux.alibaba.com>
10
-
* Tony Lu <tonylu@linux.alibaba.com>
11
-
*
12
-
*/
13
-
14
-
#ifndef _SMC_LOOPBACK_H
15
-
#define _SMC_LOOPBACK_H
16
-
17
-
#include <linux/device.h>
18
-
#include <net/smc.h>
19
-
20
-
#define SMC_LO_MAX_DMBS 5000
21
-
#define SMC_LO_DMBS_HASH_BITS 12
22
-
23
-
struct smc_lo_dmb_node {
24
-
struct hlist_node list;
25
-
u64 token;
26
-
u32 len;
27
-
u32 sba_idx;
28
-
void *cpu_addr;
29
-
dma_addr_t dma_addr;
30
-
refcount_t refcnt;
31
-
};
32
-
33
-
struct smc_lo_dev {
34
-
struct smcd_dev *smcd;
35
-
atomic_t dmb_cnt;
36
-
rwlock_t dmb_ht_lock;
37
-
DECLARE_BITMAP(sba_idx_mask, SMC_LO_MAX_DMBS);
38
-
DECLARE_HASHTABLE(dmb_ht, SMC_LO_DMBS_HASH_BITS);
39
-
wait_queue_head_t ldev_release;
40
-
};
41
-
42
-
const struct smcd_ops *smc_lo_get_smcd_ops(void);
43
-
44
-
int smc_loopback_init(struct smc_lo_dev **smc_lb);
45
-
void smc_loopback_exit(void);
46
-
47
-
#endif /* _SMC_LOOPBACK_H */