Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
RDMA/hns: Add bonding event handler
Register netdev notifier for two bonding events NETDEV_CHANGEUPPER
and NETDEV_CHANGELOWERSTATE.
In NETDEV_CHANGEUPPER event handler, check some rules about the HW
constraints when trying to link a new slave to the masteri, and
store some bonding information from the notifier. In unlinking case,
simply check the number of the rest slaves to decide whether the
bond is still supported.
In NETDEV_CHANGELOWERSTATE event handler, not much is done. It
simply sets the bond state when the bond is ready, which will be
used later.
Signed-off-by: Junxian Huang <huangjunxian6@hisilicon.com>
Link: https://patch.msgid.link/20251112093510.3696363-4-huangjunxian6@hisilicon.com
Signed-off-by: Leon Romanovsky <leon@kernel.org>
authored by
Junxian Huang
and committed by
Leon Romanovsky
d31d410b
b37ad2e2
+340
+314
drivers/infiniband/hw/hns/hns_roce_bond.c
+314
drivers/infiniband/hw/hns/hns_roce_bond.c
···
9
9
10
10
static DEFINE_XARRAY(roce_bond_xa);
11
11
12
+
static struct hns_roce_dev *hns_roce_get_hrdev_by_netdev(struct net_device *net_dev)
13
+
{
14
+
struct ib_device *ibdev =
15
+
ib_device_get_by_netdev(net_dev, RDMA_DRIVER_HNS);
16
+
17
+
if (!ibdev)
18
+
return NULL;
19
+
20
+
return container_of(ibdev, struct hns_roce_dev, ib_dev);
21
+
}
22
+
12
23
static struct net_device *get_upper_dev_from_ndev(struct net_device *net_dev)
13
24
{
14
25
struct net_device *upper_dev;
···
142
131
return 0;
143
132
}
144
133
134
+
static bool is_dev_bond_supported(struct hns_roce_bond_group *bond_grp,
135
+
struct net_device *net_dev)
136
+
{
137
+
struct hns_roce_dev *hr_dev = hns_roce_get_hrdev_by_netdev(net_dev);
138
+
bool ret = true;
139
+
140
+
if (!hr_dev) {
141
+
if (bond_grp &&
142
+
get_netdev_bond_slave_id(net_dev, bond_grp) >= 0)
143
+
return true;
144
+
else
145
+
return false;
146
+
}
147
+
148
+
if (!(hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_BOND)) {
149
+
ret = false;
150
+
goto out;
151
+
}
152
+
153
+
if (hr_dev->is_vf || pci_num_vf(hr_dev->pci_dev) > 0) {
154
+
ret = false;
155
+
goto out;
156
+
}
157
+
158
+
if (bond_grp->bus_num != get_hr_bus_num(hr_dev))
159
+
ret = false;
160
+
161
+
out:
162
+
ib_device_put(&hr_dev->ib_dev);
163
+
return ret;
164
+
}
165
+
166
+
static bool check_slave_support(struct hns_roce_bond_group *bond_grp,
167
+
struct net_device *upper_dev)
168
+
{
169
+
struct net_device *net_dev;
170
+
u8 slave_num = 0;
171
+
172
+
rcu_read_lock();
173
+
for_each_netdev_in_bond_rcu(upper_dev, net_dev) {
174
+
if (is_dev_bond_supported(bond_grp, net_dev)) {
175
+
slave_num++;
176
+
continue;
177
+
}
178
+
rcu_read_unlock();
179
+
return false;
180
+
}
181
+
rcu_read_unlock();
182
+
183
+
return (slave_num > 1 && slave_num <= ROCE_BOND_FUNC_MAX);
184
+
}
185
+
186
+
static void hns_roce_attach_bond_grp(struct hns_roce_bond_group *bond_grp,
187
+
struct hns_roce_dev *hr_dev,
188
+
struct net_device *upper_dev)
189
+
{
190
+
bond_grp->upper_dev = upper_dev;
191
+
bond_grp->main_hr_dev = hr_dev;
192
+
bond_grp->bond_state = HNS_ROCE_BOND_NOT_BONDED;
193
+
bond_grp->bond_ready = false;
194
+
}
195
+
196
+
static bool lowerstate_event_filter(struct hns_roce_bond_group *bond_grp,
197
+
struct net_device *net_dev)
198
+
{
199
+
struct hns_roce_bond_group *bond_grp_tmp;
200
+
201
+
bond_grp_tmp = hns_roce_get_bond_grp(net_dev, bond_grp->bus_num);
202
+
return bond_grp_tmp == bond_grp;
203
+
}
204
+
205
+
static void lowerstate_event_setting(struct hns_roce_bond_group *bond_grp,
206
+
struct netdev_notifier_changelowerstate_info *info)
207
+
{
208
+
mutex_lock(&bond_grp->bond_mutex);
209
+
210
+
if (bond_grp->bond_ready &&
211
+
bond_grp->bond_state == HNS_ROCE_BOND_IS_BONDED)
212
+
bond_grp->bond_state = HNS_ROCE_BOND_SLAVE_CHANGESTATE;
213
+
214
+
mutex_unlock(&bond_grp->bond_mutex);
215
+
}
216
+
217
+
static bool hns_roce_bond_lowerstate_event(struct hns_roce_bond_group *bond_grp,
218
+
struct netdev_notifier_changelowerstate_info *info)
219
+
{
220
+
struct net_device *net_dev =
221
+
netdev_notifier_info_to_dev((struct netdev_notifier_info *)info);
222
+
223
+
if (!netif_is_lag_port(net_dev))
224
+
return false;
225
+
226
+
if (!lowerstate_event_filter(bond_grp, net_dev))
227
+
return false;
228
+
229
+
lowerstate_event_setting(bond_grp, info);
230
+
231
+
return true;
232
+
}
233
+
234
+
static bool is_bond_setting_supported(struct netdev_lag_upper_info *bond_info)
235
+
{
236
+
if (!bond_info)
237
+
return false;
238
+
239
+
if (bond_info->tx_type != NETDEV_LAG_TX_TYPE_ACTIVEBACKUP &&
240
+
bond_info->tx_type != NETDEV_LAG_TX_TYPE_HASH)
241
+
return false;
242
+
243
+
if (bond_info->tx_type == NETDEV_LAG_TX_TYPE_HASH &&
244
+
bond_info->hash_type > NETDEV_LAG_HASH_L23)
245
+
return false;
246
+
247
+
return true;
248
+
}
249
+
250
+
static void upper_event_setting(struct hns_roce_bond_group *bond_grp,
251
+
struct netdev_notifier_changeupper_info *info)
252
+
{
253
+
struct netdev_lag_upper_info *bond_upper_info = NULL;
254
+
bool slave_inc = info->linking;
255
+
256
+
if (slave_inc)
257
+
bond_upper_info = info->upper_info;
258
+
259
+
if (bond_upper_info) {
260
+
bond_grp->tx_type = bond_upper_info->tx_type;
261
+
bond_grp->hash_type = bond_upper_info->hash_type;
262
+
}
263
+
}
264
+
265
+
static bool check_unlinking_bond_support(struct hns_roce_bond_group *bond_grp)
266
+
{
267
+
struct net_device *net_dev;
268
+
u8 slave_num = 0;
269
+
270
+
rcu_read_lock();
271
+
for_each_netdev_in_bond_rcu(bond_grp->upper_dev, net_dev) {
272
+
if (get_netdev_bond_slave_id(net_dev, bond_grp) >= 0)
273
+
slave_num++;
274
+
}
275
+
rcu_read_unlock();
276
+
277
+
return (slave_num > 1);
278
+
}
279
+
280
+
static bool check_linking_bond_support(struct netdev_lag_upper_info *bond_info,
281
+
struct hns_roce_bond_group *bond_grp,
282
+
struct net_device *upper_dev)
283
+
{
284
+
if (!is_bond_setting_supported(bond_info))
285
+
return false;
286
+
287
+
return check_slave_support(bond_grp, upper_dev);
288
+
}
289
+
290
+
static enum bond_support_type
291
+
check_bond_support(struct hns_roce_bond_group *bond_grp,
292
+
struct net_device *upper_dev,
293
+
struct netdev_notifier_changeupper_info *info)
294
+
{
295
+
bool bond_grp_exist = false;
296
+
bool support;
297
+
298
+
if (upper_dev == bond_grp->upper_dev)
299
+
bond_grp_exist = true;
300
+
301
+
if (!info->linking && !bond_grp_exist)
302
+
return BOND_NOT_SUPPORT;
303
+
304
+
if (info->linking)
305
+
support = check_linking_bond_support(info->upper_info, bond_grp,
306
+
upper_dev);
307
+
else
308
+
support = check_unlinking_bond_support(bond_grp);
309
+
310
+
if (support)
311
+
return BOND_SUPPORT;
312
+
313
+
return bond_grp_exist ? BOND_EXISTING_NOT_SUPPORT : BOND_NOT_SUPPORT;
314
+
}
315
+
316
+
static bool upper_event_filter(struct netdev_notifier_changeupper_info *info,
317
+
struct hns_roce_bond_group *bond_grp,
318
+
struct net_device *net_dev)
319
+
{
320
+
struct net_device *upper_dev = info->upper_dev;
321
+
struct hns_roce_bond_group *bond_grp_tmp;
322
+
struct hns_roce_dev *hr_dev;
323
+
bool ret = true;
324
+
u8 bus_num;
325
+
326
+
if (!info->linking ||
327
+
bond_grp->bond_state != HNS_ROCE_BOND_NOT_ATTACHED)
328
+
return bond_grp->upper_dev == upper_dev;
329
+
330
+
hr_dev = hns_roce_get_hrdev_by_netdev(net_dev);
331
+
if (!hr_dev)
332
+
return false;
333
+
334
+
bus_num = get_hr_bus_num(hr_dev);
335
+
if (bond_grp->bus_num != bus_num) {
336
+
ret = false;
337
+
goto out;
338
+
}
339
+
340
+
bond_grp_tmp = hns_roce_get_bond_grp(net_dev, bus_num);
341
+
if (bond_grp_tmp && bond_grp_tmp != bond_grp)
342
+
ret = false;
343
+
out:
344
+
ib_device_put(&hr_dev->ib_dev);
345
+
return ret;
346
+
}
347
+
348
+
static bool hns_roce_bond_upper_event(struct hns_roce_bond_group *bond_grp,
349
+
struct netdev_notifier_changeupper_info *info)
350
+
{
351
+
struct net_device *net_dev =
352
+
netdev_notifier_info_to_dev((struct netdev_notifier_info *)info);
353
+
struct net_device *upper_dev = info->upper_dev;
354
+
enum bond_support_type support = BOND_SUPPORT;
355
+
struct hns_roce_dev *hr_dev;
356
+
int slave_id;
357
+
358
+
if (!upper_dev || !netif_is_lag_master(upper_dev))
359
+
return false;
360
+
361
+
if (!upper_event_filter(info, bond_grp, net_dev))
362
+
return false;
363
+
364
+
mutex_lock(&bond_grp->bond_mutex);
365
+
support = check_bond_support(bond_grp, upper_dev, info);
366
+
if (support == BOND_NOT_SUPPORT) {
367
+
mutex_unlock(&bond_grp->bond_mutex);
368
+
return false;
369
+
}
370
+
371
+
if (bond_grp->bond_state == HNS_ROCE_BOND_NOT_ATTACHED) {
372
+
hr_dev = hns_roce_get_hrdev_by_netdev(net_dev);
373
+
if (!hr_dev) {
374
+
mutex_unlock(&bond_grp->bond_mutex);
375
+
return false;
376
+
}
377
+
hns_roce_attach_bond_grp(bond_grp, hr_dev, upper_dev);
378
+
ib_device_put(&hr_dev->ib_dev);
379
+
}
380
+
381
+
/* In the case of netdev being unregistered, the roce
382
+
* instance shouldn't be inited.
383
+
*/
384
+
if (net_dev->reg_state >= NETREG_UNREGISTERING) {
385
+
slave_id = get_netdev_bond_slave_id(net_dev, bond_grp);
386
+
if (slave_id >= 0) {
387
+
bond_grp->bond_func_info[slave_id].net_dev = NULL;
388
+
bond_grp->bond_func_info[slave_id].handle = NULL;
389
+
}
390
+
}
391
+
392
+
if (support == BOND_SUPPORT) {
393
+
bond_grp->bond_ready = true;
394
+
if (bond_grp->bond_state != HNS_ROCE_BOND_NOT_BONDED)
395
+
bond_grp->bond_state = HNS_ROCE_BOND_SLAVE_CHANGE_NUM;
396
+
}
397
+
mutex_unlock(&bond_grp->bond_mutex);
398
+
if (support == BOND_SUPPORT)
399
+
upper_event_setting(bond_grp, info);
400
+
401
+
return true;
402
+
}
403
+
404
+
static int hns_roce_bond_event(struct notifier_block *self,
405
+
unsigned long event, void *ptr)
406
+
{
407
+
struct hns_roce_bond_group *bond_grp =
408
+
container_of(self, struct hns_roce_bond_group, bond_nb);
409
+
bool changed = false;
410
+
411
+
if (event == NETDEV_CHANGEUPPER)
412
+
changed = hns_roce_bond_upper_event(bond_grp, ptr);
413
+
if (event == NETDEV_CHANGELOWERSTATE)
414
+
changed = hns_roce_bond_lowerstate_event(bond_grp, ptr);
415
+
416
+
return NOTIFY_DONE;
417
+
}
418
+
145
419
int hns_roce_alloc_bond_grp(struct hns_roce_dev *hr_dev)
146
420
{
147
421
struct hns_roce_bond_group *bgrps[ROCE_BOND_NUM_MAX];
···
445
149
goto mem_err;
446
150
}
447
151
152
+
mutex_init(&bond_grp->bond_mutex);
153
+
154
+
bond_grp->bond_ready = false;
155
+
bond_grp->bond_state = HNS_ROCE_BOND_NOT_ATTACHED;
448
156
bond_grp->bus_num = bus_num;
449
157
450
158
ret = alloc_bond_id(bond_grp);
···
458
158
goto alloc_id_err;
459
159
}
460
160
161
+
bond_grp->bond_nb.notifier_call = hns_roce_bond_event;
162
+
ret = register_netdevice_notifier(&bond_grp->bond_nb);
163
+
if (ret) {
164
+
ibdev_err(&hr_dev->ib_dev,
165
+
"failed to register bond nb, ret = %d.\n", ret);
166
+
goto register_nb_err;
167
+
}
461
168
bgrps[i] = bond_grp;
462
169
}
463
170
464
171
return 0;
465
172
173
+
register_nb_err:
174
+
remove_bond_id(bond_grp->bus_num, bond_grp->bond_id);
466
175
alloc_id_err:
176
+
mutex_destroy(&bond_grp->bond_mutex);
467
177
kvfree(bond_grp);
468
178
mem_err:
469
179
for (i--; i >= 0; i--) {
180
+
unregister_netdevice_notifier(&bgrps[i]->bond_nb);
470
181
remove_bond_id(bgrps[i]->bus_num, bgrps[i]->bond_id);
182
+
mutex_destroy(&bgrps[i]->bond_mutex);
471
183
kvfree(bgrps[i]);
472
184
}
473
185
return ret;
···
497
185
bond_grp = die_info->bgrps[i];
498
186
if (!bond_grp)
499
187
continue;
188
+
unregister_netdevice_notifier(&bond_grp->bond_nb);
500
189
remove_bond_id(bond_grp->bus_num, bond_grp->bond_id);
190
+
mutex_destroy(&bond_grp->bond_mutex);
501
191
kvfree(bond_grp);
502
192
}
503
193
}
+26
drivers/infiniband/hw/hns/hns_roce_bond.h
+26
drivers/infiniband/hw/hns/hns_roce_bond.h
···
14
14
15
15
#define BOND_ID(id) BIT(id)
16
16
17
+
enum bond_support_type {
18
+
BOND_NOT_SUPPORT,
19
+
/*
20
+
* bond_grp already exists, but in the current
21
+
* conditions it's no longer supported
22
+
*/
23
+
BOND_EXISTING_NOT_SUPPORT,
24
+
BOND_SUPPORT,
25
+
};
26
+
27
+
enum hns_roce_bond_state {
28
+
HNS_ROCE_BOND_NOT_ATTACHED,
29
+
HNS_ROCE_BOND_NOT_BONDED,
30
+
HNS_ROCE_BOND_IS_BONDED,
31
+
HNS_ROCE_BOND_SLAVE_CHANGE_NUM,
32
+
HNS_ROCE_BOND_SLAVE_CHANGESTATE,
33
+
};
34
+
17
35
struct hns_roce_func_info {
18
36
struct net_device *net_dev;
37
+
struct hnae3_handle *handle;
19
38
};
20
39
21
40
struct hns_roce_bond_group {
22
41
struct net_device *upper_dev;
42
+
struct hns_roce_dev *main_hr_dev;
23
43
u8 bond_id;
24
44
u8 bus_num;
25
45
struct hns_roce_func_info bond_func_info[ROCE_BOND_FUNC_MAX];
46
+
bool bond_ready;
47
+
enum hns_roce_bond_state bond_state;
48
+
enum netdev_lag_tx_type tx_type;
49
+
enum netdev_lag_hash hash_type;
50
+
struct mutex bond_mutex;
51
+
struct notifier_block bond_nb;
26
52
};
27
53
28
54
struct hns_roce_die_info {