Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Copyright (c) 2016 Mellanox Technologies. All rights reserved.
4 * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
5 */
6
7#include <net/genetlink.h>
8#define CREATE_TRACE_POINTS
9#include <trace/events/devlink.h>
10
11#include "devl_internal.h"
12
13EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_hwmsg);
14EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_hwerr);
15EXPORT_TRACEPOINT_SYMBOL_GPL(devlink_trap_report);
16
17DEFINE_XARRAY_FLAGS(devlinks, XA_FLAGS_ALLOC);
18
19static struct devlink *devlinks_xa_get(unsigned long index)
20{
21 struct devlink *devlink;
22
23 rcu_read_lock();
24 devlink = xa_find(&devlinks, &index, index, DEVLINK_REGISTERED);
25 if (!devlink || !devlink_try_get(devlink))
26 devlink = NULL;
27 rcu_read_unlock();
28 return devlink;
29}
30
31/* devlink_rels xarray contains 1:1 relationships between
32 * devlink object and related nested devlink instance.
33 * The xarray index is used to get the nested object from
34 * the nested-in object code.
35 */
36static DEFINE_XARRAY_FLAGS(devlink_rels, XA_FLAGS_ALLOC1);
37
38#define DEVLINK_REL_IN_USE XA_MARK_0
39
40struct devlink_rel {
41 u32 index;
42 refcount_t refcount;
43 u32 devlink_index;
44 struct {
45 u32 devlink_index;
46 u32 obj_index;
47 devlink_rel_notify_cb_t *notify_cb;
48 devlink_rel_cleanup_cb_t *cleanup_cb;
49 struct delayed_work notify_work;
50 } nested_in;
51};
52
53static void devlink_rel_free(struct devlink_rel *rel)
54{
55 xa_erase(&devlink_rels, rel->index);
56 kfree(rel);
57}
58
59static void __devlink_rel_get(struct devlink_rel *rel)
60{
61 refcount_inc(&rel->refcount);
62}
63
64static void __devlink_rel_put(struct devlink_rel *rel)
65{
66 if (refcount_dec_and_test(&rel->refcount))
67 devlink_rel_free(rel);
68}
69
70static void devlink_rel_nested_in_notify_work(struct work_struct *work)
71{
72 struct devlink_rel *rel = container_of(work, struct devlink_rel,
73 nested_in.notify_work.work);
74 struct devlink *devlink;
75
76 devlink = devlinks_xa_get(rel->nested_in.devlink_index);
77 if (!devlink)
78 goto rel_put;
79 if (!devl_trylock(devlink)) {
80 devlink_put(devlink);
81 goto reschedule_work;
82 }
83 if (!devl_is_registered(devlink)) {
84 devl_unlock(devlink);
85 devlink_put(devlink);
86 goto rel_put;
87 }
88 if (!xa_get_mark(&devlink_rels, rel->index, DEVLINK_REL_IN_USE))
89 rel->nested_in.cleanup_cb(devlink, rel->nested_in.obj_index, rel->index);
90 rel->nested_in.notify_cb(devlink, rel->nested_in.obj_index);
91 devl_unlock(devlink);
92 devlink_put(devlink);
93
94rel_put:
95 __devlink_rel_put(rel);
96 return;
97
98reschedule_work:
99 schedule_delayed_work(&rel->nested_in.notify_work, 1);
100}
101
102static void devlink_rel_nested_in_notify_work_schedule(struct devlink_rel *rel)
103{
104 __devlink_rel_get(rel);
105 schedule_delayed_work(&rel->nested_in.notify_work, 0);
106}
107
108static struct devlink_rel *devlink_rel_alloc(void)
109{
110 struct devlink_rel *rel;
111 static u32 next;
112 int err;
113
114 rel = kzalloc_obj(*rel);
115 if (!rel)
116 return ERR_PTR(-ENOMEM);
117
118 err = xa_alloc_cyclic(&devlink_rels, &rel->index, rel,
119 xa_limit_32b, &next, GFP_KERNEL);
120 if (err < 0) {
121 kfree(rel);
122 return ERR_PTR(err);
123 }
124
125 refcount_set(&rel->refcount, 1);
126 INIT_DELAYED_WORK(&rel->nested_in.notify_work,
127 &devlink_rel_nested_in_notify_work);
128 return rel;
129}
130
131static void devlink_rel_put(struct devlink *devlink)
132{
133 struct devlink_rel *rel = devlink->rel;
134
135 if (!rel)
136 return;
137 xa_clear_mark(&devlink_rels, rel->index, DEVLINK_REL_IN_USE);
138 devlink_rel_nested_in_notify_work_schedule(rel);
139 __devlink_rel_put(rel);
140 devlink->rel = NULL;
141}
142
143void devlink_rel_nested_in_clear(u32 rel_index)
144{
145 xa_clear_mark(&devlink_rels, rel_index, DEVLINK_REL_IN_USE);
146}
147
148int devlink_rel_nested_in_add(u32 *rel_index, u32 devlink_index,
149 u32 obj_index, devlink_rel_notify_cb_t *notify_cb,
150 devlink_rel_cleanup_cb_t *cleanup_cb,
151 struct devlink *devlink)
152{
153 struct devlink_rel *rel = devlink_rel_alloc();
154
155 ASSERT_DEVLINK_NOT_REGISTERED(devlink);
156
157 if (IS_ERR(rel))
158 return PTR_ERR(rel);
159
160 rel->devlink_index = devlink->index;
161 rel->nested_in.devlink_index = devlink_index;
162 rel->nested_in.obj_index = obj_index;
163 rel->nested_in.notify_cb = notify_cb;
164 rel->nested_in.cleanup_cb = cleanup_cb;
165 *rel_index = rel->index;
166 xa_set_mark(&devlink_rels, rel->index, DEVLINK_REL_IN_USE);
167 devlink->rel = rel;
168 return 0;
169}
170
171/**
172 * devlink_rel_nested_in_notify - Notify the object this devlink
173 * instance is nested in.
174 * @devlink: devlink
175 *
176 * This is called upon network namespace change of devlink instance.
177 * In case this devlink instance is nested in another devlink object,
178 * a notification of a change of this object should be sent
179 * over netlink. The parent devlink instance lock needs to be
180 * taken during the notification preparation.
181 * Since the parent may or may not be locked, 'work' is utilized.
182 */
183void devlink_rel_nested_in_notify(struct devlink *devlink)
184{
185 struct devlink_rel *rel = devlink->rel;
186
187 if (!rel)
188 return;
189 devlink_rel_nested_in_notify_work_schedule(rel);
190}
191
192static struct devlink_rel *devlink_rel_find(unsigned long rel_index)
193{
194 return xa_find(&devlink_rels, &rel_index, rel_index,
195 DEVLINK_REL_IN_USE);
196}
197
198static struct devlink *devlink_rel_devlink_get(u32 rel_index)
199{
200 struct devlink_rel *rel;
201 u32 devlink_index;
202
203 if (!rel_index)
204 return NULL;
205 xa_lock(&devlink_rels);
206 rel = devlink_rel_find(rel_index);
207 if (rel)
208 devlink_index = rel->devlink_index;
209 xa_unlock(&devlink_rels);
210 if (!rel)
211 return NULL;
212 return devlinks_xa_get(devlink_index);
213}
214
215int devlink_rel_devlink_handle_put(struct sk_buff *msg, struct devlink *devlink,
216 u32 rel_index, int attrtype,
217 bool *msg_updated)
218{
219 struct net *net = devlink_net(devlink);
220 struct devlink *rel_devlink;
221 int err;
222
223 rel_devlink = devlink_rel_devlink_get(rel_index);
224 if (!rel_devlink)
225 return 0;
226 err = devlink_nl_put_nested_handle(msg, net, rel_devlink, attrtype);
227 devlink_put(rel_devlink);
228 if (!err && msg_updated)
229 *msg_updated = true;
230 return err;
231}
232
233void *devlink_priv(struct devlink *devlink)
234{
235 return &devlink->priv;
236}
237EXPORT_SYMBOL_GPL(devlink_priv);
238
239struct devlink *priv_to_devlink(void *priv)
240{
241 return container_of(priv, struct devlink, priv);
242}
243EXPORT_SYMBOL_GPL(priv_to_devlink);
244
245struct device *devlink_to_dev(const struct devlink *devlink)
246{
247 return devlink->dev;
248}
249EXPORT_SYMBOL_GPL(devlink_to_dev);
250
251const char *devlink_bus_name(const struct devlink *devlink)
252{
253 return devlink->dev ? devlink->dev->bus->name : DEVLINK_INDEX_BUS_NAME;
254}
255EXPORT_SYMBOL_GPL(devlink_bus_name);
256
257const char *devlink_dev_name(const struct devlink *devlink)
258{
259 return devlink->dev ? dev_name(devlink->dev) : devlink->dev_name_index;
260}
261EXPORT_SYMBOL_GPL(devlink_dev_name);
262
263const char *devlink_dev_driver_name(const struct devlink *devlink)
264{
265 return devlink->dev_driver->name;
266}
267EXPORT_SYMBOL_GPL(devlink_dev_driver_name);
268
269struct net *devlink_net(const struct devlink *devlink)
270{
271 return read_pnet(&devlink->_net);
272}
273EXPORT_SYMBOL_GPL(devlink_net);
274
275void devl_assert_locked(struct devlink *devlink)
276{
277 lockdep_assert_held(&devlink->lock);
278}
279EXPORT_SYMBOL_GPL(devl_assert_locked);
280
281#ifdef CONFIG_LOCKDEP
282/* For use in conjunction with LOCKDEP only e.g. rcu_dereference_protected() */
283bool devl_lock_is_held(struct devlink *devlink)
284{
285 return lockdep_is_held(&devlink->lock);
286}
287EXPORT_SYMBOL_GPL(devl_lock_is_held);
288#endif
289
290void devl_lock(struct devlink *devlink)
291{
292 mutex_lock(&devlink->lock);
293}
294EXPORT_SYMBOL_GPL(devl_lock);
295
296int devl_trylock(struct devlink *devlink)
297{
298 return mutex_trylock(&devlink->lock);
299}
300EXPORT_SYMBOL_GPL(devl_trylock);
301
302void devl_unlock(struct devlink *devlink)
303{
304 mutex_unlock(&devlink->lock);
305}
306EXPORT_SYMBOL_GPL(devl_unlock);
307
308/**
309 * devlink_try_get() - try to obtain a reference on a devlink instance
310 * @devlink: instance to reference
311 *
312 * Obtain a reference on a devlink instance. A reference on a devlink instance
313 * only implies that it's safe to take the instance lock. It does not imply
314 * that the instance is registered, use devl_is_registered() after taking
315 * the instance lock to check registration status.
316 */
317struct devlink *__must_check devlink_try_get(struct devlink *devlink)
318{
319 if (refcount_inc_not_zero(&devlink->refcount))
320 return devlink;
321 return NULL;
322}
323
324static void devlink_release(struct work_struct *work)
325{
326 struct devlink *devlink;
327
328 devlink = container_of(to_rcu_work(work), struct devlink, rwork);
329
330 mutex_destroy(&devlink->lock);
331 lockdep_unregister_key(&devlink->lock_key);
332 if (devlink->dev)
333 put_device(devlink->dev);
334 else
335 kfree(devlink->dev_name_index);
336 kvfree(devlink);
337}
338
339void devlink_put(struct devlink *devlink)
340{
341 if (refcount_dec_and_test(&devlink->refcount))
342 queue_rcu_work(system_percpu_wq, &devlink->rwork);
343}
344
345static struct devlink *__devlinks_xa_find_get(struct net *net,
346 unsigned long *indexp,
347 unsigned long end)
348{
349 struct devlink *devlink = NULL;
350
351 rcu_read_lock();
352retry:
353 devlink = xa_find(&devlinks, indexp, end, DEVLINK_REGISTERED);
354 if (!devlink)
355 goto unlock;
356
357 if (!devlink_try_get(devlink))
358 goto next;
359 if (!net_eq(devlink_net(devlink), net)) {
360 devlink_put(devlink);
361 goto next;
362 }
363unlock:
364 rcu_read_unlock();
365 return devlink;
366
367next:
368 (*indexp)++;
369 goto retry;
370}
371
372struct devlink *devlinks_xa_find_get(struct net *net, unsigned long *indexp)
373{
374 return __devlinks_xa_find_get(net, indexp, ULONG_MAX);
375}
376
377struct devlink *devlinks_xa_lookup_get(struct net *net, unsigned long index)
378{
379 return __devlinks_xa_find_get(net, &index, index);
380}
381
382/**
383 * devl_register - Register devlink instance
384 * @devlink: devlink
385 */
386int devl_register(struct devlink *devlink)
387{
388 ASSERT_DEVLINK_NOT_REGISTERED(devlink);
389 devl_assert_locked(devlink);
390
391 xa_set_mark(&devlinks, devlink->index, DEVLINK_REGISTERED);
392 devlink_notify_register(devlink);
393 devlink_rel_nested_in_notify(devlink);
394
395 return 0;
396}
397EXPORT_SYMBOL_GPL(devl_register);
398
399void devlink_register(struct devlink *devlink)
400{
401 devl_lock(devlink);
402 devl_register(devlink);
403 devl_unlock(devlink);
404}
405EXPORT_SYMBOL_GPL(devlink_register);
406
407/**
408 * devl_unregister - Unregister devlink instance
409 * @devlink: devlink
410 */
411void devl_unregister(struct devlink *devlink)
412{
413 ASSERT_DEVLINK_REGISTERED(devlink);
414 devl_assert_locked(devlink);
415
416 devlink_notify_unregister(devlink);
417 xa_clear_mark(&devlinks, devlink->index, DEVLINK_REGISTERED);
418 devlink_rel_put(devlink);
419}
420EXPORT_SYMBOL_GPL(devl_unregister);
421
422void devlink_unregister(struct devlink *devlink)
423{
424 devl_lock(devlink);
425 devl_unregister(devlink);
426 devl_unlock(devlink);
427}
428EXPORT_SYMBOL_GPL(devlink_unregister);
429
430struct devlink *__devlink_alloc(const struct devlink_ops *ops, size_t priv_size,
431 struct net *net, struct device *dev,
432 const struct device_driver *dev_driver)
433{
434 struct devlink *devlink;
435 static u32 last_id;
436 int ret;
437
438 WARN_ON(!ops || !dev_driver);
439 if (!devlink_reload_actions_valid(ops))
440 return NULL;
441
442 devlink = kvzalloc_flex(*devlink, priv, priv_size);
443 if (!devlink)
444 return NULL;
445
446 ret = xa_alloc_cyclic(&devlinks, &devlink->index, devlink, xa_limit_31b,
447 &last_id, GFP_KERNEL);
448 if (ret < 0)
449 goto err_xa_alloc;
450
451 if (dev) {
452 devlink->dev = get_device(dev);
453 } else {
454 devlink->dev_name_index = kasprintf(GFP_KERNEL, "%u", devlink->index);
455 if (!devlink->dev_name_index)
456 goto err_kasprintf;
457 }
458
459 devlink->ops = ops;
460 devlink->dev_driver = dev_driver;
461 xa_init_flags(&devlink->ports, XA_FLAGS_ALLOC);
462 xa_init_flags(&devlink->params, XA_FLAGS_ALLOC);
463 xa_init_flags(&devlink->snapshot_ids, XA_FLAGS_ALLOC);
464 xa_init_flags(&devlink->nested_rels, XA_FLAGS_ALLOC);
465 write_pnet(&devlink->_net, net);
466 INIT_LIST_HEAD(&devlink->rate_list);
467 INIT_LIST_HEAD(&devlink->linecard_list);
468 INIT_LIST_HEAD(&devlink->sb_list);
469 INIT_LIST_HEAD_RCU(&devlink->dpipe_table_list);
470 INIT_LIST_HEAD(&devlink->resource_list);
471 INIT_LIST_HEAD(&devlink->region_list);
472 INIT_LIST_HEAD(&devlink->reporter_list);
473 INIT_LIST_HEAD(&devlink->trap_list);
474 INIT_LIST_HEAD(&devlink->trap_group_list);
475 INIT_LIST_HEAD(&devlink->trap_policer_list);
476 INIT_RCU_WORK(&devlink->rwork, devlink_release);
477 lockdep_register_key(&devlink->lock_key);
478 mutex_init(&devlink->lock);
479 lockdep_set_class(&devlink->lock, &devlink->lock_key);
480 refcount_set(&devlink->refcount, 1);
481
482 return devlink;
483
484err_kasprintf:
485 xa_erase(&devlinks, devlink->index);
486err_xa_alloc:
487 kvfree(devlink);
488 return NULL;
489}
490
491/**
492 * devlink_alloc_ns - Allocate new devlink instance resources
493 * in specific namespace
494 *
495 * @ops: ops
496 * @priv_size: size of user private data
497 * @net: net namespace
498 * @dev: parent device
499 *
500 * Allocate new devlink instance resources, including devlink index
501 * and name.
502 */
503struct devlink *devlink_alloc_ns(const struct devlink_ops *ops,
504 size_t priv_size, struct net *net,
505 struct device *dev)
506{
507 WARN_ON(!dev);
508 return __devlink_alloc(ops, priv_size, net, dev, dev->driver);
509}
510EXPORT_SYMBOL_GPL(devlink_alloc_ns);
511
512/**
513 * devlink_free - Free devlink instance resources
514 *
515 * @devlink: devlink
516 */
517void devlink_free(struct devlink *devlink)
518{
519 ASSERT_DEVLINK_NOT_REGISTERED(devlink);
520
521 WARN_ON(!list_empty(&devlink->trap_policer_list));
522 WARN_ON(!list_empty(&devlink->trap_group_list));
523 WARN_ON(!list_empty(&devlink->trap_list));
524 WARN_ON(!list_empty(&devlink->reporter_list));
525 WARN_ON(!list_empty(&devlink->region_list));
526 WARN_ON(!list_empty(&devlink->resource_list));
527 WARN_ON(!list_empty(&devlink->dpipe_table_list));
528 WARN_ON(!list_empty(&devlink->sb_list));
529 WARN_ON(devlink_rates_check(devlink, NULL, NULL));
530 WARN_ON(!list_empty(&devlink->linecard_list));
531 WARN_ON(!xa_empty(&devlink->ports));
532
533 xa_destroy(&devlink->nested_rels);
534 xa_destroy(&devlink->snapshot_ids);
535 xa_destroy(&devlink->params);
536 xa_destroy(&devlink->ports);
537
538 xa_erase(&devlinks, devlink->index);
539
540 devlink_put(devlink);
541}
542EXPORT_SYMBOL_GPL(devlink_free);
543
544static void __net_exit devlink_pernet_pre_exit(struct net *net)
545{
546 struct devlink *devlink;
547 u32 actions_performed;
548 unsigned long index;
549 int err;
550
551 /* In case network namespace is getting destroyed, reload
552 * all devlink instances from this namespace into init_net.
553 */
554 devlinks_xa_for_each_registered_get(net, index, devlink) {
555 devl_dev_lock(devlink, true);
556 err = 0;
557 if (devl_is_registered(devlink))
558 err = devlink_reload(devlink, &init_net,
559 DEVLINK_RELOAD_ACTION_DRIVER_REINIT,
560 DEVLINK_RELOAD_LIMIT_UNSPEC,
561 &actions_performed, NULL);
562 devl_dev_unlock(devlink, true);
563 devlink_put(devlink);
564 if (err && err != -EOPNOTSUPP)
565 pr_warn("Failed to reload devlink instance into init_net\n");
566 }
567}
568
569static struct pernet_operations devlink_pernet_ops __net_initdata = {
570 .pre_exit = devlink_pernet_pre_exit,
571};
572
573static struct notifier_block devlink_port_netdevice_nb = {
574 .notifier_call = devlink_port_netdevice_event,
575};
576
577static int __init devlink_init(void)
578{
579 int err;
580
581 err = register_pernet_subsys(&devlink_pernet_ops);
582 if (err)
583 goto out;
584 err = genl_register_family(&devlink_nl_family);
585 if (err)
586 goto out_unreg_pernet_subsys;
587 err = register_netdevice_notifier(&devlink_port_netdevice_nb);
588 if (!err)
589 return 0;
590
591 genl_unregister_family(&devlink_nl_family);
592
593out_unreg_pernet_subsys:
594 unregister_pernet_subsys(&devlink_pernet_ops);
595out:
596 WARN_ON(err);
597 return err;
598}
599
600subsys_initcall(devlink_init);