Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1// SPDX-License-Identifier: GPL-2.0-or-later
2#include <linux/compat.h>
3#include <linux/dma-mapping.h>
4#include <linux/iommu.h>
5#include <linux/module.h>
6#include <linux/poll.h>
7#include <linux/slab.h>
8#include <linux/uacce.h>
9
10static dev_t uacce_devt;
11static DEFINE_XARRAY_ALLOC(uacce_xa);
12
13static const struct class uacce_class = {
14 .name = UACCE_NAME,
15};
16
17/*
18 * If the parent driver or the device disappears, the queue state is invalid and
19 * ops are not usable anymore.
20 */
21static bool uacce_queue_is_valid(struct uacce_queue *q)
22{
23 return q->state == UACCE_Q_INIT || q->state == UACCE_Q_STARTED;
24}
25
26static int uacce_start_queue(struct uacce_queue *q)
27{
28 int ret;
29
30 if (q->state != UACCE_Q_INIT)
31 return -EINVAL;
32
33 if (q->uacce->ops->start_queue) {
34 ret = q->uacce->ops->start_queue(q);
35 if (ret < 0)
36 return ret;
37 }
38
39 q->state = UACCE_Q_STARTED;
40 return 0;
41}
42
43static int uacce_stop_queue(struct uacce_queue *q)
44{
45 struct uacce_device *uacce = q->uacce;
46
47 if (q->state != UACCE_Q_STARTED)
48 return 0;
49
50 if (uacce->ops->stop_queue)
51 uacce->ops->stop_queue(q);
52
53 q->state = UACCE_Q_INIT;
54
55 return 0;
56}
57
58static void uacce_put_queue(struct uacce_queue *q)
59{
60 struct uacce_device *uacce = q->uacce;
61
62 uacce_stop_queue(q);
63
64 if (q->state != UACCE_Q_INIT)
65 return;
66
67 if (uacce->ops->put_queue)
68 uacce->ops->put_queue(q);
69
70 q->state = UACCE_Q_ZOMBIE;
71}
72
73static long uacce_fops_unl_ioctl(struct file *filep,
74 unsigned int cmd, unsigned long arg)
75{
76 struct uacce_queue *q = filep->private_data;
77 struct uacce_device *uacce = q->uacce;
78 long ret = -ENXIO;
79
80 /*
81 * uacce->ops->ioctl() may take the mmap_lock when copying arg to/from
82 * user. Avoid a circular lock dependency with uacce_fops_mmap(), which
83 * gets called with mmap_lock held, by taking uacce->mutex instead of
84 * q->mutex. Doing this in uacce_fops_mmap() is not possible because
85 * uacce_fops_open() calls iommu_sva_bind_device(), which takes
86 * mmap_lock, while holding uacce->mutex.
87 */
88 mutex_lock(&uacce->mutex);
89 if (!uacce_queue_is_valid(q))
90 goto out_unlock;
91
92 switch (cmd) {
93 case UACCE_CMD_START_Q:
94 ret = uacce_start_queue(q);
95 break;
96 case UACCE_CMD_PUT_Q:
97 ret = uacce_stop_queue(q);
98 break;
99 default:
100 if (uacce->ops->ioctl)
101 ret = uacce->ops->ioctl(q, cmd, arg);
102 else
103 ret = -EINVAL;
104 }
105out_unlock:
106 mutex_unlock(&uacce->mutex);
107 return ret;
108}
109
110#ifdef CONFIG_COMPAT
111static long uacce_fops_compat_ioctl(struct file *filep,
112 unsigned int cmd, unsigned long arg)
113{
114 arg = (unsigned long)compat_ptr(arg);
115
116 return uacce_fops_unl_ioctl(filep, cmd, arg);
117}
118#endif
119
120static int uacce_bind_queue(struct uacce_device *uacce, struct uacce_queue *q)
121{
122 u32 pasid;
123 struct iommu_sva *handle;
124
125 if (!(uacce->flags & UACCE_DEV_SVA))
126 return 0;
127
128 handle = iommu_sva_bind_device(uacce->parent, current->mm);
129 if (IS_ERR(handle))
130 return PTR_ERR(handle);
131
132 pasid = iommu_sva_get_pasid(handle);
133 if (pasid == IOMMU_PASID_INVALID) {
134 iommu_sva_unbind_device(handle);
135 return -ENODEV;
136 }
137
138 q->handle = handle;
139 q->pasid = pasid;
140 return 0;
141}
142
143static void uacce_unbind_queue(struct uacce_queue *q)
144{
145 if (!q->handle)
146 return;
147 iommu_sva_unbind_device(q->handle);
148 q->handle = NULL;
149}
150
151static int uacce_fops_open(struct inode *inode, struct file *filep)
152{
153 struct uacce_device *uacce;
154 struct uacce_queue *q;
155 int ret;
156
157 uacce = xa_load(&uacce_xa, iminor(inode));
158 if (!uacce)
159 return -ENODEV;
160
161 q = kzalloc_obj(struct uacce_queue);
162 if (!q)
163 return -ENOMEM;
164
165 mutex_lock(&uacce->mutex);
166
167 if (!uacce->parent) {
168 ret = -EINVAL;
169 goto out_with_mem;
170 }
171
172 ret = uacce_bind_queue(uacce, q);
173 if (ret)
174 goto out_with_mem;
175
176 q->uacce = uacce;
177
178 if (uacce->ops->get_queue) {
179 ret = uacce->ops->get_queue(uacce, q->pasid, q);
180 if (ret < 0)
181 goto out_with_bond;
182 }
183
184 init_waitqueue_head(&q->wait);
185 filep->private_data = q;
186 q->state = UACCE_Q_INIT;
187 q->mapping = filep->f_mapping;
188 mutex_init(&q->mutex);
189 list_add(&q->list, &uacce->queues);
190 mutex_unlock(&uacce->mutex);
191
192 return 0;
193
194out_with_bond:
195 uacce_unbind_queue(q);
196out_with_mem:
197 kfree(q);
198 mutex_unlock(&uacce->mutex);
199 return ret;
200}
201
202static int uacce_fops_release(struct inode *inode, struct file *filep)
203{
204 struct uacce_queue *q = filep->private_data;
205 struct uacce_device *uacce = q->uacce;
206
207 mutex_lock(&uacce->mutex);
208 uacce_put_queue(q);
209 uacce_unbind_queue(q);
210 list_del(&q->list);
211 mutex_unlock(&uacce->mutex);
212 kfree(q);
213
214 return 0;
215}
216
217static void uacce_vma_close(struct vm_area_struct *vma)
218{
219 struct uacce_queue *q = vma->vm_private_data;
220
221 if (vma->vm_pgoff < UACCE_MAX_REGION) {
222 struct uacce_qfile_region *qfr = q->qfrs[vma->vm_pgoff];
223
224 mutex_lock(&q->mutex);
225 q->qfrs[vma->vm_pgoff] = NULL;
226 mutex_unlock(&q->mutex);
227 kfree(qfr);
228 }
229}
230
231static int uacce_vma_mremap(struct vm_area_struct *area)
232{
233 return -EPERM;
234}
235
236static const struct vm_operations_struct uacce_vm_ops = {
237 .close = uacce_vma_close,
238 .mremap = uacce_vma_mremap,
239};
240
241static int uacce_fops_mmap(struct file *filep, struct vm_area_struct *vma)
242{
243 struct uacce_queue *q = filep->private_data;
244 struct uacce_device *uacce = q->uacce;
245 struct uacce_qfile_region *qfr;
246 enum uacce_qfrt type = UACCE_MAX_REGION;
247 int ret = 0;
248
249 if (vma->vm_pgoff < UACCE_MAX_REGION)
250 type = vma->vm_pgoff;
251 else
252 return -EINVAL;
253
254 qfr = kzalloc_obj(*qfr);
255 if (!qfr)
256 return -ENOMEM;
257
258 vm_flags_set(vma, VM_DONTCOPY | VM_DONTEXPAND | VM_WIPEONFORK);
259 vma->vm_ops = &uacce_vm_ops;
260 vma->vm_private_data = q;
261 qfr->type = type;
262
263 mutex_lock(&q->mutex);
264 if (!uacce_queue_is_valid(q)) {
265 ret = -ENXIO;
266 goto out_with_lock;
267 }
268
269 if (q->qfrs[type]) {
270 ret = -EEXIST;
271 goto out_with_lock;
272 }
273
274 switch (type) {
275 case UACCE_QFRT_MMIO:
276 case UACCE_QFRT_DUS:
277 if (!uacce->ops->mmap) {
278 ret = -EINVAL;
279 goto out_with_lock;
280 }
281
282 ret = uacce->ops->mmap(q, vma, qfr);
283 if (ret)
284 goto out_with_lock;
285 break;
286
287 default:
288 ret = -EINVAL;
289 goto out_with_lock;
290 }
291
292 q->qfrs[type] = qfr;
293 mutex_unlock(&q->mutex);
294
295 return ret;
296
297out_with_lock:
298 mutex_unlock(&q->mutex);
299 kfree(qfr);
300 return ret;
301}
302
303static __poll_t uacce_fops_poll(struct file *file, poll_table *wait)
304{
305 struct uacce_queue *q = file->private_data;
306 struct uacce_device *uacce = q->uacce;
307 __poll_t ret = 0;
308
309 mutex_lock(&q->mutex);
310 if (!uacce_queue_is_valid(q))
311 goto out_unlock;
312
313 poll_wait(file, &q->wait, wait);
314
315 if (uacce->ops->is_q_updated && uacce->ops->is_q_updated(q))
316 ret = EPOLLIN | EPOLLRDNORM;
317
318out_unlock:
319 mutex_unlock(&q->mutex);
320 return ret;
321}
322
323static const struct file_operations uacce_fops = {
324 .owner = THIS_MODULE,
325 .open = uacce_fops_open,
326 .release = uacce_fops_release,
327 .unlocked_ioctl = uacce_fops_unl_ioctl,
328#ifdef CONFIG_COMPAT
329 .compat_ioctl = uacce_fops_compat_ioctl,
330#endif
331 .mmap = uacce_fops_mmap,
332 .poll = uacce_fops_poll,
333};
334
335#define to_uacce_device(dev) container_of(dev, struct uacce_device, dev)
336
337static ssize_t api_show(struct device *dev,
338 struct device_attribute *attr, char *buf)
339{
340 struct uacce_device *uacce = to_uacce_device(dev);
341
342 return sysfs_emit(buf, "%s\n", uacce->api_ver);
343}
344
345static ssize_t flags_show(struct device *dev,
346 struct device_attribute *attr, char *buf)
347{
348 struct uacce_device *uacce = to_uacce_device(dev);
349
350 return sysfs_emit(buf, "%u\n", uacce->flags);
351}
352
353static ssize_t available_instances_show(struct device *dev,
354 struct device_attribute *attr,
355 char *buf)
356{
357 struct uacce_device *uacce = to_uacce_device(dev);
358
359 if (!uacce->ops->get_available_instances)
360 return -ENODEV;
361
362 return sysfs_emit(buf, "%d\n",
363 uacce->ops->get_available_instances(uacce));
364}
365
366static ssize_t algorithms_show(struct device *dev,
367 struct device_attribute *attr, char *buf)
368{
369 struct uacce_device *uacce = to_uacce_device(dev);
370
371 return sysfs_emit(buf, "%s\n", uacce->algs);
372}
373
374static ssize_t region_mmio_size_show(struct device *dev,
375 struct device_attribute *attr, char *buf)
376{
377 struct uacce_device *uacce = to_uacce_device(dev);
378
379 return sysfs_emit(buf, "%lu\n",
380 uacce->qf_pg_num[UACCE_QFRT_MMIO] << PAGE_SHIFT);
381}
382
383static ssize_t region_dus_size_show(struct device *dev,
384 struct device_attribute *attr, char *buf)
385{
386 struct uacce_device *uacce = to_uacce_device(dev);
387
388 return sysfs_emit(buf, "%lu\n",
389 uacce->qf_pg_num[UACCE_QFRT_DUS] << PAGE_SHIFT);
390}
391
392static ssize_t isolate_show(struct device *dev,
393 struct device_attribute *attr, char *buf)
394{
395 struct uacce_device *uacce = to_uacce_device(dev);
396
397 return sysfs_emit(buf, "%d\n", uacce->ops->get_isolate_state(uacce));
398}
399
400static ssize_t isolate_strategy_show(struct device *dev, struct device_attribute *attr, char *buf)
401{
402 struct uacce_device *uacce = to_uacce_device(dev);
403 u32 val;
404
405 if (!uacce->ops->isolate_err_threshold_read)
406 return -ENOENT;
407
408 val = uacce->ops->isolate_err_threshold_read(uacce);
409
410 return sysfs_emit(buf, "%u\n", val);
411}
412
413static ssize_t isolate_strategy_store(struct device *dev, struct device_attribute *attr,
414 const char *buf, size_t count)
415{
416 struct uacce_device *uacce = to_uacce_device(dev);
417 unsigned long val;
418 int ret;
419
420 if (!uacce->ops->isolate_err_threshold_write)
421 return -ENOENT;
422
423 if (kstrtoul(buf, 0, &val) < 0)
424 return -EINVAL;
425
426 if (val > UACCE_MAX_ERR_THRESHOLD)
427 return -EINVAL;
428
429 ret = uacce->ops->isolate_err_threshold_write(uacce, val);
430 if (ret)
431 return ret;
432
433 return count;
434}
435
436static DEVICE_ATTR_RO(api);
437static DEVICE_ATTR_RO(flags);
438static DEVICE_ATTR_RO(available_instances);
439static DEVICE_ATTR_RO(algorithms);
440static DEVICE_ATTR_RO(region_mmio_size);
441static DEVICE_ATTR_RO(region_dus_size);
442static DEVICE_ATTR_RO(isolate);
443static DEVICE_ATTR_RW(isolate_strategy);
444
445static struct attribute *uacce_dev_attrs[] = {
446 &dev_attr_api.attr,
447 &dev_attr_flags.attr,
448 &dev_attr_available_instances.attr,
449 &dev_attr_algorithms.attr,
450 &dev_attr_region_mmio_size.attr,
451 &dev_attr_region_dus_size.attr,
452 &dev_attr_isolate.attr,
453 &dev_attr_isolate_strategy.attr,
454 NULL,
455};
456
457static umode_t uacce_dev_is_visible(struct kobject *kobj,
458 struct attribute *attr, int n)
459{
460 struct device *dev = kobj_to_dev(kobj);
461 struct uacce_device *uacce = to_uacce_device(dev);
462
463 if (((attr == &dev_attr_region_mmio_size.attr) &&
464 (!uacce->qf_pg_num[UACCE_QFRT_MMIO])) ||
465 ((attr == &dev_attr_region_dus_size.attr) &&
466 (!uacce->qf_pg_num[UACCE_QFRT_DUS])))
467 return 0;
468
469 if (attr == &dev_attr_isolate_strategy.attr &&
470 (!uacce->ops->isolate_err_threshold_read &&
471 !uacce->ops->isolate_err_threshold_write))
472 return 0;
473
474 if (attr == &dev_attr_isolate.attr && !uacce->ops->get_isolate_state)
475 return 0;
476
477 return attr->mode;
478}
479
480static struct attribute_group uacce_dev_group = {
481 .is_visible = uacce_dev_is_visible,
482 .attrs = uacce_dev_attrs,
483};
484
485__ATTRIBUTE_GROUPS(uacce_dev);
486
487static void uacce_release(struct device *dev)
488{
489 struct uacce_device *uacce = to_uacce_device(dev);
490
491 kfree(uacce);
492}
493
494/**
495 * uacce_alloc() - alloc an accelerator
496 * @parent: pointer of uacce parent device
497 * @interface: pointer of uacce_interface for register
498 *
499 * Returns uacce pointer if success and ERR_PTR if not
500 * Need check returned negotiated uacce->flags
501 */
502struct uacce_device *uacce_alloc(struct device *parent,
503 struct uacce_interface *interface)
504{
505 unsigned int flags = interface->flags;
506 struct uacce_device *uacce;
507 int ret;
508
509 uacce = kzalloc_obj(struct uacce_device);
510 if (!uacce)
511 return ERR_PTR(-ENOMEM);
512
513 uacce->parent = parent;
514 uacce->flags = flags;
515 uacce->ops = interface->ops;
516
517 ret = xa_alloc(&uacce_xa, &uacce->dev_id, uacce, xa_limit_32b,
518 GFP_KERNEL);
519 if (ret < 0)
520 goto err_with_uacce;
521
522 INIT_LIST_HEAD(&uacce->queues);
523 mutex_init(&uacce->mutex);
524 device_initialize(&uacce->dev);
525 uacce->dev.devt = MKDEV(MAJOR(uacce_devt), uacce->dev_id);
526 uacce->dev.class = &uacce_class;
527 uacce->dev.groups = uacce_dev_groups;
528 uacce->dev.parent = uacce->parent;
529 uacce->dev.release = uacce_release;
530 dev_set_name(&uacce->dev, "%s-%d", interface->name, uacce->dev_id);
531
532 return uacce;
533
534err_with_uacce:
535 kfree(uacce);
536 return ERR_PTR(ret);
537}
538EXPORT_SYMBOL_GPL(uacce_alloc);
539
540/**
541 * uacce_register() - add the accelerator to cdev and export to user space
542 * @uacce: The initialized uacce device
543 *
544 * Return 0 if register succeeded, or an error.
545 */
546int uacce_register(struct uacce_device *uacce)
547{
548 int ret;
549
550 if (!uacce)
551 return -ENODEV;
552
553 uacce->cdev = cdev_alloc();
554 if (!uacce->cdev)
555 return -ENOMEM;
556
557 uacce->cdev->ops = &uacce_fops;
558 uacce->cdev->owner = THIS_MODULE;
559
560 ret = cdev_device_add(uacce->cdev, &uacce->dev);
561 if (ret)
562 uacce->cdev = NULL;
563
564 return ret;
565}
566EXPORT_SYMBOL_GPL(uacce_register);
567
568/**
569 * uacce_remove() - remove the accelerator
570 * @uacce: the accelerator to remove
571 */
572void uacce_remove(struct uacce_device *uacce)
573{
574 struct uacce_queue *q, *next_q;
575
576 if (!uacce)
577 return;
578
579 /*
580 * uacce_fops_open() may be running concurrently, even after we remove
581 * the cdev. Holding uacce->mutex ensures that open() does not obtain a
582 * removed uacce device.
583 */
584 mutex_lock(&uacce->mutex);
585 /* ensure no open queue remains */
586 list_for_each_entry_safe(q, next_q, &uacce->queues, list) {
587 /*
588 * Taking q->mutex ensures that fops do not use the defunct
589 * uacce->ops after the queue is disabled.
590 */
591 mutex_lock(&q->mutex);
592 uacce_put_queue(q);
593 mutex_unlock(&q->mutex);
594 uacce_unbind_queue(q);
595
596 /*
597 * unmap remaining mapping from user space, preventing user still
598 * access the mmaped area while parent device is already removed
599 */
600 unmap_mapping_range(q->mapping, 0, 0, 1);
601 }
602
603 if (uacce->cdev)
604 cdev_device_del(uacce->cdev, &uacce->dev);
605 xa_erase(&uacce_xa, uacce->dev_id);
606 /*
607 * uacce exists as long as there are open fds, but ops will be freed
608 * now. Ensure that bugs cause NULL deref rather than use-after-free.
609 */
610 uacce->ops = NULL;
611 uacce->parent = NULL;
612 mutex_unlock(&uacce->mutex);
613 put_device(&uacce->dev);
614}
615EXPORT_SYMBOL_GPL(uacce_remove);
616
617static int __init uacce_init(void)
618{
619 int ret;
620
621 ret = class_register(&uacce_class);
622 if (ret)
623 return ret;
624
625 ret = alloc_chrdev_region(&uacce_devt, 0, MINORMASK, UACCE_NAME);
626 if (ret)
627 class_unregister(&uacce_class);
628
629 return ret;
630}
631
632static __exit void uacce_exit(void)
633{
634 unregister_chrdev_region(uacce_devt, MINORMASK);
635 class_unregister(&uacce_class);
636}
637
638subsys_initcall(uacce_init);
639module_exit(uacce_exit);
640
641MODULE_LICENSE("GPL");
642MODULE_AUTHOR("HiSilicon Tech. Co., Ltd.");
643MODULE_DESCRIPTION("Accelerator interface for Userland applications");