Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0 or MIT */
2/* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */
3/* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
4/* Copyright 2023 Collabora ltd. */
5
6#ifndef __PANTHOR_DEVICE_H__
7#define __PANTHOR_DEVICE_H__
8
9#include <linux/atomic.h>
10#include <linux/io-pgtable.h>
11#include <linux/regulator/consumer.h>
12#include <linux/pm_runtime.h>
13#include <linux/sched.h>
14#include <linux/spinlock.h>
15
16#include <drm/drm_device.h>
17#include <drm/drm_mm.h>
18#include <drm/gpu_scheduler.h>
19#include <drm/panthor_drm.h>
20
21struct panthor_csf;
22struct panthor_csf_ctx;
23struct panthor_device;
24struct panthor_gpu;
25struct panthor_group_pool;
26struct panthor_heap_pool;
27struct panthor_hw;
28struct panthor_job;
29struct panthor_mmu;
30struct panthor_fw;
31struct panthor_perfcnt;
32struct panthor_pwr;
33struct panthor_vm;
34struct panthor_vm_pool;
35
36/**
37 * struct panthor_soc_data - Panthor SoC Data
38 */
39struct panthor_soc_data {
40 /** @asn_hash_enable: True if GPU_L2_CONFIG_ASN_HASH_ENABLE must be set. */
41 bool asn_hash_enable;
42
43 /** @asn_hash: ASN_HASH values when asn_hash_enable is true. */
44 u32 asn_hash[3];
45};
46
47/**
48 * enum panthor_device_pm_state - PM state
49 */
50enum panthor_device_pm_state {
51 /** @PANTHOR_DEVICE_PM_STATE_SUSPENDED: Device is suspended. */
52 PANTHOR_DEVICE_PM_STATE_SUSPENDED = 0,
53
54 /** @PANTHOR_DEVICE_PM_STATE_RESUMING: Device is being resumed. */
55 PANTHOR_DEVICE_PM_STATE_RESUMING,
56
57 /** @PANTHOR_DEVICE_PM_STATE_ACTIVE: Device is active. */
58 PANTHOR_DEVICE_PM_STATE_ACTIVE,
59
60 /** @PANTHOR_DEVICE_PM_STATE_SUSPENDING: Device is being suspended. */
61 PANTHOR_DEVICE_PM_STATE_SUSPENDING,
62};
63
64enum panthor_irq_state {
65 /** @PANTHOR_IRQ_STATE_ACTIVE: IRQ is active and ready to process events. */
66 PANTHOR_IRQ_STATE_ACTIVE = 0,
67 /** @PANTHOR_IRQ_STATE_PROCESSING: IRQ is currently processing events. */
68 PANTHOR_IRQ_STATE_PROCESSING,
69 /** @PANTHOR_IRQ_STATE_SUSPENDED: IRQ is suspended. */
70 PANTHOR_IRQ_STATE_SUSPENDED,
71 /** @PANTHOR_IRQ_STATE_SUSPENDING: IRQ is being suspended. */
72 PANTHOR_IRQ_STATE_SUSPENDING,
73};
74
75/**
76 * struct panthor_irq - IRQ data
77 *
78 * Used to automate IRQ handling for the 3 different IRQs we have in this driver.
79 */
80struct panthor_irq {
81 /** @ptdev: Panthor device */
82 struct panthor_device *ptdev;
83
84 /** @irq: IRQ number. */
85 int irq;
86
87 /** @mask: Values to write to xxx_INT_MASK if active. */
88 u32 mask;
89
90 /**
91 * @mask_lock: protects modifications to _INT_MASK and @mask.
92 *
93 * In paths where _INT_MASK is updated based on a state
94 * transition/check, it's crucial for the state update/check to be
95 * inside the locked section, otherwise it introduces a race window
96 * leading to potential _INT_MASK inconsistencies.
97 */
98 spinlock_t mask_lock;
99
100 /** @state: one of &enum panthor_irq_state reflecting the current state. */
101 atomic_t state;
102};
103
104/**
105 * enum panthor_device_profiling_mode - Profiling state
106 */
107enum panthor_device_profiling_flags {
108 /** @PANTHOR_DEVICE_PROFILING_DISABLED: Profiling is disabled. */
109 PANTHOR_DEVICE_PROFILING_DISABLED = 0,
110
111 /** @PANTHOR_DEVICE_PROFILING_CYCLES: Sampling job cycles. */
112 PANTHOR_DEVICE_PROFILING_CYCLES = BIT(0),
113
114 /** @PANTHOR_DEVICE_PROFILING_TIMESTAMP: Sampling job timestamp. */
115 PANTHOR_DEVICE_PROFILING_TIMESTAMP = BIT(1),
116
117 /** @PANTHOR_DEVICE_PROFILING_ALL: Sampling everything. */
118 PANTHOR_DEVICE_PROFILING_ALL =
119 PANTHOR_DEVICE_PROFILING_CYCLES |
120 PANTHOR_DEVICE_PROFILING_TIMESTAMP,
121};
122
123/**
124 * struct panthor_device - Panthor device
125 */
126struct panthor_device {
127 /** @base: Base drm_device. */
128 struct drm_device base;
129
130 /** @soc_data: Optional SoC data. */
131 const struct panthor_soc_data *soc_data;
132
133 /** @phys_addr: Physical address of the iomem region. */
134 phys_addr_t phys_addr;
135
136 /** @iomem: CPU mapping of the IOMEM region. */
137 void __iomem *iomem;
138
139 /** @clks: GPU clocks. */
140 struct {
141 /** @core: Core clock. */
142 struct clk *core;
143
144 /** @stacks: Stacks clock. This clock is optional. */
145 struct clk *stacks;
146
147 /** @coregroup: Core group clock. This clock is optional. */
148 struct clk *coregroup;
149 } clks;
150
151 /** @coherent: True if the CPU/GPU are memory coherent. */
152 bool coherent;
153
154 /** @gpu_info: GPU information. */
155 struct drm_panthor_gpu_info gpu_info;
156
157 /** @csif_info: Command stream interface information. */
158 struct drm_panthor_csif_info csif_info;
159
160 /** @hw: GPU-specific data. */
161 struct panthor_hw *hw;
162
163 /** @pwr: Power control management data. */
164 struct panthor_pwr *pwr;
165
166 /** @gpu: GPU management data. */
167 struct panthor_gpu *gpu;
168
169 /** @fw: FW management data. */
170 struct panthor_fw *fw;
171
172 /** @mmu: MMU management data. */
173 struct panthor_mmu *mmu;
174
175 /** @scheduler: Scheduler management data. */
176 struct panthor_scheduler *scheduler;
177
178 /** @devfreq: Device frequency scaling management data. */
179 struct panthor_devfreq *devfreq;
180
181 /** @unplug: Device unplug related fields. */
182 struct {
183 /** @lock: Lock used to serialize unplug operations. */
184 struct mutex lock;
185
186 /**
187 * @done: Completion object signaled when the unplug
188 * operation is done.
189 */
190 struct completion done;
191 } unplug;
192
193 /** @reset: Reset related fields. */
194 struct {
195 /** @wq: Ordered worqueud used to schedule reset operations. */
196 struct workqueue_struct *wq;
197
198 /** @work: Reset work. */
199 struct work_struct work;
200
201 /** @pending: Set to true if a reset is pending. */
202 atomic_t pending;
203
204 /**
205 * @fast: True if the post_reset logic can proceed with a fast reset.
206 *
207 * A fast reset is just a reset where the driver doesn't reload the FW sections.
208 *
209 * Any time the firmware is properly suspended, a fast reset can take place.
210 * On the other hand, if the halt operation failed, the driver will reload
211 * all FW sections to make sure we start from a fresh state.
212 */
213 bool fast;
214 } reset;
215
216 /** @pm: Power management related data. */
217 struct {
218 /** @state: Power state. */
219 atomic_t state;
220
221 /**
222 * @mmio_lock: Lock protecting MMIO userspace CPU mappings.
223 *
224 * This is needed to ensure we map the dummy IO pages when
225 * the device is being suspended, and the real IO pages when
226 * the device is being resumed. We can't just do with the
227 * state atomicity to deal with this race.
228 */
229 struct mutex mmio_lock;
230
231 /**
232 * @dummy_latest_flush: Dummy LATEST_FLUSH page.
233 *
234 * Used to replace the real LATEST_FLUSH page when the GPU
235 * is suspended.
236 */
237 struct page *dummy_latest_flush;
238
239 /** @recovery_needed: True when a resume attempt failed. */
240 atomic_t recovery_needed;
241 } pm;
242
243 /** @profile_mask: User-set profiling flags for job accounting. */
244 u32 profile_mask;
245
246 /** @fast_rate: Maximum device clock frequency. Set by DVFS */
247 unsigned long fast_rate;
248
249#ifdef CONFIG_DEBUG_FS
250 /** @gems: Device-wide list of GEM objects owned by at least one file. */
251 struct {
252 /** @gems.lock: Protects the device-wide list of GEM objects. */
253 struct mutex lock;
254
255 /** @node: Used to keep track of all the device's DRM objects */
256 struct list_head node;
257 } gems;
258#endif
259};
260
261struct panthor_gpu_usage {
262 u64 time;
263 u64 cycles;
264};
265
266/**
267 * struct panthor_file - Panthor file
268 */
269struct panthor_file {
270 /** @ptdev: Device attached to this file. */
271 struct panthor_device *ptdev;
272
273 /** @user_mmio: User MMIO related fields. */
274 struct {
275 /**
276 * @offset: Offset used for user MMIO mappings.
277 *
278 * This offset should not be used to check the type of mapping
279 * except in panthor_mmap(). After that point, MMIO mapping
280 * offsets have been adjusted to match
281 * DRM_PANTHOR_USER_MMIO_OFFSET and that macro should be used
282 * instead.
283 * Make sure this rule is followed at all times, because
284 * userspace is in control of the offset, and can change the
285 * value behind our back. Otherwise it can lead to erroneous
286 * branching happening in kernel space.
287 */
288 u64 offset;
289 } user_mmio;
290
291 /** @vms: VM pool attached to this file. */
292 struct panthor_vm_pool *vms;
293
294 /** @groups: Scheduling group pool attached to this file. */
295 struct panthor_group_pool *groups;
296
297 /** @stats: cycle and timestamp measures for job execution. */
298 struct panthor_gpu_usage stats;
299};
300
301int panthor_device_init(struct panthor_device *ptdev);
302void panthor_device_unplug(struct panthor_device *ptdev);
303
304/**
305 * panthor_device_schedule_reset() - Schedules a reset operation
306 */
307static inline void panthor_device_schedule_reset(struct panthor_device *ptdev)
308{
309 if (!atomic_cmpxchg(&ptdev->reset.pending, 0, 1) &&
310 atomic_read(&ptdev->pm.state) == PANTHOR_DEVICE_PM_STATE_ACTIVE)
311 queue_work(ptdev->reset.wq, &ptdev->reset.work);
312}
313
314/**
315 * panthor_device_reset_is_pending() - Checks if a reset is pending.
316 *
317 * Return: true if a reset is pending, false otherwise.
318 */
319static inline bool panthor_device_reset_is_pending(struct panthor_device *ptdev)
320{
321 return atomic_read(&ptdev->reset.pending) != 0;
322}
323
324int panthor_device_mmap_io(struct panthor_device *ptdev,
325 struct vm_area_struct *vma);
326
327int panthor_device_resume(struct device *dev);
328int panthor_device_suspend(struct device *dev);
329
330static inline int panthor_device_resume_and_get(struct panthor_device *ptdev)
331{
332 int ret = pm_runtime_resume_and_get(ptdev->base.dev);
333
334 /* If the resume failed, we need to clear the runtime_error, which
335 * can done by forcing the RPM state to suspended. If multiple
336 * threads called panthor_device_resume_and_get(), we only want
337 * one of them to update the state, hence the cmpxchg. Note that a
338 * thread might enter panthor_device_resume_and_get() and call
339 * pm_runtime_resume_and_get() after another thread had attempted
340 * to resume and failed. This means we will end up with an error
341 * without even attempting a resume ourselves. The only risk here
342 * is to report an error when the second resume attempt might have
343 * succeeded. Given resume errors are not expected, this is probably
344 * something we can live with.
345 */
346 if (ret && atomic_cmpxchg(&ptdev->pm.recovery_needed, 1, 0) == 1)
347 pm_runtime_set_suspended(ptdev->base.dev);
348
349 return ret;
350}
351
352enum drm_panthor_exception_type {
353 DRM_PANTHOR_EXCEPTION_OK = 0x00,
354 DRM_PANTHOR_EXCEPTION_TERMINATED = 0x04,
355 DRM_PANTHOR_EXCEPTION_KABOOM = 0x05,
356 DRM_PANTHOR_EXCEPTION_EUREKA = 0x06,
357 DRM_PANTHOR_EXCEPTION_ACTIVE = 0x08,
358 DRM_PANTHOR_EXCEPTION_CS_RES_TERM = 0x0f,
359 DRM_PANTHOR_EXCEPTION_MAX_NON_FAULT = 0x3f,
360 DRM_PANTHOR_EXCEPTION_CS_CONFIG_FAULT = 0x40,
361 DRM_PANTHOR_EXCEPTION_CS_UNRECOVERABLE = 0x41,
362 DRM_PANTHOR_EXCEPTION_CS_ENDPOINT_FAULT = 0x44,
363 DRM_PANTHOR_EXCEPTION_CS_BUS_FAULT = 0x48,
364 DRM_PANTHOR_EXCEPTION_CS_INSTR_INVALID = 0x49,
365 DRM_PANTHOR_EXCEPTION_CS_CALL_STACK_OVERFLOW = 0x4a,
366 DRM_PANTHOR_EXCEPTION_CS_INHERIT_FAULT = 0x4b,
367 DRM_PANTHOR_EXCEPTION_INSTR_INVALID_PC = 0x50,
368 DRM_PANTHOR_EXCEPTION_INSTR_INVALID_ENC = 0x51,
369 DRM_PANTHOR_EXCEPTION_INSTR_BARRIER_FAULT = 0x55,
370 DRM_PANTHOR_EXCEPTION_DATA_INVALID_FAULT = 0x58,
371 DRM_PANTHOR_EXCEPTION_TILE_RANGE_FAULT = 0x59,
372 DRM_PANTHOR_EXCEPTION_ADDR_RANGE_FAULT = 0x5a,
373 DRM_PANTHOR_EXCEPTION_IMPRECISE_FAULT = 0x5b,
374 DRM_PANTHOR_EXCEPTION_OOM = 0x60,
375 DRM_PANTHOR_EXCEPTION_CSF_FW_INTERNAL_ERROR = 0x68,
376 DRM_PANTHOR_EXCEPTION_CSF_RES_EVICTION_TIMEOUT = 0x69,
377 DRM_PANTHOR_EXCEPTION_GPU_BUS_FAULT = 0x80,
378 DRM_PANTHOR_EXCEPTION_GPU_SHAREABILITY_FAULT = 0x88,
379 DRM_PANTHOR_EXCEPTION_SYS_SHAREABILITY_FAULT = 0x89,
380 DRM_PANTHOR_EXCEPTION_GPU_CACHEABILITY_FAULT = 0x8a,
381 DRM_PANTHOR_EXCEPTION_TRANSLATION_FAULT_0 = 0xc0,
382 DRM_PANTHOR_EXCEPTION_TRANSLATION_FAULT_1 = 0xc1,
383 DRM_PANTHOR_EXCEPTION_TRANSLATION_FAULT_2 = 0xc2,
384 DRM_PANTHOR_EXCEPTION_TRANSLATION_FAULT_3 = 0xc3,
385 DRM_PANTHOR_EXCEPTION_TRANSLATION_FAULT_4 = 0xc4,
386 DRM_PANTHOR_EXCEPTION_PERM_FAULT_0 = 0xc8,
387 DRM_PANTHOR_EXCEPTION_PERM_FAULT_1 = 0xc9,
388 DRM_PANTHOR_EXCEPTION_PERM_FAULT_2 = 0xca,
389 DRM_PANTHOR_EXCEPTION_PERM_FAULT_3 = 0xcb,
390 DRM_PANTHOR_EXCEPTION_ACCESS_FLAG_1 = 0xd9,
391 DRM_PANTHOR_EXCEPTION_ACCESS_FLAG_2 = 0xda,
392 DRM_PANTHOR_EXCEPTION_ACCESS_FLAG_3 = 0xdb,
393 DRM_PANTHOR_EXCEPTION_ADDR_SIZE_FAULT_IN = 0xe0,
394 DRM_PANTHOR_EXCEPTION_ADDR_SIZE_FAULT_OUT0 = 0xe4,
395 DRM_PANTHOR_EXCEPTION_ADDR_SIZE_FAULT_OUT1 = 0xe5,
396 DRM_PANTHOR_EXCEPTION_ADDR_SIZE_FAULT_OUT2 = 0xe6,
397 DRM_PANTHOR_EXCEPTION_ADDR_SIZE_FAULT_OUT3 = 0xe7,
398 DRM_PANTHOR_EXCEPTION_MEM_ATTR_FAULT_0 = 0xe8,
399 DRM_PANTHOR_EXCEPTION_MEM_ATTR_FAULT_1 = 0xe9,
400 DRM_PANTHOR_EXCEPTION_MEM_ATTR_FAULT_2 = 0xea,
401 DRM_PANTHOR_EXCEPTION_MEM_ATTR_FAULT_3 = 0xeb,
402};
403
404/**
405 * panthor_exception_is_fault() - Checks if an exception is a fault.
406 *
407 * Return: true if the exception is a fault, false otherwise.
408 */
409static inline bool
410panthor_exception_is_fault(u32 exception_code)
411{
412 return exception_code > DRM_PANTHOR_EXCEPTION_MAX_NON_FAULT;
413}
414
415const char *panthor_exception_name(struct panthor_device *ptdev,
416 u32 exception_code);
417
418/**
419 * PANTHOR_IRQ_HANDLER() - Define interrupt handlers and the interrupt
420 * registration function.
421 *
422 * The boiler-plate to gracefully deal with shared interrupts is
423 * auto-generated. All you have to do is call PANTHOR_IRQ_HANDLER()
424 * just after the actual handler. The handler prototype is:
425 *
426 * void (*handler)(struct panthor_device *, u32 status);
427 */
428#define PANTHOR_IRQ_HANDLER(__name, __reg_prefix, __handler) \
429static irqreturn_t panthor_ ## __name ## _irq_raw_handler(int irq, void *data) \
430{ \
431 struct panthor_irq *pirq = data; \
432 struct panthor_device *ptdev = pirq->ptdev; \
433 enum panthor_irq_state old_state; \
434 \
435 if (!gpu_read(ptdev, __reg_prefix ## _INT_STAT)) \
436 return IRQ_NONE; \
437 \
438 guard(spinlock_irqsave)(&pirq->mask_lock); \
439 old_state = atomic_cmpxchg(&pirq->state, \
440 PANTHOR_IRQ_STATE_ACTIVE, \
441 PANTHOR_IRQ_STATE_PROCESSING); \
442 if (old_state != PANTHOR_IRQ_STATE_ACTIVE) \
443 return IRQ_NONE; \
444 \
445 gpu_write(ptdev, __reg_prefix ## _INT_MASK, 0); \
446 return IRQ_WAKE_THREAD; \
447} \
448 \
449static irqreturn_t panthor_ ## __name ## _irq_threaded_handler(int irq, void *data) \
450{ \
451 struct panthor_irq *pirq = data; \
452 struct panthor_device *ptdev = pirq->ptdev; \
453 irqreturn_t ret = IRQ_NONE; \
454 \
455 while (true) { \
456 /* It's safe to access pirq->mask without the lock held here. If a new \
457 * event gets added to the mask and the corresponding IRQ is pending, \
458 * we'll process it right away instead of adding an extra raw -> threaded \
459 * round trip. If an event is removed and the status bit is set, it will \
460 * be ignored, just like it would have been if the mask had been adjusted \
461 * right before the HW event kicks in. TLDR; it's all expected races we're \
462 * covered for. \
463 */ \
464 u32 status = gpu_read(ptdev, __reg_prefix ## _INT_RAWSTAT) & pirq->mask; \
465 \
466 if (!status) \
467 break; \
468 \
469 __handler(ptdev, status); \
470 ret = IRQ_HANDLED; \
471 } \
472 \
473 scoped_guard(spinlock_irqsave, &pirq->mask_lock) { \
474 enum panthor_irq_state old_state; \
475 \
476 old_state = atomic_cmpxchg(&pirq->state, \
477 PANTHOR_IRQ_STATE_PROCESSING, \
478 PANTHOR_IRQ_STATE_ACTIVE); \
479 if (old_state == PANTHOR_IRQ_STATE_PROCESSING) \
480 gpu_write(ptdev, __reg_prefix ## _INT_MASK, pirq->mask); \
481 } \
482 \
483 return ret; \
484} \
485 \
486static inline void panthor_ ## __name ## _irq_suspend(struct panthor_irq *pirq) \
487{ \
488 scoped_guard(spinlock_irqsave, &pirq->mask_lock) { \
489 atomic_set(&pirq->state, PANTHOR_IRQ_STATE_SUSPENDING); \
490 gpu_write(pirq->ptdev, __reg_prefix ## _INT_MASK, 0); \
491 } \
492 synchronize_irq(pirq->irq); \
493 atomic_set(&pirq->state, PANTHOR_IRQ_STATE_SUSPENDED); \
494} \
495 \
496static inline void panthor_ ## __name ## _irq_resume(struct panthor_irq *pirq) \
497{ \
498 guard(spinlock_irqsave)(&pirq->mask_lock); \
499 \
500 atomic_set(&pirq->state, PANTHOR_IRQ_STATE_ACTIVE); \
501 gpu_write(pirq->ptdev, __reg_prefix ## _INT_CLEAR, pirq->mask); \
502 gpu_write(pirq->ptdev, __reg_prefix ## _INT_MASK, pirq->mask); \
503} \
504 \
505static int panthor_request_ ## __name ## _irq(struct panthor_device *ptdev, \
506 struct panthor_irq *pirq, \
507 int irq, u32 mask) \
508{ \
509 pirq->ptdev = ptdev; \
510 pirq->irq = irq; \
511 pirq->mask = mask; \
512 spin_lock_init(&pirq->mask_lock); \
513 panthor_ ## __name ## _irq_resume(pirq); \
514 \
515 return devm_request_threaded_irq(ptdev->base.dev, irq, \
516 panthor_ ## __name ## _irq_raw_handler, \
517 panthor_ ## __name ## _irq_threaded_handler, \
518 IRQF_SHARED, KBUILD_MODNAME "-" # __name, \
519 pirq); \
520} \
521 \
522static inline void panthor_ ## __name ## _irq_enable_events(struct panthor_irq *pirq, u32 mask) \
523{ \
524 guard(spinlock_irqsave)(&pirq->mask_lock); \
525 pirq->mask |= mask; \
526 \
527 /* The only situation where we need to write the new mask is if the IRQ is active. \
528 * If it's being processed, the mask will be restored for us in _irq_threaded_handler() \
529 * on the PROCESSING -> ACTIVE transition. \
530 * If the IRQ is suspended/suspending, the mask is restored at resume time. \
531 */ \
532 if (atomic_read(&pirq->state) == PANTHOR_IRQ_STATE_ACTIVE) \
533 gpu_write(pirq->ptdev, __reg_prefix ## _INT_MASK, pirq->mask); \
534} \
535 \
536static inline void panthor_ ## __name ## _irq_disable_events(struct panthor_irq *pirq, u32 mask)\
537{ \
538 guard(spinlock_irqsave)(&pirq->mask_lock); \
539 pirq->mask &= ~mask; \
540 \
541 /* The only situation where we need to write the new mask is if the IRQ is active. \
542 * If it's being processed, the mask will be restored for us in _irq_threaded_handler() \
543 * on the PROCESSING -> ACTIVE transition. \
544 * If the IRQ is suspended/suspending, the mask is restored at resume time. \
545 */ \
546 if (atomic_read(&pirq->state) == PANTHOR_IRQ_STATE_ACTIVE) \
547 gpu_write(pirq->ptdev, __reg_prefix ## _INT_MASK, pirq->mask); \
548}
549
550extern struct workqueue_struct *panthor_cleanup_wq;
551
552static inline void gpu_write(struct panthor_device *ptdev, u32 reg, u32 data)
553{
554 writel(data, ptdev->iomem + reg);
555}
556
557static inline u32 gpu_read(struct panthor_device *ptdev, u32 reg)
558{
559 return readl(ptdev->iomem + reg);
560}
561
562static inline u32 gpu_read_relaxed(struct panthor_device *ptdev, u32 reg)
563{
564 return readl_relaxed(ptdev->iomem + reg);
565}
566
567static inline void gpu_write64(struct panthor_device *ptdev, u32 reg, u64 data)
568{
569 gpu_write(ptdev, reg, lower_32_bits(data));
570 gpu_write(ptdev, reg + 4, upper_32_bits(data));
571}
572
573static inline u64 gpu_read64(struct panthor_device *ptdev, u32 reg)
574{
575 return (gpu_read(ptdev, reg) | ((u64)gpu_read(ptdev, reg + 4) << 32));
576}
577
578static inline u64 gpu_read64_relaxed(struct panthor_device *ptdev, u32 reg)
579{
580 return (gpu_read_relaxed(ptdev, reg) |
581 ((u64)gpu_read_relaxed(ptdev, reg + 4) << 32));
582}
583
584static inline u64 gpu_read64_counter(struct panthor_device *ptdev, u32 reg)
585{
586 u32 lo, hi1, hi2;
587 do {
588 hi1 = gpu_read(ptdev, reg + 4);
589 lo = gpu_read(ptdev, reg);
590 hi2 = gpu_read(ptdev, reg + 4);
591 } while (hi1 != hi2);
592 return lo | ((u64)hi2 << 32);
593}
594
595#define gpu_read_poll_timeout(dev, reg, val, cond, delay_us, timeout_us) \
596 read_poll_timeout(gpu_read, val, cond, delay_us, timeout_us, false, \
597 dev, reg)
598
599#define gpu_read_poll_timeout_atomic(dev, reg, val, cond, delay_us, \
600 timeout_us) \
601 read_poll_timeout_atomic(gpu_read, val, cond, delay_us, timeout_us, \
602 false, dev, reg)
603
604#define gpu_read64_poll_timeout(dev, reg, val, cond, delay_us, timeout_us) \
605 read_poll_timeout(gpu_read64, val, cond, delay_us, timeout_us, false, \
606 dev, reg)
607
608#define gpu_read64_poll_timeout_atomic(dev, reg, val, cond, delay_us, \
609 timeout_us) \
610 read_poll_timeout_atomic(gpu_read64, val, cond, delay_us, timeout_us, \
611 false, dev, reg)
612
613#define gpu_read_relaxed_poll_timeout_atomic(dev, reg, val, cond, delay_us, \
614 timeout_us) \
615 read_poll_timeout_atomic(gpu_read_relaxed, val, cond, delay_us, \
616 timeout_us, false, dev, reg)
617
618#define gpu_read64_relaxed_poll_timeout(dev, reg, val, cond, delay_us, \
619 timeout_us) \
620 read_poll_timeout(gpu_read64_relaxed, val, cond, delay_us, timeout_us, \
621 false, dev, reg)
622
623#endif