Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2020 - 2024 Intel Corporation
4 */
5
6#include "ivpu_drv.h"
7#include "ivpu_hw.h"
8#include "ivpu_hw_btrs.h"
9#include "ivpu_hw_ip.h"
10
11#include <asm/msr-index.h>
12#include <asm/msr.h>
13#include <linux/dmi.h>
14#include <linux/fault-inject.h>
15#include <linux/pm_runtime.h>
16
17#ifdef CONFIG_FAULT_INJECTION
18DECLARE_FAULT_ATTR(ivpu_hw_failure);
19
20static char *ivpu_fail_hw;
21module_param_named_unsafe(fail_hw, ivpu_fail_hw, charp, 0444);
22MODULE_PARM_DESC(fail_hw, "<interval>,<probability>,<space>,<times>");
23#endif
24
25#define FW_SHARED_MEM_ALIGNMENT SZ_512K /* VPU MTRR limitation */
26
27#define ECC_MCA_SIGNAL_ENABLE_MASK 0xff
28
29static char *platform_to_str(u32 platform)
30{
31 switch (platform) {
32 case IVPU_PLATFORM_SILICON:
33 return "SILICON";
34 case IVPU_PLATFORM_SIMICS:
35 return "SIMICS";
36 case IVPU_PLATFORM_FPGA:
37 return "FPGA";
38 case IVPU_PLATFORM_HSLE:
39 return "HSLE";
40 default:
41 return "Invalid platform";
42 }
43}
44
45static void platform_init(struct ivpu_device *vdev)
46{
47 int platform = ivpu_hw_btrs_platform_read(vdev);
48
49 ivpu_dbg(vdev, MISC, "Platform type: %s (%d)\n", platform_to_str(platform), platform);
50
51 switch (platform) {
52 case IVPU_PLATFORM_SILICON:
53 case IVPU_PLATFORM_SIMICS:
54 case IVPU_PLATFORM_FPGA:
55 case IVPU_PLATFORM_HSLE:
56 vdev->platform = platform;
57 break;
58
59 default:
60 ivpu_err(vdev, "Invalid platform type: %d\n", platform);
61 break;
62 }
63}
64
65static void wa_init(struct ivpu_device *vdev)
66{
67 vdev->wa.punit_disabled = false;
68 vdev->wa.clear_runtime_mem = false;
69
70 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL)
71 vdev->wa.interrupt_clear_with_0 = ivpu_hw_btrs_irqs_clear_with_0_mtl(vdev);
72
73 if ((ivpu_device_id(vdev) == PCI_DEVICE_ID_LNL &&
74 ivpu_revision(vdev) < IVPU_HW_IP_REV_LNL_B0) ||
75 (ivpu_device_id(vdev) == PCI_DEVICE_ID_NVL &&
76 ivpu_revision(vdev) == IVPU_HW_IP_REV_NVL_A0))
77 vdev->wa.disable_clock_relinquish = true;
78
79 if (ivpu_test_mode & IVPU_TEST_MODE_CLK_RELINQ_ENABLE)
80 vdev->wa.disable_clock_relinquish = false;
81
82 if (ivpu_test_mode & IVPU_TEST_MODE_CLK_RELINQ_DISABLE)
83 vdev->wa.disable_clock_relinquish = true;
84
85 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
86 vdev->wa.wp0_during_power_up = true;
87
88 if (ivpu_test_mode & IVPU_TEST_MODE_D0I2_DISABLE)
89 vdev->wa.disable_d0i2 = true;
90
91 IVPU_PRINT_WA(punit_disabled);
92 IVPU_PRINT_WA(clear_runtime_mem);
93 IVPU_PRINT_WA(interrupt_clear_with_0);
94 IVPU_PRINT_WA(disable_clock_relinquish);
95 IVPU_PRINT_WA(wp0_during_power_up);
96 IVPU_PRINT_WA(disable_d0i2);
97}
98
99static void timeouts_init(struct ivpu_device *vdev)
100{
101 if (ivpu_test_mode & IVPU_TEST_MODE_DISABLE_TIMEOUTS) {
102 vdev->timeout.boot = -1;
103 vdev->timeout.jsm = -1;
104 vdev->timeout.tdr = -1;
105 vdev->timeout.inference = -1;
106 vdev->timeout.autosuspend = -1;
107 vdev->timeout.d0i3_entry_msg = -1;
108 } else if (ivpu_is_fpga(vdev)) {
109 vdev->timeout.boot = 50;
110 vdev->timeout.jsm = 15000;
111 vdev->timeout.tdr = 30000;
112 vdev->timeout.inference = 900000;
113 vdev->timeout.autosuspend = -1;
114 vdev->timeout.d0i3_entry_msg = 500;
115 vdev->timeout.state_dump_msg = 10000;
116 } else if (ivpu_is_simics(vdev)) {
117 vdev->timeout.boot = 50;
118 vdev->timeout.jsm = 500;
119 vdev->timeout.tdr = 10000;
120 vdev->timeout.inference = 300000;
121 vdev->timeout.autosuspend = 100;
122 vdev->timeout.d0i3_entry_msg = 100;
123 vdev->timeout.state_dump_msg = 10;
124 } else {
125 vdev->timeout.boot = 1000;
126 vdev->timeout.jsm = 500;
127 vdev->timeout.tdr = 2000;
128 vdev->timeout.inference = 60000;
129 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
130 vdev->timeout.autosuspend = 10;
131 else
132 vdev->timeout.autosuspend = 100;
133 vdev->timeout.d0i3_entry_msg = 5;
134 vdev->timeout.state_dump_msg = 100;
135 }
136}
137
138static void priority_bands_init(struct ivpu_device *vdev)
139{
140 /* Idle */
141 vdev->hw->hws.grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_IDLE] = 0;
142 vdev->hw->hws.process_grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_IDLE] = 50000;
143 vdev->hw->hws.process_quantum[VPU_JOB_SCHEDULING_PRIORITY_BAND_IDLE] = 160000;
144 /* Normal */
145 vdev->hw->hws.grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_NORMAL] = 50000;
146 vdev->hw->hws.process_grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_NORMAL] = 50000;
147 vdev->hw->hws.process_quantum[VPU_JOB_SCHEDULING_PRIORITY_BAND_NORMAL] = 300000;
148 /* Focus */
149 vdev->hw->hws.grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_FOCUS] = 50000;
150 vdev->hw->hws.process_grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_FOCUS] = 50000;
151 vdev->hw->hws.process_quantum[VPU_JOB_SCHEDULING_PRIORITY_BAND_FOCUS] = 200000;
152 /* Realtime */
153 vdev->hw->hws.grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_REALTIME] = 0;
154 vdev->hw->hws.process_grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_REALTIME] = 50000;
155 vdev->hw->hws.process_quantum[VPU_JOB_SCHEDULING_PRIORITY_BAND_REALTIME] = 200000;
156}
157
158int ivpu_hw_range_init(struct ivpu_device *vdev, struct ivpu_addr_range *range, u64 start, u64 size)
159{
160 u64 end;
161
162 if (!range || check_add_overflow(start, size, &end)) {
163 ivpu_err(vdev, "Invalid range: start 0x%llx size %llu\n", start, size);
164 return -EINVAL;
165 }
166
167 range->start = start;
168 range->end = end;
169
170 return 0;
171}
172
173static void memory_ranges_init(struct ivpu_device *vdev)
174{
175 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) {
176 ivpu_hw_range_init(vdev, &vdev->hw->ranges.runtime, 0x84800000, SZ_64M);
177 ivpu_hw_range_init(vdev, &vdev->hw->ranges.global, 0x90000000, SZ_256M);
178 ivpu_hw_range_init(vdev, &vdev->hw->ranges.user, 0xa0000000, 511 * SZ_1M);
179 ivpu_hw_range_init(vdev, &vdev->hw->ranges.shave, 0x180000000, SZ_2G);
180 ivpu_hw_range_init(vdev, &vdev->hw->ranges.dma, 0x200000000, SZ_128G);
181 } else {
182 ivpu_hw_range_init(vdev, &vdev->hw->ranges.runtime, 0x80000000, SZ_64M);
183 ivpu_hw_range_init(vdev, &vdev->hw->ranges.global, 0x90000000, SZ_256M);
184 ivpu_hw_range_init(vdev, &vdev->hw->ranges.shave, 0x80000000, SZ_2G);
185 ivpu_hw_range_init(vdev, &vdev->hw->ranges.user, 0x100000000, SZ_256G);
186 vdev->hw->ranges.dma = vdev->hw->ranges.user;
187 }
188
189 drm_WARN_ON(&vdev->drm, !IS_ALIGNED(vdev->hw->ranges.global.start,
190 FW_SHARED_MEM_ALIGNMENT));
191}
192
193static int wp_enable(struct ivpu_device *vdev)
194{
195 return ivpu_hw_btrs_wp_drive(vdev, true);
196}
197
198static int wp_disable(struct ivpu_device *vdev)
199{
200 return ivpu_hw_btrs_wp_drive(vdev, false);
201}
202
203int ivpu_hw_power_up(struct ivpu_device *vdev)
204{
205 int ret;
206
207 if (IVPU_WA(wp0_during_power_up)) {
208 /* WP requests may fail when powering down, so issue WP 0 here */
209 ret = wp_disable(vdev);
210 if (ret)
211 ivpu_warn(vdev, "Failed to disable workpoint: %d\n", ret);
212 }
213
214 ret = ivpu_hw_btrs_d0i3_disable(vdev);
215 if (ret)
216 ivpu_warn(vdev, "Failed to disable D0I3: %d\n", ret);
217
218 ret = wp_enable(vdev);
219 if (ret) {
220 ivpu_err(vdev, "Failed to enable workpoint: %d\n", ret);
221 return ret;
222 }
223
224 if (ivpu_hw_btrs_gen(vdev) >= IVPU_HW_BTRS_LNL) {
225 if (IVPU_WA(disable_clock_relinquish))
226 ivpu_hw_btrs_clock_relinquish_disable_lnl(vdev);
227 ivpu_hw_btrs_profiling_freq_reg_set_lnl(vdev);
228 ivpu_hw_btrs_ats_print_lnl(vdev);
229 }
230
231 ret = ivpu_hw_ip_host_ss_configure(vdev);
232 if (ret) {
233 ivpu_err(vdev, "Failed to configure host SS: %d\n", ret);
234 return ret;
235 }
236
237 ivpu_hw_ip_idle_gen_disable(vdev);
238
239 ret = ivpu_hw_btrs_wait_for_clock_res_own_ack(vdev);
240 if (ret) {
241 ivpu_err(vdev, "Timed out waiting for clock resource own ACK\n");
242 return ret;
243 }
244
245 ret = ivpu_hw_ip_pwr_domain_enable(vdev);
246 if (ret) {
247 ivpu_err(vdev, "Failed to enable power domain: %d\n", ret);
248 return ret;
249 }
250
251 ret = ivpu_hw_ip_host_ss_axi_enable(vdev);
252 if (ret) {
253 ivpu_err(vdev, "Failed to enable AXI: %d\n", ret);
254 return ret;
255 }
256
257 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_LNL)
258 ivpu_hw_btrs_set_port_arbitration_weights_lnl(vdev);
259
260 ret = ivpu_hw_ip_top_noc_enable(vdev);
261 if (ret)
262 ivpu_err(vdev, "Failed to enable TOP NOC: %d\n", ret);
263
264 return ret;
265}
266
267static void save_d0i3_entry_timestamp(struct ivpu_device *vdev)
268{
269 vdev->hw->d0i3_entry_host_ts = ktime_get_boottime();
270 vdev->hw->d0i3_entry_vpu_ts = ivpu_hw_ip_read_perf_timer_counter(vdev);
271}
272
273int ivpu_hw_reset(struct ivpu_device *vdev)
274{
275 int ret = 0;
276
277 if (ivpu_hw_btrs_ip_reset(vdev)) {
278 ivpu_err(vdev, "Failed to reset NPU IP\n");
279 ret = -EIO;
280 }
281
282 if (wp_disable(vdev)) {
283 ivpu_err(vdev, "Failed to disable workpoint\n");
284 ret = -EIO;
285 }
286
287 return ret;
288}
289
290int ivpu_hw_power_down(struct ivpu_device *vdev)
291{
292 int ret = 0;
293
294 save_d0i3_entry_timestamp(vdev);
295
296 if (!ivpu_hw_is_idle(vdev))
297 ivpu_warn(vdev, "NPU not idle during power down\n");
298
299 if (ivpu_hw_reset(vdev)) {
300 ivpu_err(vdev, "Failed to reset NPU\n");
301 ret = -EIO;
302 }
303
304 if (ivpu_hw_btrs_d0i3_enable(vdev)) {
305 ivpu_err(vdev, "Failed to enter D0I3\n");
306 ret = -EIO;
307 }
308
309 return ret;
310}
311
312int ivpu_hw_init(struct ivpu_device *vdev)
313{
314 ivpu_hw_btrs_info_init(vdev);
315 ivpu_hw_btrs_freq_ratios_init(vdev);
316 priority_bands_init(vdev);
317 memory_ranges_init(vdev);
318 platform_init(vdev);
319 wa_init(vdev);
320 timeouts_init(vdev);
321 atomic_set(&vdev->hw->firewall_irq_counter, 0);
322
323#ifdef CONFIG_FAULT_INJECTION
324 if (ivpu_fail_hw)
325 setup_fault_attr(&ivpu_hw_failure, ivpu_fail_hw);
326#endif
327
328 return 0;
329}
330
331int ivpu_hw_boot_fw(struct ivpu_device *vdev)
332{
333 int ret;
334
335 ivpu_hw_ip_snoop_disable(vdev);
336 ivpu_hw_ip_tbu_mmu_enable(vdev);
337 ret = ivpu_hw_ip_soc_cpu_boot(vdev);
338 if (ret)
339 ivpu_err(vdev, "Failed to boot SOC CPU: %d\n", ret);
340
341 return ret;
342}
343
344void ivpu_hw_profiling_freq_drive(struct ivpu_device *vdev, bool enable)
345{
346 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) {
347 vdev->hw->pll.profiling_freq = PLL_PROFILING_FREQ_DEFAULT;
348 return;
349 }
350
351 if (enable)
352 vdev->hw->pll.profiling_freq = PLL_PROFILING_FREQ_HIGH;
353 else
354 vdev->hw->pll.profiling_freq = PLL_PROFILING_FREQ_DEFAULT;
355}
356
357void ivpu_irq_handlers_init(struct ivpu_device *vdev)
358{
359 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
360 vdev->hw->irq.ip_irq_handler = ivpu_hw_ip_irq_handler_37xx;
361 else
362 vdev->hw->irq.ip_irq_handler = ivpu_hw_ip_irq_handler_40xx;
363
364 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL)
365 vdev->hw->irq.btrs_irq_handler = ivpu_hw_btrs_irq_handler_mtl;
366 else
367 vdev->hw->irq.btrs_irq_handler = ivpu_hw_btrs_irq_handler_lnl;
368}
369
370void ivpu_hw_irq_enable(struct ivpu_device *vdev)
371{
372 ivpu_hw_ip_irq_enable(vdev);
373 ivpu_hw_btrs_irq_enable(vdev);
374}
375
376void ivpu_hw_irq_disable(struct ivpu_device *vdev)
377{
378 ivpu_hw_btrs_irq_disable(vdev);
379 ivpu_hw_ip_irq_disable(vdev);
380}
381
382irqreturn_t ivpu_hw_irq_handler(int irq, void *ptr)
383{
384 struct ivpu_device *vdev = ptr;
385 bool ip_handled, btrs_handled;
386
387 ivpu_hw_btrs_global_int_disable(vdev);
388
389 btrs_handled = ivpu_hw_btrs_irq_handler(vdev, irq);
390 if (!ivpu_hw_is_idle((vdev)) || !btrs_handled)
391 ip_handled = ivpu_hw_ip_irq_handler(vdev, irq);
392 else
393 ip_handled = false;
394
395 /* Re-enable global interrupts to re-trigger MSI for pending interrupts */
396 ivpu_hw_btrs_global_int_enable(vdev);
397
398 if (!ip_handled && !btrs_handled)
399 return IRQ_NONE;
400
401 pm_runtime_mark_last_busy(vdev->drm.dev);
402 return IRQ_HANDLED;
403}
404
405bool ivpu_hw_uses_ecc_mca_signal(struct ivpu_device *vdev)
406{
407 unsigned long long msr_integrity_caps;
408 int ret;
409
410 if (ivpu_hw_ip_gen(vdev) < IVPU_HW_IP_50XX)
411 return false;
412
413 ret = rdmsrq_safe(MSR_INTEGRITY_CAPS, &msr_integrity_caps);
414 if (ret) {
415 ivpu_warn(vdev, "Error reading MSR_INTEGRITY_CAPS: %d", ret);
416 return false;
417 }
418
419 ivpu_dbg(vdev, MISC, "MSR_INTEGRITY_CAPS: 0x%llx\n", msr_integrity_caps);
420
421 return msr_integrity_caps & ECC_MCA_SIGNAL_ENABLE_MASK;
422}