Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/*
2 * Copyright 2010 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24
25#include <linux/firmware.h>
26#include <linux/pci.h>
27#include <linux/slab.h>
28
29#include <drm/drm_edid.h>
30#include <drm/drm_vblank.h>
31#include <drm/radeon_drm.h>
32#include <drm/drm_fourcc.h>
33#include <drm/drm_framebuffer.h>
34
35#include "atom.h"
36#include "avivod.h"
37#include "cik.h"
38#include "ni.h"
39#include "rv770.h"
40#include "evergreen.h"
41#include "evergreen_blit_shaders.h"
42#include "evergreen_reg.h"
43#include "evergreend.h"
44#include "radeon.h"
45#include "radeon_asic.h"
46#include "radeon_audio.h"
47#include "radeon_ucode.h"
48#include "si.h"
49
50#define DC_HPDx_CONTROL(x) (DC_HPD1_CONTROL + (x * 0xc))
51#define DC_HPDx_INT_CONTROL(x) (DC_HPD1_INT_CONTROL + (x * 0xc))
52#define DC_HPDx_INT_STATUS_REG(x) (DC_HPD1_INT_STATUS + (x * 0xc))
53
54/*
55 * Indirect registers accessor
56 */
57u32 eg_cg_rreg(struct radeon_device *rdev, u32 reg)
58{
59 unsigned long flags;
60 u32 r;
61
62 spin_lock_irqsave(&rdev->cg_idx_lock, flags);
63 WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
64 r = RREG32(EVERGREEN_CG_IND_DATA);
65 spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
66 return r;
67}
68
69void eg_cg_wreg(struct radeon_device *rdev, u32 reg, u32 v)
70{
71 unsigned long flags;
72
73 spin_lock_irqsave(&rdev->cg_idx_lock, flags);
74 WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
75 WREG32(EVERGREEN_CG_IND_DATA, (v));
76 spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
77}
78
79u32 eg_pif_phy0_rreg(struct radeon_device *rdev, u32 reg)
80{
81 unsigned long flags;
82 u32 r;
83
84 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
85 WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
86 r = RREG32(EVERGREEN_PIF_PHY0_DATA);
87 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
88 return r;
89}
90
91void eg_pif_phy0_wreg(struct radeon_device *rdev, u32 reg, u32 v)
92{
93 unsigned long flags;
94
95 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
96 WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
97 WREG32(EVERGREEN_PIF_PHY0_DATA, (v));
98 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
99}
100
101u32 eg_pif_phy1_rreg(struct radeon_device *rdev, u32 reg)
102{
103 unsigned long flags;
104 u32 r;
105
106 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
107 WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
108 r = RREG32(EVERGREEN_PIF_PHY1_DATA);
109 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
110 return r;
111}
112
113void eg_pif_phy1_wreg(struct radeon_device *rdev, u32 reg, u32 v)
114{
115 unsigned long flags;
116
117 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
118 WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
119 WREG32(EVERGREEN_PIF_PHY1_DATA, (v));
120 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
121}
122
123static const u32 crtc_offsets[6] =
124{
125 EVERGREEN_CRTC0_REGISTER_OFFSET,
126 EVERGREEN_CRTC1_REGISTER_OFFSET,
127 EVERGREEN_CRTC2_REGISTER_OFFSET,
128 EVERGREEN_CRTC3_REGISTER_OFFSET,
129 EVERGREEN_CRTC4_REGISTER_OFFSET,
130 EVERGREEN_CRTC5_REGISTER_OFFSET
131};
132
133#include "clearstate_evergreen.h"
134
135static const u32 sumo_rlc_save_restore_register_list[] =
136{
137 0x98fc,
138 0x9830,
139 0x9834,
140 0x9838,
141 0x9870,
142 0x9874,
143 0x8a14,
144 0x8b24,
145 0x8bcc,
146 0x8b10,
147 0x8d00,
148 0x8d04,
149 0x8c00,
150 0x8c04,
151 0x8c08,
152 0x8c0c,
153 0x8d8c,
154 0x8c20,
155 0x8c24,
156 0x8c28,
157 0x8c18,
158 0x8c1c,
159 0x8cf0,
160 0x8e2c,
161 0x8e38,
162 0x8c30,
163 0x9508,
164 0x9688,
165 0x9608,
166 0x960c,
167 0x9610,
168 0x9614,
169 0x88c4,
170 0x88d4,
171 0xa008,
172 0x900c,
173 0x9100,
174 0x913c,
175 0x98f8,
176 0x98f4,
177 0x9b7c,
178 0x3f8c,
179 0x8950,
180 0x8954,
181 0x8a18,
182 0x8b28,
183 0x9144,
184 0x9148,
185 0x914c,
186 0x3f90,
187 0x3f94,
188 0x915c,
189 0x9160,
190 0x9178,
191 0x917c,
192 0x9180,
193 0x918c,
194 0x9190,
195 0x9194,
196 0x9198,
197 0x919c,
198 0x91a8,
199 0x91ac,
200 0x91b0,
201 0x91b4,
202 0x91b8,
203 0x91c4,
204 0x91c8,
205 0x91cc,
206 0x91d0,
207 0x91d4,
208 0x91e0,
209 0x91e4,
210 0x91ec,
211 0x91f0,
212 0x91f4,
213 0x9200,
214 0x9204,
215 0x929c,
216 0x9150,
217 0x802c,
218};
219
220static void evergreen_gpu_init(struct radeon_device *rdev);
221void evergreen_fini(struct radeon_device *rdev);
222void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
223void evergreen_program_aspm(struct radeon_device *rdev);
224
225static const u32 evergreen_golden_registers[] =
226{
227 0x3f90, 0xffff0000, 0xff000000,
228 0x9148, 0xffff0000, 0xff000000,
229 0x3f94, 0xffff0000, 0xff000000,
230 0x914c, 0xffff0000, 0xff000000,
231 0x9b7c, 0xffffffff, 0x00000000,
232 0x8a14, 0xffffffff, 0x00000007,
233 0x8b10, 0xffffffff, 0x00000000,
234 0x960c, 0xffffffff, 0x54763210,
235 0x88c4, 0xffffffff, 0x000000c2,
236 0x88d4, 0xffffffff, 0x00000010,
237 0x8974, 0xffffffff, 0x00000000,
238 0xc78, 0x00000080, 0x00000080,
239 0x5eb4, 0xffffffff, 0x00000002,
240 0x5e78, 0xffffffff, 0x001000f0,
241 0x6104, 0x01000300, 0x00000000,
242 0x5bc0, 0x00300000, 0x00000000,
243 0x7030, 0xffffffff, 0x00000011,
244 0x7c30, 0xffffffff, 0x00000011,
245 0x10830, 0xffffffff, 0x00000011,
246 0x11430, 0xffffffff, 0x00000011,
247 0x12030, 0xffffffff, 0x00000011,
248 0x12c30, 0xffffffff, 0x00000011,
249 0xd02c, 0xffffffff, 0x08421000,
250 0x240c, 0xffffffff, 0x00000380,
251 0x8b24, 0xffffffff, 0x00ff0fff,
252 0x28a4c, 0x06000000, 0x06000000,
253 0x10c, 0x00000001, 0x00000001,
254 0x8d00, 0xffffffff, 0x100e4848,
255 0x8d04, 0xffffffff, 0x00164745,
256 0x8c00, 0xffffffff, 0xe4000003,
257 0x8c04, 0xffffffff, 0x40600060,
258 0x8c08, 0xffffffff, 0x001c001c,
259 0x8cf0, 0xffffffff, 0x08e00620,
260 0x8c20, 0xffffffff, 0x00800080,
261 0x8c24, 0xffffffff, 0x00800080,
262 0x8c18, 0xffffffff, 0x20202078,
263 0x8c1c, 0xffffffff, 0x00001010,
264 0x28350, 0xffffffff, 0x00000000,
265 0xa008, 0xffffffff, 0x00010000,
266 0x5c4, 0xffffffff, 0x00000001,
267 0x9508, 0xffffffff, 0x00000002,
268 0x913c, 0x0000000f, 0x0000000a
269};
270
271static const u32 evergreen_golden_registers2[] =
272{
273 0x2f4c, 0xffffffff, 0x00000000,
274 0x54f4, 0xffffffff, 0x00000000,
275 0x54f0, 0xffffffff, 0x00000000,
276 0x5498, 0xffffffff, 0x00000000,
277 0x549c, 0xffffffff, 0x00000000,
278 0x5494, 0xffffffff, 0x00000000,
279 0x53cc, 0xffffffff, 0x00000000,
280 0x53c8, 0xffffffff, 0x00000000,
281 0x53c4, 0xffffffff, 0x00000000,
282 0x53c0, 0xffffffff, 0x00000000,
283 0x53bc, 0xffffffff, 0x00000000,
284 0x53b8, 0xffffffff, 0x00000000,
285 0x53b4, 0xffffffff, 0x00000000,
286 0x53b0, 0xffffffff, 0x00000000
287};
288
289static const u32 cypress_mgcg_init[] =
290{
291 0x802c, 0xffffffff, 0xc0000000,
292 0x5448, 0xffffffff, 0x00000100,
293 0x55e4, 0xffffffff, 0x00000100,
294 0x160c, 0xffffffff, 0x00000100,
295 0x5644, 0xffffffff, 0x00000100,
296 0xc164, 0xffffffff, 0x00000100,
297 0x8a18, 0xffffffff, 0x00000100,
298 0x897c, 0xffffffff, 0x06000100,
299 0x8b28, 0xffffffff, 0x00000100,
300 0x9144, 0xffffffff, 0x00000100,
301 0x9a60, 0xffffffff, 0x00000100,
302 0x9868, 0xffffffff, 0x00000100,
303 0x8d58, 0xffffffff, 0x00000100,
304 0x9510, 0xffffffff, 0x00000100,
305 0x949c, 0xffffffff, 0x00000100,
306 0x9654, 0xffffffff, 0x00000100,
307 0x9030, 0xffffffff, 0x00000100,
308 0x9034, 0xffffffff, 0x00000100,
309 0x9038, 0xffffffff, 0x00000100,
310 0x903c, 0xffffffff, 0x00000100,
311 0x9040, 0xffffffff, 0x00000100,
312 0xa200, 0xffffffff, 0x00000100,
313 0xa204, 0xffffffff, 0x00000100,
314 0xa208, 0xffffffff, 0x00000100,
315 0xa20c, 0xffffffff, 0x00000100,
316 0x971c, 0xffffffff, 0x00000100,
317 0x977c, 0xffffffff, 0x00000100,
318 0x3f80, 0xffffffff, 0x00000100,
319 0xa210, 0xffffffff, 0x00000100,
320 0xa214, 0xffffffff, 0x00000100,
321 0x4d8, 0xffffffff, 0x00000100,
322 0x9784, 0xffffffff, 0x00000100,
323 0x9698, 0xffffffff, 0x00000100,
324 0x4d4, 0xffffffff, 0x00000200,
325 0x30cc, 0xffffffff, 0x00000100,
326 0xd0c0, 0xffffffff, 0xff000100,
327 0x802c, 0xffffffff, 0x40000000,
328 0x915c, 0xffffffff, 0x00010000,
329 0x9160, 0xffffffff, 0x00030002,
330 0x9178, 0xffffffff, 0x00070000,
331 0x917c, 0xffffffff, 0x00030002,
332 0x9180, 0xffffffff, 0x00050004,
333 0x918c, 0xffffffff, 0x00010006,
334 0x9190, 0xffffffff, 0x00090008,
335 0x9194, 0xffffffff, 0x00070000,
336 0x9198, 0xffffffff, 0x00030002,
337 0x919c, 0xffffffff, 0x00050004,
338 0x91a8, 0xffffffff, 0x00010006,
339 0x91ac, 0xffffffff, 0x00090008,
340 0x91b0, 0xffffffff, 0x00070000,
341 0x91b4, 0xffffffff, 0x00030002,
342 0x91b8, 0xffffffff, 0x00050004,
343 0x91c4, 0xffffffff, 0x00010006,
344 0x91c8, 0xffffffff, 0x00090008,
345 0x91cc, 0xffffffff, 0x00070000,
346 0x91d0, 0xffffffff, 0x00030002,
347 0x91d4, 0xffffffff, 0x00050004,
348 0x91e0, 0xffffffff, 0x00010006,
349 0x91e4, 0xffffffff, 0x00090008,
350 0x91e8, 0xffffffff, 0x00000000,
351 0x91ec, 0xffffffff, 0x00070000,
352 0x91f0, 0xffffffff, 0x00030002,
353 0x91f4, 0xffffffff, 0x00050004,
354 0x9200, 0xffffffff, 0x00010006,
355 0x9204, 0xffffffff, 0x00090008,
356 0x9208, 0xffffffff, 0x00070000,
357 0x920c, 0xffffffff, 0x00030002,
358 0x9210, 0xffffffff, 0x00050004,
359 0x921c, 0xffffffff, 0x00010006,
360 0x9220, 0xffffffff, 0x00090008,
361 0x9224, 0xffffffff, 0x00070000,
362 0x9228, 0xffffffff, 0x00030002,
363 0x922c, 0xffffffff, 0x00050004,
364 0x9238, 0xffffffff, 0x00010006,
365 0x923c, 0xffffffff, 0x00090008,
366 0x9240, 0xffffffff, 0x00070000,
367 0x9244, 0xffffffff, 0x00030002,
368 0x9248, 0xffffffff, 0x00050004,
369 0x9254, 0xffffffff, 0x00010006,
370 0x9258, 0xffffffff, 0x00090008,
371 0x925c, 0xffffffff, 0x00070000,
372 0x9260, 0xffffffff, 0x00030002,
373 0x9264, 0xffffffff, 0x00050004,
374 0x9270, 0xffffffff, 0x00010006,
375 0x9274, 0xffffffff, 0x00090008,
376 0x9278, 0xffffffff, 0x00070000,
377 0x927c, 0xffffffff, 0x00030002,
378 0x9280, 0xffffffff, 0x00050004,
379 0x928c, 0xffffffff, 0x00010006,
380 0x9290, 0xffffffff, 0x00090008,
381 0x9294, 0xffffffff, 0x00000000,
382 0x929c, 0xffffffff, 0x00000001,
383 0x802c, 0xffffffff, 0x40010000,
384 0x915c, 0xffffffff, 0x00010000,
385 0x9160, 0xffffffff, 0x00030002,
386 0x9178, 0xffffffff, 0x00070000,
387 0x917c, 0xffffffff, 0x00030002,
388 0x9180, 0xffffffff, 0x00050004,
389 0x918c, 0xffffffff, 0x00010006,
390 0x9190, 0xffffffff, 0x00090008,
391 0x9194, 0xffffffff, 0x00070000,
392 0x9198, 0xffffffff, 0x00030002,
393 0x919c, 0xffffffff, 0x00050004,
394 0x91a8, 0xffffffff, 0x00010006,
395 0x91ac, 0xffffffff, 0x00090008,
396 0x91b0, 0xffffffff, 0x00070000,
397 0x91b4, 0xffffffff, 0x00030002,
398 0x91b8, 0xffffffff, 0x00050004,
399 0x91c4, 0xffffffff, 0x00010006,
400 0x91c8, 0xffffffff, 0x00090008,
401 0x91cc, 0xffffffff, 0x00070000,
402 0x91d0, 0xffffffff, 0x00030002,
403 0x91d4, 0xffffffff, 0x00050004,
404 0x91e0, 0xffffffff, 0x00010006,
405 0x91e4, 0xffffffff, 0x00090008,
406 0x91e8, 0xffffffff, 0x00000000,
407 0x91ec, 0xffffffff, 0x00070000,
408 0x91f0, 0xffffffff, 0x00030002,
409 0x91f4, 0xffffffff, 0x00050004,
410 0x9200, 0xffffffff, 0x00010006,
411 0x9204, 0xffffffff, 0x00090008,
412 0x9208, 0xffffffff, 0x00070000,
413 0x920c, 0xffffffff, 0x00030002,
414 0x9210, 0xffffffff, 0x00050004,
415 0x921c, 0xffffffff, 0x00010006,
416 0x9220, 0xffffffff, 0x00090008,
417 0x9224, 0xffffffff, 0x00070000,
418 0x9228, 0xffffffff, 0x00030002,
419 0x922c, 0xffffffff, 0x00050004,
420 0x9238, 0xffffffff, 0x00010006,
421 0x923c, 0xffffffff, 0x00090008,
422 0x9240, 0xffffffff, 0x00070000,
423 0x9244, 0xffffffff, 0x00030002,
424 0x9248, 0xffffffff, 0x00050004,
425 0x9254, 0xffffffff, 0x00010006,
426 0x9258, 0xffffffff, 0x00090008,
427 0x925c, 0xffffffff, 0x00070000,
428 0x9260, 0xffffffff, 0x00030002,
429 0x9264, 0xffffffff, 0x00050004,
430 0x9270, 0xffffffff, 0x00010006,
431 0x9274, 0xffffffff, 0x00090008,
432 0x9278, 0xffffffff, 0x00070000,
433 0x927c, 0xffffffff, 0x00030002,
434 0x9280, 0xffffffff, 0x00050004,
435 0x928c, 0xffffffff, 0x00010006,
436 0x9290, 0xffffffff, 0x00090008,
437 0x9294, 0xffffffff, 0x00000000,
438 0x929c, 0xffffffff, 0x00000001,
439 0x802c, 0xffffffff, 0xc0000000
440};
441
442static const u32 redwood_mgcg_init[] =
443{
444 0x802c, 0xffffffff, 0xc0000000,
445 0x5448, 0xffffffff, 0x00000100,
446 0x55e4, 0xffffffff, 0x00000100,
447 0x160c, 0xffffffff, 0x00000100,
448 0x5644, 0xffffffff, 0x00000100,
449 0xc164, 0xffffffff, 0x00000100,
450 0x8a18, 0xffffffff, 0x00000100,
451 0x897c, 0xffffffff, 0x06000100,
452 0x8b28, 0xffffffff, 0x00000100,
453 0x9144, 0xffffffff, 0x00000100,
454 0x9a60, 0xffffffff, 0x00000100,
455 0x9868, 0xffffffff, 0x00000100,
456 0x8d58, 0xffffffff, 0x00000100,
457 0x9510, 0xffffffff, 0x00000100,
458 0x949c, 0xffffffff, 0x00000100,
459 0x9654, 0xffffffff, 0x00000100,
460 0x9030, 0xffffffff, 0x00000100,
461 0x9034, 0xffffffff, 0x00000100,
462 0x9038, 0xffffffff, 0x00000100,
463 0x903c, 0xffffffff, 0x00000100,
464 0x9040, 0xffffffff, 0x00000100,
465 0xa200, 0xffffffff, 0x00000100,
466 0xa204, 0xffffffff, 0x00000100,
467 0xa208, 0xffffffff, 0x00000100,
468 0xa20c, 0xffffffff, 0x00000100,
469 0x971c, 0xffffffff, 0x00000100,
470 0x977c, 0xffffffff, 0x00000100,
471 0x3f80, 0xffffffff, 0x00000100,
472 0xa210, 0xffffffff, 0x00000100,
473 0xa214, 0xffffffff, 0x00000100,
474 0x4d8, 0xffffffff, 0x00000100,
475 0x9784, 0xffffffff, 0x00000100,
476 0x9698, 0xffffffff, 0x00000100,
477 0x4d4, 0xffffffff, 0x00000200,
478 0x30cc, 0xffffffff, 0x00000100,
479 0xd0c0, 0xffffffff, 0xff000100,
480 0x802c, 0xffffffff, 0x40000000,
481 0x915c, 0xffffffff, 0x00010000,
482 0x9160, 0xffffffff, 0x00030002,
483 0x9178, 0xffffffff, 0x00070000,
484 0x917c, 0xffffffff, 0x00030002,
485 0x9180, 0xffffffff, 0x00050004,
486 0x918c, 0xffffffff, 0x00010006,
487 0x9190, 0xffffffff, 0x00090008,
488 0x9194, 0xffffffff, 0x00070000,
489 0x9198, 0xffffffff, 0x00030002,
490 0x919c, 0xffffffff, 0x00050004,
491 0x91a8, 0xffffffff, 0x00010006,
492 0x91ac, 0xffffffff, 0x00090008,
493 0x91b0, 0xffffffff, 0x00070000,
494 0x91b4, 0xffffffff, 0x00030002,
495 0x91b8, 0xffffffff, 0x00050004,
496 0x91c4, 0xffffffff, 0x00010006,
497 0x91c8, 0xffffffff, 0x00090008,
498 0x91cc, 0xffffffff, 0x00070000,
499 0x91d0, 0xffffffff, 0x00030002,
500 0x91d4, 0xffffffff, 0x00050004,
501 0x91e0, 0xffffffff, 0x00010006,
502 0x91e4, 0xffffffff, 0x00090008,
503 0x91e8, 0xffffffff, 0x00000000,
504 0x91ec, 0xffffffff, 0x00070000,
505 0x91f0, 0xffffffff, 0x00030002,
506 0x91f4, 0xffffffff, 0x00050004,
507 0x9200, 0xffffffff, 0x00010006,
508 0x9204, 0xffffffff, 0x00090008,
509 0x9294, 0xffffffff, 0x00000000,
510 0x929c, 0xffffffff, 0x00000001,
511 0x802c, 0xffffffff, 0xc0000000
512};
513
514static const u32 cedar_golden_registers[] =
515{
516 0x3f90, 0xffff0000, 0xff000000,
517 0x9148, 0xffff0000, 0xff000000,
518 0x3f94, 0xffff0000, 0xff000000,
519 0x914c, 0xffff0000, 0xff000000,
520 0x9b7c, 0xffffffff, 0x00000000,
521 0x8a14, 0xffffffff, 0x00000007,
522 0x8b10, 0xffffffff, 0x00000000,
523 0x960c, 0xffffffff, 0x54763210,
524 0x88c4, 0xffffffff, 0x000000c2,
525 0x88d4, 0xffffffff, 0x00000000,
526 0x8974, 0xffffffff, 0x00000000,
527 0xc78, 0x00000080, 0x00000080,
528 0x5eb4, 0xffffffff, 0x00000002,
529 0x5e78, 0xffffffff, 0x001000f0,
530 0x6104, 0x01000300, 0x00000000,
531 0x5bc0, 0x00300000, 0x00000000,
532 0x7030, 0xffffffff, 0x00000011,
533 0x7c30, 0xffffffff, 0x00000011,
534 0x10830, 0xffffffff, 0x00000011,
535 0x11430, 0xffffffff, 0x00000011,
536 0xd02c, 0xffffffff, 0x08421000,
537 0x240c, 0xffffffff, 0x00000380,
538 0x8b24, 0xffffffff, 0x00ff0fff,
539 0x28a4c, 0x06000000, 0x06000000,
540 0x10c, 0x00000001, 0x00000001,
541 0x8d00, 0xffffffff, 0x100e4848,
542 0x8d04, 0xffffffff, 0x00164745,
543 0x8c00, 0xffffffff, 0xe4000003,
544 0x8c04, 0xffffffff, 0x40600060,
545 0x8c08, 0xffffffff, 0x001c001c,
546 0x8cf0, 0xffffffff, 0x08e00410,
547 0x8c20, 0xffffffff, 0x00800080,
548 0x8c24, 0xffffffff, 0x00800080,
549 0x8c18, 0xffffffff, 0x20202078,
550 0x8c1c, 0xffffffff, 0x00001010,
551 0x28350, 0xffffffff, 0x00000000,
552 0xa008, 0xffffffff, 0x00010000,
553 0x5c4, 0xffffffff, 0x00000001,
554 0x9508, 0xffffffff, 0x00000002
555};
556
557static const u32 cedar_mgcg_init[] =
558{
559 0x802c, 0xffffffff, 0xc0000000,
560 0x5448, 0xffffffff, 0x00000100,
561 0x55e4, 0xffffffff, 0x00000100,
562 0x160c, 0xffffffff, 0x00000100,
563 0x5644, 0xffffffff, 0x00000100,
564 0xc164, 0xffffffff, 0x00000100,
565 0x8a18, 0xffffffff, 0x00000100,
566 0x897c, 0xffffffff, 0x06000100,
567 0x8b28, 0xffffffff, 0x00000100,
568 0x9144, 0xffffffff, 0x00000100,
569 0x9a60, 0xffffffff, 0x00000100,
570 0x9868, 0xffffffff, 0x00000100,
571 0x8d58, 0xffffffff, 0x00000100,
572 0x9510, 0xffffffff, 0x00000100,
573 0x949c, 0xffffffff, 0x00000100,
574 0x9654, 0xffffffff, 0x00000100,
575 0x9030, 0xffffffff, 0x00000100,
576 0x9034, 0xffffffff, 0x00000100,
577 0x9038, 0xffffffff, 0x00000100,
578 0x903c, 0xffffffff, 0x00000100,
579 0x9040, 0xffffffff, 0x00000100,
580 0xa200, 0xffffffff, 0x00000100,
581 0xa204, 0xffffffff, 0x00000100,
582 0xa208, 0xffffffff, 0x00000100,
583 0xa20c, 0xffffffff, 0x00000100,
584 0x971c, 0xffffffff, 0x00000100,
585 0x977c, 0xffffffff, 0x00000100,
586 0x3f80, 0xffffffff, 0x00000100,
587 0xa210, 0xffffffff, 0x00000100,
588 0xa214, 0xffffffff, 0x00000100,
589 0x4d8, 0xffffffff, 0x00000100,
590 0x9784, 0xffffffff, 0x00000100,
591 0x9698, 0xffffffff, 0x00000100,
592 0x4d4, 0xffffffff, 0x00000200,
593 0x30cc, 0xffffffff, 0x00000100,
594 0xd0c0, 0xffffffff, 0xff000100,
595 0x802c, 0xffffffff, 0x40000000,
596 0x915c, 0xffffffff, 0x00010000,
597 0x9178, 0xffffffff, 0x00050000,
598 0x917c, 0xffffffff, 0x00030002,
599 0x918c, 0xffffffff, 0x00010004,
600 0x9190, 0xffffffff, 0x00070006,
601 0x9194, 0xffffffff, 0x00050000,
602 0x9198, 0xffffffff, 0x00030002,
603 0x91a8, 0xffffffff, 0x00010004,
604 0x91ac, 0xffffffff, 0x00070006,
605 0x91e8, 0xffffffff, 0x00000000,
606 0x9294, 0xffffffff, 0x00000000,
607 0x929c, 0xffffffff, 0x00000001,
608 0x802c, 0xffffffff, 0xc0000000
609};
610
611static const u32 juniper_mgcg_init[] =
612{
613 0x802c, 0xffffffff, 0xc0000000,
614 0x5448, 0xffffffff, 0x00000100,
615 0x55e4, 0xffffffff, 0x00000100,
616 0x160c, 0xffffffff, 0x00000100,
617 0x5644, 0xffffffff, 0x00000100,
618 0xc164, 0xffffffff, 0x00000100,
619 0x8a18, 0xffffffff, 0x00000100,
620 0x897c, 0xffffffff, 0x06000100,
621 0x8b28, 0xffffffff, 0x00000100,
622 0x9144, 0xffffffff, 0x00000100,
623 0x9a60, 0xffffffff, 0x00000100,
624 0x9868, 0xffffffff, 0x00000100,
625 0x8d58, 0xffffffff, 0x00000100,
626 0x9510, 0xffffffff, 0x00000100,
627 0x949c, 0xffffffff, 0x00000100,
628 0x9654, 0xffffffff, 0x00000100,
629 0x9030, 0xffffffff, 0x00000100,
630 0x9034, 0xffffffff, 0x00000100,
631 0x9038, 0xffffffff, 0x00000100,
632 0x903c, 0xffffffff, 0x00000100,
633 0x9040, 0xffffffff, 0x00000100,
634 0xa200, 0xffffffff, 0x00000100,
635 0xa204, 0xffffffff, 0x00000100,
636 0xa208, 0xffffffff, 0x00000100,
637 0xa20c, 0xffffffff, 0x00000100,
638 0x971c, 0xffffffff, 0x00000100,
639 0xd0c0, 0xffffffff, 0xff000100,
640 0x802c, 0xffffffff, 0x40000000,
641 0x915c, 0xffffffff, 0x00010000,
642 0x9160, 0xffffffff, 0x00030002,
643 0x9178, 0xffffffff, 0x00070000,
644 0x917c, 0xffffffff, 0x00030002,
645 0x9180, 0xffffffff, 0x00050004,
646 0x918c, 0xffffffff, 0x00010006,
647 0x9190, 0xffffffff, 0x00090008,
648 0x9194, 0xffffffff, 0x00070000,
649 0x9198, 0xffffffff, 0x00030002,
650 0x919c, 0xffffffff, 0x00050004,
651 0x91a8, 0xffffffff, 0x00010006,
652 0x91ac, 0xffffffff, 0x00090008,
653 0x91b0, 0xffffffff, 0x00070000,
654 0x91b4, 0xffffffff, 0x00030002,
655 0x91b8, 0xffffffff, 0x00050004,
656 0x91c4, 0xffffffff, 0x00010006,
657 0x91c8, 0xffffffff, 0x00090008,
658 0x91cc, 0xffffffff, 0x00070000,
659 0x91d0, 0xffffffff, 0x00030002,
660 0x91d4, 0xffffffff, 0x00050004,
661 0x91e0, 0xffffffff, 0x00010006,
662 0x91e4, 0xffffffff, 0x00090008,
663 0x91e8, 0xffffffff, 0x00000000,
664 0x91ec, 0xffffffff, 0x00070000,
665 0x91f0, 0xffffffff, 0x00030002,
666 0x91f4, 0xffffffff, 0x00050004,
667 0x9200, 0xffffffff, 0x00010006,
668 0x9204, 0xffffffff, 0x00090008,
669 0x9208, 0xffffffff, 0x00070000,
670 0x920c, 0xffffffff, 0x00030002,
671 0x9210, 0xffffffff, 0x00050004,
672 0x921c, 0xffffffff, 0x00010006,
673 0x9220, 0xffffffff, 0x00090008,
674 0x9224, 0xffffffff, 0x00070000,
675 0x9228, 0xffffffff, 0x00030002,
676 0x922c, 0xffffffff, 0x00050004,
677 0x9238, 0xffffffff, 0x00010006,
678 0x923c, 0xffffffff, 0x00090008,
679 0x9240, 0xffffffff, 0x00070000,
680 0x9244, 0xffffffff, 0x00030002,
681 0x9248, 0xffffffff, 0x00050004,
682 0x9254, 0xffffffff, 0x00010006,
683 0x9258, 0xffffffff, 0x00090008,
684 0x925c, 0xffffffff, 0x00070000,
685 0x9260, 0xffffffff, 0x00030002,
686 0x9264, 0xffffffff, 0x00050004,
687 0x9270, 0xffffffff, 0x00010006,
688 0x9274, 0xffffffff, 0x00090008,
689 0x9278, 0xffffffff, 0x00070000,
690 0x927c, 0xffffffff, 0x00030002,
691 0x9280, 0xffffffff, 0x00050004,
692 0x928c, 0xffffffff, 0x00010006,
693 0x9290, 0xffffffff, 0x00090008,
694 0x9294, 0xffffffff, 0x00000000,
695 0x929c, 0xffffffff, 0x00000001,
696 0x802c, 0xffffffff, 0xc0000000,
697 0x977c, 0xffffffff, 0x00000100,
698 0x3f80, 0xffffffff, 0x00000100,
699 0xa210, 0xffffffff, 0x00000100,
700 0xa214, 0xffffffff, 0x00000100,
701 0x4d8, 0xffffffff, 0x00000100,
702 0x9784, 0xffffffff, 0x00000100,
703 0x9698, 0xffffffff, 0x00000100,
704 0x4d4, 0xffffffff, 0x00000200,
705 0x30cc, 0xffffffff, 0x00000100,
706 0x802c, 0xffffffff, 0xc0000000
707};
708
709static const u32 supersumo_golden_registers[] =
710{
711 0x5eb4, 0xffffffff, 0x00000002,
712 0x5c4, 0xffffffff, 0x00000001,
713 0x7030, 0xffffffff, 0x00000011,
714 0x7c30, 0xffffffff, 0x00000011,
715 0x6104, 0x01000300, 0x00000000,
716 0x5bc0, 0x00300000, 0x00000000,
717 0x8c04, 0xffffffff, 0x40600060,
718 0x8c08, 0xffffffff, 0x001c001c,
719 0x8c20, 0xffffffff, 0x00800080,
720 0x8c24, 0xffffffff, 0x00800080,
721 0x8c18, 0xffffffff, 0x20202078,
722 0x8c1c, 0xffffffff, 0x00001010,
723 0x918c, 0xffffffff, 0x00010006,
724 0x91a8, 0xffffffff, 0x00010006,
725 0x91c4, 0xffffffff, 0x00010006,
726 0x91e0, 0xffffffff, 0x00010006,
727 0x9200, 0xffffffff, 0x00010006,
728 0x9150, 0xffffffff, 0x6e944040,
729 0x917c, 0xffffffff, 0x00030002,
730 0x9180, 0xffffffff, 0x00050004,
731 0x9198, 0xffffffff, 0x00030002,
732 0x919c, 0xffffffff, 0x00050004,
733 0x91b4, 0xffffffff, 0x00030002,
734 0x91b8, 0xffffffff, 0x00050004,
735 0x91d0, 0xffffffff, 0x00030002,
736 0x91d4, 0xffffffff, 0x00050004,
737 0x91f0, 0xffffffff, 0x00030002,
738 0x91f4, 0xffffffff, 0x00050004,
739 0x915c, 0xffffffff, 0x00010000,
740 0x9160, 0xffffffff, 0x00030002,
741 0x3f90, 0xffff0000, 0xff000000,
742 0x9178, 0xffffffff, 0x00070000,
743 0x9194, 0xffffffff, 0x00070000,
744 0x91b0, 0xffffffff, 0x00070000,
745 0x91cc, 0xffffffff, 0x00070000,
746 0x91ec, 0xffffffff, 0x00070000,
747 0x9148, 0xffff0000, 0xff000000,
748 0x9190, 0xffffffff, 0x00090008,
749 0x91ac, 0xffffffff, 0x00090008,
750 0x91c8, 0xffffffff, 0x00090008,
751 0x91e4, 0xffffffff, 0x00090008,
752 0x9204, 0xffffffff, 0x00090008,
753 0x3f94, 0xffff0000, 0xff000000,
754 0x914c, 0xffff0000, 0xff000000,
755 0x929c, 0xffffffff, 0x00000001,
756 0x8a18, 0xffffffff, 0x00000100,
757 0x8b28, 0xffffffff, 0x00000100,
758 0x9144, 0xffffffff, 0x00000100,
759 0x5644, 0xffffffff, 0x00000100,
760 0x9b7c, 0xffffffff, 0x00000000,
761 0x8030, 0xffffffff, 0x0000100a,
762 0x8a14, 0xffffffff, 0x00000007,
763 0x8b24, 0xffffffff, 0x00ff0fff,
764 0x8b10, 0xffffffff, 0x00000000,
765 0x28a4c, 0x06000000, 0x06000000,
766 0x4d8, 0xffffffff, 0x00000100,
767 0x913c, 0xffff000f, 0x0100000a,
768 0x960c, 0xffffffff, 0x54763210,
769 0x88c4, 0xffffffff, 0x000000c2,
770 0x88d4, 0xffffffff, 0x00000010,
771 0x8974, 0xffffffff, 0x00000000,
772 0xc78, 0x00000080, 0x00000080,
773 0x5e78, 0xffffffff, 0x001000f0,
774 0xd02c, 0xffffffff, 0x08421000,
775 0xa008, 0xffffffff, 0x00010000,
776 0x8d00, 0xffffffff, 0x100e4848,
777 0x8d04, 0xffffffff, 0x00164745,
778 0x8c00, 0xffffffff, 0xe4000003,
779 0x8cf0, 0x1fffffff, 0x08e00620,
780 0x28350, 0xffffffff, 0x00000000,
781 0x9508, 0xffffffff, 0x00000002
782};
783
784static const u32 sumo_golden_registers[] =
785{
786 0x900c, 0x00ffffff, 0x0017071f,
787 0x8c18, 0xffffffff, 0x10101060,
788 0x8c1c, 0xffffffff, 0x00001010,
789 0x8c30, 0x0000000f, 0x00000005,
790 0x9688, 0x0000000f, 0x00000007
791};
792
793static const u32 wrestler_golden_registers[] =
794{
795 0x5eb4, 0xffffffff, 0x00000002,
796 0x5c4, 0xffffffff, 0x00000001,
797 0x7030, 0xffffffff, 0x00000011,
798 0x7c30, 0xffffffff, 0x00000011,
799 0x6104, 0x01000300, 0x00000000,
800 0x5bc0, 0x00300000, 0x00000000,
801 0x918c, 0xffffffff, 0x00010006,
802 0x91a8, 0xffffffff, 0x00010006,
803 0x9150, 0xffffffff, 0x6e944040,
804 0x917c, 0xffffffff, 0x00030002,
805 0x9198, 0xffffffff, 0x00030002,
806 0x915c, 0xffffffff, 0x00010000,
807 0x3f90, 0xffff0000, 0xff000000,
808 0x9178, 0xffffffff, 0x00070000,
809 0x9194, 0xffffffff, 0x00070000,
810 0x9148, 0xffff0000, 0xff000000,
811 0x9190, 0xffffffff, 0x00090008,
812 0x91ac, 0xffffffff, 0x00090008,
813 0x3f94, 0xffff0000, 0xff000000,
814 0x914c, 0xffff0000, 0xff000000,
815 0x929c, 0xffffffff, 0x00000001,
816 0x8a18, 0xffffffff, 0x00000100,
817 0x8b28, 0xffffffff, 0x00000100,
818 0x9144, 0xffffffff, 0x00000100,
819 0x9b7c, 0xffffffff, 0x00000000,
820 0x8030, 0xffffffff, 0x0000100a,
821 0x8a14, 0xffffffff, 0x00000001,
822 0x8b24, 0xffffffff, 0x00ff0fff,
823 0x8b10, 0xffffffff, 0x00000000,
824 0x28a4c, 0x06000000, 0x06000000,
825 0x4d8, 0xffffffff, 0x00000100,
826 0x913c, 0xffff000f, 0x0100000a,
827 0x960c, 0xffffffff, 0x54763210,
828 0x88c4, 0xffffffff, 0x000000c2,
829 0x88d4, 0xffffffff, 0x00000010,
830 0x8974, 0xffffffff, 0x00000000,
831 0xc78, 0x00000080, 0x00000080,
832 0x5e78, 0xffffffff, 0x001000f0,
833 0xd02c, 0xffffffff, 0x08421000,
834 0xa008, 0xffffffff, 0x00010000,
835 0x8d00, 0xffffffff, 0x100e4848,
836 0x8d04, 0xffffffff, 0x00164745,
837 0x8c00, 0xffffffff, 0xe4000003,
838 0x8cf0, 0x1fffffff, 0x08e00410,
839 0x28350, 0xffffffff, 0x00000000,
840 0x9508, 0xffffffff, 0x00000002,
841 0x900c, 0xffffffff, 0x0017071f,
842 0x8c18, 0xffffffff, 0x10101060,
843 0x8c1c, 0xffffffff, 0x00001010
844};
845
846static const u32 barts_golden_registers[] =
847{
848 0x5eb4, 0xffffffff, 0x00000002,
849 0x5e78, 0x8f311ff1, 0x001000f0,
850 0x3f90, 0xffff0000, 0xff000000,
851 0x9148, 0xffff0000, 0xff000000,
852 0x3f94, 0xffff0000, 0xff000000,
853 0x914c, 0xffff0000, 0xff000000,
854 0xc78, 0x00000080, 0x00000080,
855 0xbd4, 0x70073777, 0x00010001,
856 0xd02c, 0xbfffff1f, 0x08421000,
857 0xd0b8, 0x03773777, 0x02011003,
858 0x5bc0, 0x00200000, 0x50100000,
859 0x98f8, 0x33773777, 0x02011003,
860 0x98fc, 0xffffffff, 0x76543210,
861 0x7030, 0x31000311, 0x00000011,
862 0x2f48, 0x00000007, 0x02011003,
863 0x6b28, 0x00000010, 0x00000012,
864 0x7728, 0x00000010, 0x00000012,
865 0x10328, 0x00000010, 0x00000012,
866 0x10f28, 0x00000010, 0x00000012,
867 0x11b28, 0x00000010, 0x00000012,
868 0x12728, 0x00000010, 0x00000012,
869 0x240c, 0x000007ff, 0x00000380,
870 0x8a14, 0xf000001f, 0x00000007,
871 0x8b24, 0x3fff3fff, 0x00ff0fff,
872 0x8b10, 0x0000ff0f, 0x00000000,
873 0x28a4c, 0x07ffffff, 0x06000000,
874 0x10c, 0x00000001, 0x00010003,
875 0xa02c, 0xffffffff, 0x0000009b,
876 0x913c, 0x0000000f, 0x0100000a,
877 0x8d00, 0xffff7f7f, 0x100e4848,
878 0x8d04, 0x00ffffff, 0x00164745,
879 0x8c00, 0xfffc0003, 0xe4000003,
880 0x8c04, 0xf8ff00ff, 0x40600060,
881 0x8c08, 0x00ff00ff, 0x001c001c,
882 0x8cf0, 0x1fff1fff, 0x08e00620,
883 0x8c20, 0x0fff0fff, 0x00800080,
884 0x8c24, 0x0fff0fff, 0x00800080,
885 0x8c18, 0xffffffff, 0x20202078,
886 0x8c1c, 0x0000ffff, 0x00001010,
887 0x28350, 0x00000f01, 0x00000000,
888 0x9508, 0x3700001f, 0x00000002,
889 0x960c, 0xffffffff, 0x54763210,
890 0x88c4, 0x001f3ae3, 0x000000c2,
891 0x88d4, 0x0000001f, 0x00000010,
892 0x8974, 0xffffffff, 0x00000000
893};
894
895static const u32 turks_golden_registers[] =
896{
897 0x5eb4, 0xffffffff, 0x00000002,
898 0x5e78, 0x8f311ff1, 0x001000f0,
899 0x8c8, 0x00003000, 0x00001070,
900 0x8cc, 0x000fffff, 0x00040035,
901 0x3f90, 0xffff0000, 0xfff00000,
902 0x9148, 0xffff0000, 0xfff00000,
903 0x3f94, 0xffff0000, 0xfff00000,
904 0x914c, 0xffff0000, 0xfff00000,
905 0xc78, 0x00000080, 0x00000080,
906 0xbd4, 0x00073007, 0x00010002,
907 0xd02c, 0xbfffff1f, 0x08421000,
908 0xd0b8, 0x03773777, 0x02010002,
909 0x5bc0, 0x00200000, 0x50100000,
910 0x98f8, 0x33773777, 0x00010002,
911 0x98fc, 0xffffffff, 0x33221100,
912 0x7030, 0x31000311, 0x00000011,
913 0x2f48, 0x33773777, 0x00010002,
914 0x6b28, 0x00000010, 0x00000012,
915 0x7728, 0x00000010, 0x00000012,
916 0x10328, 0x00000010, 0x00000012,
917 0x10f28, 0x00000010, 0x00000012,
918 0x11b28, 0x00000010, 0x00000012,
919 0x12728, 0x00000010, 0x00000012,
920 0x240c, 0x000007ff, 0x00000380,
921 0x8a14, 0xf000001f, 0x00000007,
922 0x8b24, 0x3fff3fff, 0x00ff0fff,
923 0x8b10, 0x0000ff0f, 0x00000000,
924 0x28a4c, 0x07ffffff, 0x06000000,
925 0x10c, 0x00000001, 0x00010003,
926 0xa02c, 0xffffffff, 0x0000009b,
927 0x913c, 0x0000000f, 0x0100000a,
928 0x8d00, 0xffff7f7f, 0x100e4848,
929 0x8d04, 0x00ffffff, 0x00164745,
930 0x8c00, 0xfffc0003, 0xe4000003,
931 0x8c04, 0xf8ff00ff, 0x40600060,
932 0x8c08, 0x00ff00ff, 0x001c001c,
933 0x8cf0, 0x1fff1fff, 0x08e00410,
934 0x8c20, 0x0fff0fff, 0x00800080,
935 0x8c24, 0x0fff0fff, 0x00800080,
936 0x8c18, 0xffffffff, 0x20202078,
937 0x8c1c, 0x0000ffff, 0x00001010,
938 0x28350, 0x00000f01, 0x00000000,
939 0x9508, 0x3700001f, 0x00000002,
940 0x960c, 0xffffffff, 0x54763210,
941 0x88c4, 0x001f3ae3, 0x000000c2,
942 0x88d4, 0x0000001f, 0x00000010,
943 0x8974, 0xffffffff, 0x00000000
944};
945
946static const u32 caicos_golden_registers[] =
947{
948 0x5eb4, 0xffffffff, 0x00000002,
949 0x5e78, 0x8f311ff1, 0x001000f0,
950 0x8c8, 0x00003420, 0x00001450,
951 0x8cc, 0x000fffff, 0x00040035,
952 0x3f90, 0xffff0000, 0xfffc0000,
953 0x9148, 0xffff0000, 0xfffc0000,
954 0x3f94, 0xffff0000, 0xfffc0000,
955 0x914c, 0xffff0000, 0xfffc0000,
956 0xc78, 0x00000080, 0x00000080,
957 0xbd4, 0x00073007, 0x00010001,
958 0xd02c, 0xbfffff1f, 0x08421000,
959 0xd0b8, 0x03773777, 0x02010001,
960 0x5bc0, 0x00200000, 0x50100000,
961 0x98f8, 0x33773777, 0x02010001,
962 0x98fc, 0xffffffff, 0x33221100,
963 0x7030, 0x31000311, 0x00000011,
964 0x2f48, 0x33773777, 0x02010001,
965 0x6b28, 0x00000010, 0x00000012,
966 0x7728, 0x00000010, 0x00000012,
967 0x10328, 0x00000010, 0x00000012,
968 0x10f28, 0x00000010, 0x00000012,
969 0x11b28, 0x00000010, 0x00000012,
970 0x12728, 0x00000010, 0x00000012,
971 0x240c, 0x000007ff, 0x00000380,
972 0x8a14, 0xf000001f, 0x00000001,
973 0x8b24, 0x3fff3fff, 0x00ff0fff,
974 0x8b10, 0x0000ff0f, 0x00000000,
975 0x28a4c, 0x07ffffff, 0x06000000,
976 0x10c, 0x00000001, 0x00010003,
977 0xa02c, 0xffffffff, 0x0000009b,
978 0x913c, 0x0000000f, 0x0100000a,
979 0x8d00, 0xffff7f7f, 0x100e4848,
980 0x8d04, 0x00ffffff, 0x00164745,
981 0x8c00, 0xfffc0003, 0xe4000003,
982 0x8c04, 0xf8ff00ff, 0x40600060,
983 0x8c08, 0x00ff00ff, 0x001c001c,
984 0x8cf0, 0x1fff1fff, 0x08e00410,
985 0x8c20, 0x0fff0fff, 0x00800080,
986 0x8c24, 0x0fff0fff, 0x00800080,
987 0x8c18, 0xffffffff, 0x20202078,
988 0x8c1c, 0x0000ffff, 0x00001010,
989 0x28350, 0x00000f01, 0x00000000,
990 0x9508, 0x3700001f, 0x00000002,
991 0x960c, 0xffffffff, 0x54763210,
992 0x88c4, 0x001f3ae3, 0x000000c2,
993 0x88d4, 0x0000001f, 0x00000010,
994 0x8974, 0xffffffff, 0x00000000
995};
996
997static void evergreen_init_golden_registers(struct radeon_device *rdev)
998{
999 switch (rdev->family) {
1000 case CHIP_CYPRESS:
1001 case CHIP_HEMLOCK:
1002 radeon_program_register_sequence(rdev,
1003 evergreen_golden_registers,
1004 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1005 radeon_program_register_sequence(rdev,
1006 evergreen_golden_registers2,
1007 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1008 radeon_program_register_sequence(rdev,
1009 cypress_mgcg_init,
1010 (const u32)ARRAY_SIZE(cypress_mgcg_init));
1011 break;
1012 case CHIP_JUNIPER:
1013 radeon_program_register_sequence(rdev,
1014 evergreen_golden_registers,
1015 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1016 radeon_program_register_sequence(rdev,
1017 evergreen_golden_registers2,
1018 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1019 radeon_program_register_sequence(rdev,
1020 juniper_mgcg_init,
1021 (const u32)ARRAY_SIZE(juniper_mgcg_init));
1022 break;
1023 case CHIP_REDWOOD:
1024 radeon_program_register_sequence(rdev,
1025 evergreen_golden_registers,
1026 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1027 radeon_program_register_sequence(rdev,
1028 evergreen_golden_registers2,
1029 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1030 radeon_program_register_sequence(rdev,
1031 redwood_mgcg_init,
1032 (const u32)ARRAY_SIZE(redwood_mgcg_init));
1033 break;
1034 case CHIP_CEDAR:
1035 radeon_program_register_sequence(rdev,
1036 cedar_golden_registers,
1037 (const u32)ARRAY_SIZE(cedar_golden_registers));
1038 radeon_program_register_sequence(rdev,
1039 evergreen_golden_registers2,
1040 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1041 radeon_program_register_sequence(rdev,
1042 cedar_mgcg_init,
1043 (const u32)ARRAY_SIZE(cedar_mgcg_init));
1044 break;
1045 case CHIP_PALM:
1046 radeon_program_register_sequence(rdev,
1047 wrestler_golden_registers,
1048 (const u32)ARRAY_SIZE(wrestler_golden_registers));
1049 break;
1050 case CHIP_SUMO:
1051 radeon_program_register_sequence(rdev,
1052 supersumo_golden_registers,
1053 (const u32)ARRAY_SIZE(supersumo_golden_registers));
1054 break;
1055 case CHIP_SUMO2:
1056 radeon_program_register_sequence(rdev,
1057 supersumo_golden_registers,
1058 (const u32)ARRAY_SIZE(supersumo_golden_registers));
1059 radeon_program_register_sequence(rdev,
1060 sumo_golden_registers,
1061 (const u32)ARRAY_SIZE(sumo_golden_registers));
1062 break;
1063 case CHIP_BARTS:
1064 radeon_program_register_sequence(rdev,
1065 barts_golden_registers,
1066 (const u32)ARRAY_SIZE(barts_golden_registers));
1067 break;
1068 case CHIP_TURKS:
1069 radeon_program_register_sequence(rdev,
1070 turks_golden_registers,
1071 (const u32)ARRAY_SIZE(turks_golden_registers));
1072 break;
1073 case CHIP_CAICOS:
1074 radeon_program_register_sequence(rdev,
1075 caicos_golden_registers,
1076 (const u32)ARRAY_SIZE(caicos_golden_registers));
1077 break;
1078 default:
1079 break;
1080 }
1081}
1082
1083/**
1084 * evergreen_get_allowed_info_register - fetch the register for the info ioctl
1085 *
1086 * @rdev: radeon_device pointer
1087 * @reg: register offset in bytes
1088 * @val: register value
1089 *
1090 * Returns 0 for success or -EINVAL for an invalid register
1091 *
1092 */
1093int evergreen_get_allowed_info_register(struct radeon_device *rdev,
1094 u32 reg, u32 *val)
1095{
1096 switch (reg) {
1097 case GRBM_STATUS:
1098 case GRBM_STATUS_SE0:
1099 case GRBM_STATUS_SE1:
1100 case SRBM_STATUS:
1101 case SRBM_STATUS2:
1102 case DMA_STATUS_REG:
1103 case UVD_STATUS:
1104 *val = RREG32(reg);
1105 return 0;
1106 default:
1107 return -EINVAL;
1108 }
1109}
1110
1111void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1112 unsigned *bankh, unsigned *mtaspect,
1113 unsigned *tile_split)
1114{
1115 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1116 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1117 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1118 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1119 switch (*bankw) {
1120 default:
1121 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1122 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1123 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1124 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1125 }
1126 switch (*bankh) {
1127 default:
1128 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1129 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1130 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1131 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1132 }
1133 switch (*mtaspect) {
1134 default:
1135 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1136 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1137 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1138 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1139 }
1140}
1141
1142static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1143 u32 cntl_reg, u32 status_reg)
1144{
1145 int r, i;
1146 struct atom_clock_dividers dividers;
1147
1148 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1149 clock, false, ÷rs);
1150 if (r)
1151 return r;
1152
1153 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1154
1155 for (i = 0; i < 100; i++) {
1156 if (RREG32(status_reg) & DCLK_STATUS)
1157 break;
1158 mdelay(10);
1159 }
1160 if (i == 100)
1161 return -ETIMEDOUT;
1162
1163 return 0;
1164}
1165
1166int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1167{
1168 int r = 0;
1169 u32 cg_scratch = RREG32(CG_SCRATCH1);
1170
1171 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1172 if (r)
1173 goto done;
1174 cg_scratch &= 0xffff0000;
1175 cg_scratch |= vclk / 100; /* Mhz */
1176
1177 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1178 if (r)
1179 goto done;
1180 cg_scratch &= 0x0000ffff;
1181 cg_scratch |= (dclk / 100) << 16; /* Mhz */
1182
1183done:
1184 WREG32(CG_SCRATCH1, cg_scratch);
1185
1186 return r;
1187}
1188
1189int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1190{
1191 /* start off with something large */
1192 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
1193 int r;
1194
1195 /* bypass vclk and dclk with bclk */
1196 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1197 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1198 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1199
1200 /* put PLL in bypass mode */
1201 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1202
1203 if (!vclk || !dclk) {
1204 /* keep the Bypass mode, put PLL to sleep */
1205 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1206 return 0;
1207 }
1208
1209 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1210 16384, 0x03FFFFFF, 0, 128, 5,
1211 &fb_div, &vclk_div, &dclk_div);
1212 if (r)
1213 return r;
1214
1215 /* set VCO_MODE to 1 */
1216 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1217
1218 /* toggle UPLL_SLEEP to 1 then back to 0 */
1219 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1220 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1221
1222 /* deassert UPLL_RESET */
1223 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1224
1225 mdelay(1);
1226
1227 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1228 if (r)
1229 return r;
1230
1231 /* assert UPLL_RESET again */
1232 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1233
1234 /* disable spread spectrum. */
1235 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1236
1237 /* set feedback divider */
1238 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
1239
1240 /* set ref divider to 0 */
1241 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1242
1243 if (fb_div < 307200)
1244 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1245 else
1246 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1247
1248 /* set PDIV_A and PDIV_B */
1249 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1250 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
1251 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1252
1253 /* give the PLL some time to settle */
1254 mdelay(15);
1255
1256 /* deassert PLL_RESET */
1257 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1258
1259 mdelay(15);
1260
1261 /* switch from bypass mode to normal mode */
1262 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1263
1264 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1265 if (r)
1266 return r;
1267
1268 /* switch VCLK and DCLK selection */
1269 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1270 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1271 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1272
1273 mdelay(100);
1274
1275 return 0;
1276}
1277
1278void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1279{
1280 int readrq;
1281 u16 v;
1282
1283 readrq = pcie_get_readrq(rdev->pdev);
1284 v = ffs(readrq) - 8;
1285 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1286 * to avoid hangs or perfomance issues
1287 */
1288 if ((v == 0) || (v == 6) || (v == 7))
1289 pcie_set_readrq(rdev->pdev, 512);
1290}
1291
1292void dce4_program_fmt(struct drm_encoder *encoder)
1293{
1294 struct drm_device *dev = encoder->dev;
1295 struct radeon_device *rdev = dev->dev_private;
1296 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
1297 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
1298 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
1299 int bpc = 0;
1300 u32 tmp = 0;
1301 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
1302
1303 if (connector) {
1304 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1305 bpc = radeon_get_monitor_bpc(connector);
1306 dither = radeon_connector->dither;
1307 }
1308
1309 /* LVDS/eDP FMT is set up by atom */
1310 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
1311 return;
1312
1313 /* not needed for analog */
1314 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
1315 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
1316 return;
1317
1318 if (bpc == 0)
1319 return;
1320
1321 switch (bpc) {
1322 case 6:
1323 if (dither == RADEON_FMT_DITHER_ENABLE)
1324 /* XXX sort out optimal dither settings */
1325 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1326 FMT_SPATIAL_DITHER_EN);
1327 else
1328 tmp |= FMT_TRUNCATE_EN;
1329 break;
1330 case 8:
1331 if (dither == RADEON_FMT_DITHER_ENABLE)
1332 /* XXX sort out optimal dither settings */
1333 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1334 FMT_RGB_RANDOM_ENABLE |
1335 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
1336 else
1337 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
1338 break;
1339 case 10:
1340 default:
1341 /* not needed */
1342 break;
1343 }
1344
1345 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
1346}
1347
1348static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1349{
1350 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1351 return true;
1352 else
1353 return false;
1354}
1355
1356static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1357{
1358 u32 pos1, pos2;
1359
1360 pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1361 pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1362
1363 if (pos1 != pos2)
1364 return true;
1365 else
1366 return false;
1367}
1368
1369/**
1370 * dce4_wait_for_vblank - vblank wait asic callback.
1371 *
1372 * @rdev: radeon_device pointer
1373 * @crtc: crtc to wait for vblank on
1374 *
1375 * Wait for vblank on the requested crtc (evergreen+).
1376 */
1377void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1378{
1379 unsigned i = 0;
1380
1381 if (crtc >= rdev->num_crtc)
1382 return;
1383
1384 if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1385 return;
1386
1387 /* depending on when we hit vblank, we may be close to active; if so,
1388 * wait for another frame.
1389 */
1390 while (dce4_is_in_vblank(rdev, crtc)) {
1391 if (i++ % 100 == 0) {
1392 if (!dce4_is_counter_moving(rdev, crtc))
1393 break;
1394 }
1395 }
1396
1397 while (!dce4_is_in_vblank(rdev, crtc)) {
1398 if (i++ % 100 == 0) {
1399 if (!dce4_is_counter_moving(rdev, crtc))
1400 break;
1401 }
1402 }
1403}
1404
1405/**
1406 * evergreen_page_flip - pageflip callback.
1407 *
1408 * @rdev: radeon_device pointer
1409 * @crtc_id: crtc to cleanup pageflip on
1410 * @crtc_base: new address of the crtc (GPU MC address)
1411 * @async: asynchronous flip
1412 *
1413 * Triggers the actual pageflip by updating the primary
1414 * surface base address (evergreen+).
1415 */
1416void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base,
1417 bool async)
1418{
1419 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1420 struct drm_framebuffer *fb = radeon_crtc->base.primary->fb;
1421
1422 /* flip at hsync for async, default is vsync */
1423 WREG32(EVERGREEN_GRPH_FLIP_CONTROL + radeon_crtc->crtc_offset,
1424 async ? EVERGREEN_GRPH_SURFACE_UPDATE_H_RETRACE_EN : 0);
1425 /* update pitch */
1426 WREG32(EVERGREEN_GRPH_PITCH + radeon_crtc->crtc_offset,
1427 fb->pitches[0] / fb->format->cpp[0]);
1428 /* update the scanout addresses */
1429 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1430 upper_32_bits(crtc_base));
1431 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1432 (u32)crtc_base);
1433 /* post the write */
1434 RREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset);
1435}
1436
1437/**
1438 * evergreen_page_flip_pending - check if page flip is still pending
1439 *
1440 * @rdev: radeon_device pointer
1441 * @crtc_id: crtc to check
1442 *
1443 * Returns the current update pending status.
1444 */
1445bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id)
1446{
1447 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1448
1449 /* Return current update_pending status: */
1450 return !!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) &
1451 EVERGREEN_GRPH_SURFACE_UPDATE_PENDING);
1452}
1453
1454/* get temperature in millidegrees */
1455int evergreen_get_temp(struct radeon_device *rdev)
1456{
1457 u32 temp, toffset;
1458 int actual_temp = 0;
1459
1460 if (rdev->family == CHIP_JUNIPER) {
1461 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1462 TOFFSET_SHIFT;
1463 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1464 TS0_ADC_DOUT_SHIFT;
1465
1466 if (toffset & 0x100)
1467 actual_temp = temp / 2 - (0x200 - toffset);
1468 else
1469 actual_temp = temp / 2 + toffset;
1470
1471 actual_temp = actual_temp * 1000;
1472
1473 } else {
1474 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1475 ASIC_T_SHIFT;
1476
1477 if (temp & 0x400)
1478 actual_temp = -256;
1479 else if (temp & 0x200)
1480 actual_temp = 255;
1481 else if (temp & 0x100) {
1482 actual_temp = temp & 0x1ff;
1483 actual_temp |= ~0x1ff;
1484 } else
1485 actual_temp = temp & 0xff;
1486
1487 actual_temp = (actual_temp * 1000) / 2;
1488 }
1489
1490 return actual_temp;
1491}
1492
1493int sumo_get_temp(struct radeon_device *rdev)
1494{
1495 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
1496 int actual_temp = temp - 49;
1497
1498 return actual_temp * 1000;
1499}
1500
1501/**
1502 * sumo_pm_init_profile - Initialize power profiles callback.
1503 *
1504 * @rdev: radeon_device pointer
1505 *
1506 * Initialize the power states used in profile mode
1507 * (sumo, trinity, SI).
1508 * Used for profile mode only.
1509 */
1510void sumo_pm_init_profile(struct radeon_device *rdev)
1511{
1512 int idx;
1513
1514 /* default */
1515 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1516 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1517 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1518 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1519
1520 /* low,mid sh/mh */
1521 if (rdev->flags & RADEON_IS_MOBILITY)
1522 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1523 else
1524 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1525
1526 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1527 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1528 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1529 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1530
1531 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1532 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1533 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1534 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1535
1536 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1537 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1538 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1539 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1540
1541 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1542 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1543 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1544 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1545
1546 /* high sh/mh */
1547 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1548 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1549 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1550 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1551 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1552 rdev->pm.power_state[idx].num_clock_modes - 1;
1553
1554 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1555 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1556 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1557 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1558 rdev->pm.power_state[idx].num_clock_modes - 1;
1559}
1560
1561/**
1562 * btc_pm_init_profile - Initialize power profiles callback.
1563 *
1564 * @rdev: radeon_device pointer
1565 *
1566 * Initialize the power states used in profile mode
1567 * (BTC, cayman).
1568 * Used for profile mode only.
1569 */
1570void btc_pm_init_profile(struct radeon_device *rdev)
1571{
1572 int idx;
1573
1574 /* default */
1575 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1576 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1577 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1578 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1579 /* starting with BTC, there is one state that is used for both
1580 * MH and SH. Difference is that we always use the high clock index for
1581 * mclk.
1582 */
1583 if (rdev->flags & RADEON_IS_MOBILITY)
1584 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1585 else
1586 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1587 /* low sh */
1588 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1589 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1590 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1591 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1592 /* mid sh */
1593 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1594 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1595 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1596 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1597 /* high sh */
1598 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1599 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1600 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1601 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1602 /* low mh */
1603 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1604 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1605 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1606 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1607 /* mid mh */
1608 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1609 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1610 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1611 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1612 /* high mh */
1613 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1614 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1615 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1616 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1617}
1618
1619/**
1620 * evergreen_pm_misc - set additional pm hw parameters callback.
1621 *
1622 * @rdev: radeon_device pointer
1623 *
1624 * Set non-clock parameters associated with a power state
1625 * (voltage, etc.) (evergreen+).
1626 */
1627void evergreen_pm_misc(struct radeon_device *rdev)
1628{
1629 int req_ps_idx = rdev->pm.requested_power_state_index;
1630 int req_cm_idx = rdev->pm.requested_clock_mode_index;
1631 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1632 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
1633 struct drm_device *ddev = rdev_to_drm(rdev);
1634
1635 if (voltage->type == VOLTAGE_SW) {
1636 /* 0xff0x are flags rather then an actual voltage */
1637 if ((voltage->voltage & 0xff00) == 0xff00)
1638 return;
1639 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
1640 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
1641 rdev->pm.current_vddc = voltage->voltage;
1642 drm_dbg(ddev, "Setting: vddc: %d\n", voltage->voltage);
1643 }
1644
1645 /* starting with BTC, there is one state that is used for both
1646 * MH and SH. Difference is that we always use the high clock index for
1647 * mclk and vddci.
1648 */
1649 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1650 (rdev->family >= CHIP_BARTS) &&
1651 rdev->pm.active_crtc_count &&
1652 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1653 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1654 voltage = &rdev->pm.power_state[req_ps_idx].
1655 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1656
1657 /* 0xff0x are flags rather then an actual voltage */
1658 if ((voltage->vddci & 0xff00) == 0xff00)
1659 return;
1660 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1661 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1662 rdev->pm.current_vddci = voltage->vddci;
1663 drm_dbg(ddev, "Setting: vddci: %d\n", voltage->vddci);
1664 }
1665 }
1666}
1667
1668/**
1669 * evergreen_pm_prepare - pre-power state change callback.
1670 *
1671 * @rdev: radeon_device pointer
1672 *
1673 * Prepare for a power state change (evergreen+).
1674 */
1675void evergreen_pm_prepare(struct radeon_device *rdev)
1676{
1677 struct drm_device *ddev = rdev_to_drm(rdev);
1678 struct drm_crtc *crtc;
1679 struct radeon_crtc *radeon_crtc;
1680 u32 tmp;
1681
1682 /* disable any active CRTCs */
1683 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1684 radeon_crtc = to_radeon_crtc(crtc);
1685 if (radeon_crtc->enabled) {
1686 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1687 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1688 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1689 }
1690 }
1691}
1692
1693/**
1694 * evergreen_pm_finish - post-power state change callback.
1695 *
1696 * @rdev: radeon_device pointer
1697 *
1698 * Clean up after a power state change (evergreen+).
1699 */
1700void evergreen_pm_finish(struct radeon_device *rdev)
1701{
1702 struct drm_device *ddev = rdev_to_drm(rdev);
1703 struct drm_crtc *crtc;
1704 struct radeon_crtc *radeon_crtc;
1705 u32 tmp;
1706
1707 /* enable any active CRTCs */
1708 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1709 radeon_crtc = to_radeon_crtc(crtc);
1710 if (radeon_crtc->enabled) {
1711 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1712 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1713 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1714 }
1715 }
1716}
1717
1718/**
1719 * evergreen_hpd_sense - hpd sense callback.
1720 *
1721 * @rdev: radeon_device pointer
1722 * @hpd: hpd (hotplug detect) pin
1723 *
1724 * Checks if a digital monitor is connected (evergreen+).
1725 * Returns true if connected, false if not connected.
1726 */
1727bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1728{
1729 if (hpd == RADEON_HPD_NONE)
1730 return false;
1731
1732 return !!(RREG32(DC_HPDx_INT_STATUS_REG(hpd)) & DC_HPDx_SENSE);
1733}
1734
1735/**
1736 * evergreen_hpd_set_polarity - hpd set polarity callback.
1737 *
1738 * @rdev: radeon_device pointer
1739 * @hpd: hpd (hotplug detect) pin
1740 *
1741 * Set the polarity of the hpd pin (evergreen+).
1742 */
1743void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1744 enum radeon_hpd_id hpd)
1745{
1746 bool connected = evergreen_hpd_sense(rdev, hpd);
1747
1748 if (hpd == RADEON_HPD_NONE)
1749 return;
1750
1751 if (connected)
1752 WREG32_AND(DC_HPDx_INT_CONTROL(hpd), ~DC_HPDx_INT_POLARITY);
1753 else
1754 WREG32_OR(DC_HPDx_INT_CONTROL(hpd), DC_HPDx_INT_POLARITY);
1755}
1756
1757/**
1758 * evergreen_hpd_init - hpd setup callback.
1759 *
1760 * @rdev: radeon_device pointer
1761 *
1762 * Setup the hpd pins used by the card (evergreen+).
1763 * Enable the pin, set the polarity, and enable the hpd interrupts.
1764 */
1765void evergreen_hpd_init(struct radeon_device *rdev)
1766{
1767 struct drm_device *dev = rdev_to_drm(rdev);
1768 struct drm_connector *connector;
1769 unsigned enabled = 0;
1770 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1771 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
1772
1773 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1774 enum radeon_hpd_id hpd =
1775 to_radeon_connector(connector)->hpd.hpd;
1776
1777 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1778 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1779 /* don't try to enable hpd on eDP or LVDS avoid breaking the
1780 * aux dp channel on imac and help (but not completely fix)
1781 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1782 * also avoid interrupt storms during dpms.
1783 */
1784 continue;
1785 }
1786
1787 if (hpd == RADEON_HPD_NONE)
1788 continue;
1789
1790 WREG32(DC_HPDx_CONTROL(hpd), tmp);
1791 enabled |= 1 << hpd;
1792
1793 radeon_hpd_set_polarity(rdev, hpd);
1794 }
1795 radeon_irq_kms_enable_hpd(rdev, enabled);
1796}
1797
1798/**
1799 * evergreen_hpd_fini - hpd tear down callback.
1800 *
1801 * @rdev: radeon_device pointer
1802 *
1803 * Tear down the hpd pins used by the card (evergreen+).
1804 * Disable the hpd interrupts.
1805 */
1806void evergreen_hpd_fini(struct radeon_device *rdev)
1807{
1808 struct drm_device *dev = rdev_to_drm(rdev);
1809 struct drm_connector *connector;
1810 unsigned disabled = 0;
1811
1812 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1813 enum radeon_hpd_id hpd =
1814 to_radeon_connector(connector)->hpd.hpd;
1815
1816 if (hpd == RADEON_HPD_NONE)
1817 continue;
1818
1819 WREG32(DC_HPDx_CONTROL(hpd), 0);
1820 disabled |= 1 << hpd;
1821 }
1822 radeon_irq_kms_disable_hpd(rdev, disabled);
1823}
1824
1825/* watermark setup */
1826
1827static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1828 struct radeon_crtc *radeon_crtc,
1829 struct drm_display_mode *mode,
1830 struct drm_display_mode *other_mode)
1831{
1832 u32 tmp, buffer_alloc, i;
1833 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1834 /*
1835 * Line Buffer Setup
1836 * There are 3 line buffers, each one shared by 2 display controllers.
1837 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1838 * the display controllers. The paritioning is done via one of four
1839 * preset allocations specified in bits 2:0:
1840 * first display controller
1841 * 0 - first half of lb (3840 * 2)
1842 * 1 - first 3/4 of lb (5760 * 2)
1843 * 2 - whole lb (7680 * 2), other crtc must be disabled
1844 * 3 - first 1/4 of lb (1920 * 2)
1845 * second display controller
1846 * 4 - second half of lb (3840 * 2)
1847 * 5 - second 3/4 of lb (5760 * 2)
1848 * 6 - whole lb (7680 * 2), other crtc must be disabled
1849 * 7 - last 1/4 of lb (1920 * 2)
1850 */
1851 /* this can get tricky if we have two large displays on a paired group
1852 * of crtcs. Ideally for multiple large displays we'd assign them to
1853 * non-linked crtcs for maximum line buffer allocation.
1854 */
1855 if (radeon_crtc->base.enabled && mode) {
1856 if (other_mode) {
1857 tmp = 0; /* 1/2 */
1858 buffer_alloc = 1;
1859 } else {
1860 tmp = 2; /* whole */
1861 buffer_alloc = 2;
1862 }
1863 } else {
1864 tmp = 0;
1865 buffer_alloc = 0;
1866 }
1867
1868 /* second controller of the pair uses second half of the lb */
1869 if (radeon_crtc->crtc_id % 2)
1870 tmp += 4;
1871 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1872
1873 if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
1874 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1875 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1876 for (i = 0; i < rdev->usec_timeout; i++) {
1877 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1878 DMIF_BUFFERS_ALLOCATED_COMPLETED)
1879 break;
1880 udelay(1);
1881 }
1882 }
1883
1884 if (radeon_crtc->base.enabled && mode) {
1885 switch (tmp) {
1886 case 0:
1887 case 4:
1888 default:
1889 if (ASIC_IS_DCE5(rdev))
1890 return 4096 * 2;
1891 else
1892 return 3840 * 2;
1893 case 1:
1894 case 5:
1895 if (ASIC_IS_DCE5(rdev))
1896 return 6144 * 2;
1897 else
1898 return 5760 * 2;
1899 case 2:
1900 case 6:
1901 if (ASIC_IS_DCE5(rdev))
1902 return 8192 * 2;
1903 else
1904 return 7680 * 2;
1905 case 3:
1906 case 7:
1907 if (ASIC_IS_DCE5(rdev))
1908 return 2048 * 2;
1909 else
1910 return 1920 * 2;
1911 }
1912 }
1913
1914 /* controller not enabled, so no lb used */
1915 return 0;
1916}
1917
1918u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
1919{
1920 u32 tmp = RREG32(MC_SHARED_CHMAP);
1921
1922 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1923 case 0:
1924 default:
1925 return 1;
1926 case 1:
1927 return 2;
1928 case 2:
1929 return 4;
1930 case 3:
1931 return 8;
1932 }
1933}
1934
1935struct evergreen_wm_params {
1936 u32 dram_channels; /* number of dram channels */
1937 u32 yclk; /* bandwidth per dram data pin in kHz */
1938 u32 sclk; /* engine clock in kHz */
1939 u32 disp_clk; /* display clock in kHz */
1940 u32 src_width; /* viewport width */
1941 u32 active_time; /* active display time in ns */
1942 u32 blank_time; /* blank time in ns */
1943 bool interlaced; /* mode is interlaced */
1944 fixed20_12 vsc; /* vertical scale ratio */
1945 u32 num_heads; /* number of active crtcs */
1946 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1947 u32 lb_size; /* line buffer allocated to pipe */
1948 u32 vtaps; /* vertical scaler taps */
1949};
1950
1951static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1952{
1953 /* Calculate DRAM Bandwidth and the part allocated to display. */
1954 fixed20_12 dram_efficiency; /* 0.7 */
1955 fixed20_12 yclk, dram_channels, bandwidth;
1956 fixed20_12 a;
1957
1958 a.full = dfixed_const(1000);
1959 yclk.full = dfixed_const(wm->yclk);
1960 yclk.full = dfixed_div(yclk, a);
1961 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1962 a.full = dfixed_const(10);
1963 dram_efficiency.full = dfixed_const(7);
1964 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1965 bandwidth.full = dfixed_mul(dram_channels, yclk);
1966 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1967
1968 return dfixed_trunc(bandwidth);
1969}
1970
1971static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
1972{
1973 /* Calculate DRAM Bandwidth and the part allocated to display. */
1974 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1975 fixed20_12 yclk, dram_channels, bandwidth;
1976 fixed20_12 a;
1977
1978 a.full = dfixed_const(1000);
1979 yclk.full = dfixed_const(wm->yclk);
1980 yclk.full = dfixed_div(yclk, a);
1981 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1982 a.full = dfixed_const(10);
1983 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1984 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1985 bandwidth.full = dfixed_mul(dram_channels, yclk);
1986 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1987
1988 return dfixed_trunc(bandwidth);
1989}
1990
1991static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
1992{
1993 /* Calculate the display Data return Bandwidth */
1994 fixed20_12 return_efficiency; /* 0.8 */
1995 fixed20_12 sclk, bandwidth;
1996 fixed20_12 a;
1997
1998 a.full = dfixed_const(1000);
1999 sclk.full = dfixed_const(wm->sclk);
2000 sclk.full = dfixed_div(sclk, a);
2001 a.full = dfixed_const(10);
2002 return_efficiency.full = dfixed_const(8);
2003 return_efficiency.full = dfixed_div(return_efficiency, a);
2004 a.full = dfixed_const(32);
2005 bandwidth.full = dfixed_mul(a, sclk);
2006 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2007
2008 return dfixed_trunc(bandwidth);
2009}
2010
2011static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
2012{
2013 /* Calculate the DMIF Request Bandwidth */
2014 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2015 fixed20_12 disp_clk, bandwidth;
2016 fixed20_12 a;
2017
2018 a.full = dfixed_const(1000);
2019 disp_clk.full = dfixed_const(wm->disp_clk);
2020 disp_clk.full = dfixed_div(disp_clk, a);
2021 a.full = dfixed_const(10);
2022 disp_clk_request_efficiency.full = dfixed_const(8);
2023 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2024 a.full = dfixed_const(32);
2025 bandwidth.full = dfixed_mul(a, disp_clk);
2026 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
2027
2028 return dfixed_trunc(bandwidth);
2029}
2030
2031static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
2032{
2033 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2034 u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
2035 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
2036 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
2037
2038 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2039}
2040
2041static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2042{
2043 /* Calculate the display mode Average Bandwidth
2044 * DisplayMode should contain the source and destination dimensions,
2045 * timing, etc.
2046 */
2047 fixed20_12 bpp;
2048 fixed20_12 line_time;
2049 fixed20_12 src_width;
2050 fixed20_12 bandwidth;
2051 fixed20_12 a;
2052
2053 a.full = dfixed_const(1000);
2054 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2055 line_time.full = dfixed_div(line_time, a);
2056 bpp.full = dfixed_const(wm->bytes_per_pixel);
2057 src_width.full = dfixed_const(wm->src_width);
2058 bandwidth.full = dfixed_mul(src_width, bpp);
2059 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2060 bandwidth.full = dfixed_div(bandwidth, line_time);
2061
2062 return dfixed_trunc(bandwidth);
2063}
2064
2065static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2066{
2067 /* First calcualte the latency in ns */
2068 u32 mc_latency = 2000; /* 2000 ns. */
2069 u32 available_bandwidth = evergreen_available_bandwidth(wm);
2070 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2071 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2072 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2073 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2074 (wm->num_heads * cursor_line_pair_return_time);
2075 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2076 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2077 fixed20_12 a, b, c;
2078
2079 if (wm->num_heads == 0)
2080 return 0;
2081
2082 a.full = dfixed_const(2);
2083 b.full = dfixed_const(1);
2084 if ((wm->vsc.full > a.full) ||
2085 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2086 (wm->vtaps >= 5) ||
2087 ((wm->vsc.full >= a.full) && wm->interlaced))
2088 max_src_lines_per_dst_line = 4;
2089 else
2090 max_src_lines_per_dst_line = 2;
2091
2092 a.full = dfixed_const(available_bandwidth);
2093 b.full = dfixed_const(wm->num_heads);
2094 a.full = dfixed_div(a, b);
2095
2096 lb_fill_bw = min(dfixed_trunc(a), wm->disp_clk * wm->bytes_per_pixel / 1000);
2097
2098 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2099 b.full = dfixed_const(1000);
2100 c.full = dfixed_const(lb_fill_bw);
2101 b.full = dfixed_div(c, b);
2102 a.full = dfixed_div(a, b);
2103 line_fill_time = dfixed_trunc(a);
2104
2105 if (line_fill_time < wm->active_time)
2106 return latency;
2107 else
2108 return latency + (line_fill_time - wm->active_time);
2109
2110}
2111
2112static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2113{
2114 if (evergreen_average_bandwidth(wm) <=
2115 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2116 return true;
2117 else
2118 return false;
2119};
2120
2121static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2122{
2123 if (evergreen_average_bandwidth(wm) <=
2124 (evergreen_available_bandwidth(wm) / wm->num_heads))
2125 return true;
2126 else
2127 return false;
2128};
2129
2130static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2131{
2132 u32 lb_partitions = wm->lb_size / wm->src_width;
2133 u32 line_time = wm->active_time + wm->blank_time;
2134 u32 latency_tolerant_lines;
2135 u32 latency_hiding;
2136 fixed20_12 a;
2137
2138 a.full = dfixed_const(1);
2139 if (wm->vsc.full > a.full)
2140 latency_tolerant_lines = 1;
2141 else {
2142 if (lb_partitions <= (wm->vtaps + 1))
2143 latency_tolerant_lines = 1;
2144 else
2145 latency_tolerant_lines = 2;
2146 }
2147
2148 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2149
2150 if (evergreen_latency_watermark(wm) <= latency_hiding)
2151 return true;
2152 else
2153 return false;
2154}
2155
2156static void evergreen_program_watermarks(struct radeon_device *rdev,
2157 struct radeon_crtc *radeon_crtc,
2158 u32 lb_size, u32 num_heads)
2159{
2160 struct drm_display_mode *mode = &radeon_crtc->base.mode;
2161 struct evergreen_wm_params wm_low, wm_high;
2162 u32 dram_channels;
2163 u32 active_time;
2164 u32 line_time = 0;
2165 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2166 u32 priority_a_mark = 0, priority_b_mark = 0;
2167 u32 priority_a_cnt = PRIORITY_OFF;
2168 u32 priority_b_cnt = PRIORITY_OFF;
2169 u32 pipe_offset = radeon_crtc->crtc_id * 16;
2170 u32 tmp, arb_control3;
2171 fixed20_12 a, b, c;
2172 struct drm_device *ddev = rdev_to_drm(rdev);
2173
2174 if (radeon_crtc->base.enabled && num_heads && mode) {
2175 active_time = (u32) div_u64((u64)mode->crtc_hdisplay * 1000000,
2176 (u32)mode->clock);
2177 line_time = (u32) div_u64((u64)mode->crtc_htotal * 1000000,
2178 (u32)mode->clock);
2179 line_time = min(line_time, (u32)65535);
2180 priority_a_cnt = 0;
2181 priority_b_cnt = 0;
2182 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2183
2184 /* watermark for high clocks */
2185 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2186 wm_high.yclk =
2187 radeon_dpm_get_mclk(rdev, false) * 10;
2188 wm_high.sclk =
2189 radeon_dpm_get_sclk(rdev, false) * 10;
2190 } else {
2191 wm_high.yclk = rdev->pm.current_mclk * 10;
2192 wm_high.sclk = rdev->pm.current_sclk * 10;
2193 }
2194
2195 wm_high.disp_clk = mode->clock;
2196 wm_high.src_width = mode->crtc_hdisplay;
2197 wm_high.active_time = active_time;
2198 wm_high.blank_time = line_time - wm_high.active_time;
2199 wm_high.interlaced = false;
2200 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2201 wm_high.interlaced = true;
2202 wm_high.vsc = radeon_crtc->vsc;
2203 wm_high.vtaps = 1;
2204 if (radeon_crtc->rmx_type != RMX_OFF)
2205 wm_high.vtaps = 2;
2206 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2207 wm_high.lb_size = lb_size;
2208 wm_high.dram_channels = dram_channels;
2209 wm_high.num_heads = num_heads;
2210
2211 /* watermark for low clocks */
2212 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2213 wm_low.yclk =
2214 radeon_dpm_get_mclk(rdev, true) * 10;
2215 wm_low.sclk =
2216 radeon_dpm_get_sclk(rdev, true) * 10;
2217 } else {
2218 wm_low.yclk = rdev->pm.current_mclk * 10;
2219 wm_low.sclk = rdev->pm.current_sclk * 10;
2220 }
2221
2222 wm_low.disp_clk = mode->clock;
2223 wm_low.src_width = mode->crtc_hdisplay;
2224 wm_low.active_time = active_time;
2225 wm_low.blank_time = line_time - wm_low.active_time;
2226 wm_low.interlaced = false;
2227 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2228 wm_low.interlaced = true;
2229 wm_low.vsc = radeon_crtc->vsc;
2230 wm_low.vtaps = 1;
2231 if (radeon_crtc->rmx_type != RMX_OFF)
2232 wm_low.vtaps = 2;
2233 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2234 wm_low.lb_size = lb_size;
2235 wm_low.dram_channels = dram_channels;
2236 wm_low.num_heads = num_heads;
2237
2238 /* set for high clocks */
2239 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
2240 /* set for low clocks */
2241 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
2242
2243 /* possibly force display priority to high */
2244 /* should really do this at mode validation time... */
2245 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2246 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2247 !evergreen_check_latency_hiding(&wm_high) ||
2248 (rdev->disp_priority == 2)) {
2249 drm_dbg_kms(ddev, "force priority a to high\n");
2250 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2251 }
2252 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2253 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2254 !evergreen_check_latency_hiding(&wm_low) ||
2255 (rdev->disp_priority == 2)) {
2256 drm_dbg_kms(ddev, "force priority b to high\n");
2257 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2258 }
2259
2260 a.full = dfixed_const(1000);
2261 b.full = dfixed_const(mode->clock);
2262 b.full = dfixed_div(b, a);
2263 c.full = dfixed_const(latency_watermark_a);
2264 c.full = dfixed_mul(c, b);
2265 c.full = dfixed_mul(c, radeon_crtc->hsc);
2266 c.full = dfixed_div(c, a);
2267 a.full = dfixed_const(16);
2268 c.full = dfixed_div(c, a);
2269 priority_a_mark = dfixed_trunc(c);
2270 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2271
2272 a.full = dfixed_const(1000);
2273 b.full = dfixed_const(mode->clock);
2274 b.full = dfixed_div(b, a);
2275 c.full = dfixed_const(latency_watermark_b);
2276 c.full = dfixed_mul(c, b);
2277 c.full = dfixed_mul(c, radeon_crtc->hsc);
2278 c.full = dfixed_div(c, a);
2279 a.full = dfixed_const(16);
2280 c.full = dfixed_div(c, a);
2281 priority_b_mark = dfixed_trunc(c);
2282 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2283
2284 /* Save number of lines the linebuffer leads before the scanout */
2285 radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
2286 }
2287
2288 /* select wm A */
2289 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2290 tmp = arb_control3;
2291 tmp &= ~LATENCY_WATERMARK_MASK(3);
2292 tmp |= LATENCY_WATERMARK_MASK(1);
2293 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2294 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2295 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2296 LATENCY_HIGH_WATERMARK(line_time)));
2297 /* select wm B */
2298 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2299 tmp &= ~LATENCY_WATERMARK_MASK(3);
2300 tmp |= LATENCY_WATERMARK_MASK(2);
2301 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2302 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2303 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2304 LATENCY_HIGH_WATERMARK(line_time)));
2305 /* restore original selection */
2306 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2307
2308 /* write the priority marks */
2309 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2310 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2311
2312 /* save values for DPM */
2313 radeon_crtc->line_time = line_time;
2314 radeon_crtc->wm_high = latency_watermark_a;
2315 radeon_crtc->wm_low = latency_watermark_b;
2316}
2317
2318/**
2319 * evergreen_bandwidth_update - update display watermarks callback.
2320 *
2321 * @rdev: radeon_device pointer
2322 *
2323 * Update the display watermarks based on the requested mode(s)
2324 * (evergreen+).
2325 */
2326void evergreen_bandwidth_update(struct radeon_device *rdev)
2327{
2328 struct drm_display_mode *mode0 = NULL;
2329 struct drm_display_mode *mode1 = NULL;
2330 u32 num_heads = 0, lb_size;
2331 int i;
2332
2333 if (!rdev->mode_info.mode_config_initialized)
2334 return;
2335
2336 radeon_update_display_priority(rdev);
2337
2338 for (i = 0; i < rdev->num_crtc; i++) {
2339 if (rdev->mode_info.crtcs[i]->base.enabled)
2340 num_heads++;
2341 }
2342 for (i = 0; i < rdev->num_crtc; i += 2) {
2343 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2344 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2345 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2346 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2347 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2348 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2349 }
2350}
2351
2352/**
2353 * evergreen_mc_wait_for_idle - wait for MC idle callback.
2354 *
2355 * @rdev: radeon_device pointer
2356 *
2357 * Wait for the MC (memory controller) to be idle.
2358 * (evergreen+).
2359 * Returns 0 if the MC is idle, -1 if not.
2360 */
2361int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
2362{
2363 unsigned i;
2364 u32 tmp;
2365
2366 for (i = 0; i < rdev->usec_timeout; i++) {
2367 /* read MC_STATUS */
2368 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2369 if (!tmp)
2370 return 0;
2371 udelay(1);
2372 }
2373 return -1;
2374}
2375
2376/*
2377 * GART
2378 */
2379void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2380{
2381 unsigned i;
2382 u32 tmp;
2383
2384 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2385
2386 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2387 for (i = 0; i < rdev->usec_timeout; i++) {
2388 /* read MC_STATUS */
2389 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2390 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2391 if (tmp == 2) {
2392 pr_warn("[drm] r600 flush TLB failed\n");
2393 return;
2394 }
2395 if (tmp) {
2396 return;
2397 }
2398 udelay(1);
2399 }
2400}
2401
2402static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
2403{
2404 u32 tmp;
2405 int r;
2406 struct drm_device *ddev = rdev_to_drm(rdev);
2407
2408 if (rdev->gart.robj == NULL) {
2409 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2410 return -EINVAL;
2411 }
2412 r = radeon_gart_table_vram_pin(rdev);
2413 if (r)
2414 return r;
2415 /* Setup L2 cache */
2416 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2417 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2418 EFFECTIVE_L2_QUEUE_SIZE(7));
2419 WREG32(VM_L2_CNTL2, 0);
2420 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2421 /* Setup TLB control */
2422 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2423 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2424 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2425 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2426 if (rdev->flags & RADEON_IS_IGP) {
2427 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2428 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2429 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2430 } else {
2431 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2432 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2433 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2434 if ((rdev->family == CHIP_JUNIPER) ||
2435 (rdev->family == CHIP_CYPRESS) ||
2436 (rdev->family == CHIP_HEMLOCK) ||
2437 (rdev->family == CHIP_BARTS))
2438 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
2439 }
2440 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2441 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2442 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2443 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2444 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2445 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2446 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2447 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2448 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2449 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2450 (u32)(rdev->dummy_page.addr >> 12));
2451 WREG32(VM_CONTEXT1_CNTL, 0);
2452
2453 evergreen_pcie_gart_tlb_flush(rdev);
2454 drm_info(ddev, "PCIE GART of %uM enabled (table at 0x%016llX).\n",
2455 (unsigned)(rdev->mc.gtt_size >> 20),
2456 (unsigned long long)rdev->gart.table_addr);
2457 rdev->gart.ready = true;
2458 return 0;
2459}
2460
2461static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
2462{
2463 u32 tmp;
2464
2465 /* Disable all tables */
2466 WREG32(VM_CONTEXT0_CNTL, 0);
2467 WREG32(VM_CONTEXT1_CNTL, 0);
2468
2469 /* Setup L2 cache */
2470 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2471 EFFECTIVE_L2_QUEUE_SIZE(7));
2472 WREG32(VM_L2_CNTL2, 0);
2473 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2474 /* Setup TLB control */
2475 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2476 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2477 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2478 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2479 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2480 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2481 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2482 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2483 radeon_gart_table_vram_unpin(rdev);
2484}
2485
2486static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
2487{
2488 evergreen_pcie_gart_disable(rdev);
2489 radeon_gart_table_vram_free(rdev);
2490 radeon_gart_fini(rdev);
2491}
2492
2493
2494static void evergreen_agp_enable(struct radeon_device *rdev)
2495{
2496 u32 tmp;
2497
2498 /* Setup L2 cache */
2499 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2500 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2501 EFFECTIVE_L2_QUEUE_SIZE(7));
2502 WREG32(VM_L2_CNTL2, 0);
2503 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2504 /* Setup TLB control */
2505 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2506 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2507 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2508 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2509 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2510 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2511 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2512 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2513 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2514 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2515 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2516 WREG32(VM_CONTEXT0_CNTL, 0);
2517 WREG32(VM_CONTEXT1_CNTL, 0);
2518}
2519
2520static const unsigned ni_dig_offsets[] = {
2521 NI_DIG0_REGISTER_OFFSET,
2522 NI_DIG1_REGISTER_OFFSET,
2523 NI_DIG2_REGISTER_OFFSET,
2524 NI_DIG3_REGISTER_OFFSET,
2525 NI_DIG4_REGISTER_OFFSET,
2526 NI_DIG5_REGISTER_OFFSET
2527};
2528
2529static const unsigned ni_tx_offsets[] = {
2530 NI_DCIO_UNIPHY0_UNIPHY_TX_CONTROL1,
2531 NI_DCIO_UNIPHY1_UNIPHY_TX_CONTROL1,
2532 NI_DCIO_UNIPHY2_UNIPHY_TX_CONTROL1,
2533 NI_DCIO_UNIPHY3_UNIPHY_TX_CONTROL1,
2534 NI_DCIO_UNIPHY4_UNIPHY_TX_CONTROL1,
2535 NI_DCIO_UNIPHY5_UNIPHY_TX_CONTROL1
2536};
2537
2538static const unsigned evergreen_dp_offsets[] = {
2539 EVERGREEN_DP0_REGISTER_OFFSET,
2540 EVERGREEN_DP1_REGISTER_OFFSET,
2541 EVERGREEN_DP2_REGISTER_OFFSET,
2542 EVERGREEN_DP3_REGISTER_OFFSET,
2543 EVERGREEN_DP4_REGISTER_OFFSET,
2544 EVERGREEN_DP5_REGISTER_OFFSET
2545};
2546
2547static const unsigned evergreen_disp_int_status[] = {
2548 DISP_INTERRUPT_STATUS,
2549 DISP_INTERRUPT_STATUS_CONTINUE,
2550 DISP_INTERRUPT_STATUS_CONTINUE2,
2551 DISP_INTERRUPT_STATUS_CONTINUE3,
2552 DISP_INTERRUPT_STATUS_CONTINUE4,
2553 DISP_INTERRUPT_STATUS_CONTINUE5
2554};
2555
2556/*
2557 * Assumption is that EVERGREEN_CRTC_MASTER_EN enable for requested crtc
2558 * We go from crtc to connector and it is not relible since it
2559 * should be an opposite direction .If crtc is enable then
2560 * find the dig_fe which selects this crtc and insure that it enable.
2561 * if such dig_fe is found then find dig_be which selects found dig_be and
2562 * insure that it enable and in DP_SST mode.
2563 * if UNIPHY_PLL_CONTROL1.enable then we should disconnect timing
2564 * from dp symbols clocks .
2565 */
2566static bool evergreen_is_dp_sst_stream_enabled(struct radeon_device *rdev,
2567 unsigned crtc_id, unsigned *ret_dig_fe)
2568{
2569 unsigned i;
2570 unsigned dig_fe;
2571 unsigned dig_be;
2572 unsigned dig_en_be;
2573 unsigned uniphy_pll;
2574 unsigned digs_fe_selected;
2575 unsigned dig_be_mode;
2576 unsigned dig_fe_mask;
2577 bool is_enabled = false;
2578 bool found_crtc = false;
2579
2580 /* loop through all running dig_fe to find selected crtc */
2581 for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2582 dig_fe = RREG32(NI_DIG_FE_CNTL + ni_dig_offsets[i]);
2583 if (dig_fe & NI_DIG_FE_CNTL_SYMCLK_FE_ON &&
2584 crtc_id == NI_DIG_FE_CNTL_SOURCE_SELECT(dig_fe)) {
2585 /* found running pipe */
2586 found_crtc = true;
2587 dig_fe_mask = 1 << i;
2588 dig_fe = i;
2589 break;
2590 }
2591 }
2592
2593 if (found_crtc) {
2594 /* loop through all running dig_be to find selected dig_fe */
2595 for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2596 dig_be = RREG32(NI_DIG_BE_CNTL + ni_dig_offsets[i]);
2597 /* if dig_fe_selected by dig_be? */
2598 digs_fe_selected = NI_DIG_BE_CNTL_FE_SOURCE_SELECT(dig_be);
2599 dig_be_mode = NI_DIG_FE_CNTL_MODE(dig_be);
2600 if (dig_fe_mask & digs_fe_selected &&
2601 /* if dig_be in sst mode? */
2602 dig_be_mode == NI_DIG_BE_DPSST) {
2603 dig_en_be = RREG32(NI_DIG_BE_EN_CNTL +
2604 ni_dig_offsets[i]);
2605 uniphy_pll = RREG32(NI_DCIO_UNIPHY0_PLL_CONTROL1 +
2606 ni_tx_offsets[i]);
2607 /* dig_be enable and tx is running */
2608 if (dig_en_be & NI_DIG_BE_EN_CNTL_ENABLE &&
2609 dig_en_be & NI_DIG_BE_EN_CNTL_SYMBCLK_ON &&
2610 uniphy_pll & NI_DCIO_UNIPHY0_PLL_CONTROL1_ENABLE) {
2611 is_enabled = true;
2612 *ret_dig_fe = dig_fe;
2613 break;
2614 }
2615 }
2616 }
2617 }
2618
2619 return is_enabled;
2620}
2621
2622/*
2623 * Blank dig when in dp sst mode
2624 * Dig ignores crtc timing
2625 */
2626static void evergreen_blank_dp_output(struct radeon_device *rdev,
2627 unsigned dig_fe)
2628{
2629 unsigned stream_ctrl;
2630 unsigned fifo_ctrl;
2631 unsigned counter = 0;
2632 struct drm_device *ddev = rdev_to_drm(rdev);
2633
2634 if (dig_fe >= ARRAY_SIZE(evergreen_dp_offsets)) {
2635 drm_err(ddev, "invalid dig_fe %d\n", dig_fe);
2636 return;
2637 }
2638
2639 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2640 evergreen_dp_offsets[dig_fe]);
2641 if (!(stream_ctrl & EVERGREEN_DP_VID_STREAM_CNTL_ENABLE)) {
2642 drm_err(ddev, "dig %d , should be enable\n", dig_fe);
2643 return;
2644 }
2645
2646 stream_ctrl &= ~EVERGREEN_DP_VID_STREAM_CNTL_ENABLE;
2647 WREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2648 evergreen_dp_offsets[dig_fe], stream_ctrl);
2649
2650 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2651 evergreen_dp_offsets[dig_fe]);
2652 while (counter < 32 && stream_ctrl & EVERGREEN_DP_VID_STREAM_STATUS) {
2653 msleep(1);
2654 counter++;
2655 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2656 evergreen_dp_offsets[dig_fe]);
2657 }
2658 if (counter >= 32)
2659 drm_err(ddev, "counter exceeds %d\n", counter);
2660
2661 fifo_ctrl = RREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe]);
2662 fifo_ctrl |= EVERGREEN_DP_STEER_FIFO_RESET;
2663 WREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe], fifo_ctrl);
2664
2665}
2666
2667void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
2668{
2669 u32 crtc_enabled, tmp, frame_count, blackout;
2670 int i, j;
2671 unsigned dig_fe;
2672
2673 if (!ASIC_IS_NODCE(rdev)) {
2674 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2675 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
2676
2677 /* disable VGA render */
2678 WREG32(VGA_RENDER_CONTROL, 0);
2679 }
2680 /* blank the display controllers */
2681 for (i = 0; i < rdev->num_crtc; i++) {
2682 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2683 if (crtc_enabled) {
2684 save->crtc_enabled[i] = true;
2685 if (ASIC_IS_DCE6(rdev)) {
2686 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2687 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2688 radeon_wait_for_vblank(rdev, i);
2689 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2690 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2691 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2692 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2693 }
2694 } else {
2695 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2696 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2697 radeon_wait_for_vblank(rdev, i);
2698 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2699 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2700 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2701 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2702 }
2703 }
2704 /* wait for the next frame */
2705 frame_count = radeon_get_vblank_counter(rdev, i);
2706 for (j = 0; j < rdev->usec_timeout; j++) {
2707 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2708 break;
2709 udelay(1);
2710 }
2711 /*we should disable dig if it drives dp sst*/
2712 /*but we are in radeon_device_init and the topology is unknown*/
2713 /*and it is available after radeon_modeset_init*/
2714 /*the following method radeon_atom_encoder_dpms_dig*/
2715 /*does the job if we initialize it properly*/
2716 /*for now we do it this manually*/
2717 /**/
2718 if (ASIC_IS_DCE5(rdev) &&
2719 evergreen_is_dp_sst_stream_enabled(rdev, i, &dig_fe))
2720 evergreen_blank_dp_output(rdev, dig_fe);
2721 /*we could remove 6 lines below*/
2722 /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2723 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2724 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2725 tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2726 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2727 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2728 save->crtc_enabled[i] = false;
2729 /* ***** */
2730 } else {
2731 save->crtc_enabled[i] = false;
2732 }
2733 }
2734
2735 radeon_mc_wait_for_idle(rdev);
2736
2737 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2738 if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2739 /* Block CPU access */
2740 WREG32(BIF_FB_EN, 0);
2741 /* blackout the MC */
2742 blackout &= ~BLACKOUT_MODE_MASK;
2743 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
2744 }
2745 /* wait for the MC to settle */
2746 udelay(100);
2747
2748 /* lock double buffered regs */
2749 for (i = 0; i < rdev->num_crtc; i++) {
2750 if (save->crtc_enabled[i]) {
2751 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2752 if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2753 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2754 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2755 }
2756 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2757 if (!(tmp & 1)) {
2758 tmp |= 1;
2759 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2760 }
2761 }
2762 }
2763}
2764
2765void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
2766{
2767 u32 tmp, frame_count;
2768 int i, j;
2769
2770 /* update crtc base addresses */
2771 for (i = 0; i < rdev->num_crtc; i++) {
2772 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2773 upper_32_bits(rdev->mc.vram_start));
2774 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2775 upper_32_bits(rdev->mc.vram_start));
2776 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
2777 (u32)rdev->mc.vram_start);
2778 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
2779 (u32)rdev->mc.vram_start);
2780 }
2781
2782 if (!ASIC_IS_NODCE(rdev)) {
2783 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2784 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2785 }
2786
2787 /* unlock regs and wait for update */
2788 for (i = 0; i < rdev->num_crtc; i++) {
2789 if (save->crtc_enabled[i]) {
2790 tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2791 if ((tmp & 0x7) != 0) {
2792 tmp &= ~0x7;
2793 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2794 }
2795 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2796 if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2797 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2798 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2799 }
2800 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2801 if (tmp & 1) {
2802 tmp &= ~1;
2803 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2804 }
2805 for (j = 0; j < rdev->usec_timeout; j++) {
2806 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2807 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2808 break;
2809 udelay(1);
2810 }
2811 }
2812 }
2813
2814 /* unblackout the MC */
2815 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2816 tmp &= ~BLACKOUT_MODE_MASK;
2817 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2818 /* allow CPU access */
2819 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2820
2821 for (i = 0; i < rdev->num_crtc; i++) {
2822 if (save->crtc_enabled[i]) {
2823 if (ASIC_IS_DCE6(rdev)) {
2824 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2825 tmp &= ~EVERGREEN_CRTC_BLANK_DATA_EN;
2826 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2827 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2828 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2829 } else {
2830 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2831 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2832 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2833 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2834 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2835 }
2836 /* wait for the next frame */
2837 frame_count = radeon_get_vblank_counter(rdev, i);
2838 for (j = 0; j < rdev->usec_timeout; j++) {
2839 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2840 break;
2841 udelay(1);
2842 }
2843 }
2844 }
2845 if (!ASIC_IS_NODCE(rdev)) {
2846 /* Unlock vga access */
2847 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2848 mdelay(1);
2849 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2850 }
2851}
2852
2853void evergreen_mc_program(struct radeon_device *rdev)
2854{
2855 struct evergreen_mc_save save;
2856 u32 tmp;
2857 int i, j;
2858
2859 /* Initialize HDP */
2860 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2861 WREG32((0x2c14 + j), 0x00000000);
2862 WREG32((0x2c18 + j), 0x00000000);
2863 WREG32((0x2c1c + j), 0x00000000);
2864 WREG32((0x2c20 + j), 0x00000000);
2865 WREG32((0x2c24 + j), 0x00000000);
2866 }
2867 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2868
2869 evergreen_mc_stop(rdev, &save);
2870 if (evergreen_mc_wait_for_idle(rdev)) {
2871 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2872 }
2873 /* Lockout access through VGA aperture*/
2874 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2875 /* Update configuration */
2876 if (rdev->flags & RADEON_IS_AGP) {
2877 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2878 /* VRAM before AGP */
2879 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2880 rdev->mc.vram_start >> 12);
2881 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2882 rdev->mc.gtt_end >> 12);
2883 } else {
2884 /* VRAM after AGP */
2885 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2886 rdev->mc.gtt_start >> 12);
2887 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2888 rdev->mc.vram_end >> 12);
2889 }
2890 } else {
2891 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2892 rdev->mc.vram_start >> 12);
2893 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2894 rdev->mc.vram_end >> 12);
2895 }
2896 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
2897 /* llano/ontario only */
2898 if ((rdev->family == CHIP_PALM) ||
2899 (rdev->family == CHIP_SUMO) ||
2900 (rdev->family == CHIP_SUMO2)) {
2901 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2902 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2903 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2904 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2905 }
2906 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2907 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2908 WREG32(MC_VM_FB_LOCATION, tmp);
2909 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
2910 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
2911 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
2912 if (rdev->flags & RADEON_IS_AGP) {
2913 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2914 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2915 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2916 } else {
2917 WREG32(MC_VM_AGP_BASE, 0);
2918 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2919 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2920 }
2921 if (evergreen_mc_wait_for_idle(rdev)) {
2922 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2923 }
2924 evergreen_mc_resume(rdev, &save);
2925 /* we need to own VRAM, so turn off the VGA renderer here
2926 * to stop it overwriting our objects */
2927 rv515_vga_render_disable(rdev);
2928}
2929
2930/*
2931 * CP.
2932 */
2933void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2934{
2935 struct radeon_ring *ring = &rdev->ring[ib->ring];
2936 u32 next_rptr;
2937
2938 /* set to DX10/11 mode */
2939 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2940 radeon_ring_write(ring, 1);
2941
2942 if (ring->rptr_save_reg) {
2943 next_rptr = ring->wptr + 3 + 4;
2944 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2945 radeon_ring_write(ring, ((ring->rptr_save_reg -
2946 PACKET3_SET_CONFIG_REG_START) >> 2));
2947 radeon_ring_write(ring, next_rptr);
2948 } else if (rdev->wb.enabled) {
2949 next_rptr = ring->wptr + 5 + 4;
2950 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2951 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2952 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2953 radeon_ring_write(ring, next_rptr);
2954 radeon_ring_write(ring, 0);
2955 }
2956
2957 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2958 radeon_ring_write(ring,
2959#ifdef __BIG_ENDIAN
2960 (2 << 0) |
2961#endif
2962 (ib->gpu_addr & 0xFFFFFFFC));
2963 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2964 radeon_ring_write(ring, ib->length_dw);
2965}
2966
2967
2968static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2969{
2970 const __be32 *fw_data;
2971 int i;
2972
2973 if (!rdev->me_fw || !rdev->pfp_fw)
2974 return -EINVAL;
2975
2976 r700_cp_stop(rdev);
2977 WREG32(CP_RB_CNTL,
2978#ifdef __BIG_ENDIAN
2979 BUF_SWAP_32BIT |
2980#endif
2981 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
2982
2983 fw_data = (const __be32 *)rdev->pfp_fw->data;
2984 WREG32(CP_PFP_UCODE_ADDR, 0);
2985 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2986 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2987 WREG32(CP_PFP_UCODE_ADDR, 0);
2988
2989 fw_data = (const __be32 *)rdev->me_fw->data;
2990 WREG32(CP_ME_RAM_WADDR, 0);
2991 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2992 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2993
2994 WREG32(CP_PFP_UCODE_ADDR, 0);
2995 WREG32(CP_ME_RAM_WADDR, 0);
2996 WREG32(CP_ME_RAM_RADDR, 0);
2997 return 0;
2998}
2999
3000static int evergreen_cp_start(struct radeon_device *rdev)
3001{
3002 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3003 int r, i;
3004 uint32_t cp_me;
3005 struct drm_device *ddev = rdev_to_drm(rdev);
3006
3007 r = radeon_ring_lock(rdev, ring, 7);
3008 if (r) {
3009 drm_err(ddev, "radeon: cp failed to lock ring (%d).\n", r);
3010 return r;
3011 }
3012 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3013 radeon_ring_write(ring, 0x1);
3014 radeon_ring_write(ring, 0x0);
3015 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
3016 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3017 radeon_ring_write(ring, 0);
3018 radeon_ring_write(ring, 0);
3019 radeon_ring_unlock_commit(rdev, ring, false);
3020
3021 cp_me = 0xff;
3022 WREG32(CP_ME_CNTL, cp_me);
3023
3024 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
3025 if (r) {
3026 drm_err(ddev, "radeon: cp failed to lock ring (%d).\n", r);
3027 return r;
3028 }
3029
3030 /* setup clear context state */
3031 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3032 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3033
3034 for (i = 0; i < evergreen_default_size; i++)
3035 radeon_ring_write(ring, evergreen_default_state[i]);
3036
3037 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3038 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3039
3040 /* set clear context state */
3041 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3042 radeon_ring_write(ring, 0);
3043
3044 /* SQ_VTX_BASE_VTX_LOC */
3045 radeon_ring_write(ring, 0xc0026f00);
3046 radeon_ring_write(ring, 0x00000000);
3047 radeon_ring_write(ring, 0x00000000);
3048 radeon_ring_write(ring, 0x00000000);
3049
3050 /* Clear consts */
3051 radeon_ring_write(ring, 0xc0036f00);
3052 radeon_ring_write(ring, 0x00000bc4);
3053 radeon_ring_write(ring, 0xffffffff);
3054 radeon_ring_write(ring, 0xffffffff);
3055 radeon_ring_write(ring, 0xffffffff);
3056
3057 radeon_ring_write(ring, 0xc0026900);
3058 radeon_ring_write(ring, 0x00000316);
3059 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3060 radeon_ring_write(ring, 0x00000010); /* */
3061
3062 radeon_ring_unlock_commit(rdev, ring, false);
3063
3064 return 0;
3065}
3066
3067static int evergreen_cp_resume(struct radeon_device *rdev)
3068{
3069 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3070 u32 tmp;
3071 u32 rb_bufsz;
3072 int r;
3073
3074 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
3075 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
3076 SOFT_RESET_PA |
3077 SOFT_RESET_SH |
3078 SOFT_RESET_VGT |
3079 SOFT_RESET_SPI |
3080 SOFT_RESET_SX));
3081 RREG32(GRBM_SOFT_RESET);
3082 mdelay(15);
3083 WREG32(GRBM_SOFT_RESET, 0);
3084 RREG32(GRBM_SOFT_RESET);
3085
3086 /* Set ring buffer size */
3087 rb_bufsz = order_base_2(ring->ring_size / 8);
3088 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3089#ifdef __BIG_ENDIAN
3090 tmp |= BUF_SWAP_32BIT;
3091#endif
3092 WREG32(CP_RB_CNTL, tmp);
3093 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3094 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3095
3096 /* Set the write pointer delay */
3097 WREG32(CP_RB_WPTR_DELAY, 0);
3098
3099 /* Initialize the ring buffer's read and write pointers */
3100 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
3101 WREG32(CP_RB_RPTR_WR, 0);
3102 ring->wptr = 0;
3103 WREG32(CP_RB_WPTR, ring->wptr);
3104
3105 /* set the wb address whether it's enabled or not */
3106 WREG32(CP_RB_RPTR_ADDR,
3107 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
3108 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3109 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3110
3111 if (rdev->wb.enabled)
3112 WREG32(SCRATCH_UMSK, 0xff);
3113 else {
3114 tmp |= RB_NO_UPDATE;
3115 WREG32(SCRATCH_UMSK, 0);
3116 }
3117
3118 mdelay(1);
3119 WREG32(CP_RB_CNTL, tmp);
3120
3121 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
3122 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
3123
3124 evergreen_cp_start(rdev);
3125 ring->ready = true;
3126 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
3127 if (r) {
3128 ring->ready = false;
3129 return r;
3130 }
3131 return 0;
3132}
3133
3134/*
3135 * Core functions
3136 */
3137static void evergreen_gpu_init(struct radeon_device *rdev)
3138{
3139 u32 gb_addr_config;
3140 u32 mc_arb_ramcfg;
3141 u32 sx_debug_1;
3142 u32 smx_dc_ctl0;
3143 u32 sq_config;
3144 u32 sq_lds_resource_mgmt;
3145 u32 sq_gpr_resource_mgmt_1;
3146 u32 sq_gpr_resource_mgmt_2;
3147 u32 sq_gpr_resource_mgmt_3;
3148 u32 sq_thread_resource_mgmt;
3149 u32 sq_thread_resource_mgmt_2;
3150 u32 sq_stack_resource_mgmt_1;
3151 u32 sq_stack_resource_mgmt_2;
3152 u32 sq_stack_resource_mgmt_3;
3153 u32 vgt_cache_invalidation;
3154 u32 hdp_host_path_cntl, tmp;
3155 u32 disabled_rb_mask;
3156 int i, j, ps_thread_count;
3157
3158 switch (rdev->family) {
3159 case CHIP_CYPRESS:
3160 case CHIP_HEMLOCK:
3161 rdev->config.evergreen.num_ses = 2;
3162 rdev->config.evergreen.max_pipes = 4;
3163 rdev->config.evergreen.max_tile_pipes = 8;
3164 rdev->config.evergreen.max_simds = 10;
3165 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3166 rdev->config.evergreen.max_gprs = 256;
3167 rdev->config.evergreen.max_threads = 248;
3168 rdev->config.evergreen.max_gs_threads = 32;
3169 rdev->config.evergreen.max_stack_entries = 512;
3170 rdev->config.evergreen.sx_num_of_sets = 4;
3171 rdev->config.evergreen.sx_max_export_size = 256;
3172 rdev->config.evergreen.sx_max_export_pos_size = 64;
3173 rdev->config.evergreen.sx_max_export_smx_size = 192;
3174 rdev->config.evergreen.max_hw_contexts = 8;
3175 rdev->config.evergreen.sq_num_cf_insts = 2;
3176
3177 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3178 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3179 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3180 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
3181 break;
3182 case CHIP_JUNIPER:
3183 rdev->config.evergreen.num_ses = 1;
3184 rdev->config.evergreen.max_pipes = 4;
3185 rdev->config.evergreen.max_tile_pipes = 4;
3186 rdev->config.evergreen.max_simds = 10;
3187 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3188 rdev->config.evergreen.max_gprs = 256;
3189 rdev->config.evergreen.max_threads = 248;
3190 rdev->config.evergreen.max_gs_threads = 32;
3191 rdev->config.evergreen.max_stack_entries = 512;
3192 rdev->config.evergreen.sx_num_of_sets = 4;
3193 rdev->config.evergreen.sx_max_export_size = 256;
3194 rdev->config.evergreen.sx_max_export_pos_size = 64;
3195 rdev->config.evergreen.sx_max_export_smx_size = 192;
3196 rdev->config.evergreen.max_hw_contexts = 8;
3197 rdev->config.evergreen.sq_num_cf_insts = 2;
3198
3199 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3200 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3201 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3202 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
3203 break;
3204 case CHIP_REDWOOD:
3205 rdev->config.evergreen.num_ses = 1;
3206 rdev->config.evergreen.max_pipes = 4;
3207 rdev->config.evergreen.max_tile_pipes = 4;
3208 rdev->config.evergreen.max_simds = 5;
3209 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3210 rdev->config.evergreen.max_gprs = 256;
3211 rdev->config.evergreen.max_threads = 248;
3212 rdev->config.evergreen.max_gs_threads = 32;
3213 rdev->config.evergreen.max_stack_entries = 256;
3214 rdev->config.evergreen.sx_num_of_sets = 4;
3215 rdev->config.evergreen.sx_max_export_size = 256;
3216 rdev->config.evergreen.sx_max_export_pos_size = 64;
3217 rdev->config.evergreen.sx_max_export_smx_size = 192;
3218 rdev->config.evergreen.max_hw_contexts = 8;
3219 rdev->config.evergreen.sq_num_cf_insts = 2;
3220
3221 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3222 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3223 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3224 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
3225 break;
3226 case CHIP_CEDAR:
3227 default:
3228 rdev->config.evergreen.num_ses = 1;
3229 rdev->config.evergreen.max_pipes = 2;
3230 rdev->config.evergreen.max_tile_pipes = 2;
3231 rdev->config.evergreen.max_simds = 2;
3232 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3233 rdev->config.evergreen.max_gprs = 256;
3234 rdev->config.evergreen.max_threads = 192;
3235 rdev->config.evergreen.max_gs_threads = 16;
3236 rdev->config.evergreen.max_stack_entries = 256;
3237 rdev->config.evergreen.sx_num_of_sets = 4;
3238 rdev->config.evergreen.sx_max_export_size = 128;
3239 rdev->config.evergreen.sx_max_export_pos_size = 32;
3240 rdev->config.evergreen.sx_max_export_smx_size = 96;
3241 rdev->config.evergreen.max_hw_contexts = 4;
3242 rdev->config.evergreen.sq_num_cf_insts = 1;
3243
3244 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3245 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3246 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3247 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3248 break;
3249 case CHIP_PALM:
3250 rdev->config.evergreen.num_ses = 1;
3251 rdev->config.evergreen.max_pipes = 2;
3252 rdev->config.evergreen.max_tile_pipes = 2;
3253 rdev->config.evergreen.max_simds = 2;
3254 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3255 rdev->config.evergreen.max_gprs = 256;
3256 rdev->config.evergreen.max_threads = 192;
3257 rdev->config.evergreen.max_gs_threads = 16;
3258 rdev->config.evergreen.max_stack_entries = 256;
3259 rdev->config.evergreen.sx_num_of_sets = 4;
3260 rdev->config.evergreen.sx_max_export_size = 128;
3261 rdev->config.evergreen.sx_max_export_pos_size = 32;
3262 rdev->config.evergreen.sx_max_export_smx_size = 96;
3263 rdev->config.evergreen.max_hw_contexts = 4;
3264 rdev->config.evergreen.sq_num_cf_insts = 1;
3265
3266 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3267 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3268 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3269 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3270 break;
3271 case CHIP_SUMO:
3272 rdev->config.evergreen.num_ses = 1;
3273 rdev->config.evergreen.max_pipes = 4;
3274 rdev->config.evergreen.max_tile_pipes = 4;
3275 if (rdev->pdev->device == 0x9648)
3276 rdev->config.evergreen.max_simds = 3;
3277 else if ((rdev->pdev->device == 0x9647) ||
3278 (rdev->pdev->device == 0x964a))
3279 rdev->config.evergreen.max_simds = 4;
3280 else
3281 rdev->config.evergreen.max_simds = 5;
3282 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3283 rdev->config.evergreen.max_gprs = 256;
3284 rdev->config.evergreen.max_threads = 248;
3285 rdev->config.evergreen.max_gs_threads = 32;
3286 rdev->config.evergreen.max_stack_entries = 256;
3287 rdev->config.evergreen.sx_num_of_sets = 4;
3288 rdev->config.evergreen.sx_max_export_size = 256;
3289 rdev->config.evergreen.sx_max_export_pos_size = 64;
3290 rdev->config.evergreen.sx_max_export_smx_size = 192;
3291 rdev->config.evergreen.max_hw_contexts = 8;
3292 rdev->config.evergreen.sq_num_cf_insts = 2;
3293
3294 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3295 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3296 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3297 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
3298 break;
3299 case CHIP_SUMO2:
3300 rdev->config.evergreen.num_ses = 1;
3301 rdev->config.evergreen.max_pipes = 4;
3302 rdev->config.evergreen.max_tile_pipes = 4;
3303 rdev->config.evergreen.max_simds = 2;
3304 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3305 rdev->config.evergreen.max_gprs = 256;
3306 rdev->config.evergreen.max_threads = 248;
3307 rdev->config.evergreen.max_gs_threads = 32;
3308 rdev->config.evergreen.max_stack_entries = 512;
3309 rdev->config.evergreen.sx_num_of_sets = 4;
3310 rdev->config.evergreen.sx_max_export_size = 256;
3311 rdev->config.evergreen.sx_max_export_pos_size = 64;
3312 rdev->config.evergreen.sx_max_export_smx_size = 192;
3313 rdev->config.evergreen.max_hw_contexts = 4;
3314 rdev->config.evergreen.sq_num_cf_insts = 2;
3315
3316 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3317 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3318 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3319 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
3320 break;
3321 case CHIP_BARTS:
3322 rdev->config.evergreen.num_ses = 2;
3323 rdev->config.evergreen.max_pipes = 4;
3324 rdev->config.evergreen.max_tile_pipes = 8;
3325 rdev->config.evergreen.max_simds = 7;
3326 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3327 rdev->config.evergreen.max_gprs = 256;
3328 rdev->config.evergreen.max_threads = 248;
3329 rdev->config.evergreen.max_gs_threads = 32;
3330 rdev->config.evergreen.max_stack_entries = 512;
3331 rdev->config.evergreen.sx_num_of_sets = 4;
3332 rdev->config.evergreen.sx_max_export_size = 256;
3333 rdev->config.evergreen.sx_max_export_pos_size = 64;
3334 rdev->config.evergreen.sx_max_export_smx_size = 192;
3335 rdev->config.evergreen.max_hw_contexts = 8;
3336 rdev->config.evergreen.sq_num_cf_insts = 2;
3337
3338 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3339 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3340 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3341 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
3342 break;
3343 case CHIP_TURKS:
3344 rdev->config.evergreen.num_ses = 1;
3345 rdev->config.evergreen.max_pipes = 4;
3346 rdev->config.evergreen.max_tile_pipes = 4;
3347 rdev->config.evergreen.max_simds = 6;
3348 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3349 rdev->config.evergreen.max_gprs = 256;
3350 rdev->config.evergreen.max_threads = 248;
3351 rdev->config.evergreen.max_gs_threads = 32;
3352 rdev->config.evergreen.max_stack_entries = 256;
3353 rdev->config.evergreen.sx_num_of_sets = 4;
3354 rdev->config.evergreen.sx_max_export_size = 256;
3355 rdev->config.evergreen.sx_max_export_pos_size = 64;
3356 rdev->config.evergreen.sx_max_export_smx_size = 192;
3357 rdev->config.evergreen.max_hw_contexts = 8;
3358 rdev->config.evergreen.sq_num_cf_insts = 2;
3359
3360 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3361 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3362 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3363 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
3364 break;
3365 case CHIP_CAICOS:
3366 rdev->config.evergreen.num_ses = 1;
3367 rdev->config.evergreen.max_pipes = 2;
3368 rdev->config.evergreen.max_tile_pipes = 2;
3369 rdev->config.evergreen.max_simds = 2;
3370 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3371 rdev->config.evergreen.max_gprs = 256;
3372 rdev->config.evergreen.max_threads = 192;
3373 rdev->config.evergreen.max_gs_threads = 16;
3374 rdev->config.evergreen.max_stack_entries = 256;
3375 rdev->config.evergreen.sx_num_of_sets = 4;
3376 rdev->config.evergreen.sx_max_export_size = 128;
3377 rdev->config.evergreen.sx_max_export_pos_size = 32;
3378 rdev->config.evergreen.sx_max_export_smx_size = 96;
3379 rdev->config.evergreen.max_hw_contexts = 4;
3380 rdev->config.evergreen.sq_num_cf_insts = 1;
3381
3382 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3383 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3384 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3385 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
3386 break;
3387 }
3388
3389 /* Initialize HDP */
3390 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3391 WREG32((0x2c14 + j), 0x00000000);
3392 WREG32((0x2c18 + j), 0x00000000);
3393 WREG32((0x2c1c + j), 0x00000000);
3394 WREG32((0x2c20 + j), 0x00000000);
3395 WREG32((0x2c24 + j), 0x00000000);
3396 }
3397
3398 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3399 WREG32(SRBM_INT_CNTL, 0x1);
3400 WREG32(SRBM_INT_ACK, 0x1);
3401
3402 evergreen_fix_pci_max_read_req_size(rdev);
3403
3404 RREG32(MC_SHARED_CHMAP);
3405 if ((rdev->family == CHIP_PALM) ||
3406 (rdev->family == CHIP_SUMO) ||
3407 (rdev->family == CHIP_SUMO2))
3408 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3409 else
3410 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3411
3412 /* setup tiling info dword. gb_addr_config is not adequate since it does
3413 * not have bank info, so create a custom tiling dword.
3414 * bits 3:0 num_pipes
3415 * bits 7:4 num_banks
3416 * bits 11:8 group_size
3417 * bits 15:12 row_size
3418 */
3419 rdev->config.evergreen.tile_config = 0;
3420 switch (rdev->config.evergreen.max_tile_pipes) {
3421 case 1:
3422 default:
3423 rdev->config.evergreen.tile_config |= (0 << 0);
3424 break;
3425 case 2:
3426 rdev->config.evergreen.tile_config |= (1 << 0);
3427 break;
3428 case 4:
3429 rdev->config.evergreen.tile_config |= (2 << 0);
3430 break;
3431 case 8:
3432 rdev->config.evergreen.tile_config |= (3 << 0);
3433 break;
3434 }
3435 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
3436 if (rdev->flags & RADEON_IS_IGP)
3437 rdev->config.evergreen.tile_config |= 1 << 4;
3438 else {
3439 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3440 case 0: /* four banks */
3441 rdev->config.evergreen.tile_config |= 0 << 4;
3442 break;
3443 case 1: /* eight banks */
3444 rdev->config.evergreen.tile_config |= 1 << 4;
3445 break;
3446 case 2: /* sixteen banks */
3447 default:
3448 rdev->config.evergreen.tile_config |= 2 << 4;
3449 break;
3450 }
3451 }
3452 rdev->config.evergreen.tile_config |= 0 << 8;
3453 rdev->config.evergreen.tile_config |=
3454 ((gb_addr_config & 0x30000000) >> 28) << 12;
3455
3456 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3457 u32 efuse_straps_4;
3458 u32 efuse_straps_3;
3459
3460 efuse_straps_4 = RREG32_RCU(0x204);
3461 efuse_straps_3 = RREG32_RCU(0x203);
3462 tmp = (((efuse_straps_4 & 0xf) << 4) |
3463 ((efuse_straps_3 & 0xf0000000) >> 28));
3464 } else {
3465 tmp = 0;
3466 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3467 u32 rb_disable_bitmap;
3468
3469 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3470 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3471 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3472 tmp <<= 4;
3473 tmp |= rb_disable_bitmap;
3474 }
3475 }
3476 /* enabled rb are just the one not disabled :) */
3477 disabled_rb_mask = tmp;
3478 tmp = 0;
3479 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3480 tmp |= (1 << i);
3481 /* if all the backends are disabled, fix it up here */
3482 if ((disabled_rb_mask & tmp) == tmp) {
3483 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3484 disabled_rb_mask &= ~(1 << i);
3485 }
3486
3487 for (i = 0; i < rdev->config.evergreen.num_ses; i++) {
3488 u32 simd_disable_bitmap;
3489
3490 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3491 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3492 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
3493 simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds;
3494 tmp <<= 16;
3495 tmp |= simd_disable_bitmap;
3496 }
3497 rdev->config.evergreen.active_simds = hweight32(~tmp);
3498
3499 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3500 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3501
3502 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3503 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3504 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3505 WREG32(DMA_TILING_CONFIG, gb_addr_config);
3506 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3507 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3508 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3509
3510 if ((rdev->config.evergreen.max_backends == 1) &&
3511 (rdev->flags & RADEON_IS_IGP)) {
3512 if ((disabled_rb_mask & 3) == 1) {
3513 /* RB0 disabled, RB1 enabled */
3514 tmp = 0x11111111;
3515 } else {
3516 /* RB1 disabled, RB0 enabled */
3517 tmp = 0x00000000;
3518 }
3519 } else {
3520 tmp = gb_addr_config & NUM_PIPES_MASK;
3521 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3522 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3523 }
3524 rdev->config.evergreen.backend_map = tmp;
3525 WREG32(GB_BACKEND_MAP, tmp);
3526
3527 WREG32(CGTS_SYS_TCC_DISABLE, 0);
3528 WREG32(CGTS_TCC_DISABLE, 0);
3529 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3530 WREG32(CGTS_USER_TCC_DISABLE, 0);
3531
3532 /* set HW defaults for 3D engine */
3533 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3534 ROQ_IB2_START(0x2b)));
3535
3536 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3537
3538 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3539 SYNC_GRADIENT |
3540 SYNC_WALKER |
3541 SYNC_ALIGNER));
3542
3543 sx_debug_1 = RREG32(SX_DEBUG_1);
3544 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3545 WREG32(SX_DEBUG_1, sx_debug_1);
3546
3547
3548 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3549 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3550 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3551 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3552
3553 if (rdev->family <= CHIP_SUMO2)
3554 WREG32(SMX_SAR_CTL0, 0x00010000);
3555
3556 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3557 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3558 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3559
3560 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3561 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3562 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3563
3564 WREG32(VGT_NUM_INSTANCES, 1);
3565 WREG32(SPI_CONFIG_CNTL, 0);
3566 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3567 WREG32(CP_PERFMON_CNTL, 0);
3568
3569 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3570 FETCH_FIFO_HIWATER(0x4) |
3571 DONE_FIFO_HIWATER(0xe0) |
3572 ALU_UPDATE_FIFO_HIWATER(0x8)));
3573
3574 sq_config = RREG32(SQ_CONFIG);
3575 sq_config &= ~(PS_PRIO(3) |
3576 VS_PRIO(3) |
3577 GS_PRIO(3) |
3578 ES_PRIO(3));
3579 sq_config |= (VC_ENABLE |
3580 EXPORT_SRC_C |
3581 PS_PRIO(0) |
3582 VS_PRIO(1) |
3583 GS_PRIO(2) |
3584 ES_PRIO(3));
3585
3586 switch (rdev->family) {
3587 case CHIP_CEDAR:
3588 case CHIP_PALM:
3589 case CHIP_SUMO:
3590 case CHIP_SUMO2:
3591 case CHIP_CAICOS:
3592 /* no vertex cache */
3593 sq_config &= ~VC_ENABLE;
3594 break;
3595 default:
3596 break;
3597 }
3598
3599 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3600
3601 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 12 / 32);
3602 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3603 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3604 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3605 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3606 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3607 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3608
3609 switch (rdev->family) {
3610 case CHIP_CEDAR:
3611 case CHIP_PALM:
3612 case CHIP_SUMO:
3613 case CHIP_SUMO2:
3614 ps_thread_count = 96;
3615 break;
3616 default:
3617 ps_thread_count = 128;
3618 break;
3619 }
3620
3621 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
3622 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3623 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3624 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3625 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3626 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3627
3628 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3629 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3630 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3631 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3632 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3633 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3634
3635 WREG32(SQ_CONFIG, sq_config);
3636 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3637 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3638 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3639 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3640 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3641 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3642 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3643 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3644 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3645 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3646
3647 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3648 FORCE_EOV_MAX_REZ_CNT(255)));
3649
3650 switch (rdev->family) {
3651 case CHIP_CEDAR:
3652 case CHIP_PALM:
3653 case CHIP_SUMO:
3654 case CHIP_SUMO2:
3655 case CHIP_CAICOS:
3656 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
3657 break;
3658 default:
3659 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
3660 break;
3661 }
3662 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3663 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3664
3665 WREG32(VGT_GS_VERTEX_REUSE, 16);
3666 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
3667 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3668
3669 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3670 WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3671
3672 WREG32(CB_PERF_CTR0_SEL_0, 0);
3673 WREG32(CB_PERF_CTR0_SEL_1, 0);
3674 WREG32(CB_PERF_CTR1_SEL_0, 0);
3675 WREG32(CB_PERF_CTR1_SEL_1, 0);
3676 WREG32(CB_PERF_CTR2_SEL_0, 0);
3677 WREG32(CB_PERF_CTR2_SEL_1, 0);
3678 WREG32(CB_PERF_CTR3_SEL_0, 0);
3679 WREG32(CB_PERF_CTR3_SEL_1, 0);
3680
3681 /* clear render buffer base addresses */
3682 WREG32(CB_COLOR0_BASE, 0);
3683 WREG32(CB_COLOR1_BASE, 0);
3684 WREG32(CB_COLOR2_BASE, 0);
3685 WREG32(CB_COLOR3_BASE, 0);
3686 WREG32(CB_COLOR4_BASE, 0);
3687 WREG32(CB_COLOR5_BASE, 0);
3688 WREG32(CB_COLOR6_BASE, 0);
3689 WREG32(CB_COLOR7_BASE, 0);
3690 WREG32(CB_COLOR8_BASE, 0);
3691 WREG32(CB_COLOR9_BASE, 0);
3692 WREG32(CB_COLOR10_BASE, 0);
3693 WREG32(CB_COLOR11_BASE, 0);
3694
3695 /* set the shader const cache sizes to 0 */
3696 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3697 WREG32(i, 0);
3698 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3699 WREG32(i, 0);
3700
3701 tmp = RREG32(HDP_MISC_CNTL);
3702 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3703 WREG32(HDP_MISC_CNTL, tmp);
3704
3705 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3706 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3707
3708 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3709
3710 udelay(50);
3711
3712}
3713
3714int evergreen_mc_init(struct radeon_device *rdev)
3715{
3716 u32 tmp;
3717 int chansize, numchan;
3718
3719 /* Get VRAM informations */
3720 rdev->mc.vram_is_ddr = true;
3721 if ((rdev->family == CHIP_PALM) ||
3722 (rdev->family == CHIP_SUMO) ||
3723 (rdev->family == CHIP_SUMO2))
3724 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3725 else
3726 tmp = RREG32(MC_ARB_RAMCFG);
3727 if (tmp & CHANSIZE_OVERRIDE) {
3728 chansize = 16;
3729 } else if (tmp & CHANSIZE_MASK) {
3730 chansize = 64;
3731 } else {
3732 chansize = 32;
3733 }
3734 tmp = RREG32(MC_SHARED_CHMAP);
3735 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3736 case 0:
3737 default:
3738 numchan = 1;
3739 break;
3740 case 1:
3741 numchan = 2;
3742 break;
3743 case 2:
3744 numchan = 4;
3745 break;
3746 case 3:
3747 numchan = 8;
3748 break;
3749 }
3750 rdev->mc.vram_width = numchan * chansize;
3751 /* Could aper size report 0 ? */
3752 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3753 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3754 /* Setup GPU memory space */
3755 if ((rdev->family == CHIP_PALM) ||
3756 (rdev->family == CHIP_SUMO) ||
3757 (rdev->family == CHIP_SUMO2)) {
3758 /* size in bytes on fusion */
3759 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3760 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3761 } else {
3762 /* size in MB on evergreen/cayman/tn */
3763 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3764 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3765 }
3766 rdev->mc.visible_vram_size = rdev->mc.aper_size;
3767 r700_vram_gtt_location(rdev, &rdev->mc);
3768 radeon_update_bandwidth_info(rdev);
3769
3770 return 0;
3771}
3772
3773void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
3774{
3775 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n",
3776 RREG32(GRBM_STATUS));
3777 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n",
3778 RREG32(GRBM_STATUS_SE0));
3779 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n",
3780 RREG32(GRBM_STATUS_SE1));
3781 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n",
3782 RREG32(SRBM_STATUS));
3783 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n",
3784 RREG32(SRBM_STATUS2));
3785 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3786 RREG32(CP_STALLED_STAT1));
3787 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3788 RREG32(CP_STALLED_STAT2));
3789 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
3790 RREG32(CP_BUSY_STAT));
3791 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
3792 RREG32(CP_STAT));
3793 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
3794 RREG32(DMA_STATUS_REG));
3795 if (rdev->family >= CHIP_CAYMAN) {
3796 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n",
3797 RREG32(DMA_STATUS_REG + 0x800));
3798 }
3799}
3800
3801bool evergreen_is_display_hung(struct radeon_device *rdev)
3802{
3803 u32 crtc_hung = 0;
3804 u32 crtc_status[6];
3805 u32 i, j, tmp;
3806
3807 for (i = 0; i < rdev->num_crtc; i++) {
3808 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3809 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3810 crtc_hung |= (1 << i);
3811 }
3812 }
3813
3814 for (j = 0; j < 10; j++) {
3815 for (i = 0; i < rdev->num_crtc; i++) {
3816 if (crtc_hung & (1 << i)) {
3817 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3818 if (tmp != crtc_status[i])
3819 crtc_hung &= ~(1 << i);
3820 }
3821 }
3822 if (crtc_hung == 0)
3823 return false;
3824 udelay(100);
3825 }
3826
3827 return true;
3828}
3829
3830u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3831{
3832 u32 reset_mask = 0;
3833 u32 tmp;
3834 struct drm_device *ddev = rdev_to_drm(rdev);
3835
3836 /* GRBM_STATUS */
3837 tmp = RREG32(GRBM_STATUS);
3838 if (tmp & (PA_BUSY | SC_BUSY |
3839 SH_BUSY | SX_BUSY |
3840 TA_BUSY | VGT_BUSY |
3841 DB_BUSY | CB_BUSY |
3842 SPI_BUSY | VGT_BUSY_NO_DMA))
3843 reset_mask |= RADEON_RESET_GFX;
3844
3845 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3846 CP_BUSY | CP_COHERENCY_BUSY))
3847 reset_mask |= RADEON_RESET_CP;
3848
3849 if (tmp & GRBM_EE_BUSY)
3850 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3851
3852 /* DMA_STATUS_REG */
3853 tmp = RREG32(DMA_STATUS_REG);
3854 if (!(tmp & DMA_IDLE))
3855 reset_mask |= RADEON_RESET_DMA;
3856
3857 /* SRBM_STATUS2 */
3858 tmp = RREG32(SRBM_STATUS2);
3859 if (tmp & DMA_BUSY)
3860 reset_mask |= RADEON_RESET_DMA;
3861
3862 /* SRBM_STATUS */
3863 tmp = RREG32(SRBM_STATUS);
3864 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3865 reset_mask |= RADEON_RESET_RLC;
3866
3867 if (tmp & IH_BUSY)
3868 reset_mask |= RADEON_RESET_IH;
3869
3870 if (tmp & SEM_BUSY)
3871 reset_mask |= RADEON_RESET_SEM;
3872
3873 if (tmp & GRBM_RQ_PENDING)
3874 reset_mask |= RADEON_RESET_GRBM;
3875
3876 if (tmp & VMC_BUSY)
3877 reset_mask |= RADEON_RESET_VMC;
3878
3879 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3880 MCC_BUSY | MCD_BUSY))
3881 reset_mask |= RADEON_RESET_MC;
3882
3883 if (evergreen_is_display_hung(rdev))
3884 reset_mask |= RADEON_RESET_DISPLAY;
3885
3886 /* VM_L2_STATUS */
3887 tmp = RREG32(VM_L2_STATUS);
3888 if (tmp & L2_BUSY)
3889 reset_mask |= RADEON_RESET_VMC;
3890
3891 /* Skip MC reset as it's mostly likely not hung, just busy */
3892 if (reset_mask & RADEON_RESET_MC) {
3893 drm_dbg(ddev, "MC busy: 0x%08X, clearing.\n", reset_mask);
3894 reset_mask &= ~RADEON_RESET_MC;
3895 }
3896
3897 return reset_mask;
3898}
3899
3900static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3901{
3902 struct evergreen_mc_save save;
3903 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3904 u32 tmp;
3905
3906 if (reset_mask == 0)
3907 return;
3908
3909 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3910
3911 evergreen_print_gpu_status_regs(rdev);
3912
3913 /* Disable CP parsing/prefetching */
3914 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3915
3916 if (reset_mask & RADEON_RESET_DMA) {
3917 /* Disable DMA */
3918 tmp = RREG32(DMA_RB_CNTL);
3919 tmp &= ~DMA_RB_ENABLE;
3920 WREG32(DMA_RB_CNTL, tmp);
3921 }
3922
3923 udelay(50);
3924
3925 evergreen_mc_stop(rdev, &save);
3926 if (evergreen_mc_wait_for_idle(rdev)) {
3927 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3928 }
3929
3930 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3931 grbm_soft_reset |= SOFT_RESET_DB |
3932 SOFT_RESET_CB |
3933 SOFT_RESET_PA |
3934 SOFT_RESET_SC |
3935 SOFT_RESET_SPI |
3936 SOFT_RESET_SX |
3937 SOFT_RESET_SH |
3938 SOFT_RESET_TC |
3939 SOFT_RESET_TA |
3940 SOFT_RESET_VC |
3941 SOFT_RESET_VGT;
3942 }
3943
3944 if (reset_mask & RADEON_RESET_CP) {
3945 grbm_soft_reset |= SOFT_RESET_CP |
3946 SOFT_RESET_VGT;
3947
3948 srbm_soft_reset |= SOFT_RESET_GRBM;
3949 }
3950
3951 if (reset_mask & RADEON_RESET_DMA)
3952 srbm_soft_reset |= SOFT_RESET_DMA;
3953
3954 if (reset_mask & RADEON_RESET_DISPLAY)
3955 srbm_soft_reset |= SOFT_RESET_DC;
3956
3957 if (reset_mask & RADEON_RESET_RLC)
3958 srbm_soft_reset |= SOFT_RESET_RLC;
3959
3960 if (reset_mask & RADEON_RESET_SEM)
3961 srbm_soft_reset |= SOFT_RESET_SEM;
3962
3963 if (reset_mask & RADEON_RESET_IH)
3964 srbm_soft_reset |= SOFT_RESET_IH;
3965
3966 if (reset_mask & RADEON_RESET_GRBM)
3967 srbm_soft_reset |= SOFT_RESET_GRBM;
3968
3969 if (reset_mask & RADEON_RESET_VMC)
3970 srbm_soft_reset |= SOFT_RESET_VMC;
3971
3972 if (!(rdev->flags & RADEON_IS_IGP)) {
3973 if (reset_mask & RADEON_RESET_MC)
3974 srbm_soft_reset |= SOFT_RESET_MC;
3975 }
3976
3977 if (grbm_soft_reset) {
3978 tmp = RREG32(GRBM_SOFT_RESET);
3979 tmp |= grbm_soft_reset;
3980 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3981 WREG32(GRBM_SOFT_RESET, tmp);
3982 tmp = RREG32(GRBM_SOFT_RESET);
3983
3984 udelay(50);
3985
3986 tmp &= ~grbm_soft_reset;
3987 WREG32(GRBM_SOFT_RESET, tmp);
3988 tmp = RREG32(GRBM_SOFT_RESET);
3989 }
3990
3991 if (srbm_soft_reset) {
3992 tmp = RREG32(SRBM_SOFT_RESET);
3993 tmp |= srbm_soft_reset;
3994 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3995 WREG32(SRBM_SOFT_RESET, tmp);
3996 tmp = RREG32(SRBM_SOFT_RESET);
3997
3998 udelay(50);
3999
4000 tmp &= ~srbm_soft_reset;
4001 WREG32(SRBM_SOFT_RESET, tmp);
4002 tmp = RREG32(SRBM_SOFT_RESET);
4003 }
4004
4005 /* Wait a little for things to settle down */
4006 udelay(50);
4007
4008 evergreen_mc_resume(rdev, &save);
4009 udelay(50);
4010
4011 evergreen_print_gpu_status_regs(rdev);
4012}
4013
4014void evergreen_gpu_pci_config_reset(struct radeon_device *rdev)
4015{
4016 struct evergreen_mc_save save;
4017 u32 tmp, i;
4018
4019 dev_info(rdev->dev, "GPU pci config reset\n");
4020
4021 /* disable dpm? */
4022
4023 /* Disable CP parsing/prefetching */
4024 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
4025 udelay(50);
4026 /* Disable DMA */
4027 tmp = RREG32(DMA_RB_CNTL);
4028 tmp &= ~DMA_RB_ENABLE;
4029 WREG32(DMA_RB_CNTL, tmp);
4030 /* XXX other engines? */
4031
4032 /* halt the rlc */
4033 r600_rlc_stop(rdev);
4034
4035 udelay(50);
4036
4037 /* set mclk/sclk to bypass */
4038 rv770_set_clk_bypass_mode(rdev);
4039 /* disable BM */
4040 pci_clear_master(rdev->pdev);
4041 /* disable mem access */
4042 evergreen_mc_stop(rdev, &save);
4043 if (evergreen_mc_wait_for_idle(rdev)) {
4044 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
4045 }
4046 /* reset */
4047 radeon_pci_config_reset(rdev);
4048 /* wait for asic to come out of reset */
4049 for (i = 0; i < rdev->usec_timeout; i++) {
4050 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
4051 break;
4052 udelay(1);
4053 }
4054}
4055
4056int evergreen_asic_reset(struct radeon_device *rdev, bool hard)
4057{
4058 u32 reset_mask;
4059
4060 if (hard) {
4061 evergreen_gpu_pci_config_reset(rdev);
4062 return 0;
4063 }
4064
4065 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4066
4067 if (reset_mask)
4068 r600_set_bios_scratch_engine_hung(rdev, true);
4069
4070 /* try soft reset */
4071 evergreen_gpu_soft_reset(rdev, reset_mask);
4072
4073 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4074
4075 /* try pci config reset */
4076 if (reset_mask && radeon_hard_reset)
4077 evergreen_gpu_pci_config_reset(rdev);
4078
4079 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4080
4081 if (!reset_mask)
4082 r600_set_bios_scratch_engine_hung(rdev, false);
4083
4084 return 0;
4085}
4086
4087/**
4088 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
4089 *
4090 * @rdev: radeon_device pointer
4091 * @ring: radeon_ring structure holding ring information
4092 *
4093 * Check if the GFX engine is locked up.
4094 * Returns true if the engine appears to be locked up, false if not.
4095 */
4096bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4097{
4098 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4099
4100 if (!(reset_mask & (RADEON_RESET_GFX |
4101 RADEON_RESET_COMPUTE |
4102 RADEON_RESET_CP))) {
4103 radeon_ring_lockup_update(rdev, ring);
4104 return false;
4105 }
4106 return radeon_ring_test_lockup(rdev, ring);
4107}
4108
4109/*
4110 * RLC
4111 */
4112#define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000
4113#define RLC_CLEAR_STATE_END_MARKER 0x00000001
4114
4115void sumo_rlc_fini(struct radeon_device *rdev)
4116{
4117 int r;
4118
4119 /* save restore block */
4120 if (rdev->rlc.save_restore_obj) {
4121 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4122 if (unlikely(r != 0))
4123 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
4124 radeon_bo_unpin(rdev->rlc.save_restore_obj);
4125 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4126
4127 radeon_bo_unref(&rdev->rlc.save_restore_obj);
4128 rdev->rlc.save_restore_obj = NULL;
4129 }
4130
4131 /* clear state block */
4132 if (rdev->rlc.clear_state_obj) {
4133 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4134 if (unlikely(r != 0))
4135 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
4136 radeon_bo_unpin(rdev->rlc.clear_state_obj);
4137 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4138
4139 radeon_bo_unref(&rdev->rlc.clear_state_obj);
4140 rdev->rlc.clear_state_obj = NULL;
4141 }
4142
4143 /* clear state block */
4144 if (rdev->rlc.cp_table_obj) {
4145 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4146 if (unlikely(r != 0))
4147 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4148 radeon_bo_unpin(rdev->rlc.cp_table_obj);
4149 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4150
4151 radeon_bo_unref(&rdev->rlc.cp_table_obj);
4152 rdev->rlc.cp_table_obj = NULL;
4153 }
4154}
4155
4156#define CP_ME_TABLE_SIZE 96
4157
4158int sumo_rlc_init(struct radeon_device *rdev)
4159{
4160 const u32 *src_ptr;
4161 volatile u32 *dst_ptr;
4162 u32 dws, data, i, j, k, reg_num;
4163 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
4164 u64 reg_list_mc_addr;
4165 const struct cs_section_def *cs_data;
4166 int r;
4167
4168 src_ptr = rdev->rlc.reg_list;
4169 dws = rdev->rlc.reg_list_size;
4170 if (rdev->family >= CHIP_BONAIRE) {
4171 dws += (5 * 16) + 48 + 48 + 64;
4172 }
4173 cs_data = rdev->rlc.cs_data;
4174
4175 if (src_ptr) {
4176 /* save restore block */
4177 if (rdev->rlc.save_restore_obj == NULL) {
4178 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4179 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4180 NULL, &rdev->rlc.save_restore_obj);
4181 if (r) {
4182 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4183 return r;
4184 }
4185 }
4186
4187 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4188 if (unlikely(r != 0)) {
4189 sumo_rlc_fini(rdev);
4190 return r;
4191 }
4192 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4193 &rdev->rlc.save_restore_gpu_addr);
4194 if (r) {
4195 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4196 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4197 sumo_rlc_fini(rdev);
4198 return r;
4199 }
4200
4201 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
4202 if (r) {
4203 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
4204 sumo_rlc_fini(rdev);
4205 return r;
4206 }
4207 /* write the sr buffer */
4208 dst_ptr = rdev->rlc.sr_ptr;
4209 if (rdev->family >= CHIP_TAHITI) {
4210 /* SI */
4211 for (i = 0; i < rdev->rlc.reg_list_size; i++)
4212 dst_ptr[i] = cpu_to_le32(src_ptr[i]);
4213 } else {
4214 /* ON/LN/TN */
4215 /* format:
4216 * dw0: (reg2 << 16) | reg1
4217 * dw1: reg1 save space
4218 * dw2: reg2 save space
4219 */
4220 for (i = 0; i < dws; i++) {
4221 data = src_ptr[i] >> 2;
4222 i++;
4223 if (i < dws)
4224 data |= (src_ptr[i] >> 2) << 16;
4225 j = (((i - 1) * 3) / 2);
4226 dst_ptr[j] = cpu_to_le32(data);
4227 }
4228 j = ((i * 3) / 2);
4229 dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER);
4230 }
4231 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
4232 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4233 }
4234
4235 if (cs_data) {
4236 /* clear state block */
4237 if (rdev->family >= CHIP_BONAIRE) {
4238 rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
4239 } else if (rdev->family >= CHIP_TAHITI) {
4240 rdev->rlc.clear_state_size = si_get_csb_size(rdev);
4241 dws = rdev->rlc.clear_state_size + (256 / 4);
4242 } else {
4243 reg_list_num = 0;
4244 dws = 0;
4245 for (i = 0; cs_data[i].section != NULL; i++) {
4246 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4247 reg_list_num++;
4248 dws += cs_data[i].section[j].reg_count;
4249 }
4250 }
4251 reg_list_blk_index = (3 * reg_list_num + 2);
4252 dws += reg_list_blk_index;
4253 rdev->rlc.clear_state_size = dws;
4254 }
4255
4256 if (rdev->rlc.clear_state_obj == NULL) {
4257 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4258 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4259 NULL, &rdev->rlc.clear_state_obj);
4260 if (r) {
4261 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4262 sumo_rlc_fini(rdev);
4263 return r;
4264 }
4265 }
4266 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4267 if (unlikely(r != 0)) {
4268 sumo_rlc_fini(rdev);
4269 return r;
4270 }
4271 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4272 &rdev->rlc.clear_state_gpu_addr);
4273 if (r) {
4274 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4275 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
4276 sumo_rlc_fini(rdev);
4277 return r;
4278 }
4279
4280 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
4281 if (r) {
4282 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4283 sumo_rlc_fini(rdev);
4284 return r;
4285 }
4286 /* set up the cs buffer */
4287 dst_ptr = rdev->rlc.cs_ptr;
4288 if (rdev->family >= CHIP_BONAIRE) {
4289 cik_get_csb_buffer(rdev, dst_ptr);
4290 } else if (rdev->family >= CHIP_TAHITI) {
4291 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
4292 dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr));
4293 dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr));
4294 dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size);
4295 si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
4296 } else {
4297 reg_list_hdr_blk_index = 0;
4298 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4299 data = upper_32_bits(reg_list_mc_addr);
4300 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4301 reg_list_hdr_blk_index++;
4302 for (i = 0; cs_data[i].section != NULL; i++) {
4303 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4304 reg_num = cs_data[i].section[j].reg_count;
4305 data = reg_list_mc_addr & 0xffffffff;
4306 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4307 reg_list_hdr_blk_index++;
4308
4309 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4310 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4311 reg_list_hdr_blk_index++;
4312
4313 data = 0x08000000 | (reg_num * 4);
4314 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4315 reg_list_hdr_blk_index++;
4316
4317 for (k = 0; k < reg_num; k++) {
4318 data = cs_data[i].section[j].extent[k];
4319 dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data);
4320 }
4321 reg_list_mc_addr += reg_num * 4;
4322 reg_list_blk_index += reg_num;
4323 }
4324 }
4325 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER);
4326 }
4327 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4328 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4329 }
4330
4331 if (rdev->rlc.cp_table_size) {
4332 if (rdev->rlc.cp_table_obj == NULL) {
4333 r = radeon_bo_create(rdev, rdev->rlc.cp_table_size,
4334 PAGE_SIZE, true,
4335 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4336 NULL, &rdev->rlc.cp_table_obj);
4337 if (r) {
4338 dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
4339 sumo_rlc_fini(rdev);
4340 return r;
4341 }
4342 }
4343
4344 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4345 if (unlikely(r != 0)) {
4346 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4347 sumo_rlc_fini(rdev);
4348 return r;
4349 }
4350 r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
4351 &rdev->rlc.cp_table_gpu_addr);
4352 if (r) {
4353 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4354 dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
4355 sumo_rlc_fini(rdev);
4356 return r;
4357 }
4358 r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);
4359 if (r) {
4360 dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
4361 sumo_rlc_fini(rdev);
4362 return r;
4363 }
4364
4365 cik_init_cp_pg_table(rdev);
4366
4367 radeon_bo_kunmap(rdev->rlc.cp_table_obj);
4368 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4369
4370 }
4371
4372 return 0;
4373}
4374
4375static void evergreen_rlc_start(struct radeon_device *rdev)
4376{
4377 u32 mask = RLC_ENABLE;
4378
4379 if (rdev->flags & RADEON_IS_IGP) {
4380 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
4381 }
4382
4383 WREG32(RLC_CNTL, mask);
4384}
4385
4386int evergreen_rlc_resume(struct radeon_device *rdev)
4387{
4388 u32 i;
4389 const __be32 *fw_data;
4390
4391 if (!rdev->rlc_fw)
4392 return -EINVAL;
4393
4394 r600_rlc_stop(rdev);
4395
4396 WREG32(RLC_HB_CNTL, 0);
4397
4398 if (rdev->flags & RADEON_IS_IGP) {
4399 if (rdev->family == CHIP_ARUBA) {
4400 u32 always_on_bitmap =
4401 3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4402 /* find out the number of active simds */
4403 u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4404 tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4405 tmp = hweight32(~tmp);
4406 if (tmp == rdev->config.cayman.max_simds_per_se) {
4407 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4408 WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4409 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4410 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4411 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4412 }
4413 } else {
4414 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4415 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4416 }
4417 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4418 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4419 } else {
4420 WREG32(RLC_HB_BASE, 0);
4421 WREG32(RLC_HB_RPTR, 0);
4422 WREG32(RLC_HB_WPTR, 0);
4423 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4424 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4425 }
4426 WREG32(RLC_MC_CNTL, 0);
4427 WREG32(RLC_UCODE_CNTL, 0);
4428
4429 fw_data = (const __be32 *)rdev->rlc_fw->data;
4430 if (rdev->family >= CHIP_ARUBA) {
4431 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4432 WREG32(RLC_UCODE_ADDR, i);
4433 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4434 }
4435 } else if (rdev->family >= CHIP_CAYMAN) {
4436 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4437 WREG32(RLC_UCODE_ADDR, i);
4438 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4439 }
4440 } else {
4441 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4442 WREG32(RLC_UCODE_ADDR, i);
4443 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4444 }
4445 }
4446 WREG32(RLC_UCODE_ADDR, 0);
4447
4448 evergreen_rlc_start(rdev);
4449
4450 return 0;
4451}
4452
4453/* Interrupts */
4454
4455u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4456{
4457 if (crtc >= rdev->num_crtc)
4458 return 0;
4459 else
4460 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
4461}
4462
4463void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4464{
4465 int i;
4466 u32 tmp;
4467
4468 if (rdev->family >= CHIP_CAYMAN) {
4469 cayman_cp_int_cntl_setup(rdev, 0,
4470 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4471 cayman_cp_int_cntl_setup(rdev, 1, 0);
4472 cayman_cp_int_cntl_setup(rdev, 2, 0);
4473 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4474 WREG32(CAYMAN_DMA1_CNTL, tmp);
4475 } else
4476 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4477 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4478 WREG32(DMA_CNTL, tmp);
4479 WREG32(GRBM_INT_CNTL, 0);
4480 WREG32(SRBM_INT_CNTL, 0);
4481 for (i = 0; i < rdev->num_crtc; i++)
4482 WREG32(INT_MASK + crtc_offsets[i], 0);
4483 for (i = 0; i < rdev->num_crtc; i++)
4484 WREG32(GRPH_INT_CONTROL + crtc_offsets[i], 0);
4485
4486 /* only one DAC on DCE5 */
4487 if (!ASIC_IS_DCE5(rdev))
4488 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4489 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4490
4491 for (i = 0; i < 6; i++)
4492 WREG32_AND(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_POLARITY);
4493}
4494
4495/* Note that the order we write back regs here is important */
4496int evergreen_irq_set(struct radeon_device *rdev)
4497{
4498 int i;
4499 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4500 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
4501 u32 grbm_int_cntl = 0;
4502 u32 dma_cntl, dma_cntl1 = 0;
4503 u32 thermal_int = 0;
4504 struct drm_device *ddev = rdev_to_drm(rdev);
4505
4506 if (!rdev->irq.installed) {
4507 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4508 return -EINVAL;
4509 }
4510 /* don't enable anything if the ih is disabled */
4511 if (!rdev->ih.enabled) {
4512 r600_disable_interrupts(rdev);
4513 /* force the active interrupt state to all disabled */
4514 evergreen_disable_interrupt_state(rdev);
4515 return 0;
4516 }
4517
4518 if (rdev->family == CHIP_ARUBA)
4519 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4520 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4521 else
4522 thermal_int = RREG32(CG_THERMAL_INT) &
4523 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4524
4525 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4526
4527 if (rdev->family >= CHIP_CAYMAN) {
4528 /* enable CP interrupts on all rings */
4529 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4530 drm_dbg(ddev, "%s : sw int gfx\n", __func__);
4531 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4532 }
4533 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
4534 drm_dbg(ddev, "%s : sw int cp1\n", __func__);
4535 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4536 }
4537 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
4538 drm_dbg(ddev, "%s : sw int cp2\n", __func__);
4539 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4540 }
4541 } else {
4542 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4543 drm_dbg(ddev, "%s : sw int gfx\n", __func__);
4544 cp_int_cntl |= RB_INT_ENABLE;
4545 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4546 }
4547 }
4548
4549 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4550 drm_dbg(ddev, "r600_irq_set: sw int dma\n");
4551 dma_cntl |= TRAP_ENABLE;
4552 }
4553
4554 if (rdev->family >= CHIP_CAYMAN) {
4555 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4556 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4557 drm_dbg(ddev, "r600_irq_set: sw int dma1\n");
4558 dma_cntl1 |= TRAP_ENABLE;
4559 }
4560 }
4561
4562 if (rdev->irq.dpm_thermal) {
4563 drm_dbg(ddev, "dpm thermal\n");
4564 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4565 }
4566
4567 if (rdev->family >= CHIP_CAYMAN) {
4568 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4569 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4570 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4571 } else
4572 WREG32(CP_INT_CNTL, cp_int_cntl);
4573
4574 WREG32(DMA_CNTL, dma_cntl);
4575
4576 if (rdev->family >= CHIP_CAYMAN)
4577 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4578
4579 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4580
4581 for (i = 0; i < rdev->num_crtc; i++) {
4582 radeon_irq_kms_set_irq_n_enabled(
4583 rdev, INT_MASK + crtc_offsets[i],
4584 VBLANK_INT_MASK,
4585 rdev->irq.crtc_vblank_int[i] ||
4586 atomic_read(&rdev->irq.pflip[i]), "vblank", i);
4587 }
4588
4589 for (i = 0; i < rdev->num_crtc; i++)
4590 WREG32(GRPH_INT_CONTROL + crtc_offsets[i], GRPH_PFLIP_INT_MASK);
4591
4592 for (i = 0; i < 6; i++) {
4593 radeon_irq_kms_set_irq_n_enabled(
4594 rdev, DC_HPDx_INT_CONTROL(i),
4595 DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN,
4596 rdev->irq.hpd[i], "HPD", i);
4597 }
4598
4599 if (rdev->family == CHIP_ARUBA)
4600 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4601 else
4602 WREG32(CG_THERMAL_INT, thermal_int);
4603
4604 for (i = 0; i < 6; i++) {
4605 radeon_irq_kms_set_irq_n_enabled(
4606 rdev, AFMT_AUDIO_PACKET_CONTROL + crtc_offsets[i],
4607 AFMT_AZ_FORMAT_WTRIG_MASK,
4608 rdev->irq.afmt[i], "HDMI", i);
4609 }
4610
4611 /* posting read */
4612 RREG32(SRBM_STATUS);
4613
4614 return 0;
4615}
4616
4617/* Note that the order we write back regs here is important */
4618static void evergreen_irq_ack(struct radeon_device *rdev)
4619{
4620 int i, j;
4621 u32 *grph_int = rdev->irq.stat_regs.evergreen.grph_int;
4622 u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
4623 u32 *afmt_status = rdev->irq.stat_regs.evergreen.afmt_status;
4624
4625 for (i = 0; i < 6; i++) {
4626 disp_int[i] = RREG32(evergreen_disp_int_status[i]);
4627 afmt_status[i] = RREG32(AFMT_STATUS + crtc_offsets[i]);
4628 if (i < rdev->num_crtc)
4629 grph_int[i] = RREG32(GRPH_INT_STATUS + crtc_offsets[i]);
4630 }
4631
4632 /* We write back each interrupt register in pairs of two */
4633 for (i = 0; i < rdev->num_crtc; i += 2) {
4634 for (j = i; j < (i + 2); j++) {
4635 if (grph_int[j] & GRPH_PFLIP_INT_OCCURRED)
4636 WREG32(GRPH_INT_STATUS + crtc_offsets[j],
4637 GRPH_PFLIP_INT_CLEAR);
4638 }
4639
4640 for (j = i; j < (i + 2); j++) {
4641 if (disp_int[j] & LB_D1_VBLANK_INTERRUPT)
4642 WREG32(VBLANK_STATUS + crtc_offsets[j],
4643 VBLANK_ACK);
4644 if (disp_int[j] & LB_D1_VLINE_INTERRUPT)
4645 WREG32(VLINE_STATUS + crtc_offsets[j],
4646 VLINE_ACK);
4647 }
4648 }
4649
4650 for (i = 0; i < 6; i++) {
4651 if (disp_int[i] & DC_HPD1_INTERRUPT)
4652 WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_ACK);
4653 }
4654
4655 for (i = 0; i < 6; i++) {
4656 if (disp_int[i] & DC_HPD1_RX_INTERRUPT)
4657 WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_RX_INT_ACK);
4658 }
4659
4660 for (i = 0; i < 6; i++) {
4661 if (afmt_status[i] & AFMT_AZ_FORMAT_WTRIG)
4662 WREG32_OR(AFMT_AUDIO_PACKET_CONTROL + crtc_offsets[i],
4663 AFMT_AZ_FORMAT_WTRIG_ACK);
4664 }
4665}
4666
4667static void evergreen_irq_disable(struct radeon_device *rdev)
4668{
4669 r600_disable_interrupts(rdev);
4670 /* Wait and acknowledge irq */
4671 mdelay(1);
4672 evergreen_irq_ack(rdev);
4673 evergreen_disable_interrupt_state(rdev);
4674}
4675
4676void evergreen_irq_suspend(struct radeon_device *rdev)
4677{
4678 evergreen_irq_disable(rdev);
4679 r600_rlc_stop(rdev);
4680}
4681
4682static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
4683{
4684 u32 wptr, tmp;
4685
4686 if (rdev->wb.enabled)
4687 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4688 else
4689 wptr = RREG32(IH_RB_WPTR);
4690
4691 if (wptr & RB_OVERFLOW) {
4692 wptr &= ~RB_OVERFLOW;
4693 /* When a ring buffer overflow happen start parsing interrupt
4694 * from the last not overwritten vector (wptr + 16). Hopefully
4695 * this should allow us to catchup.
4696 */
4697 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
4698 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
4699 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4700 tmp = RREG32(IH_RB_CNTL);
4701 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4702 WREG32(IH_RB_CNTL, tmp);
4703 }
4704 return (wptr & rdev->ih.ptr_mask);
4705}
4706
4707int evergreen_irq_process(struct radeon_device *rdev)
4708{
4709 u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
4710 u32 *afmt_status = rdev->irq.stat_regs.evergreen.afmt_status;
4711 u32 crtc_idx, hpd_idx, afmt_idx;
4712 u32 mask;
4713 u32 wptr;
4714 u32 rptr;
4715 u32 src_id, src_data;
4716 u32 ring_index;
4717 bool queue_hotplug = false;
4718 bool queue_hdmi = false;
4719 bool queue_dp = false;
4720 bool queue_thermal = false;
4721 u32 status, addr;
4722 const char *event_name;
4723 struct drm_device *ddev = rdev_to_drm(rdev);
4724
4725 if (!rdev->ih.enabled || rdev->shutdown)
4726 return IRQ_NONE;
4727
4728 wptr = evergreen_get_ih_wptr(rdev);
4729
4730restart_ih:
4731 /* is somebody else already processing irqs? */
4732 if (atomic_xchg(&rdev->ih.lock, 1))
4733 return IRQ_NONE;
4734
4735 rptr = rdev->ih.rptr;
4736 drm_dbg(ddev, "%s start: rptr %d, wptr %d\n", __func__, rptr, wptr);
4737
4738 /* Order reading of wptr vs. reading of IH ring data */
4739 rmb();
4740
4741 /* display interrupts */
4742 evergreen_irq_ack(rdev);
4743
4744 while (rptr != wptr) {
4745 /* wptr/rptr are in bytes! */
4746 ring_index = rptr / 4;
4747 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4748 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4749
4750 switch (src_id) {
4751 case 1: /* D1 vblank/vline */
4752 case 2: /* D2 vblank/vline */
4753 case 3: /* D3 vblank/vline */
4754 case 4: /* D4 vblank/vline */
4755 case 5: /* D5 vblank/vline */
4756 case 6: /* D6 vblank/vline */
4757 crtc_idx = src_id - 1;
4758
4759 if (src_data == 0) { /* vblank */
4760 mask = LB_D1_VBLANK_INTERRUPT;
4761 event_name = "vblank";
4762
4763 if (rdev->irq.crtc_vblank_int[crtc_idx]) {
4764 drm_handle_vblank(rdev_to_drm(rdev), crtc_idx);
4765 rdev->pm.vblank_sync = true;
4766 wake_up(&rdev->irq.vblank_queue);
4767 }
4768 if (atomic_read(&rdev->irq.pflip[crtc_idx])) {
4769 radeon_crtc_handle_vblank(rdev,
4770 crtc_idx);
4771 }
4772
4773 } else if (src_data == 1) { /* vline */
4774 mask = LB_D1_VLINE_INTERRUPT;
4775 event_name = "vline";
4776 } else {
4777 drm_dbg(ddev, "Unhandled interrupt: %d %d\n",
4778 src_id, src_data);
4779 break;
4780 }
4781
4782 if (!(disp_int[crtc_idx] & mask)) {
4783 drm_dbg(ddev, "IH: D%d %s - IH event w/o asserted irq bit?\n",
4784 crtc_idx + 1, event_name);
4785 }
4786
4787 disp_int[crtc_idx] &= ~mask;
4788 drm_dbg(ddev, "IH: D%d %s\n", crtc_idx + 1, event_name);
4789
4790 break;
4791 case 8: /* D1 page flip */
4792 case 10: /* D2 page flip */
4793 case 12: /* D3 page flip */
4794 case 14: /* D4 page flip */
4795 case 16: /* D5 page flip */
4796 case 18: /* D6 page flip */
4797 drm_dbg(ddev, "IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
4798 if (radeon_use_pflipirq > 0)
4799 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
4800 break;
4801 case 42: /* HPD hotplug */
4802 if (src_data <= 5) {
4803 hpd_idx = src_data;
4804 mask = DC_HPD1_INTERRUPT;
4805 queue_hotplug = true;
4806 event_name = "HPD";
4807
4808 } else if (src_data <= 11) {
4809 hpd_idx = src_data - 6;
4810 mask = DC_HPD1_RX_INTERRUPT;
4811 queue_dp = true;
4812 event_name = "HPD_RX";
4813
4814 } else {
4815 drm_dbg(ddev, "Unhandled interrupt: %d %d\n",
4816 src_id, src_data);
4817 break;
4818 }
4819
4820 if (!(disp_int[hpd_idx] & mask))
4821 drm_dbg(ddev, "IH: IH event w/o asserted irq bit?\n");
4822
4823 disp_int[hpd_idx] &= ~mask;
4824 drm_dbg(ddev, "IH: %s%d\n", event_name, hpd_idx + 1);
4825
4826 break;
4827 case 44: /* hdmi */
4828 afmt_idx = src_data;
4829 if (afmt_idx > 5) {
4830 drm_err(ddev, "Unhandled interrupt: %d %d\n",
4831 src_id, src_data);
4832 break;
4833 }
4834
4835 if (!(afmt_status[afmt_idx] & AFMT_AZ_FORMAT_WTRIG))
4836 drm_dbg(ddev, "IH: IH event w/o asserted irq bit?\n");
4837
4838 afmt_status[afmt_idx] &= ~AFMT_AZ_FORMAT_WTRIG;
4839 queue_hdmi = true;
4840 drm_dbg(ddev, "IH: HDMI%d\n", afmt_idx + 1);
4841 break;
4842 case 96:
4843 drm_err(ddev, "SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
4844 WREG32(SRBM_INT_ACK, 0x1);
4845 break;
4846 case 124: /* UVD */
4847 drm_dbg(ddev, "IH: UVD int: 0x%08x\n", src_data);
4848 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
4849 break;
4850 case 146:
4851 case 147:
4852 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
4853 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
4854 /* reset addr and status */
4855 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
4856 if (addr == 0x0 && status == 0x0)
4857 break;
4858 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
4859 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4860 addr);
4861 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4862 status);
4863 cayman_vm_decode_fault(rdev, status, addr);
4864 break;
4865 case 176: /* CP_INT in ring buffer */
4866 case 177: /* CP_INT in IB1 */
4867 case 178: /* CP_INT in IB2 */
4868 drm_dbg(ddev, "IH: CP int: 0x%08x\n", src_data);
4869 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4870 break;
4871 case 181: /* CP EOP event */
4872 drm_dbg(ddev, "IH: CP EOP\n");
4873 if (rdev->family >= CHIP_CAYMAN) {
4874 switch (src_data) {
4875 case 0:
4876 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4877 break;
4878 case 1:
4879 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
4880 break;
4881 case 2:
4882 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
4883 break;
4884 }
4885 } else
4886 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4887 break;
4888 case 224: /* DMA trap event */
4889 drm_dbg(ddev, "IH: DMA trap\n");
4890 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
4891 break;
4892 case 230: /* thermal low to high */
4893 drm_dbg(ddev, "IH: thermal low to high\n");
4894 rdev->pm.dpm.thermal.high_to_low = false;
4895 queue_thermal = true;
4896 break;
4897 case 231: /* thermal high to low */
4898 drm_dbg(ddev, "IH: thermal high to low\n");
4899 rdev->pm.dpm.thermal.high_to_low = true;
4900 queue_thermal = true;
4901 break;
4902 case 233: /* GUI IDLE */
4903 drm_dbg(ddev, "IH: GUI idle\n");
4904 break;
4905 case 244: /* DMA trap event */
4906 if (rdev->family >= CHIP_CAYMAN) {
4907 drm_dbg(ddev, "IH: DMA1 trap\n");
4908 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
4909 }
4910 break;
4911 default:
4912 drm_dbg(ddev, "Unhandled interrupt: %d %d\n", src_id, src_data);
4913 break;
4914 }
4915
4916 /* wptr/rptr are in bytes! */
4917 rptr += 16;
4918 rptr &= rdev->ih.ptr_mask;
4919 WREG32(IH_RB_RPTR, rptr);
4920 }
4921 if (queue_dp)
4922 schedule_work(&rdev->dp_work);
4923 if (queue_hotplug)
4924 schedule_delayed_work(&rdev->hotplug_work, 0);
4925 if (queue_hdmi)
4926 schedule_work(&rdev->audio_work);
4927 if (queue_thermal && rdev->pm.dpm_enabled)
4928 schedule_work(&rdev->pm.dpm.thermal.work);
4929 rdev->ih.rptr = rptr;
4930 atomic_set(&rdev->ih.lock, 0);
4931
4932 /* make sure wptr hasn't changed while processing */
4933 wptr = evergreen_get_ih_wptr(rdev);
4934 if (wptr != rptr)
4935 goto restart_ih;
4936
4937 return IRQ_HANDLED;
4938}
4939
4940static void evergreen_uvd_init(struct radeon_device *rdev)
4941{
4942 int r;
4943
4944 if (!rdev->has_uvd)
4945 return;
4946
4947 r = radeon_uvd_init(rdev);
4948 if (r) {
4949 dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
4950 /*
4951 * At this point rdev->uvd.vcpu_bo is NULL which trickles down
4952 * to early fails uvd_v2_2_resume() and thus nothing happens
4953 * there. So it is pointless to try to go through that code
4954 * hence why we disable uvd here.
4955 */
4956 rdev->has_uvd = false;
4957 return;
4958 }
4959 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
4960 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
4961}
4962
4963static void evergreen_uvd_start(struct radeon_device *rdev)
4964{
4965 int r;
4966
4967 if (!rdev->has_uvd)
4968 return;
4969
4970 r = uvd_v2_2_resume(rdev);
4971 if (r) {
4972 dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
4973 goto error;
4974 }
4975 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
4976 if (r) {
4977 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
4978 goto error;
4979 }
4980 return;
4981
4982error:
4983 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
4984}
4985
4986static void evergreen_uvd_resume(struct radeon_device *rdev)
4987{
4988 struct radeon_ring *ring;
4989 int r;
4990
4991 if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
4992 return;
4993
4994 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
4995 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
4996 if (r) {
4997 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
4998 return;
4999 }
5000 r = uvd_v1_0_init(rdev);
5001 if (r) {
5002 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
5003 return;
5004 }
5005}
5006
5007static int evergreen_startup(struct radeon_device *rdev)
5008{
5009 struct radeon_ring *ring;
5010 int r;
5011 struct drm_device *ddev = rdev_to_drm(rdev);
5012
5013 /* enable pcie gen2 link */
5014 evergreen_pcie_gen2_enable(rdev);
5015 /* enable aspm */
5016 evergreen_program_aspm(rdev);
5017
5018 /* scratch needs to be initialized before MC */
5019 r = r600_vram_scratch_init(rdev);
5020 if (r)
5021 return r;
5022
5023 evergreen_mc_program(rdev);
5024
5025 if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) {
5026 r = ni_mc_load_microcode(rdev);
5027 if (r) {
5028 drm_err(ddev, "Failed to load MC firmware!\n");
5029 return r;
5030 }
5031 }
5032
5033 if (rdev->flags & RADEON_IS_AGP) {
5034 evergreen_agp_enable(rdev);
5035 } else {
5036 r = evergreen_pcie_gart_enable(rdev);
5037 if (r)
5038 return r;
5039 }
5040 evergreen_gpu_init(rdev);
5041
5042 /* allocate rlc buffers */
5043 if (rdev->flags & RADEON_IS_IGP) {
5044 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
5045 rdev->rlc.reg_list_size =
5046 (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
5047 rdev->rlc.cs_data = evergreen_cs_data;
5048 r = sumo_rlc_init(rdev);
5049 if (r) {
5050 drm_err(ddev, "Failed to init rlc BOs!\n");
5051 return r;
5052 }
5053 }
5054
5055 /* allocate wb buffer */
5056 r = radeon_wb_init(rdev);
5057 if (r)
5058 return r;
5059
5060 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5061 if (r) {
5062 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5063 return r;
5064 }
5065
5066 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5067 if (r) {
5068 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5069 return r;
5070 }
5071
5072 evergreen_uvd_start(rdev);
5073
5074 /* Enable IRQ */
5075 if (!rdev->irq.installed) {
5076 r = radeon_irq_kms_init(rdev);
5077 if (r)
5078 return r;
5079 }
5080
5081 r = r600_irq_init(rdev);
5082 if (r) {
5083 drm_err(ddev, "radeon: IH init failed (%d).\n", r);
5084 radeon_irq_kms_fini(rdev);
5085 return r;
5086 }
5087 evergreen_irq_set(rdev);
5088
5089 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5090 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5091 RADEON_CP_PACKET2);
5092 if (r)
5093 return r;
5094
5095 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5096 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5097 DMA_PACKET(DMA_PACKET_NOP, 0, 0));
5098 if (r)
5099 return r;
5100
5101 r = evergreen_cp_load_microcode(rdev);
5102 if (r)
5103 return r;
5104 r = evergreen_cp_resume(rdev);
5105 if (r)
5106 return r;
5107 r = r600_dma_resume(rdev);
5108 if (r)
5109 return r;
5110
5111 evergreen_uvd_resume(rdev);
5112
5113 r = radeon_ib_pool_init(rdev);
5114 if (r) {
5115 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
5116 return r;
5117 }
5118
5119 r = radeon_audio_init(rdev);
5120 if (r) {
5121 drm_err(ddev, "radeon: audio init failed\n");
5122 return r;
5123 }
5124
5125 return 0;
5126}
5127
5128int evergreen_resume(struct radeon_device *rdev)
5129{
5130 int r;
5131 struct drm_device *ddev = rdev_to_drm(rdev);
5132
5133 /* reset the asic, the gfx blocks are often in a bad state
5134 * after the driver is unloaded or after a resume
5135 */
5136 if (radeon_asic_reset(rdev))
5137 dev_warn(rdev->dev, "GPU reset failed !\n");
5138 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5139 * posting will perform necessary task to bring back GPU into good
5140 * shape.
5141 */
5142 /* post card */
5143 atom_asic_init(rdev->mode_info.atom_context);
5144
5145 /* init golden registers */
5146 evergreen_init_golden_registers(rdev);
5147
5148 if (rdev->pm.pm_method == PM_METHOD_DPM)
5149 radeon_pm_resume(rdev);
5150
5151 rdev->accel_working = true;
5152 r = evergreen_startup(rdev);
5153 if (r) {
5154 drm_err(ddev, "evergreen startup failed on resume\n");
5155 rdev->accel_working = false;
5156 return r;
5157 }
5158
5159 return r;
5160
5161}
5162
5163int evergreen_suspend(struct radeon_device *rdev)
5164{
5165 radeon_pm_suspend(rdev);
5166 radeon_audio_fini(rdev);
5167 if (rdev->has_uvd) {
5168 radeon_uvd_suspend(rdev);
5169 uvd_v1_0_fini(rdev);
5170 }
5171 r700_cp_stop(rdev);
5172 r600_dma_stop(rdev);
5173 evergreen_irq_suspend(rdev);
5174 radeon_wb_disable(rdev);
5175 evergreen_pcie_gart_disable(rdev);
5176
5177 return 0;
5178}
5179
5180/* Plan is to move initialization in that function and use
5181 * helper function so that radeon_device_init pretty much
5182 * do nothing more than calling asic specific function. This
5183 * should also allow to remove a bunch of callback function
5184 * like vram_info.
5185 */
5186int evergreen_init(struct radeon_device *rdev)
5187{
5188 int r;
5189 struct drm_device *ddev = rdev_to_drm(rdev);
5190
5191 /* Read BIOS */
5192 if (!radeon_get_bios(rdev)) {
5193 if (ASIC_IS_AVIVO(rdev))
5194 return -EINVAL;
5195 }
5196 /* Must be an ATOMBIOS */
5197 if (!rdev->is_atom_bios) {
5198 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
5199 return -EINVAL;
5200 }
5201 r = radeon_atombios_init(rdev);
5202 if (r)
5203 return r;
5204 /* reset the asic, the gfx blocks are often in a bad state
5205 * after the driver is unloaded or after a resume
5206 */
5207 if (radeon_asic_reset(rdev))
5208 dev_warn(rdev->dev, "GPU reset failed !\n");
5209 /* Post card if necessary */
5210 if (!radeon_card_posted(rdev)) {
5211 if (!rdev->bios) {
5212 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5213 return -EINVAL;
5214 }
5215 drm_info(ddev, "GPU not posted. posting now...\n");
5216 atom_asic_init(rdev->mode_info.atom_context);
5217 }
5218 /* init golden registers */
5219 evergreen_init_golden_registers(rdev);
5220 /* Initialize scratch registers */
5221 r600_scratch_init(rdev);
5222 /* Initialize surface registers */
5223 radeon_surface_init(rdev);
5224 /* Initialize clocks */
5225 radeon_get_clock_info(rdev_to_drm(rdev));
5226 /* Fence driver */
5227 radeon_fence_driver_init(rdev);
5228 /* initialize AGP */
5229 if (rdev->flags & RADEON_IS_AGP) {
5230 r = radeon_agp_init(rdev);
5231 if (r)
5232 radeon_agp_disable(rdev);
5233 }
5234 /* initialize memory controller */
5235 r = evergreen_mc_init(rdev);
5236 if (r)
5237 return r;
5238 /* Memory manager */
5239 r = radeon_bo_init(rdev);
5240 if (r)
5241 return r;
5242
5243 if (ASIC_IS_DCE5(rdev)) {
5244 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5245 r = ni_init_microcode(rdev);
5246 if (r) {
5247 drm_err(ddev, "Failed to load firmware!\n");
5248 return r;
5249 }
5250 }
5251 } else {
5252 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5253 r = r600_init_microcode(rdev);
5254 if (r) {
5255 drm_err(ddev, "Failed to load firmware!\n");
5256 return r;
5257 }
5258 }
5259 }
5260
5261 /* Initialize power management */
5262 radeon_pm_init(rdev);
5263
5264 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5265 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
5266
5267 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5268 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5269
5270 evergreen_uvd_init(rdev);
5271
5272 rdev->ih.ring_obj = NULL;
5273 r600_ih_ring_init(rdev, 64 * 1024);
5274
5275 r = r600_pcie_gart_init(rdev);
5276 if (r)
5277 return r;
5278
5279 rdev->accel_working = true;
5280 r = evergreen_startup(rdev);
5281 if (r) {
5282 dev_err(rdev->dev, "disabling GPU acceleration\n");
5283 r700_cp_fini(rdev);
5284 r600_dma_fini(rdev);
5285 r600_irq_fini(rdev);
5286 if (rdev->flags & RADEON_IS_IGP)
5287 sumo_rlc_fini(rdev);
5288 radeon_wb_fini(rdev);
5289 radeon_ib_pool_fini(rdev);
5290 radeon_irq_kms_fini(rdev);
5291 evergreen_pcie_gart_fini(rdev);
5292 rdev->accel_working = false;
5293 }
5294
5295 /* Don't start up if the MC ucode is missing on BTC parts.
5296 * The default clocks and voltages before the MC ucode
5297 * is loaded are not suffient for advanced operations.
5298 */
5299 if (ASIC_IS_DCE5(rdev)) {
5300 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5301 drm_err(ddev, "radeon: MC ucode required for NI+.\n");
5302 return -EINVAL;
5303 }
5304 }
5305
5306 return 0;
5307}
5308
5309void evergreen_fini(struct radeon_device *rdev)
5310{
5311 radeon_pm_fini(rdev);
5312 radeon_audio_fini(rdev);
5313 r700_cp_fini(rdev);
5314 r600_dma_fini(rdev);
5315 r600_irq_fini(rdev);
5316 if (rdev->flags & RADEON_IS_IGP)
5317 sumo_rlc_fini(rdev);
5318 radeon_wb_fini(rdev);
5319 radeon_ib_pool_fini(rdev);
5320 radeon_irq_kms_fini(rdev);
5321 uvd_v1_0_fini(rdev);
5322 radeon_uvd_fini(rdev);
5323 evergreen_pcie_gart_fini(rdev);
5324 r600_vram_scratch_fini(rdev);
5325 radeon_gem_fini(rdev);
5326 radeon_fence_driver_fini(rdev);
5327 radeon_agp_fini(rdev);
5328 radeon_bo_fini(rdev);
5329 radeon_atombios_fini(rdev);
5330 kfree(rdev->bios);
5331 rdev->bios = NULL;
5332}
5333
5334void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
5335{
5336 u32 link_width_cntl, speed_cntl;
5337 struct drm_device *ddev = rdev_to_drm(rdev);
5338
5339 if (radeon_pcie_gen2 == 0)
5340 return;
5341
5342 if (rdev->flags & RADEON_IS_IGP)
5343 return;
5344
5345 if (!(rdev->flags & RADEON_IS_PCIE))
5346 return;
5347
5348 /* x2 cards have a special sequence */
5349 if (ASIC_IS_X2(rdev))
5350 return;
5351
5352 if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5353 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
5354 return;
5355
5356 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5357 if (speed_cntl & LC_CURRENT_DATA_RATE) {
5358 drm_info(ddev, "PCIE gen 2 link speeds already enabled\n");
5359 return;
5360 }
5361
5362 drm_info(ddev, "enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5363
5364 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5365 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5366
5367 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5368 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5369 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5370
5371 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5372 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
5373 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5374
5375 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5376 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
5377 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5378
5379 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5380 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
5381 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5382
5383 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5384 speed_cntl |= LC_GEN2_EN_STRAP;
5385 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5386
5387 } else {
5388 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5389 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5390 if (1)
5391 link_width_cntl |= LC_UPCONFIGURE_DIS;
5392 else
5393 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5394 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5395 }
5396}
5397
5398void evergreen_program_aspm(struct radeon_device *rdev)
5399{
5400 u32 data, orig;
5401 u32 pcie_lc_cntl, pcie_lc_cntl_old;
5402 bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5403 /* fusion_platform = true
5404 * if the system is a fusion system
5405 * (APU or DGPU in a fusion system).
5406 * todo: check if the system is a fusion platform.
5407 */
5408 bool fusion_platform = false;
5409
5410 if (radeon_aspm == 0)
5411 return;
5412
5413 if (!(rdev->flags & RADEON_IS_PCIE))
5414 return;
5415
5416 switch (rdev->family) {
5417 case CHIP_CYPRESS:
5418 case CHIP_HEMLOCK:
5419 case CHIP_JUNIPER:
5420 case CHIP_REDWOOD:
5421 case CHIP_CEDAR:
5422 case CHIP_SUMO:
5423 case CHIP_SUMO2:
5424 case CHIP_PALM:
5425 case CHIP_ARUBA:
5426 disable_l0s = true;
5427 break;
5428 default:
5429 disable_l0s = false;
5430 break;
5431 }
5432
5433 if (rdev->flags & RADEON_IS_IGP)
5434 fusion_platform = true; /* XXX also dGPUs in a fusion system */
5435
5436 data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5437 if (fusion_platform)
5438 data &= ~MULTI_PIF;
5439 else
5440 data |= MULTI_PIF;
5441 if (data != orig)
5442 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5443
5444 data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5445 if (fusion_platform)
5446 data &= ~MULTI_PIF;
5447 else
5448 data |= MULTI_PIF;
5449 if (data != orig)
5450 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5451
5452 pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5453 pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5454 if (!disable_l0s) {
5455 if (rdev->family >= CHIP_BARTS)
5456 pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5457 else
5458 pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5459 }
5460
5461 if (!disable_l1) {
5462 if (rdev->family >= CHIP_BARTS)
5463 pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5464 else
5465 pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5466
5467 if (!disable_plloff_in_l1) {
5468 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5469 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5470 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5471 if (data != orig)
5472 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5473
5474 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5475 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5476 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5477 if (data != orig)
5478 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5479
5480 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5481 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5482 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5483 if (data != orig)
5484 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5485
5486 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5487 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5488 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5489 if (data != orig)
5490 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5491
5492 if (rdev->family >= CHIP_BARTS) {
5493 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5494 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5495 data |= PLL_RAMP_UP_TIME_0(4);
5496 if (data != orig)
5497 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5498
5499 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5500 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5501 data |= PLL_RAMP_UP_TIME_1(4);
5502 if (data != orig)
5503 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5504
5505 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5506 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5507 data |= PLL_RAMP_UP_TIME_0(4);
5508 if (data != orig)
5509 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5510
5511 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5512 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5513 data |= PLL_RAMP_UP_TIME_1(4);
5514 if (data != orig)
5515 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5516 }
5517
5518 data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5519 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5520 data |= LC_DYN_LANES_PWR_STATE(3);
5521 if (data != orig)
5522 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5523
5524 if (rdev->family >= CHIP_BARTS) {
5525 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5526 data &= ~LS2_EXIT_TIME_MASK;
5527 data |= LS2_EXIT_TIME(1);
5528 if (data != orig)
5529 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5530
5531 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5532 data &= ~LS2_EXIT_TIME_MASK;
5533 data |= LS2_EXIT_TIME(1);
5534 if (data != orig)
5535 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5536 }
5537 }
5538 }
5539
5540 /* evergreen parts only */
5541 if (rdev->family < CHIP_BARTS)
5542 pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5543
5544 if (pcie_lc_cntl != pcie_lc_cntl_old)
5545 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
5546}