drivers/gpu/drm/radeon/cik.c

   1 /*
   2  * Copyright 2012 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/slab.h>
  26 #include <linux/module.h>
  27 #include "drmP.h"
  28 #include "radeon.h"
  29 #include "radeon_asic.h"
  30 #include "cikd.h"
  31 #include "atom.h"
  32 #include "cik_blit_shaders.h"
  33 #include "radeon_ucode.h"
  34 #include "clearstate_ci.h"
  35
  36 MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
  37 MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
  38 MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
  39 MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
  40 MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
  41 MODULE_FIRMWARE("radeon/BONAIRE_mc2.bin");
  42 MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
  43 MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
  44 MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
  45 MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
  46 MODULE_FIRMWARE("radeon/HAWAII_me.bin");
  47 MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
  48 MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
  49 MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
  50 MODULE_FIRMWARE("radeon/HAWAII_mc2.bin");
  51 MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
  52 MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
  53 MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
  54 MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
  55 MODULE_FIRMWARE("radeon/KAVERI_me.bin");
  56 MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
  57 MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
  58 MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
  59 MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
  60 MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
  61 MODULE_FIRMWARE("radeon/KABINI_me.bin");
  62 MODULE_FIRMWARE("radeon/KABINI_ce.bin");
  63 MODULE_FIRMWARE("radeon/KABINI_mec.bin");
  64 MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
  65 MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
  66
  67 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
  68 extern void r600_ih_ring_fini(struct radeon_device *rdev);
  69 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
  70 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
  71 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
  72 extern void sumo_rlc_fini(struct radeon_device *rdev);
  73 extern int sumo_rlc_init(struct radeon_device *rdev);
  74 extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
  75 extern void si_rlc_reset(struct radeon_device *rdev);
  76 extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
  77 extern int cik_sdma_resume(struct radeon_device *rdev);
  78 extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
  79 extern void cik_sdma_fini(struct radeon_device *rdev);
  80 extern void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable);
  81 static void cik_rlc_stop(struct radeon_device *rdev);
  82 static void cik_pcie_gen3_enable(struct radeon_device *rdev);
  83 static void cik_program_aspm(struct radeon_device *rdev);
  84 static void cik_init_pg(struct radeon_device *rdev);
  85 static void cik_init_cg(struct radeon_device *rdev);
  86 static void cik_fini_pg(struct radeon_device *rdev);
  87 static void cik_fini_cg(struct radeon_device *rdev);
  88 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
  89                                           bool enable);
  90
  91 /* get temperature in millidegrees */
  92 int ci_get_temp(struct radeon_device *rdev)
  93 {
  94         u32 temp;
  95         int actual_temp = 0;
  96
  97         temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
  98                 CTF_TEMP_SHIFT;
  99
 100         if (temp & 0x200)
 101                 actual_temp = 255;
 102         else
 103                 actual_temp = temp & 0x1ff;
 104
 105         actual_temp = actual_temp * 1000;
 106
 107         return actual_temp;
 108 }
 109
 110 /* get temperature in millidegrees */
 111 int kv_get_temp(struct radeon_device *rdev)
 112 {
 113         u32 temp;
 114         int actual_temp = 0;
 115
 116         temp = RREG32_SMC(0xC0300E0C);
 117
 118         if (temp)
 119                 actual_temp = (temp / 8) - 49;
 120         else
 121                 actual_temp = 0;
 122
 123         actual_temp = actual_temp * 1000;
 124
 125         return actual_temp;
 126 }
 127
 128 /*
 129  * Indirect registers accessor
 130  */
 131 u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
 132 {
 133         unsigned long flags;
 134         u32 r;
 135
 136         spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
 137         WREG32(PCIE_INDEX, reg);
 138         (void)RREG32(PCIE_INDEX);
 139         r = RREG32(PCIE_DATA);
 140         spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
 141         return r;
 142 }
 143
 144 void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
 145 {
 146         unsigned long flags;
 147
 148         spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
 149         WREG32(PCIE_INDEX, reg);
 150         (void)RREG32(PCIE_INDEX);
 151         WREG32(PCIE_DATA, v);
 152         (void)RREG32(PCIE_DATA);
 153         spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
 154 }
 155
 156 static const u32 spectre_rlc_save_restore_register_list[] =
 157 {
 158         (0x0e00 << 16) | (0xc12c >> 2),
 159         0x00000000,
 160         (0x0e00 << 16) | (0xc140 >> 2),
 161         0x00000000,
 162         (0x0e00 << 16) | (0xc150 >> 2),
 163         0x00000000,
 164         (0x0e00 << 16) | (0xc15c >> 2),
 165         0x00000000,
 166         (0x0e00 << 16) | (0xc168 >> 2),
 167         0x00000000,
 168         (0x0e00 << 16) | (0xc170 >> 2),
 169         0x00000000,
 170         (0x0e00 << 16) | (0xc178 >> 2),
 171         0x00000000,
 172         (0x0e00 << 16) | (0xc204 >> 2),
 173         0x00000000,
 174         (0x0e00 << 16) | (0xc2b4 >> 2),
 175         0x00000000,
 176         (0x0e00 << 16) | (0xc2b8 >> 2),
 177         0x00000000,
 178         (0x0e00 << 16) | (0xc2bc >> 2),
 179         0x00000000,
 180         (0x0e00 << 16) | (0xc2c0 >> 2),
 181         0x00000000,
 182         (0x0e00 << 16) | (0x8228 >> 2),
 183         0x00000000,
 184         (0x0e00 << 16) | (0x829c >> 2),
 185         0x00000000,
 186         (0x0e00 << 16) | (0x869c >> 2),
 187         0x00000000,
 188         (0x0600 << 16) | (0x98f4 >> 2),
 189         0x00000000,
 190         (0x0e00 << 16) | (0x98f8 >> 2),
 191         0x00000000,
 192         (0x0e00 << 16) | (0x9900 >> 2),
 193         0x00000000,
 194         (0x0e00 << 16) | (0xc260 >> 2),
 195         0x00000000,
 196         (0x0e00 << 16) | (0x90e8 >> 2),
 197         0x00000000,
 198         (0x0e00 << 16) | (0x3c000 >> 2),
 199         0x00000000,
 200         (0x0e00 << 16) | (0x3c00c >> 2),
 201         0x00000000,
 202         (0x0e00 << 16) | (0x8c1c >> 2),
 203         0x00000000,
 204         (0x0e00 << 16) | (0x9700 >> 2),
 205         0x00000000,
 206         (0x0e00 << 16) | (0xcd20 >> 2),
 207         0x00000000,
 208         (0x4e00 << 16) | (0xcd20 >> 2),
 209         0x00000000,
 210         (0x5e00 << 16) | (0xcd20 >> 2),
 211         0x00000000,
 212         (0x6e00 << 16) | (0xcd20 >> 2),
 213         0x00000000,
 214         (0x7e00 << 16) | (0xcd20 >> 2),
 215         0x00000000,
 216         (0x8e00 << 16) | (0xcd20 >> 2),
 217         0x00000000,
 218         (0x9e00 << 16) | (0xcd20 >> 2),
 219         0x00000000,
 220         (0xae00 << 16) | (0xcd20 >> 2),
 221         0x00000000,
 222         (0xbe00 << 16) | (0xcd20 >> 2),
 223         0x00000000,
 224         (0x0e00 << 16) | (0x89bc >> 2),
 225         0x00000000,
 226         (0x0e00 << 16) | (0x8900 >> 2),
 227         0x00000000,
 228         0x3,
 229         (0x0e00 << 16) | (0xc130 >> 2),
 230         0x00000000,
 231         (0x0e00 << 16) | (0xc134 >> 2),
 232         0x00000000,
 233         (0x0e00 << 16) | (0xc1fc >> 2),
 234         0x00000000,
 235         (0x0e00 << 16) | (0xc208 >> 2),
 236         0x00000000,
 237         (0x0e00 << 16) | (0xc264 >> 2),
 238         0x00000000,
 239         (0x0e00 << 16) | (0xc268 >> 2),
 240         0x00000000,
 241         (0x0e00 << 16) | (0xc26c >> 2),
 242         0x00000000,
 243         (0x0e00 << 16) | (0xc270 >> 2),
 244         0x00000000,
 245         (0x0e00 << 16) | (0xc274 >> 2),
 246         0x00000000,
 247         (0x0e00 << 16) | (0xc278 >> 2),
 248         0x00000000,
 249         (0x0e00 << 16) | (0xc27c >> 2),
 250         0x00000000,
 251         (0x0e00 << 16) | (0xc280 >> 2),
 252         0x00000000,
 253         (0x0e00 << 16) | (0xc284 >> 2),
 254         0x00000000,
 255         (0x0e00 << 16) | (0xc288 >> 2),
 256         0x00000000,
 257         (0x0e00 << 16) | (0xc28c >> 2),
 258         0x00000000,
 259         (0x0e00 << 16) | (0xc290 >> 2),
 260         0x00000000,
 261         (0x0e00 << 16) | (0xc294 >> 2),
 262         0x00000000,
 263         (0x0e00 << 16) | (0xc298 >> 2),
 264         0x00000000,
 265         (0x0e00 << 16) | (0xc29c >> 2),
 266         0x00000000,
 267         (0x0e00 << 16) | (0xc2a0 >> 2),
 268         0x00000000,
 269         (0x0e00 << 16) | (0xc2a4 >> 2),
 270         0x00000000,
 271         (0x0e00 << 16) | (0xc2a8 >> 2),
 272         0x00000000,
 273         (0x0e00 << 16) | (0xc2ac  >> 2),
 274         0x00000000,
 275         (0x0e00 << 16) | (0xc2b0 >> 2),
 276         0x00000000,
 277         (0x0e00 << 16) | (0x301d0 >> 2),
 278         0x00000000,
 279         (0x0e00 << 16) | (0x30238 >> 2),
 280         0x00000000,
 281         (0x0e00 << 16) | (0x30250 >> 2),
 282         0x00000000,
 283         (0x0e00 << 16) | (0x30254 >> 2),
 284         0x00000000,
 285         (0x0e00 << 16) | (0x30258 >> 2),
 286         0x00000000,
 287         (0x0e00 << 16) | (0x3025c >> 2),
 288         0x00000000,
 289         (0x4e00 << 16) | (0xc900 >> 2),
 290         0x00000000,
 291         (0x5e00 << 16) | (0xc900 >> 2),
 292         0x00000000,
 293         (0x6e00 << 16) | (0xc900 >> 2),
 294         0x00000000,
 295         (0x7e00 << 16) | (0xc900 >> 2),
 296         0x00000000,
 297         (0x8e00 << 16) | (0xc900 >> 2),
 298         0x00000000,
 299         (0x9e00 << 16) | (0xc900 >> 2),
 300         0x00000000,
 301         (0xae00 << 16) | (0xc900 >> 2),
 302         0x00000000,
 303         (0xbe00 << 16) | (0xc900 >> 2),
 304         0x00000000,
 305         (0x4e00 << 16) | (0xc904 >> 2),
 306         0x00000000,
 307         (0x5e00 << 16) | (0xc904 >> 2),
 308         0x00000000,
 309         (0x6e00 << 16) | (0xc904 >> 2),
 310         0x00000000,
 311         (0x7e00 << 16) | (0xc904 >> 2),
 312         0x00000000,
 313         (0x8e00 << 16) | (0xc904 >> 2),
 314         0x00000000,
 315         (0x9e00 << 16) | (0xc904 >> 2),
 316         0x00000000,
 317         (0xae00 << 16) | (0xc904 >> 2),
 318         0x00000000,
 319         (0xbe00 << 16) | (0xc904 >> 2),
 320         0x00000000,
 321         (0x4e00 << 16) | (0xc908 >> 2),
 322         0x00000000,
 323         (0x5e00 << 16) | (0xc908 >> 2),
 324         0x00000000,
 325         (0x6e00 << 16) | (0xc908 >> 2),
 326         0x00000000,
 327         (0x7e00 << 16) | (0xc908 >> 2),
 328         0x00000000,
 329         (0x8e00 << 16) | (0xc908 >> 2),
 330         0x00000000,
 331         (0x9e00 << 16) | (0xc908 >> 2),
 332         0x00000000,
 333         (0xae00 << 16) | (0xc908 >> 2),
 334         0x00000000,
 335         (0xbe00 << 16) | (0xc908 >> 2),
 336         0x00000000,
 337         (0x4e00 << 16) | (0xc90c >> 2),
 338         0x00000000,
 339         (0x5e00 << 16) | (0xc90c >> 2),
 340         0x00000000,
 341         (0x6e00 << 16) | (0xc90c >> 2),
 342         0x00000000,
 343         (0x7e00 << 16) | (0xc90c >> 2),
 344         0x00000000,
 345         (0x8e00 << 16) | (0xc90c >> 2),
 346         0x00000000,
 347         (0x9e00 << 16) | (0xc90c >> 2),
 348         0x00000000,
 349         (0xae00 << 16) | (0xc90c >> 2),
 350         0x00000000,
 351         (0xbe00 << 16) | (0xc90c >> 2),
 352         0x00000000,
 353         (0x4e00 << 16) | (0xc910 >> 2),
 354         0x00000000,
 355         (0x5e00 << 16) | (0xc910 >> 2),
 356         0x00000000,
 357         (0x6e00 << 16) | (0xc910 >> 2),
 358         0x00000000,
 359         (0x7e00 << 16) | (0xc910 >> 2),
 360         0x00000000,
 361         (0x8e00 << 16) | (0xc910 >> 2),
 362         0x00000000,
 363         (0x9e00 << 16) | (0xc910 >> 2),
 364         0x00000000,
 365         (0xae00 << 16) | (0xc910 >> 2),
 366         0x00000000,
 367         (0xbe00 << 16) | (0xc910 >> 2),
 368         0x00000000,
 369         (0x0e00 << 16) | (0xc99c >> 2),
 370         0x00000000,
 371         (0x0e00 << 16) | (0x9834 >> 2),
 372         0x00000000,
 373         (0x0000 << 16) | (0x30f00 >> 2),
 374         0x00000000,
 375         (0x0001 << 16) | (0x30f00 >> 2),
 376         0x00000000,
 377         (0x0000 << 16) | (0x30f04 >> 2),
 378         0x00000000,
 379         (0x0001 << 16) | (0x30f04 >> 2),
 380         0x00000000,
 381         (0x0000 << 16) | (0x30f08 >> 2),
 382         0x00000000,
 383         (0x0001 << 16) | (0x30f08 >> 2),
 384         0x00000000,
 385         (0x0000 << 16) | (0x30f0c >> 2),
 386         0x00000000,
 387         (0x0001 << 16) | (0x30f0c >> 2),
 388         0x00000000,
 389         (0x0600 << 16) | (0x9b7c >> 2),
 390         0x00000000,
 391         (0x0e00 << 16) | (0x8a14 >> 2),
 392         0x00000000,
 393         (0x0e00 << 16) | (0x8a18 >> 2),
 394         0x00000000,
 395         (0x0600 << 16) | (0x30a00 >> 2),
 396         0x00000000,
 397         (0x0e00 << 16) | (0x8bf0 >> 2),
 398         0x00000000,
 399         (0x0e00 << 16) | (0x8bcc >> 2),
 400         0x00000000,
 401         (0x0e00 << 16) | (0x8b24 >> 2),
 402         0x00000000,
 403         (0x0e00 << 16) | (0x30a04 >> 2),
 404         0x00000000,
 405         (0x0600 << 16) | (0x30a10 >> 2),
 406         0x00000000,
 407         (0x0600 << 16) | (0x30a14 >> 2),
 408         0x00000000,
 409         (0x0600 << 16) | (0x30a18 >> 2),
 410         0x00000000,
 411         (0x0600 << 16) | (0x30a2c >> 2),
 412         0x00000000,
 413         (0x0e00 << 16) | (0xc700 >> 2),
 414         0x00000000,
 415         (0x0e00 << 16) | (0xc704 >> 2),
 416         0x00000000,
 417         (0x0e00 << 16) | (0xc708 >> 2),
 418         0x00000000,
 419         (0x0e00 << 16) | (0xc768 >> 2),
 420         0x00000000,
 421         (0x0400 << 16) | (0xc770 >> 2),
 422         0x00000000,
 423         (0x0400 << 16) | (0xc774 >> 2),
 424         0x00000000,
 425         (0x0400 << 16) | (0xc778 >> 2),
 426         0x00000000,
 427         (0x0400 << 16) | (0xc77c >> 2),
 428         0x00000000,
 429         (0x0400 << 16) | (0xc780 >> 2),
 430         0x00000000,
 431         (0x0400 << 16) | (0xc784 >> 2),
 432         0x00000000,
 433         (0x0400 << 16) | (0xc788 >> 2),
 434         0x00000000,
 435         (0x0400 << 16) | (0xc78c >> 2),
 436         0x00000000,
 437         (0x0400 << 16) | (0xc798 >> 2),
 438         0x00000000,
 439         (0x0400 << 16) | (0xc79c >> 2),
 440         0x00000000,
 441         (0x0400 << 16) | (0xc7a0 >> 2),
 442         0x00000000,
 443         (0x0400 << 16) | (0xc7a4 >> 2),
 444         0x00000000,
 445         (0x0400 << 16) | (0xc7a8 >> 2),
 446         0x00000000,
 447         (0x0400 << 16) | (0xc7ac >> 2),
 448         0x00000000,
 449         (0x0400 << 16) | (0xc7b0 >> 2),
 450         0x00000000,
 451         (0x0400 << 16) | (0xc7b4 >> 2),
 452         0x00000000,
 453         (0x0e00 << 16) | (0x9100 >> 2),
 454         0x00000000,
 455         (0x0e00 << 16) | (0x3c010 >> 2),
 456         0x00000000,
 457         (0x0e00 << 16) | (0x92a8 >> 2),
 458         0x00000000,
 459         (0x0e00 << 16) | (0x92ac >> 2),
 460         0x00000000,
 461         (0x0e00 << 16) | (0x92b4 >> 2),
 462         0x00000000,
 463         (0x0e00 << 16) | (0x92b8 >> 2),
 464         0x00000000,
 465         (0x0e00 << 16) | (0x92bc >> 2),
 466         0x00000000,
 467         (0x0e00 << 16) | (0x92c0 >> 2),
 468         0x00000000,
 469         (0x0e00 << 16) | (0x92c4 >> 2),
 470         0x00000000,
 471         (0x0e00 << 16) | (0x92c8 >> 2),
 472         0x00000000,
 473         (0x0e00 << 16) | (0x92cc >> 2),
 474         0x00000000,
 475         (0x0e00 << 16) | (0x92d0 >> 2),
 476         0x00000000,
 477         (0x0e00 << 16) | (0x8c00 >> 2),
 478         0x00000000,
 479         (0x0e00 << 16) | (0x8c04 >> 2),
 480         0x00000000,
 481         (0x0e00 << 16) | (0x8c20 >> 2),
 482         0x00000000,
 483         (0x0e00 << 16) | (0x8c38 >> 2),
 484         0x00000000,
 485         (0x0e00 << 16) | (0x8c3c >> 2),
 486         0x00000000,
 487         (0x0e00 << 16) | (0xae00 >> 2),
 488         0x00000000,
 489         (0x0e00 << 16) | (0x9604 >> 2),
 490         0x00000000,
 491         (0x0e00 << 16) | (0xac08 >> 2),
 492         0x00000000,
 493         (0x0e00 << 16) | (0xac0c >> 2),
 494         0x00000000,
 495         (0x0e00 << 16) | (0xac10 >> 2),
 496         0x00000000,
 497         (0x0e00 << 16) | (0xac14 >> 2),
 498         0x00000000,
 499         (0x0e00 << 16) | (0xac58 >> 2),
 500         0x00000000,
 501         (0x0e00 << 16) | (0xac68 >> 2),
 502         0x00000000,
 503         (0x0e00 << 16) | (0xac6c >> 2),
 504         0x00000000,
 505         (0x0e00 << 16) | (0xac70 >> 2),
 506         0x00000000,
 507         (0x0e00 << 16) | (0xac74 >> 2),
 508         0x00000000,
 509         (0x0e00 << 16) | (0xac78 >> 2),
 510         0x00000000,
 511         (0x0e00 << 16) | (0xac7c >> 2),
 512         0x00000000,
 513         (0x0e00 << 16) | (0xac80 >> 2),
 514         0x00000000,
 515         (0x0e00 << 16) | (0xac84 >> 2),
 516         0x00000000,
 517         (0x0e00 << 16) | (0xac88 >> 2),
 518         0x00000000,
 519         (0x0e00 << 16) | (0xac8c >> 2),
 520         0x00000000,
 521         (0x0e00 << 16) | (0x970c >> 2),
 522         0x00000000,
 523         (0x0e00 << 16) | (0x9714 >> 2),
 524         0x00000000,
 525         (0x0e00 << 16) | (0x9718 >> 2),
 526         0x00000000,
 527         (0x0e00 << 16) | (0x971c >> 2),
 528         0x00000000,
 529         (0x0e00 << 16) | (0x31068 >> 2),
 530         0x00000000,
 531         (0x4e00 << 16) | (0x31068 >> 2),
 532         0x00000000,
 533         (0x5e00 << 16) | (0x31068 >> 2),
 534         0x00000000,
 535         (0x6e00 << 16) | (0x31068 >> 2),
 536         0x00000000,
 537         (0x7e00 << 16) | (0x31068 >> 2),
 538         0x00000000,
 539         (0x8e00 << 16) | (0x31068 >> 2),
 540         0x00000000,
 541         (0x9e00 << 16) | (0x31068 >> 2),
 542         0x00000000,
 543         (0xae00 << 16) | (0x31068 >> 2),
 544         0x00000000,
 545         (0xbe00 << 16) | (0x31068 >> 2),
 546         0x00000000,
 547         (0x0e00 << 16) | (0xcd10 >> 2),
 548         0x00000000,
 549         (0x0e00 << 16) | (0xcd14 >> 2),
 550         0x00000000,
 551         (0x0e00 << 16) | (0x88b0 >> 2),
 552         0x00000000,
 553         (0x0e00 << 16) | (0x88b4 >> 2),
 554         0x00000000,
 555         (0x0e00 << 16) | (0x88b8 >> 2),
 556         0x00000000,
 557         (0x0e00 << 16) | (0x88bc >> 2),
 558         0x00000000,
 559         (0x0400 << 16) | (0x89c0 >> 2),
 560         0x00000000,
 561         (0x0e00 << 16) | (0x88c4 >> 2),
 562         0x00000000,
 563         (0x0e00 << 16) | (0x88c8 >> 2),
 564         0x00000000,
 565         (0x0e00 << 16) | (0x88d0 >> 2),
 566         0x00000000,
 567         (0x0e00 << 16) | (0x88d4 >> 2),
 568         0x00000000,
 569         (0x0e00 << 16) | (0x88d8 >> 2),
 570         0x00000000,
 571         (0x0e00 << 16) | (0x8980 >> 2),
 572         0x00000000,
 573         (0x0e00 << 16) | (0x30938 >> 2),
 574         0x00000000,
 575         (0x0e00 << 16) | (0x3093c >> 2),
 576         0x00000000,
 577         (0x0e00 << 16) | (0x30940 >> 2),
 578         0x00000000,
 579         (0x0e00 << 16) | (0x89a0 >> 2),
 580         0x00000000,
 581         (0x0e00 << 16) | (0x30900 >> 2),
 582         0x00000000,
 583         (0x0e00 << 16) | (0x30904 >> 2),
 584         0x00000000,
 585         (0x0e00 << 16) | (0x89b4 >> 2),
 586         0x00000000,
 587         (0x0e00 << 16) | (0x3c210 >> 2),
 588         0x00000000,
 589         (0x0e00 << 16) | (0x3c214 >> 2),
 590         0x00000000,
 591         (0x0e00 << 16) | (0x3c218 >> 2),
 592         0x00000000,
 593         (0x0e00 << 16) | (0x8904 >> 2),
 594         0x00000000,
 595         0x5,
 596         (0x0e00 << 16) | (0x8c28 >> 2),
 597         (0x0e00 << 16) | (0x8c2c >> 2),
 598         (0x0e00 << 16) | (0x8c30 >> 2),
 599         (0x0e00 << 16) | (0x8c34 >> 2),
 600         (0x0e00 << 16) | (0x9600 >> 2),
 601 };
 602
 603 static const u32 kalindi_rlc_save_restore_register_list[] =
 604 {
 605         (0x0e00 << 16) | (0xc12c >> 2),
 606         0x00000000,
 607         (0x0e00 << 16) | (0xc140 >> 2),
 608         0x00000000,
 609         (0x0e00 << 16) | (0xc150 >> 2),
 610         0x00000000,
 611         (0x0e00 << 16) | (0xc15c >> 2),
 612         0x00000000,
 613         (0x0e00 << 16) | (0xc168 >> 2),
 614         0x00000000,
 615         (0x0e00 << 16) | (0xc170 >> 2),
 616         0x00000000,
 617         (0x0e00 << 16) | (0xc204 >> 2),
 618         0x00000000,
 619         (0x0e00 << 16) | (0xc2b4 >> 2),
 620         0x00000000,
 621         (0x0e00 << 16) | (0xc2b8 >> 2),
 622         0x00000000,
 623         (0x0e00 << 16) | (0xc2bc >> 2),
 624         0x00000000,
 625         (0x0e00 << 16) | (0xc2c0 >> 2),
 626         0x00000000,
 627         (0x0e00 << 16) | (0x8228 >> 2),
 628         0x00000000,
 629         (0x0e00 << 16) | (0x829c >> 2),
 630         0x00000000,
 631         (0x0e00 << 16) | (0x869c >> 2),
 632         0x00000000,
 633         (0x0600 << 16) | (0x98f4 >> 2),
 634         0x00000000,
 635         (0x0e00 << 16) | (0x98f8 >> 2),
 636         0x00000000,
 637         (0x0e00 << 16) | (0x9900 >> 2),
 638         0x00000000,
 639         (0x0e00 << 16) | (0xc260 >> 2),
 640         0x00000000,
 641         (0x0e00 << 16) | (0x90e8 >> 2),
 642         0x00000000,
 643         (0x0e00 << 16) | (0x3c000 >> 2),
 644         0x00000000,
 645         (0x0e00 << 16) | (0x3c00c >> 2),
 646         0x00000000,
 647         (0x0e00 << 16) | (0x8c1c >> 2),
 648         0x00000000,
 649         (0x0e00 << 16) | (0x9700 >> 2),
 650         0x00000000,
 651         (0x0e00 << 16) | (0xcd20 >> 2),
 652         0x00000000,
 653         (0x4e00 << 16) | (0xcd20 >> 2),
 654         0x00000000,
 655         (0x5e00 << 16) | (0xcd20 >> 2),
 656         0x00000000,
 657         (0x6e00 << 16) | (0xcd20 >> 2),
 658         0x00000000,
 659         (0x7e00 << 16) | (0xcd20 >> 2),
 660         0x00000000,
 661         (0x0e00 << 16) | (0x89bc >> 2),
 662         0x00000000,
 663         (0x0e00 << 16) | (0x8900 >> 2),
 664         0x00000000,
 665         0x3,
 666         (0x0e00 << 16) | (0xc130 >> 2),
 667         0x00000000,
 668         (0x0e00 << 16) | (0xc134 >> 2),
 669         0x00000000,
 670         (0x0e00 << 16) | (0xc1fc >> 2),
 671         0x00000000,
 672         (0x0e00 << 16) | (0xc208 >> 2),
 673         0x00000000,
 674         (0x0e00 << 16) | (0xc264 >> 2),
 675         0x00000000,
 676         (0x0e00 << 16) | (0xc268 >> 2),
 677         0x00000000,
 678         (0x0e00 << 16) | (0xc26c >> 2),
 679         0x00000000,
 680         (0x0e00 << 16) | (0xc270 >> 2),
 681         0x00000000,
 682         (0x0e00 << 16) | (0xc274 >> 2),
 683         0x00000000,
 684         (0x0e00 << 16) | (0xc28c >> 2),
 685         0x00000000,
 686         (0x0e00 << 16) | (0xc290 >> 2),
 687         0x00000000,
 688         (0x0e00 << 16) | (0xc294 >> 2),
 689         0x00000000,
 690         (0x0e00 << 16) | (0xc298 >> 2),
 691         0x00000000,
 692         (0x0e00 << 16) | (0xc2a0 >> 2),
 693         0x00000000,
 694         (0x0e00 << 16) | (0xc2a4 >> 2),
 695         0x00000000,
 696         (0x0e00 << 16) | (0xc2a8 >> 2),
 697         0x00000000,
 698         (0x0e00 << 16) | (0xc2ac >> 2),
 699         0x00000000,
 700         (0x0e00 << 16) | (0x301d0 >> 2),
 701         0x00000000,
 702         (0x0e00 << 16) | (0x30238 >> 2),
 703         0x00000000,
 704         (0x0e00 << 16) | (0x30250 >> 2),
 705         0x00000000,
 706         (0x0e00 << 16) | (0x30254 >> 2),
 707         0x00000000,
 708         (0x0e00 << 16) | (0x30258 >> 2),
 709         0x00000000,
 710         (0x0e00 << 16) | (0x3025c >> 2),
 711         0x00000000,
 712         (0x4e00 << 16) | (0xc900 >> 2),
 713         0x00000000,
 714         (0x5e00 << 16) | (0xc900 >> 2),
 715         0x00000000,
 716         (0x6e00 << 16) | (0xc900 >> 2),
 717         0x00000000,
 718         (0x7e00 << 16) | (0xc900 >> 2),
 719         0x00000000,
 720         (0x4e00 << 16) | (0xc904 >> 2),
 721         0x00000000,
 722         (0x5e00 << 16) | (0xc904 >> 2),
 723         0x00000000,
 724         (0x6e00 << 16) | (0xc904 >> 2),
 725         0x00000000,
 726         (0x7e00 << 16) | (0xc904 >> 2),
 727         0x00000000,
 728         (0x4e00 << 16) | (0xc908 >> 2),
 729         0x00000000,
 730         (0x5e00 << 16) | (0xc908 >> 2),
 731         0x00000000,
 732         (0x6e00 << 16) | (0xc908 >> 2),
 733         0x00000000,
 734         (0x7e00 << 16) | (0xc908 >> 2),
 735         0x00000000,
 736         (0x4e00 << 16) | (0xc90c >> 2),
 737         0x00000000,
 738         (0x5e00 << 16) | (0xc90c >> 2),
 739         0x00000000,
 740         (0x6e00 << 16) | (0xc90c >> 2),
 741         0x00000000,
 742         (0x7e00 << 16) | (0xc90c >> 2),
 743         0x00000000,
 744         (0x4e00 << 16) | (0xc910 >> 2),
 745         0x00000000,
 746         (0x5e00 << 16) | (0xc910 >> 2),
 747         0x00000000,
 748         (0x6e00 << 16) | (0xc910 >> 2),
 749         0x00000000,
 750         (0x7e00 << 16) | (0xc910 >> 2),
 751         0x00000000,
 752         (0x0e00 << 16) | (0xc99c >> 2),
 753         0x00000000,
 754         (0x0e00 << 16) | (0x9834 >> 2),
 755         0x00000000,
 756         (0x0000 << 16) | (0x30f00 >> 2),
 757         0x00000000,
 758         (0x0000 << 16) | (0x30f04 >> 2),
 759         0x00000000,
 760         (0x0000 << 16) | (0x30f08 >> 2),
 761         0x00000000,
 762         (0x0000 << 16) | (0x30f0c >> 2),
 763         0x00000000,
 764         (0x0600 << 16) | (0x9b7c >> 2),
 765         0x00000000,
 766         (0x0e00 << 16) | (0x8a14 >> 2),
 767         0x00000000,
 768         (0x0e00 << 16) | (0x8a18 >> 2),
 769         0x00000000,
 770         (0x0600 << 16) | (0x30a00 >> 2),
 771         0x00000000,
 772         (0x0e00 << 16) | (0x8bf0 >> 2),
 773         0x00000000,
 774         (0x0e00 << 16) | (0x8bcc >> 2),
 775         0x00000000,
 776         (0x0e00 << 16) | (0x8b24 >> 2),
 777         0x00000000,
 778         (0x0e00 << 16) | (0x30a04 >> 2),
 779         0x00000000,
 780         (0x0600 << 16) | (0x30a10 >> 2),
 781         0x00000000,
 782         (0x0600 << 16) | (0x30a14 >> 2),
 783         0x00000000,
 784         (0x0600 << 16) | (0x30a18 >> 2),
 785         0x00000000,
 786         (0x0600 << 16) | (0x30a2c >> 2),
 787         0x00000000,
 788         (0x0e00 << 16) | (0xc700 >> 2),
 789         0x00000000,
 790         (0x0e00 << 16) | (0xc704 >> 2),
 791         0x00000000,
 792         (0x0e00 << 16) | (0xc708 >> 2),
 793         0x00000000,
 794         (0x0e00 << 16) | (0xc768 >> 2),
 795         0x00000000,
 796         (0x0400 << 16) | (0xc770 >> 2),
 797         0x00000000,
 798         (0x0400 << 16) | (0xc774 >> 2),
 799         0x00000000,
 800         (0x0400 << 16) | (0xc798 >> 2),
 801         0x00000000,
 802         (0x0400 << 16) | (0xc79c >> 2),
 803         0x00000000,
 804         (0x0e00 << 16) | (0x9100 >> 2),
 805         0x00000000,
 806         (0x0e00 << 16) | (0x3c010 >> 2),
 807         0x00000000,
 808         (0x0e00 << 16) | (0x8c00 >> 2),
 809         0x00000000,
 810         (0x0e00 << 16) | (0x8c04 >> 2),
 811         0x00000000,
 812         (0x0e00 << 16) | (0x8c20 >> 2),
 813         0x00000000,
 814         (0x0e00 << 16) | (0x8c38 >> 2),
 815         0x00000000,
 816         (0x0e00 << 16) | (0x8c3c >> 2),
 817         0x00000000,
 818         (0x0e00 << 16) | (0xae00 >> 2),
 819         0x00000000,
 820         (0x0e00 << 16) | (0x9604 >> 2),
 821         0x00000000,
 822         (0x0e00 << 16) | (0xac08 >> 2),
 823         0x00000000,
 824         (0x0e00 << 16) | (0xac0c >> 2),
 825         0x00000000,
 826         (0x0e00 << 16) | (0xac10 >> 2),
 827         0x00000000,
 828         (0x0e00 << 16) | (0xac14 >> 2),
 829         0x00000000,
 830         (0x0e00 << 16) | (0xac58 >> 2),
 831         0x00000000,
 832         (0x0e00 << 16) | (0xac68 >> 2),
 833         0x00000000,
 834         (0x0e00 << 16) | (0xac6c >> 2),
 835         0x00000000,
 836         (0x0e00 << 16) | (0xac70 >> 2),
 837         0x00000000,
 838         (0x0e00 << 16) | (0xac74 >> 2),
 839         0x00000000,
 840         (0x0e00 << 16) | (0xac78 >> 2),
 841         0x00000000,
 842         (0x0e00 << 16) | (0xac7c >> 2),
 843         0x00000000,
 844         (0x0e00 << 16) | (0xac80 >> 2),
 845         0x00000000,
 846         (0x0e00 << 16) | (0xac84 >> 2),
 847         0x00000000,
 848         (0x0e00 << 16) | (0xac88 >> 2),
 849         0x00000000,
 850         (0x0e00 << 16) | (0xac8c >> 2),
 851         0x00000000,
 852         (0x0e00 << 16) | (0x970c >> 2),
 853         0x00000000,
 854         (0x0e00 << 16) | (0x9714 >> 2),
 855         0x00000000,
 856         (0x0e00 << 16) | (0x9718 >> 2),
 857         0x00000000,
 858         (0x0e00 << 16) | (0x971c >> 2),
 859         0x00000000,
 860         (0x0e00 << 16) | (0x31068 >> 2),
 861         0x00000000,
 862         (0x4e00 << 16) | (0x31068 >> 2),
 863         0x00000000,
 864         (0x5e00 << 16) | (0x31068 >> 2),
 865         0x00000000,
 866         (0x6e00 << 16) | (0x31068 >> 2),
 867         0x00000000,
 868         (0x7e00 << 16) | (0x31068 >> 2),
 869         0x00000000,
 870         (0x0e00 << 16) | (0xcd10 >> 2),
 871         0x00000000,
 872         (0x0e00 << 16) | (0xcd14 >> 2),
 873         0x00000000,
 874         (0x0e00 << 16) | (0x88b0 >> 2),
 875         0x00000000,
 876         (0x0e00 << 16) | (0x88b4 >> 2),
 877         0x00000000,
 878         (0x0e00 << 16) | (0x88b8 >> 2),
 879         0x00000000,
 880         (0x0e00 << 16) | (0x88bc >> 2),
 881         0x00000000,
 882         (0x0400 << 16) | (0x89c0 >> 2),
 883         0x00000000,
 884         (0x0e00 << 16) | (0x88c4 >> 2),
 885         0x00000000,
 886         (0x0e00 << 16) | (0x88c8 >> 2),
 887         0x00000000,
 888         (0x0e00 << 16) | (0x88d0 >> 2),
 889         0x00000000,
 890         (0x0e00 << 16) | (0x88d4 >> 2),
 891         0x00000000,
 892         (0x0e00 << 16) | (0x88d8 >> 2),
 893         0x00000000,
 894         (0x0e00 << 16) | (0x8980 >> 2),
 895         0x00000000,
 896         (0x0e00 << 16) | (0x30938 >> 2),
 897         0x00000000,
 898         (0x0e00 << 16) | (0x3093c >> 2),
 899         0x00000000,
 900         (0x0e00 << 16) | (0x30940 >> 2),
 901         0x00000000,
 902         (0x0e00 << 16) | (0x89a0 >> 2),
 903         0x00000000,
 904         (0x0e00 << 16) | (0x30900 >> 2),
 905         0x00000000,
 906         (0x0e00 << 16) | (0x30904 >> 2),
 907         0x00000000,
 908         (0x0e00 << 16) | (0x89b4 >> 2),
 909         0x00000000,
 910         (0x0e00 << 16) | (0x3e1fc >> 2),
 911         0x00000000,
 912         (0x0e00 << 16) | (0x3c210 >> 2),
 913         0x00000000,
 914         (0x0e00 << 16) | (0x3c214 >> 2),
 915         0x00000000,
 916         (0x0e00 << 16) | (0x3c218 >> 2),
 917         0x00000000,
 918         (0x0e00 << 16) | (0x8904 >> 2),
 919         0x00000000,
 920         0x5,
 921         (0x0e00 << 16) | (0x8c28 >> 2),
 922         (0x0e00 << 16) | (0x8c2c >> 2),
 923         (0x0e00 << 16) | (0x8c30 >> 2),
 924         (0x0e00 << 16) | (0x8c34 >> 2),
 925         (0x0e00 << 16) | (0x9600 >> 2),
 926 };
 927
 928 static const u32 bonaire_golden_spm_registers[] =
 929 {
 930         0x30800, 0xe0ffffff, 0xe0000000
 931 };
 932
 933 static const u32 bonaire_golden_common_registers[] =
 934 {
 935         0xc770, 0xffffffff, 0x00000800,
 936         0xc774, 0xffffffff, 0x00000800,
 937         0xc798, 0xffffffff, 0x00007fbf,
 938         0xc79c, 0xffffffff, 0x00007faf
 939 };
 940
 941 static const u32 bonaire_golden_registers[] =
 942 {
 943         0x3354, 0x00000333, 0x00000333,
 944         0x3350, 0x000c0fc0, 0x00040200,
 945         0x9a10, 0x00010000, 0x00058208,
 946         0x3c000, 0xffff1fff, 0x00140000,
 947         0x3c200, 0xfdfc0fff, 0x00000100,
 948         0x3c234, 0x40000000, 0x40000200,
 949         0x9830, 0xffffffff, 0x00000000,
 950         0x9834, 0xf00fffff, 0x00000400,
 951         0x9838, 0x0002021c, 0x00020200,
 952         0xc78, 0x00000080, 0x00000000,
 953         0x5bb0, 0x000000f0, 0x00000070,
 954         0x5bc0, 0xf0311fff, 0x80300000,
 955         0x98f8, 0x73773777, 0x12010001,
 956         0x350c, 0x00810000, 0x408af000,
 957         0x7030, 0x31000111, 0x00000011,
 958         0x2f48, 0x73773777, 0x12010001,
 959         0x220c, 0x00007fb6, 0x0021a1b1,
 960         0x2210, 0x00007fb6, 0x002021b1,
 961         0x2180, 0x00007fb6, 0x00002191,
 962         0x2218, 0x00007fb6, 0x002121b1,
 963         0x221c, 0x00007fb6, 0x002021b1,
 964         0x21dc, 0x00007fb6, 0x00002191,
 965         0x21e0, 0x00007fb6, 0x00002191,
 966         0x3628, 0x0000003f, 0x0000000a,
 967         0x362c, 0x0000003f, 0x0000000a,
 968         0x2ae4, 0x00073ffe, 0x000022a2,
 969         0x240c, 0x000007ff, 0x00000000,
 970         0x8a14, 0xf000003f, 0x00000007,
 971         0x8bf0, 0x00002001, 0x00000001,
 972         0x8b24, 0xffffffff, 0x00ffffff,
 973         0x30a04, 0x0000ff0f, 0x00000000,
 974         0x28a4c, 0x07ffffff, 0x06000000,
 975         0x4d8, 0x00000fff, 0x00000100,
 976         0x3e78, 0x00000001, 0x00000002,
 977         0x9100, 0x03000000, 0x0362c688,
 978         0x8c00, 0x000000ff, 0x00000001,
 979         0xe40, 0x00001fff, 0x00001fff,
 980         0x9060, 0x0000007f, 0x00000020,
 981         0x9508, 0x00010000, 0x00010000,
 982         0xac14, 0x000003ff, 0x000000f3,
 983         0xac0c, 0xffffffff, 0x00001032
 984 };
 985
 986 static const u32 bonaire_mgcg_cgcg_init[] =
 987 {
 988         0xc420, 0xffffffff, 0xfffffffc,
 989         0x30800, 0xffffffff, 0xe0000000,
 990         0x3c2a0, 0xffffffff, 0x00000100,
 991         0x3c208, 0xffffffff, 0x00000100,
 992         0x3c2c0, 0xffffffff, 0xc0000100,
 993         0x3c2c8, 0xffffffff, 0xc0000100,
 994         0x3c2c4, 0xffffffff, 0xc0000100,
 995         0x55e4, 0xffffffff, 0x00600100,
 996         0x3c280, 0xffffffff, 0x00000100,
 997         0x3c214, 0xffffffff, 0x06000100,
 998         0x3c220, 0xffffffff, 0x00000100,
 999         0x3c218, 0xffffffff, 0x06000100,
1000         0x3c204, 0xffffffff, 0x00000100,
1001         0x3c2e0, 0xffffffff, 0x00000100,
1002         0x3c224, 0xffffffff, 0x00000100,
1003         0x3c200, 0xffffffff, 0x00000100,
1004         0x3c230, 0xffffffff, 0x00000100,
1005         0x3c234, 0xffffffff, 0x00000100,
1006         0x3c250, 0xffffffff, 0x00000100,
1007         0x3c254, 0xffffffff, 0x00000100,
1008         0x3c258, 0xffffffff, 0x00000100,
1009         0x3c25c, 0xffffffff, 0x00000100,
1010         0x3c260, 0xffffffff, 0x00000100,
1011         0x3c27c, 0xffffffff, 0x00000100,
1012         0x3c278, 0xffffffff, 0x00000100,
1013         0x3c210, 0xffffffff, 0x06000100,
1014         0x3c290, 0xffffffff, 0x00000100,
1015         0x3c274, 0xffffffff, 0x00000100,
1016         0x3c2b4, 0xffffffff, 0x00000100,
1017         0x3c2b0, 0xffffffff, 0x00000100,
1018         0x3c270, 0xffffffff, 0x00000100,
1019         0x30800, 0xffffffff, 0xe0000000,
1020         0x3c020, 0xffffffff, 0x00010000,
1021         0x3c024, 0xffffffff, 0x00030002,
1022         0x3c028, 0xffffffff, 0x00040007,
1023         0x3c02c, 0xffffffff, 0x00060005,
1024         0x3c030, 0xffffffff, 0x00090008,
1025         0x3c034, 0xffffffff, 0x00010000,
1026         0x3c038, 0xffffffff, 0x00030002,
1027         0x3c03c, 0xffffffff, 0x00040007,
1028         0x3c040, 0xffffffff, 0x00060005,
1029         0x3c044, 0xffffffff, 0x00090008,
1030         0x3c048, 0xffffffff, 0x00010000,
1031         0x3c04c, 0xffffffff, 0x00030002,
1032         0x3c050, 0xffffffff, 0x00040007,
1033         0x3c054, 0xffffffff, 0x00060005,
1034         0x3c058, 0xffffffff, 0x00090008,
1035         0x3c05c, 0xffffffff, 0x00010000,
1036         0x3c060, 0xffffffff, 0x00030002,
1037         0x3c064, 0xffffffff, 0x00040007,
1038         0x3c068, 0xffffffff, 0x00060005,
1039         0x3c06c, 0xffffffff, 0x00090008,
1040         0x3c070, 0xffffffff, 0x00010000,
1041         0x3c074, 0xffffffff, 0x00030002,
1042         0x3c078, 0xffffffff, 0x00040007,
1043         0x3c07c, 0xffffffff, 0x00060005,
1044         0x3c080, 0xffffffff, 0x00090008,
1045         0x3c084, 0xffffffff, 0x00010000,
1046         0x3c088, 0xffffffff, 0x00030002,
1047         0x3c08c, 0xffffffff, 0x00040007,
1048         0x3c090, 0xffffffff, 0x00060005,
1049         0x3c094, 0xffffffff, 0x00090008,
1050         0x3c098, 0xffffffff, 0x00010000,
1051         0x3c09c, 0xffffffff, 0x00030002,
1052         0x3c0a0, 0xffffffff, 0x00040007,
1053         0x3c0a4, 0xffffffff, 0x00060005,
1054         0x3c0a8, 0xffffffff, 0x00090008,
1055         0x3c000, 0xffffffff, 0x96e00200,
1056         0x8708, 0xffffffff, 0x00900100,
1057         0xc424, 0xffffffff, 0x0020003f,
1058         0x38, 0xffffffff, 0x0140001c,
1059         0x3c, 0x000f0000, 0x000f0000,
1060         0x220, 0xffffffff, 0xC060000C,
1061         0x224, 0xc0000fff, 0x00000100,
1062         0xf90, 0xffffffff, 0x00000100,
1063         0xf98, 0x00000101, 0x00000000,
1064         0x20a8, 0xffffffff, 0x00000104,
1065         0x55e4, 0xff000fff, 0x00000100,
1066         0x30cc, 0xc0000fff, 0x00000104,
1067         0xc1e4, 0x00000001, 0x00000001,
1068         0xd00c, 0xff000ff0, 0x00000100,
1069         0xd80c, 0xff000ff0, 0x00000100
1070 };
1071
1072 static const u32 spectre_golden_spm_registers[] =
1073 {
1074         0x30800, 0xe0ffffff, 0xe0000000
1075 };
1076
1077 static const u32 spectre_golden_common_registers[] =
1078 {
1079         0xc770, 0xffffffff, 0x00000800,
1080         0xc774, 0xffffffff, 0x00000800,
1081         0xc798, 0xffffffff, 0x00007fbf,
1082         0xc79c, 0xffffffff, 0x00007faf
1083 };
1084
1085 static const u32 spectre_golden_registers[] =
1086 {
1087         0x3c000, 0xffff1fff, 0x96940200,
1088         0x3c00c, 0xffff0001, 0xff000000,
1089         0x3c200, 0xfffc0fff, 0x00000100,
1090         0x6ed8, 0x00010101, 0x00010000,
1091         0x9834, 0xf00fffff, 0x00000400,
1092         0x9838, 0xfffffffc, 0x00020200,
1093         0x5bb0, 0x000000f0, 0x00000070,
1094         0x5bc0, 0xf0311fff, 0x80300000,
1095         0x98f8, 0x73773777, 0x12010001,
1096         0x9b7c, 0x00ff0000, 0x00fc0000,
1097         0x2f48, 0x73773777, 0x12010001,
1098         0x8a14, 0xf000003f, 0x00000007,
1099         0x8b24, 0xffffffff, 0x00ffffff,
1100         0x28350, 0x3f3f3fff, 0x00000082,
1101         0x28354, 0x0000003f, 0x00000000,
1102         0x3e78, 0x00000001, 0x00000002,
1103         0x913c, 0xffff03df, 0x00000004,
1104         0xc768, 0x00000008, 0x00000008,
1105         0x8c00, 0x000008ff, 0x00000800,
1106         0x9508, 0x00010000, 0x00010000,
1107         0xac0c, 0xffffffff, 0x54763210,
1108         0x214f8, 0x01ff01ff, 0x00000002,
1109         0x21498, 0x007ff800, 0x00200000,
1110         0x2015c, 0xffffffff, 0x00000f40,
1111         0x30934, 0xffffffff, 0x00000001
1112 };
1113
1114 static const u32 spectre_mgcg_cgcg_init[] =
1115 {
1116         0xc420, 0xffffffff, 0xfffffffc,
1117         0x30800, 0xffffffff, 0xe0000000,
1118         0x3c2a0, 0xffffffff, 0x00000100,
1119         0x3c208, 0xffffffff, 0x00000100,
1120         0x3c2c0, 0xffffffff, 0x00000100,
1121         0x3c2c8, 0xffffffff, 0x00000100,
1122         0x3c2c4, 0xffffffff, 0x00000100,
1123         0x55e4, 0xffffffff, 0x00600100,
1124         0x3c280, 0xffffffff, 0x00000100,
1125         0x3c214, 0xffffffff, 0x06000100,
1126         0x3c220, 0xffffffff, 0x00000100,
1127         0x3c218, 0xffffffff, 0x06000100,
1128         0x3c204, 0xffffffff, 0x00000100,
1129         0x3c2e0, 0xffffffff, 0x00000100,
1130         0x3c224, 0xffffffff, 0x00000100,
1131         0x3c200, 0xffffffff, 0x00000100,
1132         0x3c230, 0xffffffff, 0x00000100,
1133         0x3c234, 0xffffffff, 0x00000100,
1134         0x3c250, 0xffffffff, 0x00000100,
1135         0x3c254, 0xffffffff, 0x00000100,
1136         0x3c258, 0xffffffff, 0x00000100,
1137         0x3c25c, 0xffffffff, 0x00000100,
1138         0x3c260, 0xffffffff, 0x00000100,
1139         0x3c27c, 0xffffffff, 0x00000100,
1140         0x3c278, 0xffffffff, 0x00000100,
1141         0x3c210, 0xffffffff, 0x06000100,
1142         0x3c290, 0xffffffff, 0x00000100,
1143         0x3c274, 0xffffffff, 0x00000100,
1144         0x3c2b4, 0xffffffff, 0x00000100,
1145         0x3c2b0, 0xffffffff, 0x00000100,
1146         0x3c270, 0xffffffff, 0x00000100,
1147         0x30800, 0xffffffff, 0xe0000000,
1148         0x3c020, 0xffffffff, 0x00010000,
1149         0x3c024, 0xffffffff, 0x00030002,
1150         0x3c028, 0xffffffff, 0x00040007,
1151         0x3c02c, 0xffffffff, 0x00060005,
1152         0x3c030, 0xffffffff, 0x00090008,
1153         0x3c034, 0xffffffff, 0x00010000,
1154         0x3c038, 0xffffffff, 0x00030002,
1155         0x3c03c, 0xffffffff, 0x00040007,
1156         0x3c040, 0xffffffff, 0x00060005,
1157         0x3c044, 0xffffffff, 0x00090008,
1158         0x3c048, 0xffffffff, 0x00010000,
1159         0x3c04c, 0xffffffff, 0x00030002,
1160         0x3c050, 0xffffffff, 0x00040007,
1161         0x3c054, 0xffffffff, 0x00060005,
1162         0x3c058, 0xffffffff, 0x00090008,
1163         0x3c05c, 0xffffffff, 0x00010000,
1164         0x3c060, 0xffffffff, 0x00030002,
1165         0x3c064, 0xffffffff, 0x00040007,
1166         0x3c068, 0xffffffff, 0x00060005,
1167         0x3c06c, 0xffffffff, 0x00090008,
1168         0x3c070, 0xffffffff, 0x00010000,
1169         0x3c074, 0xffffffff, 0x00030002,
1170         0x3c078, 0xffffffff, 0x00040007,
1171         0x3c07c, 0xffffffff, 0x00060005,
1172         0x3c080, 0xffffffff, 0x00090008,
1173         0x3c084, 0xffffffff, 0x00010000,
1174         0x3c088, 0xffffffff, 0x00030002,
1175         0x3c08c, 0xffffffff, 0x00040007,
1176         0x3c090, 0xffffffff, 0x00060005,
1177         0x3c094, 0xffffffff, 0x00090008,
1178         0x3c098, 0xffffffff, 0x00010000,
1179         0x3c09c, 0xffffffff, 0x00030002,
1180         0x3c0a0, 0xffffffff, 0x00040007,
1181         0x3c0a4, 0xffffffff, 0x00060005,
1182         0x3c0a8, 0xffffffff, 0x00090008,
1183         0x3c0ac, 0xffffffff, 0x00010000,
1184         0x3c0b0, 0xffffffff, 0x00030002,
1185         0x3c0b4, 0xffffffff, 0x00040007,
1186         0x3c0b8, 0xffffffff, 0x00060005,
1187         0x3c0bc, 0xffffffff, 0x00090008,
1188         0x3c000, 0xffffffff, 0x96e00200,
1189         0x8708, 0xffffffff, 0x00900100,
1190         0xc424, 0xffffffff, 0x0020003f,
1191         0x38, 0xffffffff, 0x0140001c,
1192         0x3c, 0x000f0000, 0x000f0000,
1193         0x220, 0xffffffff, 0xC060000C,
1194         0x224, 0xc0000fff, 0x00000100,
1195         0xf90, 0xffffffff, 0x00000100,
1196         0xf98, 0x00000101, 0x00000000,
1197         0x20a8, 0xffffffff, 0x00000104,
1198         0x55e4, 0xff000fff, 0x00000100,
1199         0x30cc, 0xc0000fff, 0x00000104,
1200         0xc1e4, 0x00000001, 0x00000001,
1201         0xd00c, 0xff000ff0, 0x00000100,
1202         0xd80c, 0xff000ff0, 0x00000100
1203 };
1204
1205 static const u32 kalindi_golden_spm_registers[] =
1206 {
1207         0x30800, 0xe0ffffff, 0xe0000000
1208 };
1209
1210 static const u32 kalindi_golden_common_registers[] =
1211 {
1212         0xc770, 0xffffffff, 0x00000800,
1213         0xc774, 0xffffffff, 0x00000800,
1214         0xc798, 0xffffffff, 0x00007fbf,
1215         0xc79c, 0xffffffff, 0x00007faf
1216 };
1217
1218 static const u32 kalindi_golden_registers[] =
1219 {
1220         0x3c000, 0xffffdfff, 0x6e944040,
1221         0x55e4, 0xff607fff, 0xfc000100,
1222         0x3c220, 0xff000fff, 0x00000100,
1223         0x3c224, 0xff000fff, 0x00000100,
1224         0x3c200, 0xfffc0fff, 0x00000100,
1225         0x6ed8, 0x00010101, 0x00010000,
1226         0x9830, 0xffffffff, 0x00000000,
1227         0x9834, 0xf00fffff, 0x00000400,
1228         0x5bb0, 0x000000f0, 0x00000070,
1229         0x5bc0, 0xf0311fff, 0x80300000,
1230         0x98f8, 0x73773777, 0x12010001,
1231         0x98fc, 0xffffffff, 0x00000010,
1232         0x9b7c, 0x00ff0000, 0x00fc0000,
1233         0x8030, 0x00001f0f, 0x0000100a,
1234         0x2f48, 0x73773777, 0x12010001,
1235         0x2408, 0x000fffff, 0x000c007f,
1236         0x8a14, 0xf000003f, 0x00000007,
1237         0x8b24, 0x3fff3fff, 0x00ffcfff,
1238         0x30a04, 0x0000ff0f, 0x00000000,
1239         0x28a4c, 0x07ffffff, 0x06000000,
1240         0x4d8, 0x00000fff, 0x00000100,
1241         0x3e78, 0x00000001, 0x00000002,
1242         0xc768, 0x00000008, 0x00000008,
1243         0x8c00, 0x000000ff, 0x00000003,
1244         0x214f8, 0x01ff01ff, 0x00000002,
1245         0x21498, 0x007ff800, 0x00200000,
1246         0x2015c, 0xffffffff, 0x00000f40,
1247         0x88c4, 0x001f3ae3, 0x00000082,
1248         0x88d4, 0x0000001f, 0x00000010,
1249         0x30934, 0xffffffff, 0x00000000
1250 };
1251
1252 static const u32 kalindi_mgcg_cgcg_init[] =
1253 {
1254         0xc420, 0xffffffff, 0xfffffffc,
1255         0x30800, 0xffffffff, 0xe0000000,
1256         0x3c2a0, 0xffffffff, 0x00000100,
1257         0x3c208, 0xffffffff, 0x00000100,
1258         0x3c2c0, 0xffffffff, 0x00000100,
1259         0x3c2c8, 0xffffffff, 0x00000100,
1260         0x3c2c4, 0xffffffff, 0x00000100,
1261         0x55e4, 0xffffffff, 0x00600100,
1262         0x3c280, 0xffffffff, 0x00000100,
1263         0x3c214, 0xffffffff, 0x06000100,
1264         0x3c220, 0xffffffff, 0x00000100,
1265         0x3c218, 0xffffffff, 0x06000100,
1266         0x3c204, 0xffffffff, 0x00000100,
1267         0x3c2e0, 0xffffffff, 0x00000100,
1268         0x3c224, 0xffffffff, 0x00000100,
1269         0x3c200, 0xffffffff, 0x00000100,
1270         0x3c230, 0xffffffff, 0x00000100,
1271         0x3c234, 0xffffffff, 0x00000100,
1272         0x3c250, 0xffffffff, 0x00000100,
1273         0x3c254, 0xffffffff, 0x00000100,
1274         0x3c258, 0xffffffff, 0x00000100,
1275         0x3c25c, 0xffffffff, 0x00000100,
1276         0x3c260, 0xffffffff, 0x00000100,
1277         0x3c27c, 0xffffffff, 0x00000100,
1278         0x3c278, 0xffffffff, 0x00000100,
1279         0x3c210, 0xffffffff, 0x06000100,
1280         0x3c290, 0xffffffff, 0x00000100,
1281         0x3c274, 0xffffffff, 0x00000100,
1282         0x3c2b4, 0xffffffff, 0x00000100,
1283         0x3c2b0, 0xffffffff, 0x00000100,
1284         0x3c270, 0xffffffff, 0x00000100,
1285         0x30800, 0xffffffff, 0xe0000000,
1286         0x3c020, 0xffffffff, 0x00010000,
1287         0x3c024, 0xffffffff, 0x00030002,
1288         0x3c028, 0xffffffff, 0x00040007,
1289         0x3c02c, 0xffffffff, 0x00060005,
1290         0x3c030, 0xffffffff, 0x00090008,
1291         0x3c034, 0xffffffff, 0x00010000,
1292         0x3c038, 0xffffffff, 0x00030002,
1293         0x3c03c, 0xffffffff, 0x00040007,
1294         0x3c040, 0xffffffff, 0x00060005,
1295         0x3c044, 0xffffffff, 0x00090008,
1296         0x3c000, 0xffffffff, 0x96e00200,
1297         0x8708, 0xffffffff, 0x00900100,
1298         0xc424, 0xffffffff, 0x0020003f,
1299         0x38, 0xffffffff, 0x0140001c,
1300         0x3c, 0x000f0000, 0x000f0000,
1301         0x220, 0xffffffff, 0xC060000C,
1302         0x224, 0xc0000fff, 0x00000100,
1303         0x20a8, 0xffffffff, 0x00000104,
1304         0x55e4, 0xff000fff, 0x00000100,
1305         0x30cc, 0xc0000fff, 0x00000104,
1306         0xc1e4, 0x00000001, 0x00000001,
1307         0xd00c, 0xff000ff0, 0x00000100,
1308         0xd80c, 0xff000ff0, 0x00000100
1309 };
1310
1311 static const u32 hawaii_golden_spm_registers[] =
1312 {
1313         0x30800, 0xe0ffffff, 0xe0000000
1314 };
1315
1316 static const u32 hawaii_golden_common_registers[] =
1317 {
1318         0x30800, 0xffffffff, 0xe0000000,
1319         0x28350, 0xffffffff, 0x3a00161a,
1320         0x28354, 0xffffffff, 0x0000002e,
1321         0x9a10, 0xffffffff, 0x00018208,
1322         0x98f8, 0xffffffff, 0x12011003
1323 };
1324
1325 static const u32 hawaii_golden_registers[] =
1326 {
1327         0x3354, 0x00000333, 0x00000333,
1328         0x9a10, 0x00010000, 0x00058208,
1329         0x9830, 0xffffffff, 0x00000000,
1330         0x9834, 0xf00fffff, 0x00000400,
1331         0x9838, 0x0002021c, 0x00020200,
1332         0xc78, 0x00000080, 0x00000000,
1333         0x5bb0, 0x000000f0, 0x00000070,
1334         0x5bc0, 0xf0311fff, 0x80300000,
1335         0x350c, 0x00810000, 0x408af000,
1336         0x7030, 0x31000111, 0x00000011,
1337         0x2f48, 0x73773777, 0x12010001,
1338         0x2120, 0x0000007f, 0x0000001b,
1339         0x21dc, 0x00007fb6, 0x00002191,
1340         0x3628, 0x0000003f, 0x0000000a,
1341         0x362c, 0x0000003f, 0x0000000a,
1342         0x2ae4, 0x00073ffe, 0x000022a2,
1343         0x240c, 0x000007ff, 0x00000000,
1344         0x8bf0, 0x00002001, 0x00000001,
1345         0x8b24, 0xffffffff, 0x00ffffff,
1346         0x30a04, 0x0000ff0f, 0x00000000,
1347         0x28a4c, 0x07ffffff, 0x06000000,
1348         0x3e78, 0x00000001, 0x00000002,
1349         0xc768, 0x00000008, 0x00000008,
1350         0xc770, 0x00000f00, 0x00000800,
1351         0xc774, 0x00000f00, 0x00000800,
1352         0xc798, 0x00ffffff, 0x00ff7fbf,
1353         0xc79c, 0x00ffffff, 0x00ff7faf,
1354         0x8c00, 0x000000ff, 0x00000800,
1355         0xe40, 0x00001fff, 0x00001fff,
1356         0x9060, 0x0000007f, 0x00000020,
1357         0x9508, 0x00010000, 0x00010000,
1358         0xae00, 0x00100000, 0x000ff07c,
1359         0xac14, 0x000003ff, 0x0000000f,
1360         0xac10, 0xffffffff, 0x7564fdec,
1361         0xac0c, 0xffffffff, 0x3120b9a8,
1362         0xac08, 0x20000000, 0x0f9c0000
1363 };
1364
1365 static const u32 hawaii_mgcg_cgcg_init[] =
1366 {
1367         0xc420, 0xffffffff, 0xfffffffd,
1368         0x30800, 0xffffffff, 0xe0000000,
1369         0x3c2a0, 0xffffffff, 0x00000100,
1370         0x3c208, 0xffffffff, 0x00000100,
1371         0x3c2c0, 0xffffffff, 0x00000100,
1372         0x3c2c8, 0xffffffff, 0x00000100,
1373         0x3c2c4, 0xffffffff, 0x00000100,
1374         0x55e4, 0xffffffff, 0x00200100,
1375         0x3c280, 0xffffffff, 0x00000100,
1376         0x3c214, 0xffffffff, 0x06000100,
1377         0x3c220, 0xffffffff, 0x00000100,
1378         0x3c218, 0xffffffff, 0x06000100,
1379         0x3c204, 0xffffffff, 0x00000100,
1380         0x3c2e0, 0xffffffff, 0x00000100,
1381         0x3c224, 0xffffffff, 0x00000100,
1382         0x3c200, 0xffffffff, 0x00000100,
1383         0x3c230, 0xffffffff, 0x00000100,
1384         0x3c234, 0xffffffff, 0x00000100,
1385         0x3c250, 0xffffffff, 0x00000100,
1386         0x3c254, 0xffffffff, 0x00000100,
1387         0x3c258, 0xffffffff, 0x00000100,
1388         0x3c25c, 0xffffffff, 0x00000100,
1389         0x3c260, 0xffffffff, 0x00000100,
1390         0x3c27c, 0xffffffff, 0x00000100,
1391         0x3c278, 0xffffffff, 0x00000100,
1392         0x3c210, 0xffffffff, 0x06000100,
1393         0x3c290, 0xffffffff, 0x00000100,
1394         0x3c274, 0xffffffff, 0x00000100,
1395         0x3c2b4, 0xffffffff, 0x00000100,
1396         0x3c2b0, 0xffffffff, 0x00000100,
1397         0x3c270, 0xffffffff, 0x00000100,
1398         0x30800, 0xffffffff, 0xe0000000,
1399         0x3c020, 0xffffffff, 0x00010000,
1400         0x3c024, 0xffffffff, 0x00030002,
1401         0x3c028, 0xffffffff, 0x00040007,
1402         0x3c02c, 0xffffffff, 0x00060005,
1403         0x3c030, 0xffffffff, 0x00090008,
1404         0x3c034, 0xffffffff, 0x00010000,
1405         0x3c038, 0xffffffff, 0x00030002,
1406         0x3c03c, 0xffffffff, 0x00040007,
1407         0x3c040, 0xffffffff, 0x00060005,
1408         0x3c044, 0xffffffff, 0x00090008,
1409         0x3c048, 0xffffffff, 0x00010000,
1410         0x3c04c, 0xffffffff, 0x00030002,
1411         0x3c050, 0xffffffff, 0x00040007,
1412         0x3c054, 0xffffffff, 0x00060005,
1413         0x3c058, 0xffffffff, 0x00090008,
1414         0x3c05c, 0xffffffff, 0x00010000,
1415         0x3c060, 0xffffffff, 0x00030002,
1416         0x3c064, 0xffffffff, 0x00040007,
1417         0x3c068, 0xffffffff, 0x00060005,
1418         0x3c06c, 0xffffffff, 0x00090008,
1419         0x3c070, 0xffffffff, 0x00010000,
1420         0x3c074, 0xffffffff, 0x00030002,
1421         0x3c078, 0xffffffff, 0x00040007,
1422         0x3c07c, 0xffffffff, 0x00060005,
1423         0x3c080, 0xffffffff, 0x00090008,
1424         0x3c084, 0xffffffff, 0x00010000,
1425         0x3c088, 0xffffffff, 0x00030002,
1426         0x3c08c, 0xffffffff, 0x00040007,
1427         0x3c090, 0xffffffff, 0x00060005,
1428         0x3c094, 0xffffffff, 0x00090008,
1429         0x3c098, 0xffffffff, 0x00010000,
1430         0x3c09c, 0xffffffff, 0x00030002,
1431         0x3c0a0, 0xffffffff, 0x00040007,
1432         0x3c0a4, 0xffffffff, 0x00060005,
1433         0x3c0a8, 0xffffffff, 0x00090008,
1434         0x3c0ac, 0xffffffff, 0x00010000,
1435         0x3c0b0, 0xffffffff, 0x00030002,
1436         0x3c0b4, 0xffffffff, 0x00040007,
1437         0x3c0b8, 0xffffffff, 0x00060005,
1438         0x3c0bc, 0xffffffff, 0x00090008,
1439         0x3c0c0, 0xffffffff, 0x00010000,
1440         0x3c0c4, 0xffffffff, 0x00030002,
1441         0x3c0c8, 0xffffffff, 0x00040007,
1442         0x3c0cc, 0xffffffff, 0x00060005,
1443         0x3c0d0, 0xffffffff, 0x00090008,
1444         0x3c0d4, 0xffffffff, 0x00010000,
1445         0x3c0d8, 0xffffffff, 0x00030002,
1446         0x3c0dc, 0xffffffff, 0x00040007,
1447         0x3c0e0, 0xffffffff, 0x00060005,
1448         0x3c0e4, 0xffffffff, 0x00090008,
1449         0x3c0e8, 0xffffffff, 0x00010000,
1450         0x3c0ec, 0xffffffff, 0x00030002,
1451         0x3c0f0, 0xffffffff, 0x00040007,
1452         0x3c0f4, 0xffffffff, 0x00060005,
1453         0x3c0f8, 0xffffffff, 0x00090008,
1454         0xc318, 0xffffffff, 0x00020200,
1455         0x3350, 0xffffffff, 0x00000200,
1456         0x15c0, 0xffffffff, 0x00000400,
1457         0x55e8, 0xffffffff, 0x00000000,
1458         0x2f50, 0xffffffff, 0x00000902,
1459         0x3c000, 0xffffffff, 0x96940200,
1460         0x8708, 0xffffffff, 0x00900100,
1461         0xc424, 0xffffffff, 0x0020003f,
1462         0x38, 0xffffffff, 0x0140001c,
1463         0x3c, 0x000f0000, 0x000f0000,
1464         0x220, 0xffffffff, 0xc060000c,
1465         0x224, 0xc0000fff, 0x00000100,
1466         0xf90, 0xffffffff, 0x00000100,
1467         0xf98, 0x00000101, 0x00000000,
1468         0x20a8, 0xffffffff, 0x00000104,
1469         0x55e4, 0xff000fff, 0x00000100,
1470         0x30cc, 0xc0000fff, 0x00000104,
1471         0xc1e4, 0x00000001, 0x00000001,
1472         0xd00c, 0xff000ff0, 0x00000100,
1473         0xd80c, 0xff000ff0, 0x00000100
1474 };
1475
1476 static void cik_init_golden_registers(struct radeon_device *rdev)
1477 {
1478         switch (rdev->family) {
1479         case CHIP_BONAIRE:
1480                 radeon_program_register_sequence(rdev,
1481                                                  bonaire_mgcg_cgcg_init,
1482                                                  (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1483                 radeon_program_register_sequence(rdev,
1484                                                  bonaire_golden_registers,
1485                                                  (const u32)ARRAY_SIZE(bonaire_golden_registers));
1486                 radeon_program_register_sequence(rdev,
1487                                                  bonaire_golden_common_registers,
1488                                                  (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1489                 radeon_program_register_sequence(rdev,
1490                                                  bonaire_golden_spm_registers,
1491                                                  (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1492                 break;
1493         case CHIP_KABINI:
1494                 radeon_program_register_sequence(rdev,
1495                                                  kalindi_mgcg_cgcg_init,
1496                                                  (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1497                 radeon_program_register_sequence(rdev,
1498                                                  kalindi_golden_registers,
1499                                                  (const u32)ARRAY_SIZE(kalindi_golden_registers));
1500                 radeon_program_register_sequence(rdev,
1501                                                  kalindi_golden_common_registers,
1502                                                  (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1503                 radeon_program_register_sequence(rdev,
1504                                                  kalindi_golden_spm_registers,
1505                                                  (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1506                 break;
1507         case CHIP_KAVERI:
1508                 radeon_program_register_sequence(rdev,
1509                                                  spectre_mgcg_cgcg_init,
1510                                                  (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1511                 radeon_program_register_sequence(rdev,
1512                                                  spectre_golden_registers,
1513                                                  (const u32)ARRAY_SIZE(spectre_golden_registers));
1514                 radeon_program_register_sequence(rdev,
1515                                                  spectre_golden_common_registers,
1516                                                  (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1517                 radeon_program_register_sequence(rdev,
1518                                                  spectre_golden_spm_registers,
1519                                                  (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1520                 break;
1521         case CHIP_HAWAII:
1522                 radeon_program_register_sequence(rdev,
1523                                                  hawaii_mgcg_cgcg_init,
1524                                                  (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1525                 radeon_program_register_sequence(rdev,
1526                                                  hawaii_golden_registers,
1527                                                  (const u32)ARRAY_SIZE(hawaii_golden_registers));
1528                 radeon_program_register_sequence(rdev,
1529                                                  hawaii_golden_common_registers,
1530                                                  (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1531                 radeon_program_register_sequence(rdev,
1532                                                  hawaii_golden_spm_registers,
1533                                                  (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1534                 break;
1535         default:
1536                 break;
1537         }
1538 }
1539
1540 /**
1541  * cik_get_xclk - get the xclk
1542  *
1543  * @rdev: radeon_device pointer
1544  *
1545  * Returns the reference clock used by the gfx engine
1546  * (CIK).
1547  */
1548 u32 cik_get_xclk(struct radeon_device *rdev)
1549 {
1550         u32 reference_clock = rdev->clock.spll.reference_freq;
1551
1552         if (rdev->flags & RADEON_IS_IGP) {
1553                 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1554                         return reference_clock / 2;
1555         } else {
1556                 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1557                         return reference_clock / 4;
1558         }
1559         return reference_clock;
1560 }
1561
1562 /**
1563  * cik_mm_rdoorbell - read a doorbell dword
1564  *
1565  * @rdev: radeon_device pointer
1566  * @index: doorbell index
1567  *
1568  * Returns the value in the doorbell aperture at the
1569  * requested doorbell index (CIK).
1570  */
1571 u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)
1572 {
1573         if (index < rdev->doorbell.num_doorbells) {
1574                 return readl(rdev->doorbell.ptr + index);
1575         } else {
1576                 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
1577                 return 0;
1578         }
1579 }
1580
1581 /**
1582  * cik_mm_wdoorbell - write a doorbell dword
1583  *
1584  * @rdev: radeon_device pointer
1585  * @index: doorbell index
1586  * @v: value to write
1587  *
1588  * Writes @v to the doorbell aperture at the
1589  * requested doorbell index (CIK).
1590  */
1591 void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)
1592 {
1593         if (index < rdev->doorbell.num_doorbells) {
1594                 writel(v, rdev->doorbell.ptr + index);
1595         } else {
1596                 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
1597         }
1598 }
1599
1600 #define BONAIRE_IO_MC_REGS_SIZE 36
1601
1602 static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1603 {
1604         {0x00000070, 0x04400000},
1605         {0x00000071, 0x80c01803},
1606         {0x00000072, 0x00004004},
1607         {0x00000073, 0x00000100},
1608         {0x00000074, 0x00ff0000},
1609         {0x00000075, 0x34000000},
1610         {0x00000076, 0x08000014},
1611         {0x00000077, 0x00cc08ec},
1612         {0x00000078, 0x00000400},
1613         {0x00000079, 0x00000000},
1614         {0x0000007a, 0x04090000},
1615         {0x0000007c, 0x00000000},
1616         {0x0000007e, 0x4408a8e8},
1617         {0x0000007f, 0x00000304},
1618         {0x00000080, 0x00000000},
1619         {0x00000082, 0x00000001},
1620         {0x00000083, 0x00000002},
1621         {0x00000084, 0xf3e4f400},
1622         {0x00000085, 0x052024e3},
1623         {0x00000087, 0x00000000},
1624         {0x00000088, 0x01000000},
1625         {0x0000008a, 0x1c0a0000},
1626         {0x0000008b, 0xff010000},
1627         {0x0000008d, 0xffffefff},
1628         {0x0000008e, 0xfff3efff},
1629         {0x0000008f, 0xfff3efbf},
1630         {0x00000092, 0xf7ffffff},
1631         {0x00000093, 0xffffff7f},
1632         {0x00000095, 0x00101101},
1633         {0x00000096, 0x00000fff},
1634         {0x00000097, 0x00116fff},
1635         {0x00000098, 0x60010000},
1636         {0x00000099, 0x10010000},
1637         {0x0000009a, 0x00006000},
1638         {0x0000009b, 0x00001000},
1639         {0x0000009f, 0x00b48000}
1640 };
1641
1642 #define HAWAII_IO_MC_REGS_SIZE 22
1643
1644 static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1645 {
1646         {0x0000007d, 0x40000000},
1647         {0x0000007e, 0x40180304},
1648         {0x0000007f, 0x0000ff00},
1649         {0x00000081, 0x00000000},
1650         {0x00000083, 0x00000800},
1651         {0x00000086, 0x00000000},
1652         {0x00000087, 0x00000100},
1653         {0x00000088, 0x00020100},
1654         {0x00000089, 0x00000000},
1655         {0x0000008b, 0x00040000},
1656         {0x0000008c, 0x00000100},
1657         {0x0000008e, 0xff010000},
1658         {0x00000090, 0xffffefff},
1659         {0x00000091, 0xfff3efff},
1660         {0x00000092, 0xfff3efbf},
1661         {0x00000093, 0xf7ffffff},
1662         {0x00000094, 0xffffff7f},
1663         {0x00000095, 0x00000fff},
1664         {0x00000096, 0x00116fff},
1665         {0x00000097, 0x60010000},
1666         {0x00000098, 0x10010000},
1667         {0x0000009f, 0x00c79000}
1668 };
1669
1670
1671 /**
1672  * cik_srbm_select - select specific register instances
1673  *
1674  * @rdev: radeon_device pointer
1675  * @me: selected ME (micro engine)
1676  * @pipe: pipe
1677  * @queue: queue
1678  * @vmid: VMID
1679  *
1680  * Switches the currently active registers instances.  Some
1681  * registers are instanced per VMID, others are instanced per
1682  * me/pipe/queue combination.
1683  */
1684 static void cik_srbm_select(struct radeon_device *rdev,
1685                             u32 me, u32 pipe, u32 queue, u32 vmid)
1686 {
1687         u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1688                              MEID(me & 0x3) |
1689                              VMID(vmid & 0xf) |
1690                              QUEUEID(queue & 0x7));
1691         WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1692 }
1693
1694 /* ucode loading */
1695 /**
1696  * ci_mc_load_microcode - load MC ucode into the hw
1697  *
1698  * @rdev: radeon_device pointer
1699  *
1700  * Load the GDDR MC ucode into the hw (CIK).
1701  * Returns 0 on success, error on failure.
1702  */
1703 int ci_mc_load_microcode(struct radeon_device *rdev)
1704 {
1705         const __be32 *fw_data;
1706         u32 running, blackout = 0;
1707         u32 *io_mc_regs;
1708         int i, regs_size, ucode_size;
1709
1710         if (!rdev->mc_fw)
1711                 return -EINVAL;
1712
1713         ucode_size = rdev->mc_fw->size / 4;
1714
1715         switch (rdev->family) {
1716         case CHIP_BONAIRE:
1717                 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1718                 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1719                 break;
1720         case CHIP_HAWAII:
1721                 io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1722                 regs_size = HAWAII_IO_MC_REGS_SIZE;
1723                 break;
1724         default:
1725                 return -EINVAL;
1726         }
1727
1728         running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1729
1730         if (running == 0) {
1731                 if (running) {
1732                         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1733                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1734                 }
1735
1736                 /* reset the engine and set to writable */
1737                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1738                 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1739
1740                 /* load mc io regs */
1741                 for (i = 0; i < regs_size; i++) {
1742                         WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1743                         WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1744                 }
1745                 /* load the MC ucode */
1746                 fw_data = (const __be32 *)rdev->mc_fw->data;
1747                 for (i = 0; i < ucode_size; i++)
1748                         WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1749
1750                 /* put the engine back into the active state */
1751                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1752                 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1753                 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1754
1755                 /* wait for training to complete */
1756                 for (i = 0; i < rdev->usec_timeout; i++) {
1757                         if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1758                                 break;
1759                         udelay(1);
1760                 }
1761                 for (i = 0; i < rdev->usec_timeout; i++) {
1762                         if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1763                                 break;
1764                         udelay(1);
1765                 }
1766
1767                 if (running)
1768                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1769         }
1770
1771         return 0;
1772 }
1773
1774 /**
1775  * cik_init_microcode - load ucode images from disk
1776  *
1777  * @rdev: radeon_device pointer
1778  *
1779  * Use the firmware interface to load the ucode images into
1780  * the driver (not loaded into hw).
1781  * Returns 0 on success, error on failure.
1782  */
1783 static int cik_init_microcode(struct radeon_device *rdev)
1784 {
1785         const char *chip_name;
1786         size_t pfp_req_size, me_req_size, ce_req_size,
1787                 mec_req_size, rlc_req_size, mc_req_size = 0,
1788                 sdma_req_size, smc_req_size = 0, mc2_req_size = 0;
1789         char fw_name[30];
1790         int err;
1791
1792         DRM_DEBUG("\n");
1793
1794         switch (rdev->family) {
1795         case CHIP_BONAIRE:
1796                 chip_name = "BONAIRE";
1797                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1798                 me_req_size = CIK_ME_UCODE_SIZE * 4;
1799                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1800                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1801                 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1802                 mc_req_size = BONAIRE_MC_UCODE_SIZE * 4;
1803                 mc2_req_size = BONAIRE_MC2_UCODE_SIZE * 4;
1804                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1805                 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
1806                 break;
1807         case CHIP_HAWAII:
1808                 chip_name = "HAWAII";
1809                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1810                 me_req_size = CIK_ME_UCODE_SIZE * 4;
1811                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1812                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1813                 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1814                 mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
1815                 mc2_req_size = HAWAII_MC2_UCODE_SIZE * 4;
1816                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1817                 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
1818                 break;
1819         case CHIP_KAVERI:
1820                 chip_name = "KAVERI";
1821                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1822                 me_req_size = CIK_ME_UCODE_SIZE * 4;
1823                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1824                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1825                 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
1826                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1827                 break;
1828         case CHIP_KABINI:
1829                 chip_name = "KABINI";
1830                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1831                 me_req_size = CIK_ME_UCODE_SIZE * 4;
1832                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1833                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1834                 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
1835                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1836                 break;
1837         default: BUG();
1838         }
1839
1840         DRM_INFO("Loading %s Microcode\n", chip_name);
1841
1842         snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1843         err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1844         if (err)
1845                 goto out;
1846         if (rdev->pfp_fw->size != pfp_req_size) {
1847                 printk(KERN_ERR
1848                        "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1849                        rdev->pfp_fw->size, fw_name);
1850                 err = -EINVAL;
1851                 goto out;
1852         }
1853
1854         snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1855         err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1856         if (err)
1857                 goto out;
1858         if (rdev->me_fw->size != me_req_size) {
1859                 printk(KERN_ERR
1860                        "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1861                        rdev->me_fw->size, fw_name);
1862                 err = -EINVAL;
1863         }
1864
1865         snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1866         err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1867         if (err)
1868                 goto out;
1869         if (rdev->ce_fw->size != ce_req_size) {
1870                 printk(KERN_ERR
1871                        "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1872                        rdev->ce_fw->size, fw_name);
1873                 err = -EINVAL;
1874         }
1875
1876         snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
1877         err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
1878         if (err)
1879                 goto out;
1880         if (rdev->mec_fw->size != mec_req_size) {
1881                 printk(KERN_ERR
1882                        "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1883                        rdev->mec_fw->size, fw_name);
1884                 err = -EINVAL;
1885         }
1886
1887         snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
1888         err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1889         if (err)
1890                 goto out;
1891         if (rdev->rlc_fw->size != rlc_req_size) {
1892                 printk(KERN_ERR
1893                        "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
1894                        rdev->rlc_fw->size, fw_name);
1895                 err = -EINVAL;
1896         }
1897
1898         snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
1899         err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
1900         if (err)
1901                 goto out;
1902         if (rdev->sdma_fw->size != sdma_req_size) {
1903                 printk(KERN_ERR
1904                        "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
1905                        rdev->sdma_fw->size, fw_name);
1906                 err = -EINVAL;
1907         }
1908
1909         /* No SMC, MC ucode on APUs */
1910         if (!(rdev->flags & RADEON_IS_IGP)) {
1911                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
1912                 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1913                 if (err) {
1914                         snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1915                         err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1916                         if (err)
1917                                 goto out;
1918                 }
1919                 if ((rdev->mc_fw->size != mc_req_size) &&
1920                     (rdev->mc_fw->size != mc2_req_size)){
1921                         printk(KERN_ERR
1922                                "cik_mc: Bogus length %zu in firmware \"%s\"\n",
1923                                rdev->mc_fw->size, fw_name);
1924                         err = -EINVAL;
1925                 }
1926                 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
1927
1928                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1929                 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1930                 if (err) {
1931                         printk(KERN_ERR
1932                                "smc: error loading firmware \"%s\"\n",
1933                                fw_name);
1934                         release_firmware(rdev->smc_fw);
1935                         rdev->smc_fw = NULL;
1936                         err = 0;
1937                 } else if (rdev->smc_fw->size != smc_req_size) {
1938                         printk(KERN_ERR
1939                                "cik_smc: Bogus length %zu in firmware \"%s\"\n",
1940                                rdev->smc_fw->size, fw_name);
1941                         err = -EINVAL;
1942                 }
1943         }
1944
1945 out:
1946         if (err) {
1947                 if (err != -EINVAL)
1948                         printk(KERN_ERR
1949                                "cik_cp: Failed to load firmware \"%s\"\n",
1950                                fw_name);
1951                 release_firmware(rdev->pfp_fw);
1952                 rdev->pfp_fw = NULL;
1953                 release_firmware(rdev->me_fw);
1954                 rdev->me_fw = NULL;
1955                 release_firmware(rdev->ce_fw);
1956                 rdev->ce_fw = NULL;
1957                 release_firmware(rdev->rlc_fw);
1958                 rdev->rlc_fw = NULL;
1959                 release_firmware(rdev->mc_fw);
1960                 rdev->mc_fw = NULL;
1961                 release_firmware(rdev->smc_fw);
1962                 rdev->smc_fw = NULL;
1963         }
1964         return err;
1965 }
1966
1967 /*
1968  * Core functions
1969  */
1970 /**
1971  * cik_tiling_mode_table_init - init the hw tiling table
1972  *
1973  * @rdev: radeon_device pointer
1974  *
1975  * Starting with SI, the tiling setup is done globally in a
1976  * set of 32 tiling modes.  Rather than selecting each set of
1977  * parameters per surface as on older asics, we just select
1978  * which index in the tiling table we want to use, and the
1979  * surface uses those parameters (CIK).
1980  */
1981 static void cik_tiling_mode_table_init(struct radeon_device *rdev)
1982 {
1983         const u32 num_tile_mode_states = 32;
1984         const u32 num_secondary_tile_mode_states = 16;
1985         u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
1986         u32 num_pipe_configs;
1987         u32 num_rbs = rdev->config.cik.max_backends_per_se *
1988                 rdev->config.cik.max_shader_engines;
1989
1990         switch (rdev->config.cik.mem_row_size_in_kb) {
1991         case 1:
1992                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
1993                 break;
1994         case 2:
1995         default:
1996                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
1997                 break;
1998         case 4:
1999                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2000                 break;
2001         }
2002
2003         num_pipe_configs = rdev->config.cik.max_tile_pipes;
2004         if (num_pipe_configs > 8)
2005                 num_pipe_configs = 16;
2006
2007         if (num_pipe_configs == 16) {
2008                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2009                         switch (reg_offset) {
2010                         case 0:
2011                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2012                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2013                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2014                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2015                                 break;
2016                         case 1:
2017                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2018                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2019                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2020                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2021                                 break;
2022                         case 2:
2023                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2024                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2025                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2026                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2027                                 break;
2028                         case 3:
2029                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2030                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2031                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2032                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2033                                 break;
2034                         case 4:
2035                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2036                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2037                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2038                                                  TILE_SPLIT(split_equal_to_row_size));
2039                                 break;
2040                         case 5:
2041                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2042                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2043                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2044                                 break;
2045                         case 6:
2046                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2047                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2048                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2049                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2050                                 break;
2051                         case 7:
2052                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2053                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2054                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2055                                                  TILE_SPLIT(split_equal_to_row_size));
2056                                 break;
2057                         case 8:
2058                                 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2059                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2060                                 break;
2061                         case 9:
2062                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2063                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2064                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2065                                 break;
2066                         case 10:
2067                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2068                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2069                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2070                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2071                                 break;
2072                         case 11:
2073                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2074                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2075                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2076                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2077                                 break;
2078                         case 12:
2079                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2080                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2081                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2082                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2083                                 break;
2084                         case 13:
2085                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2086                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2087                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2088                                 break;
2089                         case 14:
2090                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2091                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2092                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2093                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2094                                 break;
2095                         case 16:
2096                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2097                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2098                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2099                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2100                                 break;
2101                         case 17:
2102                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2103                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2104                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2105                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2106                                 break;
2107                         case 27:
2108                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2109                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2110                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2111                                 break;
2112                         case 28:
2113                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2114                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2115                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2116                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2117                                 break;
2118                         case 29:
2119                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2120                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2121                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2122                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2123                                 break;
2124                         case 30:
2125                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2126                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2127                                                  PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2128                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2129                                 break;
2130                         default:
2131                                 gb_tile_moden = 0;
2132                                 break;
2133                         }
2134                         rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2135                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2136                 }
2137                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2138                         switch (reg_offset) {
2139                         case 0:
2140                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2141                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2142                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2143                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2144                                 break;
2145                         case 1:
2146                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2147                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2148                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2149                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2150                                 break;
2151                         case 2:
2152                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2153                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2154                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2155                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2156                                 break;
2157                         case 3:
2158                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2159                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2160                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2161                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2162                                 break;
2163                         case 4:
2164                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2165                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2166                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2167                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2168                                 break;
2169                         case 5:
2170                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2171                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2172                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2173                                                  NUM_BANKS(ADDR_SURF_4_BANK));
2174                                 break;
2175                         case 6:
2176                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2177                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2178                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2179                                                  NUM_BANKS(ADDR_SURF_2_BANK));
2180                                 break;
2181                         case 8:
2182                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2183                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2184                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2185                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2186                                 break;
2187                         case 9:
2188                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2189                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2190                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2191                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2192                                 break;
2193                         case 10:
2194                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2195                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2196                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2197                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2198                                 break;
2199                         case 11:
2200                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2201                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2202                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2203                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2204                                 break;
2205                         case 12:
2206                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2207                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2208                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2209                                                  NUM_BANKS(ADDR_SURF_4_BANK));
2210                                 break;
2211                         case 13:
2212                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2213                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2214                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2215                                                  NUM_BANKS(ADDR_SURF_2_BANK));
2216                                 break;
2217                         case 14:
2218                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2219                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2220                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2221                                                  NUM_BANKS(ADDR_SURF_2_BANK));
2222                                 break;
2223                         default:
2224                                 gb_tile_moden = 0;
2225                                 break;
2226                         }
2227                         WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2228                 }
2229         } else if (num_pipe_configs == 8) {
2230                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2231                         switch (reg_offset) {
2232                         case 0:
2233                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2234                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2235                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2236                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2237                                 break;
2238                         case 1:
2239                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2240                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2241                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2242                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2243                                 break;
2244                         case 2:
2245                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2246                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2247                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2248                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2249                                 break;
2250                         case 3:
2251                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2252                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2253                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2254                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2255                                 break;
2256                         case 4:
2257                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2258                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2259                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2260                                                  TILE_SPLIT(split_equal_to_row_size));
2261                                 break;
2262                         case 5:
2263                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2264                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2265                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2266                                 break;
2267                         case 6:
2268                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2269                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2270                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2271                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2272                                 break;
2273                         case 7:
2274                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2275                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2276                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2277                                                  TILE_SPLIT(split_equal_to_row_size));
2278                                 break;
2279                         case 8:
2280                                 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2281                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2282                                 break;
2283                         case 9:
2284                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2285                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2286                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2287                                 break;
2288                         case 10:
2289                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2290                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2291                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2292                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2293                                 break;
2294                         case 11:
2295                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2296                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2297                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2298                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2299                                 break;
2300                         case 12:
2301                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2302                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2303                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2304                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2305                                 break;
2306                         case 13:
2307                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2308                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2309                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2310                                 break;
2311                         case 14:
2312                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2313                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2314                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2315                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2316                                 break;
2317                         case 16:
2318                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2319                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2320                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2321                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2322                                 break;
2323                         case 17:
2324                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2325                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2326                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2327                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2328                                 break;
2329                         case 27:
2330                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2331                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2332                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2333                                 break;
2334                         case 28:
2335                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2336                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2337                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2338                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2339                                 break;
2340                         case 29:
2341                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2342                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2343                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2344                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2345                                 break;
2346                         case 30:
2347                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2348                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2349                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2350                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2351                                 break;
2352                         default:
2353                                 gb_tile_moden = 0;
2354                                 break;
2355                         }
2356                         rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2357                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2358                 }
2359                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2360                         switch (reg_offset) {
2361                         case 0:
2362                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2363                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2364                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2365                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2366                                 break;
2367                         case 1:
2368                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2369                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2370                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2371                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2372                                 break;
2373                         case 2:
2374                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2375                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2376                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2377                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2378                                 break;
2379                         case 3:
2380                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2381                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2382                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2383                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2384                                 break;
2385                         case 4:
2386                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2387                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2388                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2389                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2390                                 break;
2391                         case 5:
2392                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2393                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2394                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2395                                                  NUM_BANKS(ADDR_SURF_4_BANK));
2396                                 break;
2397                         case 6:
2398                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2399                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2400                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2401                                                  NUM_BANKS(ADDR_SURF_2_BANK));
2402                                 break;
2403                         case 8:
2404                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2405                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2406                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2407                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2408                                 break;
2409                         case 9:
2410                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2411                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2412                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2413                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2414                                 break;
2415                         case 10:
2416                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2417                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2418                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2419                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2420                                 break;
2421                         case 11:
2422                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2423                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2424                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2425                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2426                                 break;
2427                         case 12:
2428                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2429                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2430                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2431                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2432                                 break;
2433                         case 13:
2434                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2435                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2436                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2437                                                  NUM_BANKS(ADDR_SURF_4_BANK));
2438                                 break;
2439                         case 14:
2440                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2441                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2442                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2443                                                  NUM_BANKS(ADDR_SURF_2_BANK));
2444                                 break;
2445                         default:
2446                                 gb_tile_moden = 0;
2447                                 break;
2448                         }
2449                         rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2450                         WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2451                 }
2452         } else if (num_pipe_configs == 4) {
2453                 if (num_rbs == 4) {
2454                         for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2455                                 switch (reg_offset) {
2456                                 case 0:
2457                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2458                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2459                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2460                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2461                                         break;
2462                                 case 1:
2463                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2464                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2465                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2466                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2467                                         break;
2468                                 case 2:
2469                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2470                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2471                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2472                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2473                                         break;
2474                                 case 3:
2475                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2476                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2477                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2478                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2479                                         break;
2480                                 case 4:
2481                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2482                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2483                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2484                                                          TILE_SPLIT(split_equal_to_row_size));
2485                                         break;
2486                                 case 5:
2487                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2488                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2489                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2490                                         break;
2491                                 case 6:
2492                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2493                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2494                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2495                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2496                                         break;
2497                                 case 7:
2498                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2499                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2500                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2501                                                          TILE_SPLIT(split_equal_to_row_size));
2502                                         break;
2503                                 case 8:
2504                                         gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2505                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16));
2506                                         break;
2507                                 case 9:
2508                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2509                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2510                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2511                                         break;
2512                                 case 10:
2513                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2514                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2515                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2516                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2517                                         break;
2518                                 case 11:
2519                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2520                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2521                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2522                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2523                                         break;
2524                                 case 12:
2525                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2526                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2527                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2528                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2529                                         break;
2530                                 case 13:
2531                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2532                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2533                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2534                                         break;
2535                                 case 14:
2536                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2537                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2538                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2539                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2540                                         break;
2541                                 case 16:
2542                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2543                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2544                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2545                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2546                                         break;
2547                                 case 17:
2548                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2549                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2550                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2551                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2552                                         break;
2553                                 case 27:
2554                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2555                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2556                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2557                                         break;
2558                                 case 28:
2559                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2560                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2561                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2562                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2563                                         break;
2564                                 case 29:
2565                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2566                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2567                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2568                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2569                                         break;
2570                                 case 30:
2571                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2572                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2573                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2574                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2575                                         break;
2576                                 default:
2577                                         gb_tile_moden = 0;
2578                                         break;
2579                                 }
2580                                 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2581                                 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2582                         }
2583                 } else if (num_rbs < 4) {
2584                         for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2585                                 switch (reg_offset) {
2586                                 case 0:
2587                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2588                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2589                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2590                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2591                                         break;
2592                                 case 1:
2593                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2594                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2595                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2596                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2597                                         break;
2598                                 case 2:
2599                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2600                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2601                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2602                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2603                                         break;
2604                                 case 3:
2605                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2606                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2607                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2608                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2609                                         break;
2610                                 case 4:
2611                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2612                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2613                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2614                                                          TILE_SPLIT(split_equal_to_row_size));
2615                                         break;
2616                                 case 5:
2617                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2618                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2619                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2620                                         break;
2621                                 case 6:
2622                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2623                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2624                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2625                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2626                                         break;
2627                                 case 7:
2628                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2629                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2630                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2631                                                          TILE_SPLIT(split_equal_to_row_size));
2632                                         break;
2633                                 case 8:
2634                                         gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2635                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16));
2636                                         break;
2637                                 case 9:
2638                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2639                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2640                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2641                                         break;
2642                                 case 10:
2643                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2644                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2645                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2646                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2647                                         break;
2648                                 case 11:
2649                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2650                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2651                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2652                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2653                                         break;
2654                                 case 12:
2655                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2656                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2657                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2658                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2659                                         break;
2660                                 case 13:
2661                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2662                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2663                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2664                                         break;
2665                                 case 14:
2666                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2667                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2668                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2669                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2670                                         break;
2671                                 case 16:
2672                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2673                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2674                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2675                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2676                                         break;
2677                                 case 17:
2678                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2679                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2680                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2681                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2682                                         break;
2683                                 case 27:
2684                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2685                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2686                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2687                                         break;
2688                                 case 28:
2689                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2690                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2691                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2692                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2693                                         break;
2694                                 case 29:
2695                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2696                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2697                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2698                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2699                                         break;
2700                                 case 30:
2701                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2702                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2703                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2704                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2705                                         break;
2706                                 default:
2707                                         gb_tile_moden = 0;
2708                                         break;
2709                                 }
2710                                 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2711                                 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2712                         }
2713                 }
2714                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2715                         switch (reg_offset) {
2716                         case 0:
2717                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2718                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2719                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2720                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2721                                 break;
2722                         case 1:
2723                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2724                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2725                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2726                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2727                                 break;
2728                         case 2:
2729                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2730                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2731                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2732                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2733                                 break;
2734                         case 3:
2735                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2736                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2737                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2738                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2739                                 break;
2740                         case 4:
2741                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2742                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2743                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2744                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2745                                 break;
2746                         case 5:
2747                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2748                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2749                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2750                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2751                                 break;
2752                         case 6:
2753                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2754                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2755                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2756                                                  NUM_BANKS(ADDR_SURF_4_BANK));
2757                                 break;
2758                         case 8:
2759                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2760                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2761                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2762                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2763                                 break;
2764                         case 9:
2765                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2766                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2767                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2768                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2769                                 break;
2770                         case 10:
2771                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2772                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2773                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2774                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2775                                 break;
2776                         case 11:
2777                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2778                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2779                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2780                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2781                                 break;
2782                         case 12:
2783                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2784                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2785                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2786                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2787                                 break;
2788                         case 13:
2789                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2790                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2791                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2792                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2793                                 break;
2794                         case 14:
2795                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2796                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2797                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2798                                                  NUM_BANKS(ADDR_SURF_4_BANK));
2799                                 break;
2800                         default:
2801                                 gb_tile_moden = 0;
2802                                 break;
2803                         }
2804                         rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2805                         WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2806                 }
2807         } else if (num_pipe_configs == 2) {
2808                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2809                         switch (reg_offset) {
2810                         case 0:
2811                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2812                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2813                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2814                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2815                                 break;
2816                         case 1:
2817                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2818                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2819                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2820                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2821                                 break;
2822                         case 2:
2823                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2824                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2825                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2826                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2827                                 break;
2828                         case 3:
2829                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2830                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2831                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2832                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2833                                 break;
2834                         case 4:
2835                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2836                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2837                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2838                                                  TILE_SPLIT(split_equal_to_row_size));
2839                                 break;
2840                         case 5:
2841                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2842                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2843                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2844                                 break;
2845                         case 6:
2846                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2847                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2848                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2849                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2850                                 break;
2851                         case 7:
2852                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2853                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2854                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2855                                                  TILE_SPLIT(split_equal_to_row_size));
2856                                 break;
2857                         case 8:
2858                                 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2859                                                 PIPE_CONFIG(ADDR_SURF_P2);
2860                                 break;
2861                         case 9:
2862                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2863                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2864                                                  PIPE_CONFIG(ADDR_SURF_P2));
2865                                 break;
2866                         case 10:
2867                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2868                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2869                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2870                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2871                                 break;
2872                         case 11:
2873                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2874                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2875                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2876                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2877                                 break;
2878                         case 12:
2879                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2880                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2881                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2882                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2883                                 break;
2884                         case 13:
2885                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2886                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2887                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2888                                 break;
2889                         case 14:
2890                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2891                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2892                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2893                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2894                                 break;
2895                         case 16:
2896                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2897                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2898                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2899                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2900                                 break;
2901                         case 17:
2902                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2903                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2904                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2905                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2906                                 break;
2907                         case 27:
2908                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2909                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2910                                                  PIPE_CONFIG(ADDR_SURF_P2));
2911                                 break;
2912                         case 28:
2913                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2914                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2915                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2916                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2917                                 break;
2918                         case 29:
2919                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2920                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2921                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2922                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2923                                 break;
2924                         case 30:
2925                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2926                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2927                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2928                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2929                                 break;
2930                         default:
2931                                 gb_tile_moden = 0;
2932                                 break;
2933                         }
2934                         rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2935                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2936                 }
2937                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2938                         switch (reg_offset) {
2939                         case 0:
2940                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2941                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2942                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2943                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2944                                 break;
2945                         case 1:
2946                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2947                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2948                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2949                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2950                                 break;
2951                         case 2:
2952                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2953                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2954                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2955                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2956                                 break;
2957                         case 3:
2958                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2959                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2960                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2961                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2962                                 break;
2963                         case 4:
2964                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2965                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2966                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2967                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2968                                 break;
2969                         case 5:
2970                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2971                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2972                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2973                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2974                                 break;
2975                         case 6:
2976                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2977                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2978                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2979                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2980                                 break;
2981                         case 8:
2982                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2983                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2984                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2985                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2986                                 break;
2987                         case 9:
2988                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2989                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2990                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2991                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2992                                 break;
2993                         case 10:
2994                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2995                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2996                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2997                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2998                                 break;
2999                         case 11:
3000                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3001                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3002                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3003                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3004                                 break;
3005                         case 12:
3006                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3007                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3008                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3009                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3010                                 break;
3011                         case 13:
3012                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3013                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3014                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3015                                                  NUM_BANKS(ADDR_SURF_16_BANK));
3016                                 break;
3017                         case 14:
3018                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3019                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3020                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3021                                                  NUM_BANKS(ADDR_SURF_8_BANK));
3022                                 break;
3023                         default:
3024                                 gb_tile_moden = 0;
3025                                 break;
3026                         }
3027                         rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
3028                         WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3029                 }
3030         } else
3031                 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
3032 }
3033
3034 /**
3035  * cik_select_se_sh - select which SE, SH to address
3036  *
3037  * @rdev: radeon_device pointer
3038  * @se_num: shader engine to address
3039  * @sh_num: sh block to address
3040  *
3041  * Select which SE, SH combinations to address. Certain
3042  * registers are instanced per SE or SH.  0xffffffff means
3043  * broadcast to all SEs or SHs (CIK).
3044  */
3045 static void cik_select_se_sh(struct radeon_device *rdev,
3046                              u32 se_num, u32 sh_num)
3047 {
3048         u32 data = INSTANCE_BROADCAST_WRITES;
3049
3050         if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
3051                 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
3052         else if (se_num == 0xffffffff)
3053                 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3054         else if (sh_num == 0xffffffff)
3055                 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3056         else
3057                 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3058         WREG32(GRBM_GFX_INDEX, data);
3059 }
3060
3061 /**
3062  * cik_create_bitmask - create a bitmask
3063  *
3064  * @bit_width: length of the mask
3065  *
3066  * create a variable length bit mask (CIK).
3067  * Returns the bitmask.
3068  */
3069 static u32 cik_create_bitmask(u32 bit_width)
3070 {
3071         u32 i, mask = 0;
3072
3073         for (i = 0; i < bit_width; i++) {
3074                 mask <<= 1;
3075                 mask |= 1;
3076         }
3077         return mask;
3078 }
3079
3080 /**
3081  * cik_get_rb_disabled - computes the mask of disabled RBs
3082  *
3083  * @rdev: radeon_device pointer
3084  * @max_rb_num: max RBs (render backends) for the asic
3085  * @se_num: number of SEs (shader engines) for the asic
3086  * @sh_per_se: number of SH blocks per SE for the asic
3087  *
3088  * Calculates the bitmask of disabled RBs (CIK).
3089  * Returns the disabled RB bitmask.
3090  */
3091 static u32 cik_get_rb_disabled(struct radeon_device *rdev,
3092                               u32 max_rb_num_per_se,
3093                               u32 sh_per_se)
3094 {
3095         u32 data, mask;
3096
3097         data = RREG32(CC_RB_BACKEND_DISABLE);
3098         if (data & 1)
3099                 data &= BACKEND_DISABLE_MASK;
3100         else
3101                 data = 0;
3102         data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3103
3104         data >>= BACKEND_DISABLE_SHIFT;
3105
3106         mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se);
3107
3108         return data & mask;
3109 }
3110
3111 /**
3112  * cik_setup_rb - setup the RBs on the asic
3113  *
3114  * @rdev: radeon_device pointer
3115  * @se_num: number of SEs (shader engines) for the asic
3116  * @sh_per_se: number of SH blocks per SE for the asic
3117  * @max_rb_num: max RBs (render backends) for the asic
3118  *
3119  * Configures per-SE/SH RB registers (CIK).
3120  */
3121 static void cik_setup_rb(struct radeon_device *rdev,
3122                          u32 se_num, u32 sh_per_se,
3123                          u32 max_rb_num_per_se)
3124 {
3125         int i, j;
3126         u32 data, mask;
3127         u32 disabled_rbs = 0;
3128         u32 enabled_rbs = 0;
3129
3130         for (i = 0; i < se_num; i++) {
3131                 for (j = 0; j < sh_per_se; j++) {
3132                         cik_select_se_sh(rdev, i, j);
3133                         data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
3134                         if (rdev->family == CHIP_HAWAII)
3135                                 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3136                         else
3137                                 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
3138                 }
3139         }
3140         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3141
3142         mask = 1;
3143         for (i = 0; i < max_rb_num_per_se * se_num; i++) {
3144                 if (!(disabled_rbs & mask))
3145                         enabled_rbs |= mask;
3146                 mask <<= 1;
3147         }
3148
3149         rdev->config.cik.backend_enable_mask = enabled_rbs;
3150
3151         for (i = 0; i < se_num; i++) {
3152                 cik_select_se_sh(rdev, i, 0xffffffff);
3153                 data = 0;
3154                 for (j = 0; j < sh_per_se; j++) {
3155                         switch (enabled_rbs & 3) {
3156                         case 0:
3157                                 if (j == 0)
3158                                         data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3159                                 else
3160                                         data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3161                                 break;
3162                         case 1:
3163                                 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3164                                 break;
3165                         case 2:
3166                                 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3167                                 break;
3168                         case 3:
3169                         default:
3170                                 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3171                                 break;
3172                         }
3173                         enabled_rbs >>= 2;
3174                 }
3175                 WREG32(PA_SC_RASTER_CONFIG, data);
3176         }
3177         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3178 }
3179
3180 /**
3181  * cik_gpu_init - setup the 3D engine
3182  *
3183  * @rdev: radeon_device pointer
3184  *
3185  * Configures the 3D engine and tiling configuration
3186  * registers so that the 3D engine is usable.
3187  */
3188 static void cik_gpu_init(struct radeon_device *rdev)
3189 {
3190         u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3191         u32 mc_shared_chmap, mc_arb_ramcfg;
3192         u32 hdp_host_path_cntl;
3193         u32 tmp;
3194         int i, j;
3195
3196         switch (rdev->family) {
3197         case CHIP_BONAIRE:
3198                 rdev->config.cik.max_shader_engines = 2;
3199                 rdev->config.cik.max_tile_pipes = 4;
3200                 rdev->config.cik.max_cu_per_sh = 7;
3201                 rdev->config.cik.max_sh_per_se = 1;
3202                 rdev->config.cik.max_backends_per_se = 2;
3203                 rdev->config.cik.max_texture_channel_caches = 4;
3204                 rdev->config.cik.max_gprs = 256;
3205                 rdev->config.cik.max_gs_threads = 32;
3206                 rdev->config.cik.max_hw_contexts = 8;
3207
3208                 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3209                 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3210                 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3211                 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3212                 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3213                 break;
3214         case CHIP_HAWAII:
3215                 rdev->config.cik.max_shader_engines = 4;
3216                 rdev->config.cik.max_tile_pipes = 16;
3217                 rdev->config.cik.max_cu_per_sh = 11;
3218                 rdev->config.cik.max_sh_per_se = 1;
3219                 rdev->config.cik.max_backends_per_se = 4;
3220                 rdev->config.cik.max_texture_channel_caches = 16;
3221                 rdev->config.cik.max_gprs = 256;
3222                 rdev->config.cik.max_gs_threads = 32;
3223                 rdev->config.cik.max_hw_contexts = 8;
3224
3225                 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3226                 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3227                 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3228                 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3229                 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3230                 break;
3231         case CHIP_KAVERI:
3232                 rdev->config.cik.max_shader_engines = 1;
3233                 rdev->config.cik.max_tile_pipes = 4;
3234                 if ((rdev->pdev->device == 0x1304) ||
3235                     (rdev->pdev->device == 0x1305) ||
3236                     (rdev->pdev->device == 0x130C) ||
3237                     (rdev->pdev->device == 0x130F) ||
3238                     (rdev->pdev->device == 0x1310) ||
3239                     (rdev->pdev->device == 0x1311) ||
3240                     (rdev->pdev->device == 0x131C)) {
3241                         rdev->config.cik.max_cu_per_sh = 8;
3242                         rdev->config.cik.max_backends_per_se = 2;
3243                 } else if ((rdev->pdev->device == 0x1309) ||
3244                            (rdev->pdev->device == 0x130A) ||
3245                            (rdev->pdev->device == 0x130D) ||
3246                            (rdev->pdev->device == 0x1313) ||
3247                            (rdev->pdev->device == 0x131D)) {
3248                         rdev->config.cik.max_cu_per_sh = 6;
3249                         rdev->config.cik.max_backends_per_se = 2;
3250                 } else if ((rdev->pdev->device == 0x1306) ||
3251                            (rdev->pdev->device == 0x1307) ||
3252                            (rdev->pdev->device == 0x130B) ||
3253                            (rdev->pdev->device == 0x130E) ||
3254                            (rdev->pdev->device == 0x1315) ||
3255                            (rdev->pdev->device == 0x131B)) {
3256                         rdev->config.cik.max_cu_per_sh = 4;
3257                         rdev->config.cik.max_backends_per_se = 1;
3258                 } else {
3259                         rdev->config.cik.max_cu_per_sh = 3;
3260                         rdev->config.cik.max_backends_per_se = 1;
3261                 }
3262                 rdev->config.cik.max_sh_per_se = 1;
3263                 rdev->config.cik.max_texture_channel_caches = 4;
3264                 rdev->config.cik.max_gprs = 256;
3265                 rdev->config.cik.max_gs_threads = 16;
3266                 rdev->config.cik.max_hw_contexts = 8;
3267
3268                 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3269                 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3270                 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3271                 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3272                 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3273                 break;
3274         case CHIP_KABINI:
3275         default:
3276                 rdev->config.cik.max_shader_engines = 1;
3277                 rdev->config.cik.max_tile_pipes = 2;
3278                 rdev->config.cik.max_cu_per_sh = 2;
3279                 rdev->config.cik.max_sh_per_se = 1;
3280                 rdev->config.cik.max_backends_per_se = 1;
3281                 rdev->config.cik.max_texture_channel_caches = 2;
3282                 rdev->config.cik.max_gprs = 256;
3283                 rdev->config.cik.max_gs_threads = 16;
3284                 rdev->config.cik.max_hw_contexts = 8;
3285
3286                 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3287                 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3288                 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3289                 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3290                 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3291                 break;
3292         }
3293
3294         /* Initialize HDP */
3295         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3296                 WREG32((0x2c14 + j), 0x00000000);
3297                 WREG32((0x2c18 + j), 0x00000000);
3298                 WREG32((0x2c1c + j), 0x00000000);
3299                 WREG32((0x2c20 + j), 0x00000000);
3300                 WREG32((0x2c24 + j), 0x00000000);
3301         }
3302
3303         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3304
3305         WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3306
3307         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3308         mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3309
3310         rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3311         rdev->config.cik.mem_max_burst_length_bytes = 256;
3312         tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3313         rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3314         if (rdev->config.cik.mem_row_size_in_kb > 4)
3315                 rdev->config.cik.mem_row_size_in_kb = 4;
3316         /* XXX use MC settings? */
3317         rdev->config.cik.shader_engine_tile_size = 32;
3318         rdev->config.cik.num_gpus = 1;
3319         rdev->config.cik.multi_gpu_tile_size = 64;
3320
3321         /* fix up row size */
3322         gb_addr_config &= ~ROW_SIZE_MASK;
3323         switch (rdev->config.cik.mem_row_size_in_kb) {
3324         case 1:
3325         default:
3326                 gb_addr_config |= ROW_SIZE(0);
3327                 break;
3328         case 2:
3329                 gb_addr_config |= ROW_SIZE(1);
3330                 break;
3331         case 4:
3332                 gb_addr_config |= ROW_SIZE(2);
3333                 break;
3334         }
3335
3336         /* setup tiling info dword.  gb_addr_config is not adequate since it does
3337          * not have bank info, so create a custom tiling dword.
3338          * bits 3:0   num_pipes
3339          * bits 7:4   num_banks
3340          * bits 11:8  group_size
3341          * bits 15:12 row_size
3342          */
3343         rdev->config.cik.tile_config = 0;
3344         switch (rdev->config.cik.num_tile_pipes) {
3345         case 1:
3346                 rdev->config.cik.tile_config |= (0 << 0);
3347                 break;
3348         case 2:
3349                 rdev->config.cik.tile_config |= (1 << 0);
3350                 break;
3351         case 4:
3352                 rdev->config.cik.tile_config |= (2 << 0);
3353                 break;
3354         case 8:
3355         default:
3356                 /* XXX what about 12? */
3357                 rdev->config.cik.tile_config |= (3 << 0);
3358                 break;
3359         }
3360         rdev->config.cik.tile_config |=
3361                 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
3362         rdev->config.cik.tile_config |=
3363                 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3364         rdev->config.cik.tile_config |=
3365                 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3366
3367         WREG32(GB_ADDR_CONFIG, gb_addr_config);
3368         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3369         WREG32(DMIF_ADDR_CALC, gb_addr_config);
3370         WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3371         WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
3372         WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3373         WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3374         WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3375
3376         cik_tiling_mode_table_init(rdev);
3377
3378         cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3379                      rdev->config.cik.max_sh_per_se,
3380                      rdev->config.cik.max_backends_per_se);
3381
3382         /* set HW defaults for 3D engine */
3383         WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3384
3385         WREG32(SX_DEBUG_1, 0x20);
3386
3387         WREG32(TA_CNTL_AUX, 0x00010000);
3388
3389         tmp = RREG32(SPI_CONFIG_CNTL);
3390         tmp |= 0x03000000;
3391         WREG32(SPI_CONFIG_CNTL, tmp);
3392
3393         WREG32(SQ_CONFIG, 1);
3394
3395         WREG32(DB_DEBUG, 0);
3396
3397         tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3398         tmp |= 0x00000400;
3399         WREG32(DB_DEBUG2, tmp);
3400
3401         tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3402         tmp |= 0x00020200;
3403         WREG32(DB_DEBUG3, tmp);
3404
3405         tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3406         tmp |= 0x00018208;
3407         WREG32(CB_HW_CONTROL, tmp);
3408
3409         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3410
3411         WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3412                                  SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3413                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3414                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3415
3416         WREG32(VGT_NUM_INSTANCES, 1);
3417
3418         WREG32(CP_PERFMON_CNTL, 0);
3419
3420         WREG32(SQ_CONFIG, 0);
3421
3422         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3423                                           FORCE_EOV_MAX_REZ_CNT(255)));
3424
3425         WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3426                AUTO_INVLD_EN(ES_AND_GS_AUTO));
3427
3428         WREG32(VGT_GS_VERTEX_REUSE, 16);
3429         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3430
3431         tmp = RREG32(HDP_MISC_CNTL);
3432         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3433         WREG32(HDP_MISC_CNTL, tmp);
3434
3435         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3436         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3437
3438         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3439         WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3440
3441         udelay(50);
3442 }
3443
3444 /*
3445  * GPU scratch registers helpers function.
3446  */
3447 /**
3448  * cik_scratch_init - setup driver info for CP scratch regs
3449  *
3450  * @rdev: radeon_device pointer
3451  *
3452  * Set up the number and offset of the CP scratch registers.
3453  * NOTE: use of CP scratch registers is a legacy inferface and
3454  * is not used by default on newer asics (r6xx+).  On newer asics,
3455  * memory buffers are used for fences rather than scratch regs.
3456  */
3457 static void cik_scratch_init(struct radeon_device *rdev)
3458 {
3459         int i;
3460
3461         rdev->scratch.num_reg = 7;
3462         rdev->scratch.reg_base = SCRATCH_REG0;
3463         for (i = 0; i < rdev->scratch.num_reg; i++) {
3464                 rdev->scratch.free[i] = true;
3465                 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3466         }
3467 }
3468
3469 /**
3470  * cik_ring_test - basic gfx ring test
3471  *
3472  * @rdev: radeon_device pointer
3473  * @ring: radeon_ring structure holding ring information
3474  *
3475  * Allocate a scratch register and write to it using the gfx ring (CIK).
3476  * Provides a basic gfx ring test to verify that the ring is working.
3477  * Used by cik_cp_gfx_resume();
3478  * Returns 0 on success, error on failure.
3479  */
3480 int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3481 {
3482         uint32_t scratch;
3483         uint32_t tmp = 0;
3484         unsigned i;
3485         int r;
3486
3487         r = radeon_scratch_get(rdev, &scratch);
3488         if (r) {
3489                 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3490                 return r;
3491         }
3492         WREG32(scratch, 0xCAFEDEAD);
3493         r = radeon_ring_lock(rdev, ring, 3);
3494         if (r) {
3495                 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3496                 radeon_scratch_free(rdev, scratch);
3497                 return r;
3498         }
3499         radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3500         radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3501         radeon_ring_write(ring, 0xDEADBEEF);
3502         radeon_ring_unlock_commit(rdev, ring);
3503
3504         for (i = 0; i < rdev->usec_timeout; i++) {
3505                 tmp = RREG32(scratch);
3506                 if (tmp == 0xDEADBEEF)
3507                         break;
3508                 DRM_UDELAY(1);
3509         }
3510         if (i < rdev->usec_timeout) {
3511                 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3512         } else {
3513                 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3514                           ring->idx, scratch, tmp);
3515                 r = -EINVAL;
3516         }
3517         radeon_scratch_free(rdev, scratch);
3518         return r;
3519 }
3520
3521 /**
3522  * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp
3523  *
3524  * @rdev: radeon_device pointer
3525  * @ridx: radeon ring index
3526  *
3527  * Emits an hdp flush on the cp.
3528  */
3529 static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev,
3530                                        int ridx)
3531 {
3532         struct radeon_ring *ring = &rdev->ring[ridx];
3533         u32 ref_and_mask;
3534
3535         switch (ring->idx) {
3536         case CAYMAN_RING_TYPE_CP1_INDEX:
3537         case CAYMAN_RING_TYPE_CP2_INDEX:
3538         default:
3539                 switch (ring->me) {
3540                 case 0:
3541                         ref_and_mask = CP2 << ring->pipe;
3542                         break;
3543                 case 1:
3544                         ref_and_mask = CP6 << ring->pipe;
3545                         break;
3546                 default:
3547                         return;
3548                 }
3549                 break;
3550         case RADEON_RING_TYPE_GFX_INDEX:
3551                 ref_and_mask = CP0;
3552                 break;
3553         }
3554
3555         radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
3556         radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
3557                                  WAIT_REG_MEM_FUNCTION(3) |  /* == */
3558                                  WAIT_REG_MEM_ENGINE(1)));   /* pfp */
3559         radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2);
3560         radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2);
3561         radeon_ring_write(ring, ref_and_mask);
3562         radeon_ring_write(ring, ref_and_mask);
3563         radeon_ring_write(ring, 0x20); /* poll interval */
3564 }
3565
3566 /**
3567  * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
3568  *
3569  * @rdev: radeon_device pointer
3570  * @fence: radeon fence object
3571  *
3572  * Emits a fence sequnce number on the gfx ring and flushes
3573  * GPU caches.
3574  */
3575 void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3576                              struct radeon_fence *fence)
3577 {
3578         struct radeon_ring *ring = &rdev->ring[fence->ring];
3579         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3580
3581         /* EVENT_WRITE_EOP - flush caches, send int */
3582         radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3583         radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3584                                  EOP_TC_ACTION_EN |
3585                                  EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3586                                  EVENT_INDEX(5)));
3587         radeon_ring_write(ring, addr & 0xfffffffc);
3588         radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3589         radeon_ring_write(ring, fence->seq);
3590         radeon_ring_write(ring, 0);
3591         /* HDP flush */
3592         cik_hdp_flush_cp_ring_emit(rdev, fence->ring);
3593 }
3594
3595 /**
3596  * cik_fence_compute_ring_emit - emit a fence on the compute ring
3597  *
3598  * @rdev: radeon_device pointer
3599  * @fence: radeon fence object
3600  *
3601  * Emits a fence sequnce number on the compute ring and flushes
3602  * GPU caches.
3603  */
3604 void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3605                                  struct radeon_fence *fence)
3606 {
3607         struct radeon_ring *ring = &rdev->ring[fence->ring];
3608         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3609
3610         /* RELEASE_MEM - flush caches, send int */
3611         radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3612         radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3613                                  EOP_TC_ACTION_EN |
3614                                  EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3615                                  EVENT_INDEX(5)));
3616         radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3617         radeon_ring_write(ring, addr & 0xfffffffc);
3618         radeon_ring_write(ring, upper_32_bits(addr));
3619         radeon_ring_write(ring, fence->seq);
3620         radeon_ring_write(ring, 0);
3621         /* HDP flush */
3622         cik_hdp_flush_cp_ring_emit(rdev, fence->ring);
3623 }
3624
3625 bool cik_semaphore_ring_emit(struct radeon_device *rdev,
3626                              struct radeon_ring *ring,
3627                              struct radeon_semaphore *semaphore,
3628                              bool emit_wait)
3629 {
3630         uint64_t addr = semaphore->gpu_addr;
3631         unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3632
3633         radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3634         radeon_ring_write(ring, addr & 0xffffffff);
3635         radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
3636
3637         return true;
3638 }
3639
3640 /**
3641  * cik_copy_cpdma - copy pages using the CP DMA engine
3642  *
3643  * @rdev: radeon_device pointer
3644  * @src_offset: src GPU address
3645  * @dst_offset: dst GPU address
3646  * @num_gpu_pages: number of GPU pages to xfer
3647  * @fence: radeon fence object
3648  *
3649  * Copy GPU paging using the CP DMA engine (CIK+).
3650  * Used by the radeon ttm implementation to move pages if
3651  * registered as the asic copy callback.
3652  */
3653 int cik_copy_cpdma(struct radeon_device *rdev,
3654                    uint64_t src_offset, uint64_t dst_offset,
3655                    unsigned num_gpu_pages,
3656                    struct radeon_fence **fence)
3657 {
3658         struct radeon_semaphore *sem = NULL;
3659         int ring_index = rdev->asic->copy.blit_ring_index;
3660         struct radeon_ring *ring = &rdev->ring[ring_index];
3661         u32 size_in_bytes, cur_size_in_bytes, control;
3662         int i, num_loops;
3663         int r = 0;
3664
3665         r = radeon_semaphore_create(rdev, &sem);
3666         if (r) {
3667                 DRM_ERROR("radeon: moving bo (%d).\n", r);
3668                 return r;
3669         }
3670
3671         size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3672         num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3673         r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
3674         if (r) {
3675                 DRM_ERROR("radeon: moving bo (%d).\n", r);
3676                 radeon_semaphore_free(rdev, &sem, NULL);
3677                 return r;
3678         }
3679
3680         radeon_semaphore_sync_to(sem, *fence);
3681         radeon_semaphore_sync_rings(rdev, sem, ring->idx);
3682
3683         for (i = 0; i < num_loops; i++) {
3684                 cur_size_in_bytes = size_in_bytes;
3685                 if (cur_size_in_bytes > 0x1fffff)
3686                         cur_size_in_bytes = 0x1fffff;
3687                 size_in_bytes -= cur_size_in_bytes;
3688                 control = 0;
3689                 if (size_in_bytes == 0)
3690                         control |= PACKET3_DMA_DATA_CP_SYNC;
3691                 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
3692                 radeon_ring_write(ring, control);
3693                 radeon_ring_write(ring, lower_32_bits(src_offset));
3694                 radeon_ring_write(ring, upper_32_bits(src_offset));
3695                 radeon_ring_write(ring, lower_32_bits(dst_offset));
3696                 radeon_ring_write(ring, upper_32_bits(dst_offset));
3697                 radeon_ring_write(ring, cur_size_in_bytes);
3698                 src_offset += cur_size_in_bytes;
3699                 dst_offset += cur_size_in_bytes;
3700         }
3701
3702         r = radeon_fence_emit(rdev, fence, ring->idx);
3703         if (r) {
3704                 radeon_ring_unlock_undo(rdev, ring);
3705                 return r;
3706         }
3707
3708         radeon_ring_unlock_commit(rdev, ring);
3709         radeon_semaphore_free(rdev, &sem, *fence);
3710
3711         return r;
3712 }
3713
3714 /*
3715  * IB stuff
3716  */
3717 /**
3718  * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
3719  *
3720  * @rdev: radeon_device pointer
3721  * @ib: radeon indirect buffer object
3722  *
3723  * Emits an DE (drawing engine) or CE (constant engine) IB
3724  * on the gfx ring.  IBs are usually generated by userspace
3725  * acceleration drivers and submitted to the kernel for
3726  * sheduling on the ring.  This function schedules the IB
3727  * on the gfx ring for execution by the GPU.
3728  */
3729 void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3730 {
3731         struct radeon_ring *ring = &rdev->ring[ib->ring];
3732         u32 header, control = INDIRECT_BUFFER_VALID;
3733
3734         if (ib->is_const_ib) {
3735                 /* set switch buffer packet before const IB */
3736                 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3737                 radeon_ring_write(ring, 0);
3738
3739                 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3740         } else {
3741                 u32 next_rptr;
3742                 if (ring->rptr_save_reg) {
3743                         next_rptr = ring->wptr + 3 + 4;
3744                         radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3745                         radeon_ring_write(ring, ((ring->rptr_save_reg -
3746                                                   PACKET3_SET_UCONFIG_REG_START) >> 2));
3747                         radeon_ring_write(ring, next_rptr);
3748                 } else if (rdev->wb.enabled) {
3749                         next_rptr = ring->wptr + 5 + 4;
3750                         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3751                         radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
3752                         radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3753                         radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
3754                         radeon_ring_write(ring, next_rptr);
3755                 }
3756
3757                 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3758         }
3759
3760         control |= ib->length_dw |
3761                 (ib->vm ? (ib->vm->id << 24) : 0);
3762
3763         radeon_ring_write(ring, header);
3764         radeon_ring_write(ring,
3765 #ifdef __BIG_ENDIAN
3766                           (2 << 0) |
3767 #endif
3768                           (ib->gpu_addr & 0xFFFFFFFC));
3769         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3770         radeon_ring_write(ring, control);
3771 }
3772
3773 /**
3774  * cik_ib_test - basic gfx ring IB test
3775  *
3776  * @rdev: radeon_device pointer
3777  * @ring: radeon_ring structure holding ring information
3778  *
3779  * Allocate an IB and execute it on the gfx ring (CIK).
3780  * Provides a basic gfx ring test to verify that IBs are working.
3781  * Returns 0 on success, error on failure.
3782  */
3783 int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3784 {
3785         struct radeon_ib ib;
3786         uint32_t scratch;
3787         uint32_t tmp = 0;
3788         unsigned i;
3789         int r;
3790
3791         r = radeon_scratch_get(rdev, &scratch);
3792         if (r) {
3793                 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3794                 return r;
3795         }
3796         WREG32(scratch, 0xCAFEDEAD);
3797         r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3798         if (r) {
3799                 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
3800                 radeon_scratch_free(rdev, scratch);
3801                 return r;
3802         }
3803         ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
3804         ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
3805         ib.ptr[2] = 0xDEADBEEF;
3806         ib.length_dw = 3;
3807         r = radeon_ib_schedule(rdev, &ib, NULL);
3808         if (r) {
3809                 radeon_scratch_free(rdev, scratch);
3810                 radeon_ib_free(rdev, &ib);
3811                 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3812                 return r;
3813         }
3814         r = radeon_fence_wait(ib.fence, false);
3815         if (r) {
3816                 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
3817                 radeon_scratch_free(rdev, scratch);
3818                 radeon_ib_free(rdev, &ib);
3819                 return r;
3820         }
3821         for (i = 0; i < rdev->usec_timeout; i++) {
3822                 tmp = RREG32(scratch);
3823                 if (tmp == 0xDEADBEEF)
3824                         break;
3825                 DRM_UDELAY(1);
3826         }
3827         if (i < rdev->usec_timeout) {
3828                 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3829         } else {
3830                 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3831                           scratch, tmp);
3832                 r = -EINVAL;
3833         }
3834         radeon_scratch_free(rdev, scratch);
3835         radeon_ib_free(rdev, &ib);
3836         return r;
3837 }
3838
3839 /*
3840  * CP.
3841  * On CIK, gfx and compute now have independant command processors.
3842  *
3843  * GFX
3844  * Gfx consists of a single ring and can process both gfx jobs and
3845  * compute jobs.  The gfx CP consists of three microengines (ME):
3846  * PFP - Pre-Fetch Parser
3847  * ME - Micro Engine
3848  * CE - Constant Engine
3849  * The PFP and ME make up what is considered the Drawing Engine (DE).
3850  * The CE is an asynchronous engine used for updating buffer desciptors
3851  * used by the DE so that they can be loaded into cache in parallel
3852  * while the DE is processing state update packets.
3853  *
3854  * Compute
3855  * The compute CP consists of two microengines (ME):
3856  * MEC1 - Compute MicroEngine 1
3857  * MEC2 - Compute MicroEngine 2
3858  * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
3859  * The queues are exposed to userspace and are programmed directly
3860  * by the compute runtime.
3861  */
3862 /**
3863  * cik_cp_gfx_enable - enable/disable the gfx CP MEs
3864  *
3865  * @rdev: radeon_device pointer
3866  * @enable: enable or disable the MEs
3867  *
3868  * Halts or unhalts the gfx MEs.
3869  */
3870 static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
3871 {
3872         if (enable)
3873                 WREG32(CP_ME_CNTL, 0);
3874         else {
3875                 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3876                         radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3877                 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3878                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3879         }
3880         udelay(50);
3881 }
3882
3883 /**
3884  * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
3885  *
3886  * @rdev: radeon_device pointer
3887  *
3888  * Loads the gfx PFP, ME, and CE ucode.
3889  * Returns 0 for success, -EINVAL if the ucode is not available.
3890  */
3891 static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
3892 {
3893         const __be32 *fw_data;
3894         int i;
3895
3896         if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3897                 return -EINVAL;
3898
3899         cik_cp_gfx_enable(rdev, false);
3900
3901         /* PFP */
3902         fw_data = (const __be32 *)rdev->pfp_fw->data;
3903         WREG32(CP_PFP_UCODE_ADDR, 0);
3904         for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
3905                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3906         WREG32(CP_PFP_UCODE_ADDR, 0);
3907
3908         /* CE */
3909         fw_data = (const __be32 *)rdev->ce_fw->data;
3910         WREG32(CP_CE_UCODE_ADDR, 0);
3911         for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
3912                 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3913         WREG32(CP_CE_UCODE_ADDR, 0);
3914
3915         /* ME */
3916         fw_data = (const __be32 *)rdev->me_fw->data;
3917         WREG32(CP_ME_RAM_WADDR, 0);
3918         for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
3919                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3920         WREG32(CP_ME_RAM_WADDR, 0);
3921
3922         WREG32(CP_PFP_UCODE_ADDR, 0);
3923         WREG32(CP_CE_UCODE_ADDR, 0);
3924         WREG32(CP_ME_RAM_WADDR, 0);
3925         WREG32(CP_ME_RAM_RADDR, 0);
3926         return 0;
3927 }
3928
3929 /**
3930  * cik_cp_gfx_start - start the gfx ring
3931  *
3932  * @rdev: radeon_device pointer
3933  *
3934  * Enables the ring and loads the clear state context and other
3935  * packets required to init the ring.
3936  * Returns 0 for success, error for failure.
3937  */
3938 static int cik_cp_gfx_start(struct radeon_device *rdev)
3939 {
3940         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3941         int r, i;
3942
3943         /* init the CP */
3944         WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
3945         WREG32(CP_ENDIAN_SWAP, 0);
3946         WREG32(CP_DEVICE_ID, 1);
3947
3948         cik_cp_gfx_enable(rdev, true);
3949
3950         r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
3951         if (r) {
3952                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3953                 return r;
3954         }
3955
3956         /* init the CE partitions.  CE only used for gfx on CIK */
3957         radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3958         radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3959         radeon_ring_write(ring, 0xc000);
3960         radeon_ring_write(ring, 0xc000);
3961
3962         /* setup clear context state */
3963         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3964         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3965
3966         radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3967         radeon_ring_write(ring, 0x80000000);
3968         radeon_ring_write(ring, 0x80000000);
3969
3970         for (i = 0; i < cik_default_size; i++)
3971                 radeon_ring_write(ring, cik_default_state[i]);
3972
3973         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3974         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3975
3976         /* set clear context state */
3977         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3978         radeon_ring_write(ring, 0);
3979
3980         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3981         radeon_ring_write(ring, 0x00000316);
3982         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3983         radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3984
3985         radeon_ring_unlock_commit(rdev, ring);
3986
3987         return 0;
3988 }
3989
3990 /**
3991  * cik_cp_gfx_fini - stop the gfx ring
3992  *
3993  * @rdev: radeon_device pointer
3994  *
3995  * Stop the gfx ring and tear down the driver ring
3996  * info.
3997  */
3998 static void cik_cp_gfx_fini(struct radeon_device *rdev)
3999 {
4000         cik_cp_gfx_enable(rdev, false);
4001         radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4002 }
4003
4004 /**
4005  * cik_cp_gfx_resume - setup the gfx ring buffer registers
4006  *
4007  * @rdev: radeon_device pointer
4008  *
4009  * Program the location and size of the gfx ring buffer
4010  * and test it to make sure it's working.
4011  * Returns 0 for success, error for failure.
4012  */
4013 static int cik_cp_gfx_resume(struct radeon_device *rdev)
4014 {
4015         struct radeon_ring *ring;
4016         u32 tmp;
4017         u32 rb_bufsz;
4018         u64 rb_addr;
4019         int r;
4020
4021         WREG32(CP_SEM_WAIT_TIMER, 0x0);
4022         if (rdev->family != CHIP_HAWAII)
4023                 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
4024
4025         /* Set the write pointer delay */
4026         WREG32(CP_RB_WPTR_DELAY, 0);
4027
4028         /* set the RB to use vmid 0 */
4029         WREG32(CP_RB_VMID, 0);
4030
4031         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
4032
4033         /* ring 0 - compute and gfx */
4034         /* Set ring buffer size */
4035         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4036         rb_bufsz = order_base_2(ring->ring_size / 8);
4037         tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
4038 #ifdef __BIG_ENDIAN
4039         tmp |= BUF_SWAP_32BIT;
4040 #endif
4041         WREG32(CP_RB0_CNTL, tmp);
4042
4043         /* Initialize the ring buffer's read and write pointers */
4044         WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
4045         ring->wptr = 0;
4046         WREG32(CP_RB0_WPTR, ring->wptr);
4047
4048         /* set the wb address wether it's enabled or not */
4049         WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
4050         WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
4051
4052         /* scratch register shadowing is no longer supported */
4053         WREG32(SCRATCH_UMSK, 0);
4054
4055         if (!rdev->wb.enabled)
4056                 tmp |= RB_NO_UPDATE;
4057
4058         mdelay(1);
4059         WREG32(CP_RB0_CNTL, tmp);
4060
4061         rb_addr = ring->gpu_addr >> 8;
4062         WREG32(CP_RB0_BASE, rb_addr);
4063         WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4064
4065         /* start the ring */
4066         cik_cp_gfx_start(rdev);
4067         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4068         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4069         if (r) {
4070                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4071                 return r;
4072         }
4073
4074         if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4075                 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
4076
4077         return 0;
4078 }
4079
4080 u32 cik_gfx_get_rptr(struct radeon_device *rdev,
4081                      struct radeon_ring *ring)
4082 {
4083         u32 rptr;
4084
4085         if (rdev->wb.enabled)
4086                 rptr = rdev->wb.wb[ring->rptr_offs/4];
4087         else
4088                 rptr = RREG32(CP_RB0_RPTR);
4089
4090         return rptr;
4091 }
4092
4093 u32 cik_gfx_get_wptr(struct radeon_device *rdev,
4094                      struct radeon_ring *ring)
4095 {
4096         u32 wptr;
4097
4098         wptr = RREG32(CP_RB0_WPTR);
4099
4100         return wptr;
4101 }
4102
4103 void cik_gfx_set_wptr(struct radeon_device *rdev,
4104                       struct radeon_ring *ring)
4105 {
4106         WREG32(CP_RB0_WPTR, ring->wptr);
4107         (void)RREG32(CP_RB0_WPTR);
4108 }
4109
4110 u32 cik_compute_get_rptr(struct radeon_device *rdev,
4111                          struct radeon_ring *ring)
4112 {
4113         u32 rptr;
4114
4115         if (rdev->wb.enabled) {
4116                 rptr = rdev->wb.wb[ring->rptr_offs/4];
4117         } else {
4118                 mutex_lock(&rdev->srbm_mutex);
4119                 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4120                 rptr = RREG32(CP_HQD_PQ_RPTR);
4121                 cik_srbm_select(rdev, 0, 0, 0, 0);
4122                 mutex_unlock(&rdev->srbm_mutex);
4123         }
4124
4125         return rptr;
4126 }
4127
4128 u32 cik_compute_get_wptr(struct radeon_device *rdev,
4129                          struct radeon_ring *ring)
4130 {
4131         u32 wptr;
4132
4133         if (rdev->wb.enabled) {
4134                 /* XXX check if swapping is necessary on BE */
4135                 wptr = rdev->wb.wb[ring->wptr_offs/4];
4136         } else {
4137                 mutex_lock(&rdev->srbm_mutex);
4138                 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4139                 wptr = RREG32(CP_HQD_PQ_WPTR);
4140                 cik_srbm_select(rdev, 0, 0, 0, 0);
4141                 mutex_unlock(&rdev->srbm_mutex);
4142         }
4143
4144         return wptr;
4145 }
4146
4147 void cik_compute_set_wptr(struct radeon_device *rdev,
4148                           struct radeon_ring *ring)
4149 {
4150         /* XXX check if swapping is necessary on BE */
4151         rdev->wb.wb[ring->wptr_offs/4] = ring->wptr;
4152         WDOORBELL32(ring->doorbell_index, ring->wptr);
4153 }
4154
4155 /**
4156  * cik_cp_compute_enable - enable/disable the compute CP MEs
4157  *
4158  * @rdev: radeon_device pointer
4159  * @enable: enable or disable the MEs
4160  *
4161  * Halts or unhalts the compute MEs.
4162  */
4163 static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4164 {
4165         if (enable)
4166                 WREG32(CP_MEC_CNTL, 0);
4167         else {
4168                 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
4169                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
4170                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
4171         }
4172         udelay(50);
4173 }
4174
4175 /**
4176  * cik_cp_compute_load_microcode - load the compute CP ME ucode
4177  *
4178  * @rdev: radeon_device pointer
4179  *
4180  * Loads the compute MEC1&2 ucode.
4181  * Returns 0 for success, -EINVAL if the ucode is not available.
4182  */
4183 static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4184 {
4185         const __be32 *fw_data;
4186         int i;
4187
4188         if (!rdev->mec_fw)
4189                 return -EINVAL;
4190
4191         cik_cp_compute_enable(rdev, false);
4192
4193         /* MEC1 */
4194         fw_data = (const __be32 *)rdev->mec_fw->data;
4195         WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4196         for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4197                 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4198         WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4199
4200         if (rdev->family == CHIP_KAVERI) {
4201                 /* MEC2 */
4202                 fw_data = (const __be32 *)rdev->mec_fw->data;
4203                 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4204                 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4205                         WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4206                 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4207         }
4208
4209         return 0;
4210 }
4211
4212 /**
4213  * cik_cp_compute_start - start the compute queues
4214  *
4215  * @rdev: radeon_device pointer
4216  *
4217  * Enable the compute queues.
4218  * Returns 0 for success, error for failure.
4219  */
4220 static int cik_cp_compute_start(struct radeon_device *rdev)
4221 {
4222         cik_cp_compute_enable(rdev, true);
4223
4224         return 0;
4225 }
4226
4227 /**
4228  * cik_cp_compute_fini - stop the compute queues
4229  *
4230  * @rdev: radeon_device pointer
4231  *
4232  * Stop the compute queues and tear down the driver queue
4233  * info.
4234  */
4235 static void cik_cp_compute_fini(struct radeon_device *rdev)
4236 {
4237         int i, idx, r;
4238
4239         cik_cp_compute_enable(rdev, false);
4240
4241         for (i = 0; i < 2; i++) {
4242                 if (i == 0)
4243                         idx = CAYMAN_RING_TYPE_CP1_INDEX;
4244                 else
4245                         idx = CAYMAN_RING_TYPE_CP2_INDEX;
4246
4247                 if (rdev->ring[idx].mqd_obj) {
4248                         r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4249                         if (unlikely(r != 0))
4250                                 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4251
4252                         radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4253                         radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4254
4255                         radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4256                         rdev->ring[idx].mqd_obj = NULL;
4257                 }
4258         }
4259 }
4260
4261 static void cik_mec_fini(struct radeon_device *rdev)
4262 {
4263         int r;
4264
4265         if (rdev->mec.hpd_eop_obj) {
4266                 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4267                 if (unlikely(r != 0))
4268                         dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4269                 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4270                 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4271
4272                 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4273                 rdev->mec.hpd_eop_obj = NULL;
4274         }
4275 }
4276
4277 #define MEC_HPD_SIZE 2048
4278
4279 static int cik_mec_init(struct radeon_device *rdev)
4280 {
4281         int r;
4282         u32 *hpd;
4283
4284         /*
4285          * KV:    2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4286          * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4287          */
4288         if (rdev->family == CHIP_KAVERI)
4289                 rdev->mec.num_mec = 2;
4290         else
4291                 rdev->mec.num_mec = 1;
4292         rdev->mec.num_pipe = 4;
4293         rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4294
4295         if (rdev->mec.hpd_eop_obj == NULL) {
4296                 r = radeon_bo_create(rdev,
4297                                      rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4298                                      PAGE_SIZE, true,
4299                                      RADEON_GEM_DOMAIN_GTT, NULL,
4300                                      &rdev->mec.hpd_eop_obj);
4301                 if (r) {
4302                         dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4303                         return r;
4304                 }
4305         }
4306
4307         r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4308         if (unlikely(r != 0)) {
4309                 cik_mec_fini(rdev);
4310                 return r;
4311         }
4312         r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4313                           &rdev->mec.hpd_eop_gpu_addr);
4314         if (r) {
4315                 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4316                 cik_mec_fini(rdev);
4317                 return r;
4318         }
4319         r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4320         if (r) {
4321                 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4322                 cik_mec_fini(rdev);
4323                 return r;
4324         }
4325
4326         /* clear memory.  Not sure if this is required or not */
4327         memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4328
4329         radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4330         radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4331
4332         return 0;
4333 }
4334
4335 struct hqd_registers
4336 {
4337         u32 cp_mqd_base_addr;
4338         u32 cp_mqd_base_addr_hi;
4339         u32 cp_hqd_active;
4340         u32 cp_hqd_vmid;
4341         u32 cp_hqd_persistent_state;
4342         u32 cp_hqd_pipe_priority;
4343         u32 cp_hqd_queue_priority;
4344         u32 cp_hqd_quantum;
4345         u32 cp_hqd_pq_base;
4346         u32 cp_hqd_pq_base_hi;
4347         u32 cp_hqd_pq_rptr;
4348         u32 cp_hqd_pq_rptr_report_addr;
4349         u32 cp_hqd_pq_rptr_report_addr_hi;
4350         u32 cp_hqd_pq_wptr_poll_addr;
4351         u32 cp_hqd_pq_wptr_poll_addr_hi;
4352         u32 cp_hqd_pq_doorbell_control;
4353         u32 cp_hqd_pq_wptr;
4354         u32 cp_hqd_pq_control;
4355         u32 cp_hqd_ib_base_addr;
4356         u32 cp_hqd_ib_base_addr_hi;
4357         u32 cp_hqd_ib_rptr;
4358         u32 cp_hqd_ib_control;
4359         u32 cp_hqd_iq_timer;
4360         u32 cp_hqd_iq_rptr;
4361         u32 cp_hqd_dequeue_request;
4362         u32 cp_hqd_dma_offload;
4363         u32 cp_hqd_sema_cmd;
4364         u32 cp_hqd_msg_type;
4365         u32 cp_hqd_atomic0_preop_lo;
4366         u32 cp_hqd_atomic0_preop_hi;
4367         u32 cp_hqd_atomic1_preop_lo;
4368         u32 cp_hqd_atomic1_preop_hi;
4369         u32 cp_hqd_hq_scheduler0;
4370         u32 cp_hqd_hq_scheduler1;
4371         u32 cp_mqd_control;
4372 };
4373
4374 struct bonaire_mqd
4375 {
4376         u32 header;
4377         u32 dispatch_initiator;
4378         u32 dimensions[3];
4379         u32 start_idx[3];
4380         u32 num_threads[3];
4381         u32 pipeline_stat_enable;
4382         u32 perf_counter_enable;
4383         u32 pgm[2];
4384         u32 tba[2];
4385         u32 tma[2];
4386         u32 pgm_rsrc[2];
4387         u32 vmid;
4388         u32 resource_limits;
4389         u32 static_thread_mgmt01[2];
4390         u32 tmp_ring_size;
4391         u32 static_thread_mgmt23[2];
4392         u32 restart[3];
4393         u32 thread_trace_enable;
4394         u32 reserved1;
4395         u32 user_data[16];
4396         u32 vgtcs_invoke_count[2];
4397         struct hqd_registers queue_state;
4398         u32 dequeue_cntr;
4399         u32 interrupt_queue[64];
4400 };
4401
4402 /**
4403  * cik_cp_compute_resume - setup the compute queue registers
4404  *
4405  * @rdev: radeon_device pointer
4406  *
4407  * Program the compute queues and test them to make sure they
4408  * are working.
4409  * Returns 0 for success, error for failure.
4410  */
4411 static int cik_cp_compute_resume(struct radeon_device *rdev)
4412 {
4413         int r, i, idx;
4414         u32 tmp;
4415         bool use_doorbell = true;
4416         u64 hqd_gpu_addr;
4417         u64 mqd_gpu_addr;
4418         u64 eop_gpu_addr;
4419         u64 wb_gpu_addr;
4420         u32 *buf;
4421         struct bonaire_mqd *mqd;
4422
4423         r = cik_cp_compute_start(rdev);
4424         if (r)
4425                 return r;
4426
4427         /* fix up chicken bits */
4428         tmp = RREG32(CP_CPF_DEBUG);
4429         tmp |= (1 << 23);
4430         WREG32(CP_CPF_DEBUG, tmp);
4431
4432         /* init the pipes */
4433         mutex_lock(&rdev->srbm_mutex);
4434         for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) {
4435                 int me = (i < 4) ? 1 : 2;
4436                 int pipe = (i < 4) ? i : (i - 4);
4437
4438                 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2);
4439
4440                 cik_srbm_select(rdev, me, pipe, 0, 0);
4441
4442                 /* write the EOP addr */
4443                 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4444                 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
4445
4446                 /* set the VMID assigned */
4447                 WREG32(CP_HPD_EOP_VMID, 0);
4448
4449                 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4450                 tmp = RREG32(CP_HPD_EOP_CONTROL);
4451                 tmp &= ~EOP_SIZE_MASK;
4452                 tmp |= order_base_2(MEC_HPD_SIZE / 8);
4453                 WREG32(CP_HPD_EOP_CONTROL, tmp);
4454         }
4455         cik_srbm_select(rdev, 0, 0, 0, 0);
4456         mutex_unlock(&rdev->srbm_mutex);
4457
4458         /* init the queues.  Just two for now. */
4459         for (i = 0; i < 2; i++) {
4460                 if (i == 0)
4461                         idx = CAYMAN_RING_TYPE_CP1_INDEX;
4462                 else
4463                         idx = CAYMAN_RING_TYPE_CP2_INDEX;
4464
4465                 if (rdev->ring[idx].mqd_obj == NULL) {
4466                         r = radeon_bo_create(rdev,
4467                                              sizeof(struct bonaire_mqd),
4468                                              PAGE_SIZE, true,
4469                                              RADEON_GEM_DOMAIN_GTT, NULL,
4470                                              &rdev->ring[idx].mqd_obj);
4471                         if (r) {
4472                                 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4473                                 return r;
4474                         }
4475                 }
4476
4477                 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4478                 if (unlikely(r != 0)) {
4479                         cik_cp_compute_fini(rdev);
4480                         return r;
4481                 }
4482                 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4483                                   &mqd_gpu_addr);
4484                 if (r) {
4485                         dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4486                         cik_cp_compute_fini(rdev);
4487                         return r;
4488                 }
4489                 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4490                 if (r) {
4491                         dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4492                         cik_cp_compute_fini(rdev);
4493                         return r;
4494                 }
4495
4496                 /* init the mqd struct */
4497                 memset(buf, 0, sizeof(struct bonaire_mqd));
4498
4499                 mqd = (struct bonaire_mqd *)buf;
4500                 mqd->header = 0xC0310800;
4501                 mqd->static_thread_mgmt01[0] = 0xffffffff;
4502                 mqd->static_thread_mgmt01[1] = 0xffffffff;
4503                 mqd->static_thread_mgmt23[0] = 0xffffffff;
4504                 mqd->static_thread_mgmt23[1] = 0xffffffff;
4505
4506                 mutex_lock(&rdev->srbm_mutex);
4507                 cik_srbm_select(rdev, rdev->ring[idx].me,
4508                                 rdev->ring[idx].pipe,
4509                                 rdev->ring[idx].queue, 0);
4510
4511                 /* disable wptr polling */
4512                 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4513                 tmp &= ~WPTR_POLL_EN;
4514                 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4515
4516                 /* enable doorbell? */
4517                 mqd->queue_state.cp_hqd_pq_doorbell_control =
4518                         RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4519                 if (use_doorbell)
4520                         mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4521                 else
4522                         mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4523                 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4524                        mqd->queue_state.cp_hqd_pq_doorbell_control);
4525
4526                 /* disable the queue if it's active */
4527                 mqd->queue_state.cp_hqd_dequeue_request = 0;
4528                 mqd->queue_state.cp_hqd_pq_rptr = 0;
4529                 mqd->queue_state.cp_hqd_pq_wptr= 0;
4530                 if (RREG32(CP_HQD_ACTIVE) & 1) {
4531                         WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4532                         for (i = 0; i < rdev->usec_timeout; i++) {
4533                                 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4534                                         break;
4535                                 udelay(1);
4536                         }
4537                         WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4538                         WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4539                         WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4540                 }
4541
4542                 /* set the pointer to the MQD */
4543                 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4544                 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4545                 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4546                 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4547                 /* set MQD vmid to 0 */
4548                 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4549                 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4550                 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4551
4552                 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4553                 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4554                 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4555                 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4556                 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4557                 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4558
4559                 /* set up the HQD, this is similar to CP_RB0_CNTL */
4560                 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4561                 mqd->queue_state.cp_hqd_pq_control &=
4562                         ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4563
4564                 mqd->queue_state.cp_hqd_pq_control |=
4565                         order_base_2(rdev->ring[idx].ring_size / 8);
4566                 mqd->queue_state.cp_hqd_pq_control |=
4567                         (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
4568 #ifdef __BIG_ENDIAN
4569                 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4570 #endif
4571                 mqd->queue_state.cp_hqd_pq_control &=
4572                         ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4573                 mqd->queue_state.cp_hqd_pq_control |=
4574                         PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4575                 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4576
4577                 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4578                 if (i == 0)
4579                         wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
4580                 else
4581                         wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
4582                 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
4583                 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4584                 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
4585                 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
4586                        mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
4587
4588                 /* set the wb address wether it's enabled or not */
4589                 if (i == 0)
4590                         wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
4591                 else
4592                         wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
4593                 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
4594                 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
4595                         upper_32_bits(wb_gpu_addr) & 0xffff;
4596                 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
4597                        mqd->queue_state.cp_hqd_pq_rptr_report_addr);
4598                 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4599                        mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
4600
4601                 /* enable the doorbell if requested */
4602                 if (use_doorbell) {
4603                         mqd->queue_state.cp_hqd_pq_doorbell_control =
4604                                 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4605                         mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
4606                         mqd->queue_state.cp_hqd_pq_doorbell_control |=
4607                                 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);
4608                         mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4609                         mqd->queue_state.cp_hqd_pq_doorbell_control &=
4610                                 ~(DOORBELL_SOURCE | DOORBELL_HIT);
4611
4612                 } else {
4613                         mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
4614                 }
4615                 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4616                        mqd->queue_state.cp_hqd_pq_doorbell_control);
4617
4618                 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4619                 rdev->ring[idx].wptr = 0;
4620                 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
4621                 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4622                 mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR);
4623
4624                 /* set the vmid for the queue */
4625                 mqd->queue_state.cp_hqd_vmid = 0;
4626                 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
4627
4628                 /* activate the queue */
4629                 mqd->queue_state.cp_hqd_active = 1;
4630                 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
4631
4632                 cik_srbm_select(rdev, 0, 0, 0, 0);
4633                 mutex_unlock(&rdev->srbm_mutex);
4634
4635                 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
4636                 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4637
4638                 rdev->ring[idx].ready = true;
4639                 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
4640                 if (r)
4641                         rdev->ring[idx].ready = false;
4642         }
4643
4644         return 0;
4645 }
4646
4647 static void cik_cp_enable(struct radeon_device *rdev, bool enable)
4648 {
4649         cik_cp_gfx_enable(rdev, enable);
4650         cik_cp_compute_enable(rdev, enable);
4651 }
4652
4653 static int cik_cp_load_microcode(struct radeon_device *rdev)
4654 {
4655         int r;
4656
4657         r = cik_cp_gfx_load_microcode(rdev);
4658         if (r)
4659                 return r;
4660         r = cik_cp_compute_load_microcode(rdev);
4661         if (r)
4662                 return r;
4663
4664         return 0;
4665 }
4666
4667 static void cik_cp_fini(struct radeon_device *rdev)
4668 {
4669         cik_cp_gfx_fini(rdev);
4670         cik_cp_compute_fini(rdev);
4671 }
4672
4673 static int cik_cp_resume(struct radeon_device *rdev)
4674 {
4675         int r;
4676
4677         cik_enable_gui_idle_interrupt(rdev, false);
4678
4679         r = cik_cp_load_microcode(rdev);
4680         if (r)
4681                 return r;
4682
4683         r = cik_cp_gfx_resume(rdev);
4684         if (r)
4685                 return r;
4686         r = cik_cp_compute_resume(rdev);
4687         if (r)
4688                 return r;
4689
4690         cik_enable_gui_idle_interrupt(rdev, true);
4691
4692         return 0;
4693 }
4694
4695 static void cik_print_gpu_status_regs(struct radeon_device *rdev)
4696 {
4697         dev_info(rdev->dev, "  GRBM_STATUS=0x%08X\n",
4698                 RREG32(GRBM_STATUS));
4699         dev_info(rdev->dev, "  GRBM_STATUS2=0x%08X\n",
4700                 RREG32(GRBM_STATUS2));
4701         dev_info(rdev->dev, "  GRBM_STATUS_SE0=0x%08X\n",
4702                 RREG32(GRBM_STATUS_SE0));
4703         dev_info(rdev->dev, "  GRBM_STATUS_SE1=0x%08X\n",
4704                 RREG32(GRBM_STATUS_SE1));
4705         dev_info(rdev->dev, "  GRBM_STATUS_SE2=0x%08X\n",
4706                 RREG32(GRBM_STATUS_SE2));
4707         dev_info(rdev->dev, "  GRBM_STATUS_SE3=0x%08X\n",
4708                 RREG32(GRBM_STATUS_SE3));
4709         dev_info(rdev->dev, "  SRBM_STATUS=0x%08X\n",
4710                 RREG32(SRBM_STATUS));
4711         dev_info(rdev->dev, "  SRBM_STATUS2=0x%08X\n",
4712                 RREG32(SRBM_STATUS2));
4713         dev_info(rdev->dev, "  SDMA0_STATUS_REG   = 0x%08X\n",
4714                 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
4715         dev_info(rdev->dev, "  SDMA1_STATUS_REG   = 0x%08X\n",
4716                  RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
4717         dev_info(rdev->dev, "  CP_STAT = 0x%08x\n", RREG32(CP_STAT));
4718         dev_info(rdev->dev, "  CP_STALLED_STAT1 = 0x%08x\n",
4719                  RREG32(CP_STALLED_STAT1));
4720         dev_info(rdev->dev, "  CP_STALLED_STAT2 = 0x%08x\n",
4721                  RREG32(CP_STALLED_STAT2));
4722         dev_info(rdev->dev, "  CP_STALLED_STAT3 = 0x%08x\n",
4723                  RREG32(CP_STALLED_STAT3));
4724         dev_info(rdev->dev, "  CP_CPF_BUSY_STAT = 0x%08x\n",
4725                  RREG32(CP_CPF_BUSY_STAT));
4726         dev_info(rdev->dev, "  CP_CPF_STALLED_STAT1 = 0x%08x\n",
4727                  RREG32(CP_CPF_STALLED_STAT1));
4728         dev_info(rdev->dev, "  CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
4729         dev_info(rdev->dev, "  CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
4730         dev_info(rdev->dev, "  CP_CPC_STALLED_STAT1 = 0x%08x\n",
4731                  RREG32(CP_CPC_STALLED_STAT1));
4732         dev_info(rdev->dev, "  CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
4733 }
4734
4735 /**
4736  * cik_gpu_check_soft_reset - check which blocks are busy
4737  *
4738  * @rdev: radeon_device pointer
4739  *
4740  * Check which blocks are busy and return the relevant reset
4741  * mask to be used by cik_gpu_soft_reset().
4742  * Returns a mask of the blocks to be reset.
4743  */
4744 u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
4745 {
4746         u32 reset_mask = 0;
4747         u32 tmp;
4748
4749         /* GRBM_STATUS */
4750         tmp = RREG32(GRBM_STATUS);
4751         if (tmp & (PA_BUSY | SC_BUSY |
4752                    BCI_BUSY | SX_BUSY |
4753                    TA_BUSY | VGT_BUSY |
4754                    DB_BUSY | CB_BUSY |
4755                    GDS_BUSY | SPI_BUSY |
4756                    IA_BUSY | IA_BUSY_NO_DMA))
4757                 reset_mask |= RADEON_RESET_GFX;
4758
4759         if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
4760                 reset_mask |= RADEON_RESET_CP;
4761
4762         /* GRBM_STATUS2 */
4763         tmp = RREG32(GRBM_STATUS2);
4764         if (tmp & RLC_BUSY)
4765                 reset_mask |= RADEON_RESET_RLC;
4766
4767         /* SDMA0_STATUS_REG */
4768         tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
4769         if (!(tmp & SDMA_IDLE))
4770                 reset_mask |= RADEON_RESET_DMA;
4771
4772         /* SDMA1_STATUS_REG */
4773         tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
4774         if (!(tmp & SDMA_IDLE))
4775                 reset_mask |= RADEON_RESET_DMA1;
4776
4777         /* SRBM_STATUS2 */
4778         tmp = RREG32(SRBM_STATUS2);
4779         if (tmp & SDMA_BUSY)
4780                 reset_mask |= RADEON_RESET_DMA;
4781
4782         if (tmp & SDMA1_BUSY)
4783                 reset_mask |= RADEON_RESET_DMA1;
4784
4785         /* SRBM_STATUS */
4786         tmp = RREG32(SRBM_STATUS);
4787
4788         if (tmp & IH_BUSY)
4789                 reset_mask |= RADEON_RESET_IH;
4790
4791         if (tmp & SEM_BUSY)
4792                 reset_mask |= RADEON_RESET_SEM;
4793
4794         if (tmp & GRBM_RQ_PENDING)
4795                 reset_mask |= RADEON_RESET_GRBM;
4796
4797         if (tmp & VMC_BUSY)
4798                 reset_mask |= RADEON_RESET_VMC;
4799
4800         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
4801                    MCC_BUSY | MCD_BUSY))
4802                 reset_mask |= RADEON_RESET_MC;
4803
4804         if (evergreen_is_display_hung(rdev))
4805                 reset_mask |= RADEON_RESET_DISPLAY;
4806
4807         /* Skip MC reset as it's mostly likely not hung, just busy */
4808         if (reset_mask & RADEON_RESET_MC) {
4809                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
4810                 reset_mask &= ~RADEON_RESET_MC;
4811         }
4812
4813         return reset_mask;
4814 }
4815
4816 /**
4817  * cik_gpu_soft_reset - soft reset GPU
4818  *
4819  * @rdev: radeon_device pointer
4820  * @reset_mask: mask of which blocks to reset
4821  *
4822  * Soft reset the blocks specified in @reset_mask.
4823  */
4824 static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
4825 {
4826         struct evergreen_mc_save save;
4827         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
4828         u32 tmp;
4829
4830         if (reset_mask == 0)
4831                 return;
4832
4833         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
4834
4835         cik_print_gpu_status_regs(rdev);
4836         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
4837                  RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4838         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4839                  RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
4840
4841         /* disable CG/PG */
4842         cik_fini_pg(rdev);
4843         cik_fini_cg(rdev);
4844
4845         /* stop the rlc */
4846         cik_rlc_stop(rdev);
4847
4848         /* Disable GFX parsing/prefetching */
4849         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
4850
4851         /* Disable MEC parsing/prefetching */
4852         WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
4853
4854         if (reset_mask & RADEON_RESET_DMA) {
4855                 /* sdma0 */
4856                 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
4857                 tmp |= SDMA_HALT;
4858                 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
4859         }
4860         if (reset_mask & RADEON_RESET_DMA1) {
4861                 /* sdma1 */
4862                 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
4863                 tmp |= SDMA_HALT;
4864                 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
4865         }
4866
4867         evergreen_mc_stop(rdev, &save);
4868         if (evergreen_mc_wait_for_idle(rdev)) {
4869                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4870         }
4871
4872         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
4873                 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
4874
4875         if (reset_mask & RADEON_RESET_CP) {
4876                 grbm_soft_reset |= SOFT_RESET_CP;
4877
4878                 srbm_soft_reset |= SOFT_RESET_GRBM;
4879         }
4880
4881         if (reset_mask & RADEON_RESET_DMA)
4882                 srbm_soft_reset |= SOFT_RESET_SDMA;
4883
4884         if (reset_mask & RADEON_RESET_DMA1)
4885                 srbm_soft_reset |= SOFT_RESET_SDMA1;
4886
4887         if (reset_mask & RADEON_RESET_DISPLAY)
4888                 srbm_soft_reset |= SOFT_RESET_DC;
4889
4890         if (reset_mask & RADEON_RESET_RLC)
4891                 grbm_soft_reset |= SOFT_RESET_RLC;
4892
4893         if (reset_mask & RADEON_RESET_SEM)
4894                 srbm_soft_reset |= SOFT_RESET_SEM;
4895
4896         if (reset_mask & RADEON_RESET_IH)
4897                 srbm_soft_reset |= SOFT_RESET_IH;
4898
4899         if (reset_mask & RADEON_RESET_GRBM)
4900                 srbm_soft_reset |= SOFT_RESET_GRBM;
4901
4902         if (reset_mask & RADEON_RESET_VMC)
4903                 srbm_soft_reset |= SOFT_RESET_VMC;
4904
4905         if (!(rdev->flags & RADEON_IS_IGP)) {
4906                 if (reset_mask & RADEON_RESET_MC)
4907                         srbm_soft_reset |= SOFT_RESET_MC;
4908         }
4909
4910         if (grbm_soft_reset) {
4911                 tmp = RREG32(GRBM_SOFT_RESET);
4912                 tmp |= grbm_soft_reset;
4913                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4914                 WREG32(GRBM_SOFT_RESET, tmp);
4915                 tmp = RREG32(GRBM_SOFT_RESET);
4916
4917                 udelay(50);
4918
4919                 tmp &= ~grbm_soft_reset;
4920                 WREG32(GRBM_SOFT_RESET, tmp);
4921                 tmp = RREG32(GRBM_SOFT_RESET);
4922         }
4923
4924         if (srbm_soft_reset) {
4925                 tmp = RREG32(SRBM_SOFT_RESET);
4926                 tmp |= srbm_soft_reset;
4927                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
4928                 WREG32(SRBM_SOFT_RESET, tmp);
4929                 tmp = RREG32(SRBM_SOFT_RESET);
4930
4931                 udelay(50);
4932
4933                 tmp &= ~srbm_soft_reset;
4934                 WREG32(SRBM_SOFT_RESET, tmp);
4935                 tmp = RREG32(SRBM_SOFT_RESET);
4936         }
4937
4938         /* Wait a little for things to settle down */
4939         udelay(50);
4940
4941         evergreen_mc_resume(rdev, &save);
4942         udelay(50);
4943
4944         cik_print_gpu_status_regs(rdev);
4945 }
4946
4947 struct kv_reset_save_regs {
4948         u32 gmcon_reng_execute;
4949         u32 gmcon_misc;
4950         u32 gmcon_misc3;
4951 };
4952
4953 static void kv_save_regs_for_reset(struct radeon_device *rdev,
4954                                    struct kv_reset_save_regs *save)
4955 {
4956         save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE);
4957         save->gmcon_misc = RREG32(GMCON_MISC);
4958         save->gmcon_misc3 = RREG32(GMCON_MISC3);
4959
4960         WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP);
4961         WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE |
4962                                                 STCTRL_STUTTER_EN));
4963 }
4964
4965 static void kv_restore_regs_for_reset(struct radeon_device *rdev,
4966                                       struct kv_reset_save_regs *save)
4967 {
4968         int i;
4969
4970         WREG32(GMCON_PGFSM_WRITE, 0);
4971         WREG32(GMCON_PGFSM_CONFIG, 0x200010ff);
4972
4973         for (i = 0; i < 5; i++)
4974                 WREG32(GMCON_PGFSM_WRITE, 0);
4975
4976         WREG32(GMCON_PGFSM_WRITE, 0);
4977         WREG32(GMCON_PGFSM_CONFIG, 0x300010ff);
4978
4979         for (i = 0; i < 5; i++)
4980                 WREG32(GMCON_PGFSM_WRITE, 0);
4981
4982         WREG32(GMCON_PGFSM_WRITE, 0x210000);
4983         WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff);
4984
4985         for (i = 0; i < 5; i++)
4986                 WREG32(GMCON_PGFSM_WRITE, 0);
4987
4988         WREG32(GMCON_PGFSM_WRITE, 0x21003);
4989         WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff);
4990
4991         for (i = 0; i < 5; i++)
4992                 WREG32(GMCON_PGFSM_WRITE, 0);
4993
4994         WREG32(GMCON_PGFSM_WRITE, 0x2b00);
4995         WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff);
4996
4997         for (i = 0; i < 5; i++)
4998                 WREG32(GMCON_PGFSM_WRITE, 0);
4999
5000         WREG32(GMCON_PGFSM_WRITE, 0);
5001         WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff);
5002
5003         for (i = 0; i < 5; i++)
5004                 WREG32(GMCON_PGFSM_WRITE, 0);
5005
5006         WREG32(GMCON_PGFSM_WRITE, 0x420000);
5007         WREG32(GMCON_PGFSM_CONFIG, 0x100010ff);
5008
5009         for (i = 0; i < 5; i++)
5010                 WREG32(GMCON_PGFSM_WRITE, 0);
5011
5012         WREG32(GMCON_PGFSM_WRITE, 0x120202);
5013         WREG32(GMCON_PGFSM_CONFIG, 0x500010ff);
5014
5015         for (i = 0; i < 5; i++)
5016                 WREG32(GMCON_PGFSM_WRITE, 0);
5017
5018         WREG32(GMCON_PGFSM_WRITE, 0x3e3e36);
5019         WREG32(GMCON_PGFSM_CONFIG, 0x600010ff);
5020
5021         for (i = 0; i < 5; i++)
5022                 WREG32(GMCON_PGFSM_WRITE, 0);
5023
5024         WREG32(GMCON_PGFSM_WRITE, 0x373f3e);
5025         WREG32(GMCON_PGFSM_CONFIG, 0x700010ff);
5026
5027         for (i = 0; i < 5; i++)
5028                 WREG32(GMCON_PGFSM_WRITE, 0);
5029
5030         WREG32(GMCON_PGFSM_WRITE, 0x3e1332);
5031         WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff);
5032
5033         WREG32(GMCON_MISC3, save->gmcon_misc3);
5034         WREG32(GMCON_MISC, save->gmcon_misc);
5035         WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute);
5036 }
5037
5038 static void cik_gpu_pci_config_reset(struct radeon_device *rdev)
5039 {
5040         struct evergreen_mc_save save;
5041         struct kv_reset_save_regs kv_save = { 0 };
5042         u32 tmp, i;
5043
5044         dev_info(rdev->dev, "GPU pci config reset\n");
5045
5046         /* disable dpm? */
5047
5048         /* disable cg/pg */
5049         cik_fini_pg(rdev);
5050         cik_fini_cg(rdev);
5051
5052         /* Disable GFX parsing/prefetching */
5053         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5054
5055         /* Disable MEC parsing/prefetching */
5056         WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5057
5058         /* sdma0 */
5059         tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5060         tmp |= SDMA_HALT;
5061         WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5062         /* sdma1 */
5063         tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5064         tmp |= SDMA_HALT;
5065         WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5066         /* XXX other engines? */
5067
5068         /* halt the rlc, disable cp internal ints */
5069         cik_rlc_stop(rdev);
5070
5071         udelay(50);
5072
5073         /* disable mem access */
5074         evergreen_mc_stop(rdev, &save);
5075         if (evergreen_mc_wait_for_idle(rdev)) {
5076                 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
5077         }
5078
5079         if (rdev->flags & RADEON_IS_IGP)
5080                 kv_save_regs_for_reset(rdev, &kv_save);
5081
5082         /* disable BM */
5083         pci_clear_master(rdev->pdev);
5084         /* reset */
5085         radeon_pci_config_reset(rdev);
5086
5087         udelay(100);
5088
5089         /* wait for asic to come out of reset */
5090         for (i = 0; i < rdev->usec_timeout; i++) {
5091                 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
5092                         break;
5093                 udelay(1);
5094         }
5095
5096         /* does asic init need to be run first??? */
5097         if (rdev->flags & RADEON_IS_IGP)
5098                 kv_restore_regs_for_reset(rdev, &kv_save);
5099 }
5100
5101 /**
5102  * cik_asic_reset - soft reset GPU
5103  *
5104  * @rdev: radeon_device pointer
5105  *
5106  * Look up which blocks are hung and attempt
5107  * to reset them.
5108  * Returns 0 for success.
5109  */
5110 int cik_asic_reset(struct radeon_device *rdev)
5111 {
5112         u32 reset_mask;
5113
5114         reset_mask = cik_gpu_check_soft_reset(rdev);
5115
5116         if (reset_mask)
5117                 r600_set_bios_scratch_engine_hung(rdev, true);
5118
5119         /* try soft reset */
5120         cik_gpu_soft_reset(rdev, reset_mask);
5121
5122         reset_mask = cik_gpu_check_soft_reset(rdev);
5123
5124         /* try pci config reset */
5125         if (reset_mask && radeon_hard_reset)
5126                 cik_gpu_pci_config_reset(rdev);
5127
5128         reset_mask = cik_gpu_check_soft_reset(rdev);
5129
5130         if (!reset_mask)
5131                 r600_set_bios_scratch_engine_hung(rdev, false);
5132
5133         return 0;
5134 }
5135
5136 /**
5137  * cik_gfx_is_lockup - check if the 3D engine is locked up
5138  *
5139  * @rdev: radeon_device pointer
5140  * @ring: radeon_ring structure holding ring information
5141  *
5142  * Check if the 3D engine is locked up (CIK).
5143  * Returns true if the engine is locked, false if not.
5144  */
5145 bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
5146 {
5147         u32 reset_mask = cik_gpu_check_soft_reset(rdev);
5148
5149         if (!(reset_mask & (RADEON_RESET_GFX |
5150                             RADEON_RESET_COMPUTE |
5151                             RADEON_RESET_CP))) {
5152                 radeon_ring_lockup_update(rdev, ring);
5153                 return false;
5154         }
5155         return radeon_ring_test_lockup(rdev, ring);
5156 }
5157
5158 /* MC */
5159 /**
5160  * cik_mc_program - program the GPU memory controller
5161  *
5162  * @rdev: radeon_device pointer
5163  *
5164  * Set the location of vram, gart, and AGP in the GPU's
5165  * physical address space (CIK).
5166  */
5167 static void cik_mc_program(struct radeon_device *rdev)
5168 {
5169         struct evergreen_mc_save save;
5170         u32 tmp;
5171         int i, j;
5172
5173         /* Initialize HDP */
5174         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
5175                 WREG32((0x2c14 + j), 0x00000000);
5176                 WREG32((0x2c18 + j), 0x00000000);
5177                 WREG32((0x2c1c + j), 0x00000000);
5178                 WREG32((0x2c20 + j), 0x00000000);
5179                 WREG32((0x2c24 + j), 0x00000000);
5180         }
5181         WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
5182
5183         evergreen_mc_stop(rdev, &save);
5184         if (radeon_mc_wait_for_idle(rdev)) {
5185                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5186         }
5187         /* Lockout access through VGA aperture*/
5188         WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
5189         /* Update configuration */
5190         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
5191                rdev->mc.vram_start >> 12);
5192         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
5193                rdev->mc.vram_end >> 12);
5194         WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
5195                rdev->vram_scratch.gpu_addr >> 12);
5196         tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
5197         tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
5198         WREG32(MC_VM_FB_LOCATION, tmp);
5199         /* XXX double check these! */
5200         WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
5201         WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
5202         WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
5203         WREG32(MC_VM_AGP_BASE, 0);
5204         WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
5205         WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
5206         if (radeon_mc_wait_for_idle(rdev)) {
5207                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5208         }
5209         evergreen_mc_resume(rdev, &save);
5210         /* we need to own VRAM, so turn off the VGA renderer here
5211          * to stop it overwriting our objects */
5212         rv515_vga_render_disable(rdev);
5213 }
5214
5215 /**
5216  * cik_mc_init - initialize the memory controller driver params
5217  *
5218  * @rdev: radeon_device pointer
5219  *
5220  * Look up the amount of vram, vram width, and decide how to place
5221  * vram and gart within the GPU's physical address space (CIK).
5222  * Returns 0 for success.
5223  */
5224 static int cik_mc_init(struct radeon_device *rdev)
5225 {
5226         u32 tmp;
5227         int chansize, numchan;
5228
5229         /* Get VRAM informations */
5230         rdev->mc.vram_is_ddr = true;
5231         tmp = RREG32(MC_ARB_RAMCFG);
5232         if (tmp & CHANSIZE_MASK) {
5233                 chansize = 64;
5234         } else {
5235                 chansize = 32;
5236         }
5237         tmp = RREG32(MC_SHARED_CHMAP);
5238         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
5239         case 0:
5240         default:
5241                 numchan = 1;
5242                 break;
5243         case 1:
5244                 numchan = 2;
5245                 break;
5246         case 2:
5247                 numchan = 4;
5248                 break;
5249         case 3:
5250                 numchan = 8;
5251                 break;
5252         case 4:
5253                 numchan = 3;
5254                 break;
5255         case 5:
5256                 numchan = 6;
5257                 break;
5258         case 6:
5259                 numchan = 10;
5260                 break;
5261         case 7:
5262                 numchan = 12;
5263                 break;
5264         case 8:
5265                 numchan = 16;
5266                 break;
5267         }
5268         rdev->mc.vram_width = numchan * chansize;
5269         /* Could aper size report 0 ? */
5270         rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5271         rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5272         /* size in MB on si */
5273         rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5274         rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5275         rdev->mc.visible_vram_size = rdev->mc.aper_size;
5276         si_vram_gtt_location(rdev, &rdev->mc);
5277         radeon_update_bandwidth_info(rdev);
5278
5279         return 0;
5280 }
5281
5282 /*
5283  * GART
5284  * VMID 0 is the physical GPU addresses as used by the kernel.
5285  * VMIDs 1-15 are used for userspace clients and are handled
5286  * by the radeon vm/hsa code.
5287  */
5288 /**
5289  * cik_pcie_gart_tlb_flush - gart tlb flush callback
5290  *
5291  * @rdev: radeon_device pointer
5292  *
5293  * Flush the TLB for the VMID 0 page table (CIK).
5294  */
5295 void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5296 {
5297         /* flush hdp cache */
5298         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5299
5300         /* bits 0-15 are the VM contexts0-15 */
5301         WREG32(VM_INVALIDATE_REQUEST, 0x1);
5302 }
5303
5304 /**
5305  * cik_pcie_gart_enable - gart enable
5306  *
5307  * @rdev: radeon_device pointer
5308  *
5309  * This sets up the TLBs, programs the page tables for VMID0,
5310  * sets up the hw for VMIDs 1-15 which are allocated on
5311  * demand, and sets up the global locations for the LDS, GDS,
5312  * and GPUVM for FSA64 clients (CIK).
5313  * Returns 0 for success, errors for failure.
5314  */
5315 static int cik_pcie_gart_enable(struct radeon_device *rdev)
5316 {
5317         int r, i;
5318
5319         if (rdev->gart.robj == NULL) {
5320                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5321                 return -EINVAL;
5322         }
5323         r = radeon_gart_table_vram_pin(rdev);
5324         if (r)
5325                 return r;
5326         radeon_gart_restore(rdev);
5327         /* Setup TLB control */
5328         WREG32(MC_VM_MX_L1_TLB_CNTL,
5329                (0xA << 7) |
5330                ENABLE_L1_TLB |
5331                ENABLE_L1_FRAGMENT_PROCESSING |
5332                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5333                ENABLE_ADVANCED_DRIVER_MODEL |
5334                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5335         /* Setup L2 cache */
5336         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5337                ENABLE_L2_FRAGMENT_PROCESSING |
5338                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5339                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5340                EFFECTIVE_L2_QUEUE_SIZE(7) |
5341                CONTEXT1_IDENTITY_ACCESS_MODE(1));
5342         WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5343         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5344                BANK_SELECT(4) |
5345                L2_CACHE_BIGK_FRAGMENT_SIZE(4));
5346         /* setup context0 */
5347         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
5348         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
5349         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5350         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5351                         (u32)(rdev->dummy_page.addr >> 12));
5352         WREG32(VM_CONTEXT0_CNTL2, 0);
5353         WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5354                                   RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5355
5356         WREG32(0x15D4, 0);
5357         WREG32(0x15D8, 0);
5358         WREG32(0x15DC, 0);
5359
5360         /* empty context1-15 */
5361         /* FIXME start with 4G, once using 2 level pt switch to full
5362          * vm size space
5363          */
5364         /* set vm size, must be a multiple of 4 */
5365         WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
5366         WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
5367         for (i = 1; i < 16; i++) {
5368                 if (i < 8)
5369                         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
5370                                rdev->gart.table_addr >> 12);
5371                 else
5372                         WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
5373                                rdev->gart.table_addr >> 12);
5374         }
5375
5376         /* enable context1-15 */
5377         WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5378                (u32)(rdev->dummy_page.addr >> 12));
5379         WREG32(VM_CONTEXT1_CNTL2, 4);
5380         WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
5381                                 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5382                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5383                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5384                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5385                                 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5386                                 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5387                                 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5388                                 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5389                                 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5390                                 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5391                                 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5392                                 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
5393
5394         if (rdev->family == CHIP_KAVERI) {
5395                 u32 tmp = RREG32(CHUB_CONTROL);
5396                 tmp &= ~BYPASS_VM;
5397                 WREG32(CHUB_CONTROL, tmp);
5398         }
5399
5400         /* XXX SH_MEM regs */
5401         /* where to put LDS, scratch, GPUVM in FSA64 space */
5402         mutex_lock(&rdev->srbm_mutex);
5403         for (i = 0; i < 16; i++) {
5404                 cik_srbm_select(rdev, 0, 0, 0, i);
5405                 /* CP and shaders */
5406                 WREG32(SH_MEM_CONFIG, 0);
5407                 WREG32(SH_MEM_APE1_BASE, 1);
5408                 WREG32(SH_MEM_APE1_LIMIT, 0);
5409                 WREG32(SH_MEM_BASES, 0);
5410                 /* SDMA GFX */
5411                 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5412                 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5413                 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5414                 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5415                 /* XXX SDMA RLC - todo */
5416         }
5417         cik_srbm_select(rdev, 0, 0, 0, 0);
5418         mutex_unlock(&rdev->srbm_mutex);
5419
5420         cik_pcie_gart_tlb_flush(rdev);
5421         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5422                  (unsigned)(rdev->mc.gtt_size >> 20),
5423                  (unsigned long long)rdev->gart.table_addr);
5424         rdev->gart.ready = true;
5425         return 0;
5426 }
5427
5428 /**
5429  * cik_pcie_gart_disable - gart disable
5430  *
5431  * @rdev: radeon_device pointer
5432  *
5433  * This disables all VM page table (CIK).
5434  */
5435 static void cik_pcie_gart_disable(struct radeon_device *rdev)
5436 {
5437         /* Disable all tables */
5438         WREG32(VM_CONTEXT0_CNTL, 0);
5439         WREG32(VM_CONTEXT1_CNTL, 0);
5440         /* Setup TLB control */
5441         WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5442                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5443         /* Setup L2 cache */
5444         WREG32(VM_L2_CNTL,
5445                ENABLE_L2_FRAGMENT_PROCESSING |
5446                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5447                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5448                EFFECTIVE_L2_QUEUE_SIZE(7) |
5449                CONTEXT1_IDENTITY_ACCESS_MODE(1));
5450         WREG32(VM_L2_CNTL2, 0);
5451         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5452                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5453         radeon_gart_table_vram_unpin(rdev);
5454 }
5455
5456 /**
5457  * cik_pcie_gart_fini - vm fini callback
5458  *
5459  * @rdev: radeon_device pointer
5460  *
5461  * Tears down the driver GART/VM setup (CIK).
5462  */
5463 static void cik_pcie_gart_fini(struct radeon_device *rdev)
5464 {
5465         cik_pcie_gart_disable(rdev);
5466         radeon_gart_table_vram_free(rdev);
5467         radeon_gart_fini(rdev);
5468 }
5469
5470 /* vm parser */
5471 /**
5472  * cik_ib_parse - vm ib_parse callback
5473  *
5474  * @rdev: radeon_device pointer
5475  * @ib: indirect buffer pointer
5476  *
5477  * CIK uses hw IB checking so this is a nop (CIK).
5478  */
5479 int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5480 {
5481         return 0;
5482 }
5483
5484 /*
5485  * vm
5486  * VMID 0 is the physical GPU addresses as used by the kernel.
5487  * VMIDs 1-15 are used for userspace clients and are handled
5488  * by the radeon vm/hsa code.
5489  */
5490 /**
5491  * cik_vm_init - cik vm init callback
5492  *
5493  * @rdev: radeon_device pointer
5494  *
5495  * Inits cik specific vm parameters (number of VMs, base of vram for
5496  * VMIDs 1-15) (CIK).
5497  * Returns 0 for success.
5498  */
5499 int cik_vm_init(struct radeon_device *rdev)
5500 {
5501         /* number of VMs */
5502         rdev->vm_manager.nvm = 16;
5503         /* base offset of vram pages */
5504         if (rdev->flags & RADEON_IS_IGP) {
5505                 u64 tmp = RREG32(MC_VM_FB_OFFSET);
5506                 tmp <<= 22;
5507                 rdev->vm_manager.vram_base_offset = tmp;
5508         } else
5509                 rdev->vm_manager.vram_base_offset = 0;
5510
5511         return 0;
5512 }
5513
5514 /**
5515  * cik_vm_fini - cik vm fini callback
5516  *
5517  * @rdev: radeon_device pointer
5518  *
5519  * Tear down any asic specific VM setup (CIK).
5520  */
5521 void cik_vm_fini(struct radeon_device *rdev)
5522 {
5523 }
5524
5525 /**
5526  * cik_vm_decode_fault - print human readable fault info
5527  *
5528  * @rdev: radeon_device pointer
5529  * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5530  * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5531  *
5532  * Print human readable fault information (CIK).
5533  */
5534 static void cik_vm_decode_fault(struct radeon_device *rdev,
5535                                 u32 status, u32 addr, u32 mc_client)
5536 {
5537         u32 mc_id;
5538         u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
5539         u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
5540         char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
5541                 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
5542
5543         if (rdev->family == CHIP_HAWAII)
5544                 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5545         else
5546                 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5547
5548         printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
5549                protections, vmid, addr,
5550                (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
5551                block, mc_client, mc_id);
5552 }
5553
5554 /**
5555  * cik_vm_flush - cik vm flush using the CP
5556  *
5557  * @rdev: radeon_device pointer
5558  *
5559  * Update the page table base and flush the VM TLB
5560  * using the CP (CIK).
5561  */
5562 void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
5563 {
5564         struct radeon_ring *ring = &rdev->ring[ridx];
5565
5566         if (vm == NULL)
5567                 return;
5568
5569         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5570         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5571                                  WRITE_DATA_DST_SEL(0)));
5572         if (vm->id < 8) {
5573                 radeon_ring_write(ring,
5574                                   (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
5575         } else {
5576                 radeon_ring_write(ring,
5577                                   (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
5578         }
5579         radeon_ring_write(ring, 0);
5580         radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
5581
5582         /* update SH_MEM_* regs */
5583         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5584         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5585                                  WRITE_DATA_DST_SEL(0)));
5586         radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5587         radeon_ring_write(ring, 0);
5588         radeon_ring_write(ring, VMID(vm->id));
5589
5590         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
5591         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5592                                  WRITE_DATA_DST_SEL(0)));
5593         radeon_ring_write(ring, SH_MEM_BASES >> 2);
5594         radeon_ring_write(ring, 0);
5595
5596         radeon_ring_write(ring, 0); /* SH_MEM_BASES */
5597         radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
5598         radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
5599         radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
5600
5601         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5602         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5603                                  WRITE_DATA_DST_SEL(0)));
5604         radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5605         radeon_ring_write(ring, 0);
5606         radeon_ring_write(ring, VMID(0));
5607
5608         /* HDP flush */
5609         cik_hdp_flush_cp_ring_emit(rdev, ridx);
5610
5611         /* bits 0-15 are the VM contexts0-15 */
5612         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5613         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5614                                  WRITE_DATA_DST_SEL(0)));
5615         radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5616         radeon_ring_write(ring, 0);
5617         radeon_ring_write(ring, 1 << vm->id);
5618
5619         /* compute doesn't have PFP */
5620         if (ridx == RADEON_RING_TYPE_GFX_INDEX) {
5621                 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5622                 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5623                 radeon_ring_write(ring, 0x0);
5624         }
5625 }
5626
5627 /*
5628  * RLC
5629  * The RLC is a multi-purpose microengine that handles a
5630  * variety of functions, the most important of which is
5631  * the interrupt controller.
5632  */
5633 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
5634                                           bool enable)
5635 {
5636         u32 tmp = RREG32(CP_INT_CNTL_RING0);
5637
5638         if (enable)
5639                 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5640         else
5641                 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5642         WREG32(CP_INT_CNTL_RING0, tmp);
5643 }
5644
5645 static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
5646 {
5647         u32 tmp;
5648
5649         tmp = RREG32(RLC_LB_CNTL);
5650         if (enable)
5651                 tmp |= LOAD_BALANCE_ENABLE;
5652         else
5653                 tmp &= ~LOAD_BALANCE_ENABLE;
5654         WREG32(RLC_LB_CNTL, tmp);
5655 }
5656
5657 static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
5658 {
5659         u32 i, j, k;
5660         u32 mask;
5661
5662         for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
5663                 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
5664                         cik_select_se_sh(rdev, i, j);
5665                         for (k = 0; k < rdev->usec_timeout; k++) {
5666                                 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
5667                                         break;
5668                                 udelay(1);
5669                         }
5670                 }
5671         }
5672         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5673
5674         mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
5675         for (k = 0; k < rdev->usec_timeout; k++) {
5676                 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
5677                         break;
5678                 udelay(1);
5679         }
5680 }
5681
5682 static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
5683 {
5684         u32 tmp;
5685
5686         tmp = RREG32(RLC_CNTL);
5687         if (tmp != rlc)
5688                 WREG32(RLC_CNTL, rlc);
5689 }
5690
5691 static u32 cik_halt_rlc(struct radeon_device *rdev)
5692 {
5693         u32 data, orig;
5694
5695         orig = data = RREG32(RLC_CNTL);
5696
5697         if (data & RLC_ENABLE) {
5698                 u32 i;
5699
5700                 data &= ~RLC_ENABLE;
5701                 WREG32(RLC_CNTL, data);
5702
5703                 for (i = 0; i < rdev->usec_timeout; i++) {
5704                         if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
5705                                 break;
5706                         udelay(1);
5707                 }
5708
5709                 cik_wait_for_rlc_serdes(rdev);
5710         }
5711
5712         return orig;
5713 }
5714
5715 void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
5716 {
5717         u32 tmp, i, mask;
5718
5719         tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
5720         WREG32(RLC_GPR_REG2, tmp);
5721
5722         mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
5723         for (i = 0; i < rdev->usec_timeout; i++) {
5724                 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
5725                         break;
5726                 udelay(1);
5727         }
5728
5729         for (i = 0; i < rdev->usec_timeout; i++) {
5730                 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
5731                         break;
5732                 udelay(1);
5733         }
5734 }
5735
5736 void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
5737 {
5738         u32 tmp;
5739
5740         tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
5741         WREG32(RLC_GPR_REG2, tmp);
5742 }
5743
5744 /**
5745  * cik_rlc_stop - stop the RLC ME
5746  *
5747  * @rdev: radeon_device pointer
5748  *
5749  * Halt the RLC ME (MicroEngine) (CIK).
5750  */
5751 static void cik_rlc_stop(struct radeon_device *rdev)
5752 {
5753         WREG32(RLC_CNTL, 0);
5754
5755         cik_enable_gui_idle_interrupt(rdev, false);
5756
5757         cik_wait_for_rlc_serdes(rdev);
5758 }
5759
5760 /**
5761  * cik_rlc_start - start the RLC ME
5762  *
5763  * @rdev: radeon_device pointer
5764  *
5765  * Unhalt the RLC ME (MicroEngine) (CIK).
5766  */
5767 static void cik_rlc_start(struct radeon_device *rdev)
5768 {
5769         WREG32(RLC_CNTL, RLC_ENABLE);
5770
5771         cik_enable_gui_idle_interrupt(rdev, true);
5772
5773         udelay(50);
5774 }
5775
5776 /**
5777  * cik_rlc_resume - setup the RLC hw
5778  *
5779  * @rdev: radeon_device pointer
5780  *
5781  * Initialize the RLC registers, load the ucode,
5782  * and start the RLC (CIK).
5783  * Returns 0 for success, -EINVAL if the ucode is not available.
5784  */
5785 static int cik_rlc_resume(struct radeon_device *rdev)
5786 {
5787         u32 i, size, tmp;
5788         const __be32 *fw_data;
5789
5790         if (!rdev->rlc_fw)
5791                 return -EINVAL;
5792
5793         switch (rdev->family) {
5794         case CHIP_BONAIRE:
5795         case CHIP_HAWAII:
5796         default:
5797                 size = BONAIRE_RLC_UCODE_SIZE;
5798                 break;
5799         case CHIP_KAVERI:
5800                 size = KV_RLC_UCODE_SIZE;
5801                 break;
5802         case CHIP_KABINI:
5803                 size = KB_RLC_UCODE_SIZE;
5804                 break;
5805         }
5806
5807         cik_rlc_stop(rdev);
5808
5809         /* disable CG */
5810         tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
5811         WREG32(RLC_CGCG_CGLS_CTRL, tmp);
5812
5813         si_rlc_reset(rdev);
5814
5815         cik_init_pg(rdev);
5816
5817         cik_init_cg(rdev);
5818
5819         WREG32(RLC_LB_CNTR_INIT, 0);
5820         WREG32(RLC_LB_CNTR_MAX, 0x00008000);
5821
5822         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5823         WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5824         WREG32(RLC_LB_PARAMS, 0x00600408);
5825         WREG32(RLC_LB_CNTL, 0x80000004);
5826
5827         WREG32(RLC_MC_CNTL, 0);
5828         WREG32(RLC_UCODE_CNTL, 0);
5829
5830         fw_data = (const __be32 *)rdev->rlc_fw->data;
5831                 WREG32(RLC_GPM_UCODE_ADDR, 0);
5832         for (i = 0; i < size; i++)
5833                 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
5834         WREG32(RLC_GPM_UCODE_ADDR, 0);
5835
5836         /* XXX - find out what chips support lbpw */
5837         cik_enable_lbpw(rdev, false);
5838
5839         if (rdev->family == CHIP_BONAIRE)
5840                 WREG32(RLC_DRIVER_DMA_STATUS, 0);
5841
5842         cik_rlc_start(rdev);
5843
5844         return 0;
5845 }
5846
5847 static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
5848 {
5849         u32 data, orig, tmp, tmp2;
5850
5851         orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5852
5853         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
5854                 cik_enable_gui_idle_interrupt(rdev, true);
5855
5856                 tmp = cik_halt_rlc(rdev);
5857
5858                 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5859                 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5860                 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5861                 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
5862                 WREG32(RLC_SERDES_WR_CTRL, tmp2);
5863
5864                 cik_update_rlc(rdev, tmp);
5865
5866                 data |= CGCG_EN | CGLS_EN;
5867         } else {
5868                 cik_enable_gui_idle_interrupt(rdev, false);
5869
5870                 RREG32(CB_CGTT_SCLK_CTRL);
5871                 RREG32(CB_CGTT_SCLK_CTRL);
5872                 RREG32(CB_CGTT_SCLK_CTRL);
5873                 RREG32(CB_CGTT_SCLK_CTRL);
5874
5875                 data &= ~(CGCG_EN | CGLS_EN);
5876         }
5877
5878         if (orig != data)
5879                 WREG32(RLC_CGCG_CGLS_CTRL, data);
5880
5881 }
5882
5883 static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
5884 {
5885         u32 data, orig, tmp = 0;
5886
5887         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5888                 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
5889                         if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5890                                 orig = data = RREG32(CP_MEM_SLP_CNTL);
5891                                 data |= CP_MEM_LS_EN;
5892                                 if (orig != data)
5893                                         WREG32(CP_MEM_SLP_CNTL, data);
5894                         }
5895                 }
5896
5897                 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5898                 data &= 0xfffffffd;
5899                 if (orig != data)
5900                         WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5901
5902                 tmp = cik_halt_rlc(rdev);
5903
5904                 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5905                 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5906                 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5907                 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
5908                 WREG32(RLC_SERDES_WR_CTRL, data);
5909
5910                 cik_update_rlc(rdev, tmp);
5911
5912                 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
5913                         orig = data = RREG32(CGTS_SM_CTRL_REG);
5914                         data &= ~SM_MODE_MASK;
5915                         data |= SM_MODE(0x2);
5916                         data |= SM_MODE_ENABLE;
5917                         data &= ~CGTS_OVERRIDE;
5918                         if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
5919                             (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
5920                                 data &= ~CGTS_LS_OVERRIDE;
5921                         data &= ~ON_MONITOR_ADD_MASK;
5922                         data |= ON_MONITOR_ADD_EN;
5923                         data |= ON_MONITOR_ADD(0x96);
5924                         if (orig != data)
5925                                 WREG32(CGTS_SM_CTRL_REG, data);
5926                 }
5927         } else {
5928                 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5929                 data |= 0x00000002;
5930                 if (orig != data)
5931                         WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5932
5933                 data = RREG32(RLC_MEM_SLP_CNTL);
5934                 if (data & RLC_MEM_LS_EN) {
5935                         data &= ~RLC_MEM_LS_EN;
5936                         WREG32(RLC_MEM_SLP_CNTL, data);
5937                 }
5938
5939                 data = RREG32(CP_MEM_SLP_CNTL);
5940                 if (data & CP_MEM_LS_EN) {
5941                         data &= ~CP_MEM_LS_EN;
5942                         WREG32(CP_MEM_SLP_CNTL, data);
5943                 }
5944
5945                 orig = data = RREG32(CGTS_SM_CTRL_REG);
5946                 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
5947                 if (orig != data)
5948                         WREG32(CGTS_SM_CTRL_REG, data);
5949
5950                 tmp = cik_halt_rlc(rdev);
5951
5952                 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5953                 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5954                 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5955                 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
5956                 WREG32(RLC_SERDES_WR_CTRL, data);
5957
5958                 cik_update_rlc(rdev, tmp);
5959         }
5960 }
5961
5962 static const u32 mc_cg_registers[] =
5963 {
5964         MC_HUB_MISC_HUB_CG,
5965         MC_HUB_MISC_SIP_CG,
5966         MC_HUB_MISC_VM_CG,
5967         MC_XPB_CLK_GAT,
5968         ATC_MISC_CG,
5969         MC_CITF_MISC_WR_CG,
5970         MC_CITF_MISC_RD_CG,
5971         MC_CITF_MISC_VM_CG,
5972         VM_L2_CG,
5973 };
5974
5975 static void cik_enable_mc_ls(struct radeon_device *rdev,
5976                              bool enable)
5977 {
5978         int i;
5979         u32 orig, data;
5980
5981         for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5982                 orig = data = RREG32(mc_cg_registers[i]);
5983                 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
5984                         data |= MC_LS_ENABLE;
5985                 else
5986                         data &= ~MC_LS_ENABLE;
5987                 if (data != orig)
5988                         WREG32(mc_cg_registers[i], data);
5989         }
5990 }
5991
5992 static void cik_enable_mc_mgcg(struct radeon_device *rdev,
5993                                bool enable)
5994 {
5995         int i;
5996         u32 orig, data;
5997
5998         for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5999                 orig = data = RREG32(mc_cg_registers[i]);
6000                 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
6001                         data |= MC_CG_ENABLE;
6002                 else
6003                         data &= ~MC_CG_ENABLE;
6004                 if (data != orig)
6005                         WREG32(mc_cg_registers[i], data);
6006         }
6007 }
6008
6009 static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
6010                                  bool enable)
6011 {
6012         u32 orig, data;
6013
6014         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
6015                 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
6016                 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
6017         } else {
6018                 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
6019                 data |= 0xff000000;
6020                 if (data != orig)
6021                         WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
6022
6023                 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
6024                 data |= 0xff000000;
6025                 if (data != orig)
6026                         WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
6027         }
6028 }
6029
6030 static void cik_enable_sdma_mgls(struct radeon_device *rdev,
6031                                  bool enable)
6032 {
6033         u32 orig, data;
6034
6035         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
6036                 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6037                 data |= 0x100;
6038                 if (orig != data)
6039                         WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6040
6041                 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6042                 data |= 0x100;
6043                 if (orig != data)
6044                         WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6045         } else {
6046                 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6047                 data &= ~0x100;
6048                 if (orig != data)
6049                         WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6050
6051                 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6052                 data &= ~0x100;
6053                 if (orig != data)
6054                         WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6055         }
6056 }
6057
6058 static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
6059                                 bool enable)
6060 {
6061         u32 orig, data;
6062
6063         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
6064                 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6065                 data = 0xfff;
6066                 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6067
6068                 orig = data = RREG32(UVD_CGC_CTRL);
6069                 data |= DCM;
6070                 if (orig != data)
6071                         WREG32(UVD_CGC_CTRL, data);
6072         } else {
6073                 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6074                 data &= ~0xfff;
6075                 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6076
6077                 orig = data = RREG32(UVD_CGC_CTRL);
6078                 data &= ~DCM;
6079                 if (orig != data)
6080                         WREG32(UVD_CGC_CTRL, data);
6081         }
6082 }
6083
6084 static void cik_enable_bif_mgls(struct radeon_device *rdev,
6085                                bool enable)
6086 {
6087         u32 orig, data;
6088
6089         orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
6090
6091         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
6092                 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
6093                         REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
6094         else
6095                 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
6096                           REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
6097
6098         if (orig != data)
6099                 WREG32_PCIE_PORT(PCIE_CNTL2, data);
6100 }
6101
6102 static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
6103                                 bool enable)
6104 {
6105         u32 orig, data;
6106
6107         orig = data = RREG32(HDP_HOST_PATH_CNTL);
6108
6109         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
6110                 data &= ~CLOCK_GATING_DIS;
6111         else
6112                 data |= CLOCK_GATING_DIS;
6113
6114         if (orig != data)
6115                 WREG32(HDP_HOST_PATH_CNTL, data);
6116 }
6117
6118 static void cik_enable_hdp_ls(struct radeon_device *rdev,
6119                               bool enable)
6120 {
6121         u32 orig, data;
6122
6123         orig = data = RREG32(HDP_MEM_POWER_LS);
6124
6125         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
6126                 data |= HDP_LS_ENABLE;
6127         else
6128                 data &= ~HDP_LS_ENABLE;
6129
6130         if (orig != data)
6131                 WREG32(HDP_MEM_POWER_LS, data);
6132 }
6133
6134 void cik_update_cg(struct radeon_device *rdev,
6135                    u32 block, bool enable)
6136 {
6137
6138         if (block & RADEON_CG_BLOCK_GFX) {
6139                 cik_enable_gui_idle_interrupt(rdev, false);
6140                 /* order matters! */
6141                 if (enable) {
6142                         cik_enable_mgcg(rdev, true);
6143                         cik_enable_cgcg(rdev, true);
6144                 } else {
6145                         cik_enable_cgcg(rdev, false);
6146                         cik_enable_mgcg(rdev, false);
6147                 }
6148                 cik_enable_gui_idle_interrupt(rdev, true);
6149         }
6150
6151         if (block & RADEON_CG_BLOCK_MC) {
6152                 if (!(rdev->flags & RADEON_IS_IGP)) {
6153                         cik_enable_mc_mgcg(rdev, enable);
6154                         cik_enable_mc_ls(rdev, enable);
6155                 }
6156         }
6157
6158         if (block & RADEON_CG_BLOCK_SDMA) {
6159                 cik_enable_sdma_mgcg(rdev, enable);
6160                 cik_enable_sdma_mgls(rdev, enable);
6161         }
6162
6163         if (block & RADEON_CG_BLOCK_BIF) {
6164                 cik_enable_bif_mgls(rdev, enable);
6165         }
6166
6167         if (block & RADEON_CG_BLOCK_UVD) {
6168                 if (rdev->has_uvd)
6169                         cik_enable_uvd_mgcg(rdev, enable);
6170         }
6171
6172         if (block & RADEON_CG_BLOCK_HDP) {
6173                 cik_enable_hdp_mgcg(rdev, enable);
6174                 cik_enable_hdp_ls(rdev, enable);
6175         }
6176
6177         if (block & RADEON_CG_BLOCK_VCE) {
6178                 vce_v2_0_enable_mgcg(rdev, enable);
6179         }
6180 }
6181
6182 static void cik_init_cg(struct radeon_device *rdev)
6183 {
6184
6185         cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
6186
6187         if (rdev->has_uvd)
6188                 si_init_uvd_internal_cg(rdev);
6189
6190         cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6191                              RADEON_CG_BLOCK_SDMA |
6192                              RADEON_CG_BLOCK_BIF |
6193                              RADEON_CG_BLOCK_UVD |
6194                              RADEON_CG_BLOCK_HDP), true);
6195 }
6196
6197 static void cik_fini_cg(struct radeon_device *rdev)
6198 {
6199         cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6200                              RADEON_CG_BLOCK_SDMA |
6201                              RADEON_CG_BLOCK_BIF |
6202                              RADEON_CG_BLOCK_UVD |
6203                              RADEON_CG_BLOCK_HDP), false);
6204
6205         cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
6206 }
6207
6208 static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
6209                                           bool enable)
6210 {
6211         u32 data, orig;
6212
6213         orig = data = RREG32(RLC_PG_CNTL);
6214         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6215                 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6216         else
6217                 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6218         if (orig != data)
6219                 WREG32(RLC_PG_CNTL, data);
6220 }
6221
6222 static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
6223                                           bool enable)
6224 {
6225         u32 data, orig;
6226
6227         orig = data = RREG32(RLC_PG_CNTL);
6228         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6229                 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6230         else
6231                 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6232         if (orig != data)
6233                 WREG32(RLC_PG_CNTL, data);
6234 }
6235
6236 static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
6237 {
6238         u32 data, orig;
6239
6240         orig = data = RREG32(RLC_PG_CNTL);
6241         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
6242                 data &= ~DISABLE_CP_PG;
6243         else
6244                 data |= DISABLE_CP_PG;
6245         if (orig != data)
6246                 WREG32(RLC_PG_CNTL, data);
6247 }
6248
6249 static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
6250 {
6251         u32 data, orig;
6252
6253         orig = data = RREG32(RLC_PG_CNTL);
6254         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
6255                 data &= ~DISABLE_GDS_PG;
6256         else
6257                 data |= DISABLE_GDS_PG;
6258         if (orig != data)
6259                 WREG32(RLC_PG_CNTL, data);
6260 }
6261
6262 #define CP_ME_TABLE_SIZE    96
6263 #define CP_ME_TABLE_OFFSET  2048
6264 #define CP_MEC_TABLE_OFFSET 4096
6265
6266 void cik_init_cp_pg_table(struct radeon_device *rdev)
6267 {
6268         const __be32 *fw_data;
6269         volatile u32 *dst_ptr;
6270         int me, i, max_me = 4;
6271         u32 bo_offset = 0;
6272         u32 table_offset;
6273
6274         if (rdev->family == CHIP_KAVERI)
6275                 max_me = 5;
6276
6277         if (rdev->rlc.cp_table_ptr == NULL)
6278                 return;
6279
6280         /* write the cp table buffer */
6281         dst_ptr = rdev->rlc.cp_table_ptr;
6282         for (me = 0; me < max_me; me++) {
6283                 if (me == 0) {
6284                         fw_data = (const __be32 *)rdev->ce_fw->data;
6285                         table_offset = CP_ME_TABLE_OFFSET;
6286                 } else if (me == 1) {
6287                         fw_data = (const __be32 *)rdev->pfp_fw->data;
6288                         table_offset = CP_ME_TABLE_OFFSET;
6289                 } else if (me == 2) {
6290                         fw_data = (const __be32 *)rdev->me_fw->data;
6291                         table_offset = CP_ME_TABLE_OFFSET;
6292                 } else {
6293                         fw_data = (const __be32 *)rdev->mec_fw->data;
6294                         table_offset = CP_MEC_TABLE_OFFSET;
6295                 }
6296
6297                 for (i = 0; i < CP_ME_TABLE_SIZE; i ++) {
6298                         dst_ptr[bo_offset + i] = cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
6299                 }
6300                 bo_offset += CP_ME_TABLE_SIZE;
6301         }
6302 }
6303
6304 static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6305                                 bool enable)
6306 {
6307         u32 data, orig;
6308
6309         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
6310                 orig = data = RREG32(RLC_PG_CNTL);
6311                 data |= GFX_PG_ENABLE;
6312                 if (orig != data)
6313                         WREG32(RLC_PG_CNTL, data);
6314
6315                 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6316                 data |= AUTO_PG_EN;
6317                 if (orig != data)
6318                         WREG32(RLC_AUTO_PG_CTRL, data);
6319         } else {
6320                 orig = data = RREG32(RLC_PG_CNTL);
6321                 data &= ~GFX_PG_ENABLE;
6322                 if (orig != data)
6323                         WREG32(RLC_PG_CNTL, data);
6324
6325                 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6326                 data &= ~AUTO_PG_EN;
6327                 if (orig != data)
6328                         WREG32(RLC_AUTO_PG_CTRL, data);
6329
6330                 data = RREG32(DB_RENDER_CONTROL);
6331         }
6332 }
6333
6334 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6335 {
6336         u32 mask = 0, tmp, tmp1;
6337         int i;
6338
6339         cik_select_se_sh(rdev, se, sh);
6340         tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6341         tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6342         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6343
6344         tmp &= 0xffff0000;
6345
6346         tmp |= tmp1;
6347         tmp >>= 16;
6348
6349         for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6350                 mask <<= 1;
6351                 mask |= 1;
6352         }
6353
6354         return (~tmp) & mask;
6355 }
6356
6357 static void cik_init_ao_cu_mask(struct radeon_device *rdev)
6358 {
6359         u32 i, j, k, active_cu_number = 0;
6360         u32 mask, counter, cu_bitmap;
6361         u32 tmp = 0;
6362
6363         for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6364                 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6365                         mask = 1;
6366                         cu_bitmap = 0;
6367                         counter = 0;
6368                         for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6369                                 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6370                                         if (counter < 2)
6371                                                 cu_bitmap |= mask;
6372                                         counter ++;
6373                                 }
6374                                 mask <<= 1;
6375                         }
6376
6377                         active_cu_number += counter;
6378                         tmp |= (cu_bitmap << (i * 16 + j * 8));
6379                 }
6380         }
6381
6382         WREG32(RLC_PG_AO_CU_MASK, tmp);
6383
6384         tmp = RREG32(RLC_MAX_PG_CU);
6385         tmp &= ~MAX_PU_CU_MASK;
6386         tmp |= MAX_PU_CU(active_cu_number);
6387         WREG32(RLC_MAX_PG_CU, tmp);
6388 }
6389
6390 static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6391                                        bool enable)
6392 {
6393         u32 data, orig;
6394
6395         orig = data = RREG32(RLC_PG_CNTL);
6396         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
6397                 data |= STATIC_PER_CU_PG_ENABLE;
6398         else
6399                 data &= ~STATIC_PER_CU_PG_ENABLE;
6400         if (orig != data)
6401                 WREG32(RLC_PG_CNTL, data);
6402 }
6403
6404 static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6405                                         bool enable)
6406 {
6407         u32 data, orig;
6408
6409         orig = data = RREG32(RLC_PG_CNTL);
6410         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
6411                 data |= DYN_PER_CU_PG_ENABLE;
6412         else
6413                 data &= ~DYN_PER_CU_PG_ENABLE;
6414         if (orig != data)
6415                 WREG32(RLC_PG_CNTL, data);
6416 }
6417
6418 #define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6419 #define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET    0x3D
6420
6421 static void cik_init_gfx_cgpg(struct radeon_device *rdev)
6422 {
6423         u32 data, orig;
6424         u32 i;
6425
6426         if (rdev->rlc.cs_data) {
6427                 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6428                 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
6429                 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
6430                 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
6431         } else {
6432                 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6433                 for (i = 0; i < 3; i++)
6434                         WREG32(RLC_GPM_SCRATCH_DATA, 0);
6435         }
6436         if (rdev->rlc.reg_list) {
6437                 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
6438                 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6439                         WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
6440         }
6441
6442         orig = data = RREG32(RLC_PG_CNTL);
6443         data |= GFX_PG_SRC;
6444         if (orig != data)
6445                 WREG32(RLC_PG_CNTL, data);
6446
6447         WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
6448         WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
6449
6450         data = RREG32(CP_RB_WPTR_POLL_CNTL);
6451         data &= ~IDLE_POLL_COUNT_MASK;
6452         data |= IDLE_POLL_COUNT(0x60);
6453         WREG32(CP_RB_WPTR_POLL_CNTL, data);
6454
6455         data = 0x10101010;
6456         WREG32(RLC_PG_DELAY, data);
6457
6458         data = RREG32(RLC_PG_DELAY_2);
6459         data &= ~0xff;
6460         data |= 0x3;
6461         WREG32(RLC_PG_DELAY_2, data);
6462
6463         data = RREG32(RLC_AUTO_PG_CTRL);
6464         data &= ~GRBM_REG_SGIT_MASK;
6465         data |= GRBM_REG_SGIT(0x700);
6466         WREG32(RLC_AUTO_PG_CTRL, data);
6467
6468 }
6469
6470 static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
6471 {
6472         cik_enable_gfx_cgpg(rdev, enable);
6473         cik_enable_gfx_static_mgpg(rdev, enable);
6474         cik_enable_gfx_dynamic_mgpg(rdev, enable);
6475 }
6476
6477 u32 cik_get_csb_size(struct radeon_device *rdev)
6478 {
6479         u32 count = 0;
6480         const struct cs_section_def *sect = NULL;
6481         const struct cs_extent_def *ext = NULL;
6482
6483         if (rdev->rlc.cs_data == NULL)
6484                 return 0;
6485
6486         /* begin clear state */
6487         count += 2;
6488         /* context control state */
6489         count += 3;
6490
6491         for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6492                 for (ext = sect->section; ext->extent != NULL; ++ext) {
6493                         if (sect->id == SECT_CONTEXT)
6494                                 count += 2 + ext->reg_count;
6495                         else
6496                                 return 0;
6497                 }
6498         }
6499         /* pa_sc_raster_config/pa_sc_raster_config1 */
6500         count += 4;
6501         /* end clear state */
6502         count += 2;
6503         /* clear state */
6504         count += 2;
6505
6506         return count;
6507 }
6508
6509 void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
6510 {
6511         u32 count = 0, i;
6512         const struct cs_section_def *sect = NULL;
6513         const struct cs_extent_def *ext = NULL;
6514
6515         if (rdev->rlc.cs_data == NULL)
6516                 return;
6517         if (buffer == NULL)
6518                 return;
6519
6520         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6521         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
6522
6523         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6524         buffer[count++] = cpu_to_le32(0x80000000);
6525         buffer[count++] = cpu_to_le32(0x80000000);
6526
6527         for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6528                 for (ext = sect->section; ext->extent != NULL; ++ext) {
6529                         if (sect->id == SECT_CONTEXT) {
6530                                 buffer[count++] =
6531                                         cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
6532                                 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
6533                                 for (i = 0; i < ext->reg_count; i++)
6534                                         buffer[count++] = cpu_to_le32(ext->extent[i]);
6535                         } else {
6536                                 return;
6537                         }
6538                 }
6539         }
6540
6541         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
6542         buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
6543         switch (rdev->family) {
6544         case CHIP_BONAIRE:
6545                 buffer[count++] = cpu_to_le32(0x16000012);
6546                 buffer[count++] = cpu_to_le32(0x00000000);
6547                 break;
6548         case CHIP_KAVERI:
6549                 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6550                 buffer[count++] = cpu_to_le32(0x00000000);
6551                 break;
6552         case CHIP_KABINI:
6553                 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6554                 buffer[count++] = cpu_to_le32(0x00000000);
6555                 break;
6556         case CHIP_HAWAII:
6557                 buffer[count++] = cpu_to_le32(0x3a00161a);
6558                 buffer[count++] = cpu_to_le32(0x0000002e);
6559                 break;
6560         default:
6561                 buffer[count++] = cpu_to_le32(0x00000000);
6562                 buffer[count++] = cpu_to_le32(0x00000000);
6563                 break;
6564         }
6565
6566         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6567         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
6568
6569         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
6570         buffer[count++] = cpu_to_le32(0);
6571 }
6572
6573 static void cik_init_pg(struct radeon_device *rdev)
6574 {
6575         if (rdev->pg_flags) {
6576                 cik_enable_sck_slowdown_on_pu(rdev, true);
6577                 cik_enable_sck_slowdown_on_pd(rdev, true);
6578                 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
6579                         cik_init_gfx_cgpg(rdev);
6580                         cik_enable_cp_pg(rdev, true);
6581                         cik_enable_gds_pg(rdev, true);
6582                 }
6583                 cik_init_ao_cu_mask(rdev);
6584                 cik_update_gfx_pg(rdev, true);
6585         }
6586 }
6587
6588 static void cik_fini_pg(struct radeon_device *rdev)
6589 {
6590         if (rdev->pg_flags) {
6591                 cik_update_gfx_pg(rdev, false);
6592                 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
6593                         cik_enable_cp_pg(rdev, false);
6594                         cik_enable_gds_pg(rdev, false);
6595                 }
6596         }
6597 }
6598
6599 /*
6600  * Interrupts
6601  * Starting with r6xx, interrupts are handled via a ring buffer.
6602  * Ring buffers are areas of GPU accessible memory that the GPU
6603  * writes interrupt vectors into and the host reads vectors out of.
6604  * There is a rptr (read pointer) that determines where the
6605  * host is currently reading, and a wptr (write pointer)
6606  * which determines where the GPU has written.  When the
6607  * pointers are equal, the ring is idle.  When the GPU
6608  * writes vectors to the ring buffer, it increments the
6609  * wptr.  When there is an interrupt, the host then starts
6610  * fetching commands and processing them until the pointers are
6611  * equal again at which point it updates the rptr.
6612  */
6613
6614 /**
6615  * cik_enable_interrupts - Enable the interrupt ring buffer
6616  *
6617  * @rdev: radeon_device pointer
6618  *
6619  * Enable the interrupt ring buffer (CIK).
6620  */
6621 static void cik_enable_interrupts(struct radeon_device *rdev)
6622 {
6623         u32 ih_cntl = RREG32(IH_CNTL);
6624         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6625
6626         ih_cntl |= ENABLE_INTR;
6627         ih_rb_cntl |= IH_RB_ENABLE;
6628         WREG32(IH_CNTL, ih_cntl);
6629         WREG32(IH_RB_CNTL, ih_rb_cntl);
6630         rdev->ih.enabled = true;
6631 }
6632
6633 /**
6634  * cik_disable_interrupts - Disable the interrupt ring buffer
6635  *
6636  * @rdev: radeon_device pointer
6637  *
6638  * Disable the interrupt ring buffer (CIK).
6639  */
6640 static void cik_disable_interrupts(struct radeon_device *rdev)
6641 {
6642         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6643         u32 ih_cntl = RREG32(IH_CNTL);
6644
6645         ih_rb_cntl &= ~IH_RB_ENABLE;
6646         ih_cntl &= ~ENABLE_INTR;
6647         WREG32(IH_RB_CNTL, ih_rb_cntl);
6648         WREG32(IH_CNTL, ih_cntl);
6649         /* set rptr, wptr to 0 */
6650         WREG32(IH_RB_RPTR, 0);
6651         WREG32(IH_RB_WPTR, 0);
6652         rdev->ih.enabled = false;
6653         rdev->ih.rptr = 0;
6654 }
6655
6656 /**
6657  * cik_disable_interrupt_state - Disable all interrupt sources
6658  *
6659  * @rdev: radeon_device pointer
6660  *
6661  * Clear all interrupt enable bits used by the driver (CIK).
6662  */
6663 static void cik_disable_interrupt_state(struct radeon_device *rdev)
6664 {
6665         u32 tmp;
6666
6667         /* gfx ring */
6668         tmp = RREG32(CP_INT_CNTL_RING0) &
6669                 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6670         WREG32(CP_INT_CNTL_RING0, tmp);
6671         /* sdma */
6672         tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6673         WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
6674         tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6675         WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
6676         /* compute queues */
6677         WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
6678         WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
6679         WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
6680         WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
6681         WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
6682         WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
6683         WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
6684         WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
6685         /* grbm */
6686         WREG32(GRBM_INT_CNTL, 0);
6687         /* vline/vblank, etc. */
6688         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6689         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6690         if (rdev->num_crtc >= 4) {
6691                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
6692                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
6693         }
6694         if (rdev->num_crtc >= 6) {
6695                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
6696                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
6697         }
6698
6699         /* dac hotplug */
6700         WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
6701
6702         /* digital hotplug */
6703         tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6704         WREG32(DC_HPD1_INT_CONTROL, tmp);
6705         tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6706         WREG32(DC_HPD2_INT_CONTROL, tmp);
6707         tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6708         WREG32(DC_HPD3_INT_CONTROL, tmp);
6709         tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6710         WREG32(DC_HPD4_INT_CONTROL, tmp);
6711         tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6712         WREG32(DC_HPD5_INT_CONTROL, tmp);
6713         tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6714         WREG32(DC_HPD6_INT_CONTROL, tmp);
6715
6716 }
6717
6718 /**
6719  * cik_irq_init - init and enable the interrupt ring
6720  *
6721  * @rdev: radeon_device pointer
6722  *
6723  * Allocate a ring buffer for the interrupt controller,
6724  * enable the RLC, disable interrupts, enable the IH
6725  * ring buffer and enable it (CIK).
6726  * Called at device load and reume.
6727  * Returns 0 for success, errors for failure.
6728  */
6729 static int cik_irq_init(struct radeon_device *rdev)
6730 {
6731         int ret = 0;
6732         int rb_bufsz;
6733         u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
6734
6735         /* allocate ring */
6736         ret = r600_ih_ring_alloc(rdev);
6737         if (ret)
6738                 return ret;
6739
6740         /* disable irqs */
6741         cik_disable_interrupts(rdev);
6742
6743         /* init rlc */
6744         ret = cik_rlc_resume(rdev);
6745         if (ret) {
6746                 r600_ih_ring_fini(rdev);
6747                 return ret;
6748         }
6749
6750         /* setup interrupt control */
6751         /* XXX this should actually be a bus address, not an MC address. same on older asics */
6752         WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
6753         interrupt_cntl = RREG32(INTERRUPT_CNTL);
6754         /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
6755          * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
6756          */
6757         interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
6758         /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
6759         interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
6760         WREG32(INTERRUPT_CNTL, interrupt_cntl);
6761
6762         WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
6763         rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
6764
6765         ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
6766                       IH_WPTR_OVERFLOW_CLEAR |
6767                       (rb_bufsz << 1));
6768
6769         if (rdev->wb.enabled)
6770                 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
6771
6772         /* set the writeback address whether it's enabled or not */
6773         WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
6774         WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
6775
6776         WREG32(IH_RB_CNTL, ih_rb_cntl);
6777
6778         /* set rptr, wptr to 0 */
6779         WREG32(IH_RB_RPTR, 0);
6780         WREG32(IH_RB_WPTR, 0);
6781
6782         /* Default settings for IH_CNTL (disabled at first) */
6783         ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
6784         /* RPTR_REARM only works if msi's are enabled */
6785         if (rdev->msi_enabled)
6786                 ih_cntl |= RPTR_REARM;
6787         WREG32(IH_CNTL, ih_cntl);
6788
6789         /* force the active interrupt state to all disabled */
6790         cik_disable_interrupt_state(rdev);
6791
6792         pci_set_master(rdev->pdev);
6793
6794         /* enable irqs */
6795         cik_enable_interrupts(rdev);
6796
6797         return ret;
6798 }
6799
6800 /**
6801  * cik_irq_set - enable/disable interrupt sources
6802  *
6803  * @rdev: radeon_device pointer
6804  *
6805  * Enable interrupt sources on the GPU (vblanks, hpd,
6806  * etc.) (CIK).
6807  * Returns 0 for success, errors for failure.
6808  */
6809 int cik_irq_set(struct radeon_device *rdev)
6810 {
6811         u32 cp_int_cntl;
6812         u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;
6813         u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
6814         u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
6815         u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
6816         u32 grbm_int_cntl = 0;
6817         u32 dma_cntl, dma_cntl1;
6818         u32 thermal_int;
6819
6820         if (!rdev->irq.installed) {
6821                 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6822                 return -EINVAL;
6823         }
6824         /* don't enable anything if the ih is disabled */
6825         if (!rdev->ih.enabled) {
6826                 cik_disable_interrupts(rdev);
6827                 /* force the active interrupt state to all disabled */
6828                 cik_disable_interrupt_state(rdev);
6829                 return 0;
6830         }
6831
6832         cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
6833                 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6834         cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
6835
6836         hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
6837         hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
6838         hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
6839         hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
6840         hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
6841         hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
6842
6843         dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6844         dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6845
6846         cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6847         cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6848         cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6849         cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6850         cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6851         cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6852         cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6853         cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6854
6855         if (rdev->flags & RADEON_IS_IGP)
6856                 thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) &
6857                         ~(THERM_INTH_MASK | THERM_INTL_MASK);
6858         else
6859                 thermal_int = RREG32_SMC(CG_THERMAL_INT) &
6860                         ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
6861
6862         /* enable CP interrupts on all rings */
6863         if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
6864                 DRM_DEBUG("cik_irq_set: sw int gfx\n");
6865                 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
6866         }
6867         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
6868                 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6869                 DRM_DEBUG("si_irq_set: sw int cp1\n");
6870                 if (ring->me == 1) {
6871                         switch (ring->pipe) {
6872                         case 0:
6873                                 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6874                                 break;
6875                         case 1:
6876                                 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6877                                 break;
6878                         case 2:
6879                                 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6880                                 break;
6881                         case 3:
6882                                 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6883                                 break;
6884                         default:
6885                                 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6886                                 break;
6887                         }
6888                 } else if (ring->me == 2) {
6889                         switch (ring->pipe) {
6890                         case 0:
6891                                 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6892                                 break;
6893                         case 1:
6894                                 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6895                                 break;
6896                         case 2:
6897                                 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6898                                 break;
6899                         case 3:
6900                                 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6901                                 break;
6902                         default:
6903                                 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6904                                 break;
6905                         }
6906                 } else {
6907                         DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
6908                 }
6909         }
6910         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
6911                 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6912                 DRM_DEBUG("si_irq_set: sw int cp2\n");
6913                 if (ring->me == 1) {
6914                         switch (ring->pipe) {
6915                         case 0:
6916                                 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6917                                 break;
6918                         case 1:
6919                                 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6920                                 break;
6921                         case 2:
6922                                 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6923                                 break;
6924                         case 3:
6925                                 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6926                                 break;
6927                         default:
6928                                 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6929                                 break;
6930                         }
6931                 } else if (ring->me == 2) {
6932                         switch (ring->pipe) {
6933                         case 0:
6934                                 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6935                                 break;
6936                         case 1:
6937                                 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6938                                 break;
6939                         case 2:
6940                                 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6941                                 break;
6942                         case 3:
6943                                 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6944                                 break;
6945                         default:
6946                                 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6947                                 break;
6948                         }
6949                 } else {
6950                         DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
6951                 }
6952         }
6953
6954         if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
6955                 DRM_DEBUG("cik_irq_set: sw int dma\n");
6956                 dma_cntl |= TRAP_ENABLE;
6957         }
6958
6959         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
6960                 DRM_DEBUG("cik_irq_set: sw int dma1\n");
6961                 dma_cntl1 |= TRAP_ENABLE;
6962         }
6963
6964         if (rdev->irq.crtc_vblank_int[0] ||
6965             atomic_read(&rdev->irq.pflip[0])) {
6966                 DRM_DEBUG("cik_irq_set: vblank 0\n");
6967                 crtc1 |= VBLANK_INTERRUPT_MASK;
6968         }
6969         if (rdev->irq.crtc_vblank_int[1] ||
6970             atomic_read(&rdev->irq.pflip[1])) {
6971                 DRM_DEBUG("cik_irq_set: vblank 1\n");
6972                 crtc2 |= VBLANK_INTERRUPT_MASK;
6973         }
6974         if (rdev->irq.crtc_vblank_int[2] ||
6975             atomic_read(&rdev->irq.pflip[2])) {
6976                 DRM_DEBUG("cik_irq_set: vblank 2\n");
6977                 crtc3 |= VBLANK_INTERRUPT_MASK;
6978         }
6979         if (rdev->irq.crtc_vblank_int[3] ||
6980             atomic_read(&rdev->irq.pflip[3])) {
6981                 DRM_DEBUG("cik_irq_set: vblank 3\n");
6982                 crtc4 |= VBLANK_INTERRUPT_MASK;
6983         }
6984         if (rdev->irq.crtc_vblank_int[4] ||
6985             atomic_read(&rdev->irq.pflip[4])) {
6986                 DRM_DEBUG("cik_irq_set: vblank 4\n");
6987                 crtc5 |= VBLANK_INTERRUPT_MASK;
6988         }
6989         if (rdev->irq.crtc_vblank_int[5] ||
6990             atomic_read(&rdev->irq.pflip[5])) {
6991                 DRM_DEBUG("cik_irq_set: vblank 5\n");
6992                 crtc6 |= VBLANK_INTERRUPT_MASK;
6993         }
6994         if (rdev->irq.hpd[0]) {
6995                 DRM_DEBUG("cik_irq_set: hpd 1\n");
6996                 hpd1 |= DC_HPDx_INT_EN;
6997         }
6998         if (rdev->irq.hpd[1]) {
6999                 DRM_DEBUG("cik_irq_set: hpd 2\n");
7000                 hpd2 |= DC_HPDx_INT_EN;
7001         }
7002         if (rdev->irq.hpd[2]) {
7003                 DRM_DEBUG("cik_irq_set: hpd 3\n");
7004                 hpd3 |= DC_HPDx_INT_EN;
7005         }
7006         if (rdev->irq.hpd[3]) {
7007                 DRM_DEBUG("cik_irq_set: hpd 4\n");
7008                 hpd4 |= DC_HPDx_INT_EN;
7009         }
7010         if (rdev->irq.hpd[4]) {
7011                 DRM_DEBUG("cik_irq_set: hpd 5\n");
7012                 hpd5 |= DC_HPDx_INT_EN;
7013         }
7014         if (rdev->irq.hpd[5]) {
7015                 DRM_DEBUG("cik_irq_set: hpd 6\n");
7016                 hpd6 |= DC_HPDx_INT_EN;
7017         }
7018
7019         if (rdev->irq.dpm_thermal) {
7020                 DRM_DEBUG("dpm thermal\n");
7021                 if (rdev->flags & RADEON_IS_IGP)
7022                         thermal_int |= THERM_INTH_MASK | THERM_INTL_MASK;
7023                 else
7024                         thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
7025         }
7026
7027         WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
7028
7029         WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
7030         WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
7031
7032         WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
7033         WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
7034         WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
7035         WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
7036         WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
7037         WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
7038         WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
7039         WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
7040
7041         WREG32(GRBM_INT_CNTL, grbm_int_cntl);
7042
7043         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
7044         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
7045         if (rdev->num_crtc >= 4) {
7046                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
7047                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
7048         }
7049         if (rdev->num_crtc >= 6) {
7050                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
7051                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
7052         }
7053
7054         WREG32(DC_HPD1_INT_CONTROL, hpd1);
7055         WREG32(DC_HPD2_INT_CONTROL, hpd2);
7056         WREG32(DC_HPD3_INT_CONTROL, hpd3);
7057         WREG32(DC_HPD4_INT_CONTROL, hpd4);
7058         WREG32(DC_HPD5_INT_CONTROL, hpd5);
7059         WREG32(DC_HPD6_INT_CONTROL, hpd6);
7060
7061         if (rdev->flags & RADEON_IS_IGP)
7062                 WREG32_SMC(CG_THERMAL_INT_CTRL, thermal_int);
7063         else
7064                 WREG32_SMC(CG_THERMAL_INT, thermal_int);
7065
7066         return 0;
7067 }
7068
7069 /**
7070  * cik_irq_ack - ack interrupt sources
7071  *
7072  * @rdev: radeon_device pointer
7073  *
7074  * Ack interrupt sources on the GPU (vblanks, hpd,
7075  * etc.) (CIK).  Certain interrupts sources are sw
7076  * generated and do not require an explicit ack.
7077  */
7078 static inline void cik_irq_ack(struct radeon_device *rdev)
7079 {
7080         u32 tmp;
7081
7082         rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
7083         rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
7084         rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
7085         rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
7086         rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
7087         rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
7088         rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
7089
7090         if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
7091                 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
7092         if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
7093                 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
7094         if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
7095                 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
7096         if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
7097                 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
7098
7099         if (rdev->num_crtc >= 4) {
7100                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
7101                         WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
7102                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
7103                         WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
7104                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
7105                         WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
7106                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
7107                         WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
7108         }
7109
7110         if (rdev->num_crtc >= 6) {
7111                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
7112                         WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
7113                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
7114                         WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
7115                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
7116                         WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
7117                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
7118                         WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
7119         }
7120
7121         if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7122                 tmp = RREG32(DC_HPD1_INT_CONTROL);
7123                 tmp |= DC_HPDx_INT_ACK;
7124                 WREG32(DC_HPD1_INT_CONTROL, tmp);
7125         }
7126         if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7127                 tmp = RREG32(DC_HPD2_INT_CONTROL);
7128                 tmp |= DC_HPDx_INT_ACK;
7129                 WREG32(DC_HPD2_INT_CONTROL, tmp);
7130         }
7131         if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7132                 tmp = RREG32(DC_HPD3_INT_CONTROL);
7133                 tmp |= DC_HPDx_INT_ACK;
7134                 WREG32(DC_HPD3_INT_CONTROL, tmp);
7135         }
7136         if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7137                 tmp = RREG32(DC_HPD4_INT_CONTROL);
7138                 tmp |= DC_HPDx_INT_ACK;
7139                 WREG32(DC_HPD4_INT_CONTROL, tmp);
7140         }
7141         if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7142                 tmp = RREG32(DC_HPD5_INT_CONTROL);
7143                 tmp |= DC_HPDx_INT_ACK;
7144                 WREG32(DC_HPD5_INT_CONTROL, tmp);
7145         }
7146         if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7147                 tmp = RREG32(DC_HPD5_INT_CONTROL);
7148                 tmp |= DC_HPDx_INT_ACK;
7149                 WREG32(DC_HPD6_INT_CONTROL, tmp);
7150         }
7151 }
7152
7153 /**
7154  * cik_irq_disable - disable interrupts
7155  *
7156  * @rdev: radeon_device pointer
7157  *
7158  * Disable interrupts on the hw (CIK).
7159  */
7160 static void cik_irq_disable(struct radeon_device *rdev)
7161 {
7162         cik_disable_interrupts(rdev);
7163         /* Wait and acknowledge irq */
7164         mdelay(1);
7165         cik_irq_ack(rdev);
7166         cik_disable_interrupt_state(rdev);
7167 }
7168
7169 /**
7170  * cik_irq_disable - disable interrupts for suspend
7171  *
7172  * @rdev: radeon_device pointer
7173  *
7174  * Disable interrupts and stop the RLC (CIK).
7175  * Used for suspend.
7176  */
7177 static void cik_irq_suspend(struct radeon_device *rdev)
7178 {
7179         cik_irq_disable(rdev);
7180         cik_rlc_stop(rdev);
7181 }
7182
7183 /**
7184  * cik_irq_fini - tear down interrupt support
7185  *
7186  * @rdev: radeon_device pointer
7187  *
7188  * Disable interrupts on the hw and free the IH ring
7189  * buffer (CIK).
7190  * Used for driver unload.
7191  */
7192 static void cik_irq_fini(struct radeon_device *rdev)
7193 {
7194         cik_irq_suspend(rdev);
7195         r600_ih_ring_fini(rdev);
7196 }
7197
7198 /**
7199  * cik_get_ih_wptr - get the IH ring buffer wptr
7200  *
7201  * @rdev: radeon_device pointer
7202  *
7203  * Get the IH ring buffer wptr from either the register
7204  * or the writeback memory buffer (CIK).  Also check for
7205  * ring buffer overflow and deal with it.
7206  * Used by cik_irq_process().
7207  * Returns the value of the wptr.
7208  */
7209 static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
7210 {
7211         u32 wptr, tmp;
7212
7213         if (rdev->wb.enabled)
7214                 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
7215         else
7216                 wptr = RREG32(IH_RB_WPTR);
7217
7218         if (wptr & RB_OVERFLOW) {
7219                 /* When a ring buffer overflow happen start parsing interrupt
7220                  * from the last not overwritten vector (wptr + 16). Hopefully
7221                  * this should allow us to catchup.
7222                  */
7223                 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
7224                         wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
7225                 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
7226                 tmp = RREG32(IH_RB_CNTL);
7227                 tmp |= IH_WPTR_OVERFLOW_CLEAR;
7228                 WREG32(IH_RB_CNTL, tmp);
7229         }
7230         return (wptr & rdev->ih.ptr_mask);
7231 }
7232
7233 /*        CIK IV Ring
7234  * Each IV ring entry is 128 bits:
7235  * [7:0]    - interrupt source id
7236  * [31:8]   - reserved
7237  * [59:32]  - interrupt source data
7238  * [63:60]  - reserved
7239  * [71:64]  - RINGID
7240  *            CP:
7241  *            ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
7242  *            QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7243  *                     - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7244  *            ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7245  *            PIPE_ID - ME0 0=3D
7246  *                    - ME1&2 compute dispatcher (4 pipes each)
7247  *            SDMA:
7248  *            INSTANCE_ID [1:0], QUEUE_ID[1:0]
7249  *            INSTANCE_ID - 0 = sdma0, 1 = sdma1
7250  *            QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
7251  * [79:72]  - VMID
7252  * [95:80]  - PASID
7253  * [127:96] - reserved
7254  */
7255 /**
7256  * cik_irq_process - interrupt handler
7257  *
7258  * @rdev: radeon_device pointer
7259  *
7260  * Interrupt hander (CIK).  Walk the IH ring,
7261  * ack interrupts and schedule work to handle
7262  * interrupt events.
7263  * Returns irq process return code.
7264  */
7265 int cik_irq_process(struct radeon_device *rdev)
7266 {
7267         struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7268         struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7269         u32 wptr;
7270         u32 rptr;
7271         u32 src_id, src_data, ring_id;
7272         u8 me_id, pipe_id, queue_id;
7273         u32 ring_index;
7274         bool queue_hotplug = false;
7275         bool queue_reset = false;
7276         u32 addr, status, mc_client;
7277         bool queue_thermal = false;
7278
7279         if (!rdev->ih.enabled || rdev->shutdown)
7280                 return IRQ_NONE;
7281
7282         wptr = cik_get_ih_wptr(rdev);
7283
7284 restart_ih:
7285         /* is somebody else already processing irqs? */
7286         if (atomic_xchg(&rdev->ih.lock, 1))
7287                 return IRQ_NONE;
7288
7289         rptr = rdev->ih.rptr;
7290         DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7291
7292         /* Order reading of wptr vs. reading of IH ring data */
7293         rmb();
7294
7295         /* display interrupts */
7296         cik_irq_ack(rdev);
7297
7298         while (rptr != wptr) {
7299                 /* wptr/rptr are in bytes! */
7300                 ring_index = rptr / 4;
7301                 src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7302                 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7303                 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
7304
7305                 switch (src_id) {
7306                 case 1: /* D1 vblank/vline */
7307                         switch (src_data) {
7308                         case 0: /* D1 vblank */
7309                                 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
7310                                         if (rdev->irq.crtc_vblank_int[0]) {
7311                                                 drm_handle_vblank(rdev->ddev, 0);
7312                                                 rdev->pm.vblank_sync = true;
7313                                                 wake_up(&rdev->irq.vblank_queue);
7314                                         }
7315                                         if (atomic_read(&rdev->irq.pflip[0]))
7316                                                 radeon_crtc_handle_flip(rdev, 0);
7317                                         rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7318                                         DRM_DEBUG("IH: D1 vblank\n");
7319                                 }
7320                                 break;
7321                         case 1: /* D1 vline */
7322                                 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
7323                                         rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7324                                         DRM_DEBUG("IH: D1 vline\n");
7325                                 }
7326                                 break;
7327                         default:
7328                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7329                                 break;
7330                         }
7331                         break;
7332                 case 2: /* D2 vblank/vline */
7333                         switch (src_data) {
7334                         case 0: /* D2 vblank */
7335                                 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
7336                                         if (rdev->irq.crtc_vblank_int[1]) {
7337                                                 drm_handle_vblank(rdev->ddev, 1);
7338                                                 rdev->pm.vblank_sync = true;
7339                                                 wake_up(&rdev->irq.vblank_queue);
7340                                         }
7341                                         if (atomic_read(&rdev->irq.pflip[1]))
7342                                                 radeon_crtc_handle_flip(rdev, 1);
7343                                         rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7344                                         DRM_DEBUG("IH: D2 vblank\n");
7345                                 }
7346                                 break;
7347                         case 1: /* D2 vline */
7348                                 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
7349                                         rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7350                                         DRM_DEBUG("IH: D2 vline\n");
7351                                 }
7352                                 break;
7353                         default:
7354                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7355                                 break;
7356                         }
7357                         break;
7358                 case 3: /* D3 vblank/vline */
7359                         switch (src_data) {
7360                         case 0: /* D3 vblank */
7361                                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
7362                                         if (rdev->irq.crtc_vblank_int[2]) {
7363                                                 drm_handle_vblank(rdev->ddev, 2);
7364                                                 rdev->pm.vblank_sync = true;
7365                                                 wake_up(&rdev->irq.vblank_queue);
7366                                         }
7367                                         if (atomic_read(&rdev->irq.pflip[2]))
7368                                                 radeon_crtc_handle_flip(rdev, 2);
7369                                         rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7370                                         DRM_DEBUG("IH: D3 vblank\n");
7371                                 }
7372                                 break;
7373                         case 1: /* D3 vline */
7374                                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
7375                                         rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7376                                         DRM_DEBUG("IH: D3 vline\n");
7377                                 }
7378                                 break;
7379                         default:
7380                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7381                                 break;
7382                         }
7383                         break;
7384                 case 4: /* D4 vblank/vline */
7385                         switch (src_data) {
7386                         case 0: /* D4 vblank */
7387                                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
7388                                         if (rdev->irq.crtc_vblank_int[3]) {
7389                                                 drm_handle_vblank(rdev->ddev, 3);
7390                                                 rdev->pm.vblank_sync = true;
7391                                                 wake_up(&rdev->irq.vblank_queue);
7392                                         }
7393                                         if (atomic_read(&rdev->irq.pflip[3]))
7394                                                 radeon_crtc_handle_flip(rdev, 3);
7395                                         rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7396                                         DRM_DEBUG("IH: D4 vblank\n");
7397                                 }
7398                                 break;
7399                         case 1: /* D4 vline */
7400                                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
7401                                         rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
7402                                         DRM_DEBUG("IH: D4 vline\n");
7403                                 }
7404                                 break;
7405                         default:
7406                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7407                                 break;
7408                         }
7409                         break;
7410                 case 5: /* D5 vblank/vline */
7411                         switch (src_data) {
7412                         case 0: /* D5 vblank */
7413                                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
7414                                         if (rdev->irq.crtc_vblank_int[4]) {
7415                                                 drm_handle_vblank(rdev->ddev, 4);
7416                                                 rdev->pm.vblank_sync = true;
7417                                                 wake_up(&rdev->irq.vblank_queue);
7418                                         }
7419                                         if (atomic_read(&rdev->irq.pflip[4]))
7420                                                 radeon_crtc_handle_flip(rdev, 4);
7421                                         rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
7422                                         DRM_DEBUG("IH: D5 vblank\n");
7423                                 }
7424                                 break;
7425                         case 1: /* D5 vline */
7426                                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
7427                                         rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
7428                                         DRM_DEBUG("IH: D5 vline\n");
7429                                 }
7430                                 break;
7431                         default:
7432                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7433                                 break;
7434                         }
7435                         break;
7436                 case 6: /* D6 vblank/vline */
7437                         switch (src_data) {
7438                         case 0: /* D6 vblank */
7439                                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
7440                                         if (rdev->irq.crtc_vblank_int[5]) {
7441                                                 drm_handle_vblank(rdev->ddev, 5);
7442                                                 rdev->pm.vblank_sync = true;
7443                                                 wake_up(&rdev->irq.vblank_queue);
7444                                         }
7445                                         if (atomic_read(&rdev->irq.pflip[5]))
7446                                                 radeon_crtc_handle_flip(rdev, 5);
7447                                         rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
7448                                         DRM_DEBUG("IH: D6 vblank\n");
7449                                 }
7450                                 break;
7451                         case 1: /* D6 vline */
7452                                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
7453                                         rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
7454                                         DRM_DEBUG("IH: D6 vline\n");
7455                                 }
7456                                 break;
7457                         default:
7458                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7459                                 break;
7460                         }
7461                         break;
7462                 case 42: /* HPD hotplug */
7463                         switch (src_data) {
7464                         case 0:
7465                                 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7466                                         rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
7467                                         queue_hotplug = true;
7468                                         DRM_DEBUG("IH: HPD1\n");
7469                                 }
7470                                 break;
7471                         case 1:
7472                                 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7473                                         rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
7474                                         queue_hotplug = true;
7475                                         DRM_DEBUG("IH: HPD2\n");
7476                                 }
7477                                 break;
7478                         case 2:
7479                                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7480                                         rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
7481                                         queue_hotplug = true;
7482                                         DRM_DEBUG("IH: HPD3\n");
7483                                 }
7484                                 break;
7485                         case 3:
7486                                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7487                                         rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
7488                                         queue_hotplug = true;
7489                                         DRM_DEBUG("IH: HPD4\n");
7490                                 }
7491                                 break;
7492                         case 4:
7493                                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7494                                         rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
7495                                         queue_hotplug = true;
7496                                         DRM_DEBUG("IH: HPD5\n");
7497                                 }
7498                                 break;
7499                         case 5:
7500                                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7501                                         rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
7502                                         queue_hotplug = true;
7503                                         DRM_DEBUG("IH: HPD6\n");
7504                                 }
7505                                 break;
7506                         default:
7507                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7508                                 break;
7509                         }
7510                         break;
7511                 case 124: /* UVD */
7512                         DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
7513                         radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
7514                         break;
7515                 case 146:
7516                 case 147:
7517                         addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
7518                         status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
7519                         mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
7520                         dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
7521                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
7522                                 addr);
7523                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
7524                                 status);
7525                         cik_vm_decode_fault(rdev, status, addr, mc_client);
7526                         /* reset addr and status */
7527                         WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
7528                         break;
7529                 case 167: /* VCE */
7530                         DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data);
7531                         switch (src_data) {
7532                         case 0:
7533                                 radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX);
7534                                 break;
7535                         case 1:
7536                                 radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX);
7537                                 break;
7538                         default:
7539                                 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
7540                                 break;
7541                         }
7542                         break;
7543                 case 176: /* GFX RB CP_INT */
7544                 case 177: /* GFX IB CP_INT */
7545                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7546                         break;
7547                 case 181: /* CP EOP event */
7548                         DRM_DEBUG("IH: CP EOP\n");
7549                         /* XXX check the bitfield order! */
7550                         me_id = (ring_id & 0x60) >> 5;
7551                         pipe_id = (ring_id & 0x18) >> 3;
7552                         queue_id = (ring_id & 0x7) >> 0;
7553                         switch (me_id) {
7554                         case 0:
7555                                 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7556                                 break;
7557                         case 1:
7558                         case 2:
7559                                 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
7560                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7561                                 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
7562                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7563                                 break;
7564                         }
7565                         break;
7566                 case 184: /* CP Privileged reg access */
7567                         DRM_ERROR("Illegal register access in command stream\n");
7568                         /* XXX check the bitfield order! */
7569                         me_id = (ring_id & 0x60) >> 5;
7570                         pipe_id = (ring_id & 0x18) >> 3;
7571                         queue_id = (ring_id & 0x7) >> 0;
7572                         switch (me_id) {
7573                         case 0:
7574                                 /* This results in a full GPU reset, but all we need to do is soft
7575                                  * reset the CP for gfx
7576                                  */
7577                                 queue_reset = true;
7578                                 break;
7579                         case 1:
7580                                 /* XXX compute */
7581                                 queue_reset = true;
7582                                 break;
7583                         case 2:
7584                                 /* XXX compute */
7585                                 queue_reset = true;
7586                                 break;
7587                         }
7588                         break;
7589                 case 185: /* CP Privileged inst */
7590                         DRM_ERROR("Illegal instruction in command stream\n");
7591                         /* XXX check the bitfield order! */
7592                         me_id = (ring_id & 0x60) >> 5;
7593                         pipe_id = (ring_id & 0x18) >> 3;
7594                         queue_id = (ring_id & 0x7) >> 0;
7595                         switch (me_id) {
7596                         case 0:
7597                                 /* This results in a full GPU reset, but all we need to do is soft
7598                                  * reset the CP for gfx
7599                                  */
7600                                 queue_reset = true;
7601                                 break;
7602                         case 1:
7603                                 /* XXX compute */
7604                                 queue_reset = true;
7605                                 break;
7606                         case 2:
7607                                 /* XXX compute */
7608                                 queue_reset = true;
7609                                 break;
7610                         }
7611                         break;
7612                 case 224: /* SDMA trap event */
7613                         /* XXX check the bitfield order! */
7614                         me_id = (ring_id & 0x3) >> 0;
7615                         queue_id = (ring_id & 0xc) >> 2;
7616                         DRM_DEBUG("IH: SDMA trap\n");
7617                         switch (me_id) {
7618                         case 0:
7619                                 switch (queue_id) {
7620                                 case 0:
7621                                         radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
7622                                         break;
7623                                 case 1:
7624                                         /* XXX compute */
7625                                         break;
7626                                 case 2:
7627                                         /* XXX compute */
7628                                         break;
7629                                 }
7630                                 break;
7631                         case 1:
7632                                 switch (queue_id) {
7633                                 case 0:
7634                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7635                                         break;
7636                                 case 1:
7637                                         /* XXX compute */
7638                                         break;
7639                                 case 2:
7640                                         /* XXX compute */
7641                                         break;
7642                                 }
7643                                 break;
7644                         }
7645                         break;
7646                 case 230: /* thermal low to high */
7647                         DRM_DEBUG("IH: thermal low to high\n");
7648                         rdev->pm.dpm.thermal.high_to_low = false;
7649                         queue_thermal = true;
7650                         break;
7651                 case 231: /* thermal high to low */
7652                         DRM_DEBUG("IH: thermal high to low\n");
7653                         rdev->pm.dpm.thermal.high_to_low = true;
7654                         queue_thermal = true;
7655                         break;
7656                 case 233: /* GUI IDLE */
7657                         DRM_DEBUG("IH: GUI idle\n");
7658                         break;
7659                 case 241: /* SDMA Privileged inst */
7660                 case 247: /* SDMA Privileged inst */
7661                         DRM_ERROR("Illegal instruction in SDMA command stream\n");
7662                         /* XXX check the bitfield order! */
7663                         me_id = (ring_id & 0x3) >> 0;
7664                         queue_id = (ring_id & 0xc) >> 2;
7665                         switch (me_id) {
7666                         case 0:
7667                                 switch (queue_id) {
7668                                 case 0:
7669                                         queue_reset = true;
7670                                         break;
7671                                 case 1:
7672                                         /* XXX compute */
7673                                         queue_reset = true;
7674                                         break;
7675                                 case 2:
7676                                         /* XXX compute */
7677                                         queue_reset = true;
7678                                         break;
7679                                 }
7680                                 break;
7681                         case 1:
7682                                 switch (queue_id) {
7683                                 case 0:
7684                                         queue_reset = true;
7685                                         break;
7686                                 case 1:
7687                                         /* XXX compute */
7688                                         queue_reset = true;
7689                                         break;
7690                                 case 2:
7691                                         /* XXX compute */
7692                                         queue_reset = true;
7693                                         break;
7694                                 }
7695                                 break;
7696                         }
7697                         break;
7698                 default:
7699                         DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7700                         break;
7701                 }
7702
7703                 /* wptr/rptr are in bytes! */
7704                 rptr += 16;
7705                 rptr &= rdev->ih.ptr_mask;
7706         }
7707         if (queue_hotplug)
7708                 schedule_work(&rdev->hotplug_work);
7709         if (queue_reset)
7710                 schedule_work(&rdev->reset_work);
7711         if (queue_thermal)
7712                 schedule_work(&rdev->pm.dpm.thermal.work);
7713         rdev->ih.rptr = rptr;
7714         WREG32(IH_RB_RPTR, rdev->ih.rptr);
7715         atomic_set(&rdev->ih.lock, 0);
7716
7717         /* make sure wptr hasn't changed while processing */
7718         wptr = cik_get_ih_wptr(rdev);
7719         if (wptr != rptr)
7720                 goto restart_ih;
7721
7722         return IRQ_HANDLED;
7723 }
7724
7725 /*
7726  * startup/shutdown callbacks
7727  */
7728 /**
7729  * cik_startup - program the asic to a functional state
7730  *
7731  * @rdev: radeon_device pointer
7732  *
7733  * Programs the asic to a functional state (CIK).
7734  * Called by cik_init() and cik_resume().
7735  * Returns 0 for success, error for failure.
7736  */
7737 static int cik_startup(struct radeon_device *rdev)
7738 {
7739         struct radeon_ring *ring;
7740         int r;
7741
7742         /* enable pcie gen2/3 link */
7743         cik_pcie_gen3_enable(rdev);
7744         /* enable aspm */
7745         cik_program_aspm(rdev);
7746
7747         /* scratch needs to be initialized before MC */
7748         r = r600_vram_scratch_init(rdev);
7749         if (r)
7750                 return r;
7751
7752         cik_mc_program(rdev);
7753
7754         if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
7755                 r = ci_mc_load_microcode(rdev);
7756                 if (r) {
7757                         DRM_ERROR("Failed to load MC firmware!\n");
7758                         return r;
7759                 }
7760         }
7761
7762         r = cik_pcie_gart_enable(rdev);
7763         if (r)
7764                 return r;
7765         cik_gpu_init(rdev);
7766
7767         /* allocate rlc buffers */
7768         if (rdev->flags & RADEON_IS_IGP) {
7769                 if (rdev->family == CHIP_KAVERI) {
7770                         rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
7771                         rdev->rlc.reg_list_size =
7772                                 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
7773                 } else {
7774                         rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
7775                         rdev->rlc.reg_list_size =
7776                                 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
7777                 }
7778         }
7779         rdev->rlc.cs_data = ci_cs_data;
7780         rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
7781         r = sumo_rlc_init(rdev);
7782         if (r) {
7783                 DRM_ERROR("Failed to init rlc BOs!\n");
7784                 return r;
7785         }
7786
7787         /* allocate wb buffer */
7788         r = radeon_wb_init(rdev);
7789         if (r)
7790                 return r;
7791
7792         /* allocate mec buffers */
7793         r = cik_mec_init(rdev);
7794         if (r) {
7795                 DRM_ERROR("Failed to init MEC BOs!\n");
7796                 return r;
7797         }
7798
7799         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
7800         if (r) {
7801                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7802                 return r;
7803         }
7804
7805         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7806         if (r) {
7807                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7808                 return r;
7809         }
7810
7811         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7812         if (r) {
7813                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7814                 return r;
7815         }
7816
7817         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
7818         if (r) {
7819                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7820                 return r;
7821         }
7822
7823         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7824         if (r) {
7825                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7826                 return r;
7827         }
7828
7829         r = radeon_uvd_resume(rdev);
7830         if (!r) {
7831                 r = uvd_v4_2_resume(rdev);
7832                 if (!r) {
7833                         r = radeon_fence_driver_start_ring(rdev,
7834                                                            R600_RING_TYPE_UVD_INDEX);
7835                         if (r)
7836                                 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
7837                 }
7838         }
7839         if (r)
7840                 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
7841
7842         r = radeon_vce_resume(rdev);
7843         if (!r) {
7844                 r = vce_v2_0_resume(rdev);
7845                 if (!r)
7846                         r = radeon_fence_driver_start_ring(rdev,
7847                                                            TN_RING_TYPE_VCE1_INDEX);
7848                 if (!r)
7849                         r = radeon_fence_driver_start_ring(rdev,
7850                                                            TN_RING_TYPE_VCE2_INDEX);
7851         }
7852         if (r) {
7853                 dev_err(rdev->dev, "VCE init error (%d).\n", r);
7854                 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
7855                 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
7856         }
7857
7858         /* Enable IRQ */
7859         if (!rdev->irq.installed) {
7860                 r = radeon_irq_kms_init(rdev);
7861                 if (r)
7862                         return r;
7863         }
7864
7865         r = cik_irq_init(rdev);
7866         if (r) {
7867                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
7868                 radeon_irq_kms_fini(rdev);
7869                 return r;
7870         }
7871         cik_irq_set(rdev);
7872
7873         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7874         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
7875                              PACKET3(PACKET3_NOP, 0x3FFF));
7876         if (r)
7877                 return r;
7878
7879         /* set up the compute queues */
7880         /* type-2 packets are deprecated on MEC, use type-3 instead */
7881         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7882         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
7883                              PACKET3(PACKET3_NOP, 0x3FFF));
7884         if (r)
7885                 return r;
7886         ring->me = 1; /* first MEC */
7887         ring->pipe = 0; /* first pipe */
7888         ring->queue = 0; /* first queue */
7889         ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
7890
7891         /* type-2 packets are deprecated on MEC, use type-3 instead */
7892         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7893         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
7894                              PACKET3(PACKET3_NOP, 0x3FFF));
7895         if (r)
7896                 return r;
7897         /* dGPU only have 1 MEC */
7898         ring->me = 1; /* first MEC */
7899         ring->pipe = 0; /* first pipe */
7900         ring->queue = 1; /* second queue */
7901         ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
7902
7903         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7904         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
7905                              SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7906         if (r)
7907                 return r;
7908
7909         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7910         r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
7911                              SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7912         if (r)
7913                 return r;
7914
7915         r = cik_cp_resume(rdev);
7916         if (r)
7917                 return r;
7918
7919         r = cik_sdma_resume(rdev);
7920         if (r)
7921                 return r;
7922
7923         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7924         if (ring->ring_size) {
7925                 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
7926                                      RADEON_CP_PACKET2);
7927                 if (!r)
7928                         r = uvd_v1_0_init(rdev);
7929                 if (r)
7930                         DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
7931         }
7932
7933         r = -ENOENT;
7934
7935         ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
7936         if (ring->ring_size)
7937                 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
7938                                      VCE_CMD_NO_OP);
7939
7940         ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
7941         if (ring->ring_size)
7942                 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
7943                                      VCE_CMD_NO_OP);
7944
7945         if (!r)
7946                 r = vce_v1_0_init(rdev);
7947         else if (r != -ENOENT)
7948                 DRM_ERROR("radeon: failed initializing VCE (%d).\n", r);
7949
7950         r = radeon_ib_pool_init(rdev);
7951         if (r) {
7952                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
7953                 return r;
7954         }
7955
7956         r = radeon_vm_manager_init(rdev);
7957         if (r) {
7958                 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
7959                 return r;
7960         }
7961
7962         r = dce6_audio_init(rdev);
7963         if (r)
7964                 return r;
7965
7966         return 0;
7967 }
7968
7969 /**
7970  * cik_resume - resume the asic to a functional state
7971  *
7972  * @rdev: radeon_device pointer
7973  *
7974  * Programs the asic to a functional state (CIK).
7975  * Called at resume.
7976  * Returns 0 for success, error for failure.
7977  */
7978 int cik_resume(struct radeon_device *rdev)
7979 {
7980         int r;
7981
7982         /* post card */
7983         atom_asic_init(rdev->mode_info.atom_context);
7984
7985         /* init golden registers */
7986         cik_init_golden_registers(rdev);
7987
7988         if (rdev->pm.pm_method == PM_METHOD_DPM)
7989                 radeon_pm_resume(rdev);
7990
7991         rdev->accel_working = true;
7992         r = cik_startup(rdev);
7993         if (r) {
7994                 DRM_ERROR("cik startup failed on resume\n");
7995                 rdev->accel_working = false;
7996                 return r;
7997         }
7998
7999         return r;
8000
8001 }
8002
8003 /**
8004  * cik_suspend - suspend the asic
8005  *
8006  * @rdev: radeon_device pointer
8007  *
8008  * Bring the chip into a state suitable for suspend (CIK).
8009  * Called at suspend.
8010  * Returns 0 for success.
8011  */
8012 int cik_suspend(struct radeon_device *rdev)
8013 {
8014         radeon_pm_suspend(rdev);
8015         dce6_audio_fini(rdev);
8016         radeon_vm_manager_fini(rdev);
8017         cik_cp_enable(rdev, false);
8018         cik_sdma_enable(rdev, false);
8019         uvd_v1_0_fini(rdev);
8020         radeon_uvd_suspend(rdev);
8021         radeon_vce_suspend(rdev);
8022         cik_fini_pg(rdev);
8023         cik_fini_cg(rdev);
8024         cik_irq_suspend(rdev);
8025         radeon_wb_disable(rdev);
8026         cik_pcie_gart_disable(rdev);
8027         return 0;
8028 }
8029
8030 /* Plan is to move initialization in that function and use
8031  * helper function so that radeon_device_init pretty much
8032  * do nothing more than calling asic specific function. This
8033  * should also allow to remove a bunch of callback function
8034  * like vram_info.
8035  */
8036 /**
8037  * cik_init - asic specific driver and hw init
8038  *
8039  * @rdev: radeon_device pointer
8040  *
8041  * Setup asic specific driver variables and program the hw
8042  * to a functional state (CIK).
8043  * Called at driver startup.
8044  * Returns 0 for success, errors for failure.
8045  */
8046 int cik_init(struct radeon_device *rdev)
8047 {
8048         struct radeon_ring *ring;
8049         int r;
8050
8051         /* Read BIOS */
8052         if (!radeon_get_bios(rdev)) {
8053                 if (ASIC_IS_AVIVO(rdev))
8054                         return -EINVAL;
8055         }
8056         /* Must be an ATOMBIOS */
8057         if (!rdev->is_atom_bios) {
8058                 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
8059                 return -EINVAL;
8060         }
8061         r = radeon_atombios_init(rdev);
8062         if (r)
8063                 return r;
8064
8065         /* Post card if necessary */
8066         if (!radeon_card_posted(rdev)) {
8067                 if (!rdev->bios) {
8068                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
8069                         return -EINVAL;
8070                 }
8071                 DRM_INFO("GPU not posted. posting now...\n");
8072                 atom_asic_init(rdev->mode_info.atom_context);
8073         }
8074         /* init golden registers */
8075         cik_init_golden_registers(rdev);
8076         /* Initialize scratch registers */
8077         cik_scratch_init(rdev);
8078         /* Initialize surface registers */
8079         radeon_surface_init(rdev);
8080         /* Initialize clocks */
8081         radeon_get_clock_info(rdev->ddev);
8082
8083         /* Fence driver */
8084         r = radeon_fence_driver_init(rdev);
8085         if (r)
8086                 return r;
8087
8088         /* initialize memory controller */
8089         r = cik_mc_init(rdev);
8090         if (r)
8091                 return r;
8092         /* Memory manager */
8093         r = radeon_bo_init(rdev);
8094         if (r)
8095                 return r;
8096
8097         if (rdev->flags & RADEON_IS_IGP) {
8098                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8099                     !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
8100                         r = cik_init_microcode(rdev);
8101                         if (r) {
8102                                 DRM_ERROR("Failed to load firmware!\n");
8103                                 return r;
8104                         }
8105                 }
8106         } else {
8107                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8108                     !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
8109                     !rdev->mc_fw) {
8110                         r = cik_init_microcode(rdev);
8111                         if (r) {
8112                                 DRM_ERROR("Failed to load firmware!\n");
8113                                 return r;
8114                         }
8115                 }
8116         }
8117
8118         /* Initialize power management */
8119         radeon_pm_init(rdev);
8120
8121         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8122         ring->ring_obj = NULL;
8123         r600_ring_init(rdev, ring, 1024 * 1024);
8124
8125         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8126         ring->ring_obj = NULL;
8127         r600_ring_init(rdev, ring, 1024 * 1024);
8128         r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8129         if (r)
8130                 return r;
8131
8132         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8133         ring->ring_obj = NULL;
8134         r600_ring_init(rdev, ring, 1024 * 1024);
8135         r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8136         if (r)
8137                 return r;
8138
8139         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8140         ring->ring_obj = NULL;
8141         r600_ring_init(rdev, ring, 256 * 1024);
8142
8143         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8144         ring->ring_obj = NULL;
8145         r600_ring_init(rdev, ring, 256 * 1024);
8146
8147         r = radeon_uvd_init(rdev);
8148         if (!r) {
8149                 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8150                 ring->ring_obj = NULL;
8151                 r600_ring_init(rdev, ring, 4096);
8152         }
8153
8154         r = radeon_vce_init(rdev);
8155         if (!r) {
8156                 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8157                 ring->ring_obj = NULL;
8158                 r600_ring_init(rdev, ring, 4096);
8159
8160                 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8161                 ring->ring_obj = NULL;
8162                 r600_ring_init(rdev, ring, 4096);
8163         }
8164
8165         rdev->ih.ring_obj = NULL;
8166         r600_ih_ring_init(rdev, 64 * 1024);
8167
8168         r = r600_pcie_gart_init(rdev);
8169         if (r)
8170                 return r;
8171
8172         rdev->accel_working = true;
8173         r = cik_startup(rdev);
8174         if (r) {
8175                 dev_err(rdev->dev, "disabling GPU acceleration\n");
8176                 cik_cp_fini(rdev);
8177                 cik_sdma_fini(rdev);
8178                 cik_irq_fini(rdev);
8179                 sumo_rlc_fini(rdev);
8180                 cik_mec_fini(rdev);
8181                 radeon_wb_fini(rdev);
8182                 radeon_ib_pool_fini(rdev);
8183                 radeon_vm_manager_fini(rdev);
8184                 radeon_irq_kms_fini(rdev);
8185                 cik_pcie_gart_fini(rdev);
8186                 rdev->accel_working = false;
8187         }
8188
8189         /* Don't start up if the MC ucode is missing.
8190          * The default clocks and voltages before the MC ucode
8191          * is loaded are not suffient for advanced operations.
8192          */
8193         if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
8194                 DRM_ERROR("radeon: MC ucode required for NI+.\n");
8195                 return -EINVAL;
8196         }
8197
8198         return 0;
8199 }
8200
8201 /**
8202  * cik_fini - asic specific driver and hw fini
8203  *
8204  * @rdev: radeon_device pointer
8205  *
8206  * Tear down the asic specific driver variables and program the hw
8207  * to an idle state (CIK).
8208  * Called at driver unload.
8209  */
8210 void cik_fini(struct radeon_device *rdev)
8211 {
8212         radeon_pm_fini(rdev);
8213         cik_cp_fini(rdev);
8214         cik_sdma_fini(rdev);
8215         cik_fini_pg(rdev);
8216         cik_fini_cg(rdev);
8217         cik_irq_fini(rdev);
8218         sumo_rlc_fini(rdev);
8219         cik_mec_fini(rdev);
8220         radeon_wb_fini(rdev);
8221         radeon_vm_manager_fini(rdev);
8222         radeon_ib_pool_fini(rdev);
8223         radeon_irq_kms_fini(rdev);
8224         uvd_v1_0_fini(rdev);
8225         radeon_uvd_fini(rdev);
8226         radeon_vce_fini(rdev);
8227         cik_pcie_gart_fini(rdev);
8228         r600_vram_scratch_fini(rdev);
8229         radeon_gem_fini(rdev);
8230         radeon_fence_driver_fini(rdev);
8231         radeon_bo_fini(rdev);
8232         radeon_atombios_fini(rdev);
8233         kfree(rdev->bios);
8234         rdev->bios = NULL;
8235 }
8236
8237 void dce8_program_fmt(struct drm_encoder *encoder)
8238 {
8239         struct drm_device *dev = encoder->dev;
8240         struct radeon_device *rdev = dev->dev_private;
8241         struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
8242         struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
8243         struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
8244         int bpc = 0;
8245         u32 tmp = 0;
8246         enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
8247
8248         if (connector) {
8249                 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
8250                 bpc = radeon_get_monitor_bpc(connector);
8251                 dither = radeon_connector->dither;
8252         }
8253
8254         /* LVDS/eDP FMT is set up by atom */
8255         if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
8256                 return;
8257
8258         /* not needed for analog */
8259         if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
8260             (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
8261                 return;
8262
8263         if (bpc == 0)
8264                 return;
8265
8266         switch (bpc) {
8267         case 6:
8268                 if (dither == RADEON_FMT_DITHER_ENABLE)
8269                         /* XXX sort out optimal dither settings */
8270                         tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8271                                 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
8272                 else
8273                         tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
8274                 break;
8275         case 8:
8276                 if (dither == RADEON_FMT_DITHER_ENABLE)
8277                         /* XXX sort out optimal dither settings */
8278                         tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8279                                 FMT_RGB_RANDOM_ENABLE |
8280                                 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
8281                 else
8282                         tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
8283                 break;
8284         case 10:
8285                 if (dither == RADEON_FMT_DITHER_ENABLE)
8286                         /* XXX sort out optimal dither settings */
8287                         tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8288                                 FMT_RGB_RANDOM_ENABLE |
8289                                 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
8290                 else
8291                         tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
8292                 break;
8293         default:
8294                 /* not needed */
8295                 break;
8296         }
8297
8298         WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
8299 }
8300
8301 /* display watermark setup */
8302 /**
8303  * dce8_line_buffer_adjust - Set up the line buffer
8304  *
8305  * @rdev: radeon_device pointer
8306  * @radeon_crtc: the selected display controller
8307  * @mode: the current display mode on the selected display
8308  * controller
8309  *
8310  * Setup up the line buffer allocation for
8311  * the selected display controller (CIK).
8312  * Returns the line buffer size in pixels.
8313  */
8314 static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
8315                                    struct radeon_crtc *radeon_crtc,
8316                                    struct drm_display_mode *mode)
8317 {
8318         u32 tmp, buffer_alloc, i;
8319         u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
8320         /*
8321          * Line Buffer Setup
8322          * There are 6 line buffers, one for each display controllers.
8323          * There are 3 partitions per LB. Select the number of partitions
8324          * to enable based on the display width.  For display widths larger
8325          * than 4096, you need use to use 2 display controllers and combine
8326          * them using the stereo blender.
8327          */
8328         if (radeon_crtc->base.enabled && mode) {
8329                 if (mode->crtc_hdisplay < 1920) {
8330                         tmp = 1;
8331                         buffer_alloc = 2;
8332                 } else if (mode->crtc_hdisplay < 2560) {
8333                         tmp = 2;
8334                         buffer_alloc = 2;
8335                 } else if (mode->crtc_hdisplay < 4096) {
8336                         tmp = 0;
8337                         buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8338                 } else {
8339                         DRM_DEBUG_KMS("Mode too big for LB!\n");
8340                         tmp = 0;
8341                         buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8342                 }
8343         } else {
8344                 tmp = 1;
8345                 buffer_alloc = 0;
8346         }
8347
8348         WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
8349                LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
8350
8351         WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
8352                DMIF_BUFFERS_ALLOCATED(buffer_alloc));
8353         for (i = 0; i < rdev->usec_timeout; i++) {
8354                 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
8355                     DMIF_BUFFERS_ALLOCATED_COMPLETED)
8356                         break;
8357                 udelay(1);
8358         }
8359
8360         if (radeon_crtc->base.enabled && mode) {
8361                 switch (tmp) {
8362                 case 0:
8363                 default:
8364                         return 4096 * 2;
8365                 case 1:
8366                         return 1920 * 2;
8367                 case 2:
8368                         return 2560 * 2;
8369                 }
8370         }
8371
8372         /* controller not enabled, so no lb used */
8373         return 0;
8374 }
8375
8376 /**
8377  * cik_get_number_of_dram_channels - get the number of dram channels
8378  *
8379  * @rdev: radeon_device pointer
8380  *
8381  * Look up the number of video ram channels (CIK).
8382  * Used for display watermark bandwidth calculations
8383  * Returns the number of dram channels
8384  */
8385 static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
8386 {
8387         u32 tmp = RREG32(MC_SHARED_CHMAP);
8388
8389         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
8390         case 0:
8391         default:
8392                 return 1;
8393         case 1:
8394                 return 2;
8395         case 2:
8396                 return 4;
8397         case 3:
8398                 return 8;
8399         case 4:
8400                 return 3;
8401         case 5:
8402                 return 6;
8403         case 6:
8404                 return 10;
8405         case 7:
8406                 return 12;
8407         case 8:
8408                 return 16;
8409         }
8410 }
8411
8412 struct dce8_wm_params {
8413         u32 dram_channels; /* number of dram channels */
8414         u32 yclk;          /* bandwidth per dram data pin in kHz */
8415         u32 sclk;          /* engine clock in kHz */
8416         u32 disp_clk;      /* display clock in kHz */
8417         u32 src_width;     /* viewport width */
8418         u32 active_time;   /* active display time in ns */
8419         u32 blank_time;    /* blank time in ns */
8420         bool interlaced;    /* mode is interlaced */
8421         fixed20_12 vsc;    /* vertical scale ratio */
8422         u32 num_heads;     /* number of active crtcs */
8423         u32 bytes_per_pixel; /* bytes per pixel display + overlay */
8424         u32 lb_size;       /* line buffer allocated to pipe */
8425         u32 vtaps;         /* vertical scaler taps */
8426 };
8427
8428 /**
8429  * dce8_dram_bandwidth - get the dram bandwidth
8430  *
8431  * @wm: watermark calculation data
8432  *
8433  * Calculate the raw dram bandwidth (CIK).
8434  * Used for display watermark bandwidth calculations
8435  * Returns the dram bandwidth in MBytes/s
8436  */
8437 static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
8438 {
8439         /* Calculate raw DRAM Bandwidth */
8440         fixed20_12 dram_efficiency; /* 0.7 */
8441         fixed20_12 yclk, dram_channels, bandwidth;
8442         fixed20_12 a;
8443
8444         a.full = dfixed_const(1000);
8445         yclk.full = dfixed_const(wm->yclk);
8446         yclk.full = dfixed_div(yclk, a);
8447         dram_channels.full = dfixed_const(wm->dram_channels * 4);
8448         a.full = dfixed_const(10);
8449         dram_efficiency.full = dfixed_const(7);
8450         dram_efficiency.full = dfixed_div(dram_efficiency, a);
8451         bandwidth.full = dfixed_mul(dram_channels, yclk);
8452         bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
8453
8454         return dfixed_trunc(bandwidth);
8455 }
8456
8457 /**
8458  * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
8459  *
8460  * @wm: watermark calculation data
8461  *
8462  * Calculate the dram bandwidth used for display (CIK).
8463  * Used for display watermark bandwidth calculations
8464  * Returns the dram bandwidth for display in MBytes/s
8465  */
8466 static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8467 {
8468         /* Calculate DRAM Bandwidth and the part allocated to display. */
8469         fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
8470         fixed20_12 yclk, dram_channels, bandwidth;
8471         fixed20_12 a;
8472
8473         a.full = dfixed_const(1000);
8474         yclk.full = dfixed_const(wm->yclk);
8475         yclk.full = dfixed_div(yclk, a);
8476         dram_channels.full = dfixed_const(wm->dram_channels * 4);
8477         a.full = dfixed_const(10);
8478         disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
8479         disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
8480         bandwidth.full = dfixed_mul(dram_channels, yclk);
8481         bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
8482
8483         return dfixed_trunc(bandwidth);
8484 }
8485
8486 /**
8487  * dce8_data_return_bandwidth - get the data return bandwidth
8488  *
8489  * @wm: watermark calculation data
8490  *
8491  * Calculate the data return bandwidth used for display (CIK).
8492  * Used for display watermark bandwidth calculations
8493  * Returns the data return bandwidth in MBytes/s
8494  */
8495 static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
8496 {
8497         /* Calculate the display Data return Bandwidth */
8498         fixed20_12 return_efficiency; /* 0.8 */
8499         fixed20_12 sclk, bandwidth;
8500         fixed20_12 a;
8501
8502         a.full = dfixed_const(1000);
8503         sclk.full = dfixed_const(wm->sclk);
8504         sclk.full = dfixed_div(sclk, a);
8505         a.full = dfixed_const(10);
8506         return_efficiency.full = dfixed_const(8);
8507         return_efficiency.full = dfixed_div(return_efficiency, a);
8508         a.full = dfixed_const(32);
8509         bandwidth.full = dfixed_mul(a, sclk);
8510         bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
8511
8512         return dfixed_trunc(bandwidth);
8513 }
8514
8515 /**
8516  * dce8_dmif_request_bandwidth - get the dmif bandwidth
8517  *
8518  * @wm: watermark calculation data
8519  *
8520  * Calculate the dmif bandwidth used for display (CIK).
8521  * Used for display watermark bandwidth calculations
8522  * Returns the dmif bandwidth in MBytes/s
8523  */
8524 static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
8525 {
8526         /* Calculate the DMIF Request Bandwidth */
8527         fixed20_12 disp_clk_request_efficiency; /* 0.8 */
8528         fixed20_12 disp_clk, bandwidth;
8529         fixed20_12 a, b;
8530
8531         a.full = dfixed_const(1000);
8532         disp_clk.full = dfixed_const(wm->disp_clk);
8533         disp_clk.full = dfixed_div(disp_clk, a);
8534         a.full = dfixed_const(32);
8535         b.full = dfixed_mul(a, disp_clk);
8536
8537         a.full = dfixed_const(10);
8538         disp_clk_request_efficiency.full = dfixed_const(8);
8539         disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
8540
8541         bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
8542
8543         return dfixed_trunc(bandwidth);
8544 }
8545
8546 /**
8547  * dce8_available_bandwidth - get the min available bandwidth
8548  *
8549  * @wm: watermark calculation data
8550  *
8551  * Calculate the min available bandwidth used for display (CIK).
8552  * Used for display watermark bandwidth calculations
8553  * Returns the min available bandwidth in MBytes/s
8554  */
8555 static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
8556 {
8557         /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
8558         u32 dram_bandwidth = dce8_dram_bandwidth(wm);
8559         u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
8560         u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
8561
8562         return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
8563 }
8564
8565 /**
8566  * dce8_average_bandwidth - get the average available bandwidth
8567  *
8568  * @wm: watermark calculation data
8569  *
8570  * Calculate the average available bandwidth used for display (CIK).
8571  * Used for display watermark bandwidth calculations
8572  * Returns the average available bandwidth in MBytes/s
8573  */
8574 static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
8575 {
8576         /* Calculate the display mode Average Bandwidth
8577          * DisplayMode should contain the source and destination dimensions,
8578          * timing, etc.
8579          */
8580         fixed20_12 bpp;
8581         fixed20_12 line_time;
8582         fixed20_12 src_width;
8583         fixed20_12 bandwidth;
8584         fixed20_12 a;
8585
8586         a.full = dfixed_const(1000);
8587         line_time.full = dfixed_const(wm->active_time + wm->blank_time);
8588         line_time.full = dfixed_div(line_time, a);
8589         bpp.full = dfixed_const(wm->bytes_per_pixel);
8590         src_width.full = dfixed_const(wm->src_width);
8591         bandwidth.full = dfixed_mul(src_width, bpp);
8592         bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
8593         bandwidth.full = dfixed_div(bandwidth, line_time);
8594
8595         return dfixed_trunc(bandwidth);
8596 }
8597
8598 /**
8599  * dce8_latency_watermark - get the latency watermark
8600  *
8601  * @wm: watermark calculation data
8602  *
8603  * Calculate the latency watermark (CIK).
8604  * Used for display watermark bandwidth calculations
8605  * Returns the latency watermark in ns
8606  */
8607 static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
8608 {
8609         /* First calculate the latency in ns */
8610         u32 mc_latency = 2000; /* 2000 ns. */
8611         u32 available_bandwidth = dce8_available_bandwidth(wm);
8612         u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
8613         u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
8614         u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
8615         u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
8616                 (wm->num_heads * cursor_line_pair_return_time);
8617         u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
8618         u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
8619         u32 tmp, dmif_size = 12288;
8620         fixed20_12 a, b, c;
8621
8622         if (wm->num_heads == 0)
8623                 return 0;
8624
8625         a.full = dfixed_const(2);
8626         b.full = dfixed_const(1);
8627         if ((wm->vsc.full > a.full) ||
8628             ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
8629             (wm->vtaps >= 5) ||
8630             ((wm->vsc.full >= a.full) && wm->interlaced))
8631                 max_src_lines_per_dst_line = 4;
8632         else
8633                 max_src_lines_per_dst_line = 2;
8634
8635         a.full = dfixed_const(available_bandwidth);
8636         b.full = dfixed_const(wm->num_heads);
8637         a.full = dfixed_div(a, b);
8638
8639         b.full = dfixed_const(mc_latency + 512);
8640         c.full = dfixed_const(wm->disp_clk);
8641         b.full = dfixed_div(b, c);
8642
8643         c.full = dfixed_const(dmif_size);
8644         b.full = dfixed_div(c, b);
8645
8646         tmp = min(dfixed_trunc(a), dfixed_trunc(b));
8647
8648         b.full = dfixed_const(1000);
8649         c.full = dfixed_const(wm->disp_clk);
8650         b.full = dfixed_div(c, b);
8651         c.full = dfixed_const(wm->bytes_per_pixel);
8652         b.full = dfixed_mul(b, c);
8653
8654         lb_fill_bw = min(tmp, dfixed_trunc(b));
8655
8656         a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
8657         b.full = dfixed_const(1000);
8658         c.full = dfixed_const(lb_fill_bw);
8659         b.full = dfixed_div(c, b);
8660         a.full = dfixed_div(a, b);
8661         line_fill_time = dfixed_trunc(a);
8662
8663         if (line_fill_time < wm->active_time)
8664                 return latency;
8665         else
8666                 return latency + (line_fill_time - wm->active_time);
8667
8668 }
8669
8670 /**
8671  * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
8672  * average and available dram bandwidth
8673  *
8674  * @wm: watermark calculation data
8675  *
8676  * Check if the display average bandwidth fits in the display
8677  * dram bandwidth (CIK).
8678  * Used for display watermark bandwidth calculations
8679  * Returns true if the display fits, false if not.
8680  */
8681 static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8682 {
8683         if (dce8_average_bandwidth(wm) <=
8684             (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
8685                 return true;
8686         else
8687                 return false;
8688 }
8689
8690 /**
8691  * dce8_average_bandwidth_vs_available_bandwidth - check
8692  * average and available bandwidth
8693  *
8694  * @wm: watermark calculation data
8695  *
8696  * Check if the display average bandwidth fits in the display
8697  * available bandwidth (CIK).
8698  * Used for display watermark bandwidth calculations
8699  * Returns true if the display fits, false if not.
8700  */
8701 static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
8702 {
8703         if (dce8_average_bandwidth(wm) <=
8704             (dce8_available_bandwidth(wm) / wm->num_heads))
8705                 return true;
8706         else
8707                 return false;
8708 }
8709
8710 /**
8711  * dce8_check_latency_hiding - check latency hiding
8712  *
8713  * @wm: watermark calculation data
8714  *
8715  * Check latency hiding (CIK).
8716  * Used for display watermark bandwidth calculations
8717  * Returns true if the display fits, false if not.
8718  */
8719 static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
8720 {
8721         u32 lb_partitions = wm->lb_size / wm->src_width;
8722         u32 line_time = wm->active_time + wm->blank_time;
8723         u32 latency_tolerant_lines;
8724         u32 latency_hiding;
8725         fixed20_12 a;
8726
8727         a.full = dfixed_const(1);
8728         if (wm->vsc.full > a.full)
8729                 latency_tolerant_lines = 1;
8730         else {
8731                 if (lb_partitions <= (wm->vtaps + 1))
8732                         latency_tolerant_lines = 1;
8733                 else
8734                         latency_tolerant_lines = 2;
8735         }
8736
8737         latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
8738
8739         if (dce8_latency_watermark(wm) <= latency_hiding)
8740                 return true;
8741         else
8742                 return false;
8743 }
8744
8745 /**
8746  * dce8_program_watermarks - program display watermarks
8747  *
8748  * @rdev: radeon_device pointer
8749  * @radeon_crtc: the selected display controller
8750  * @lb_size: line buffer size
8751  * @num_heads: number of display controllers in use
8752  *
8753  * Calculate and program the display watermarks for the
8754  * selected display controller (CIK).
8755  */
8756 static void dce8_program_watermarks(struct radeon_device *rdev,
8757                                     struct radeon_crtc *radeon_crtc,
8758                                     u32 lb_size, u32 num_heads)
8759 {
8760         struct drm_display_mode *mode = &radeon_crtc->base.mode;
8761         struct dce8_wm_params wm_low, wm_high;
8762         u32 pixel_period;
8763         u32 line_time = 0;
8764         u32 latency_watermark_a = 0, latency_watermark_b = 0;
8765         u32 tmp, wm_mask;
8766
8767         if (radeon_crtc->base.enabled && num_heads && mode) {
8768                 pixel_period = 1000000 / (u32)mode->clock;
8769                 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
8770
8771                 /* watermark for high clocks */
8772                 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8773                     rdev->pm.dpm_enabled) {
8774                         wm_high.yclk =
8775                                 radeon_dpm_get_mclk(rdev, false) * 10;
8776                         wm_high.sclk =
8777                                 radeon_dpm_get_sclk(rdev, false) * 10;
8778                 } else {
8779                         wm_high.yclk = rdev->pm.current_mclk * 10;
8780                         wm_high.sclk = rdev->pm.current_sclk * 10;
8781                 }
8782
8783                 wm_high.disp_clk = mode->clock;
8784                 wm_high.src_width = mode->crtc_hdisplay;
8785                 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
8786                 wm_high.blank_time = line_time - wm_high.active_time;
8787                 wm_high.interlaced = false;
8788                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
8789                         wm_high.interlaced = true;
8790                 wm_high.vsc = radeon_crtc->vsc;
8791                 wm_high.vtaps = 1;
8792                 if (radeon_crtc->rmx_type != RMX_OFF)
8793                         wm_high.vtaps = 2;
8794                 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
8795                 wm_high.lb_size = lb_size;
8796                 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
8797                 wm_high.num_heads = num_heads;
8798
8799                 /* set for high clocks */
8800                 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
8801
8802                 /* possibly force display priority to high */
8803                 /* should really do this at mode validation time... */
8804                 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
8805                     !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
8806                     !dce8_check_latency_hiding(&wm_high) ||
8807                     (rdev->disp_priority == 2)) {
8808                         DRM_DEBUG_KMS("force priority to high\n");
8809                 }
8810
8811                 /* watermark for low clocks */
8812                 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8813                     rdev->pm.dpm_enabled) {
8814                         wm_low.yclk =
8815                                 radeon_dpm_get_mclk(rdev, true) * 10;
8816                         wm_low.sclk =
8817                                 radeon_dpm_get_sclk(rdev, true) * 10;
8818                 } else {
8819                         wm_low.yclk = rdev->pm.current_mclk * 10;
8820                         wm_low.sclk = rdev->pm.current_sclk * 10;
8821                 }
8822
8823                 wm_low.disp_clk = mode->clock;
8824                 wm_low.src_width = mode->crtc_hdisplay;
8825                 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
8826                 wm_low.blank_time = line_time - wm_low.active_time;
8827                 wm_low.interlaced = false;
8828                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
8829                         wm_low.interlaced = true;
8830                 wm_low.vsc = radeon_crtc->vsc;
8831                 wm_low.vtaps = 1;
8832                 if (radeon_crtc->rmx_type != RMX_OFF)
8833                         wm_low.vtaps = 2;
8834                 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
8835                 wm_low.lb_size = lb_size;
8836                 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
8837                 wm_low.num_heads = num_heads;
8838
8839                 /* set for low clocks */
8840                 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
8841
8842                 /* possibly force display priority to high */
8843                 /* should really do this at mode validation time... */
8844                 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
8845                     !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
8846                     !dce8_check_latency_hiding(&wm_low) ||
8847                     (rdev->disp_priority == 2)) {
8848                         DRM_DEBUG_KMS("force priority to high\n");
8849                 }
8850         }
8851
8852         /* select wm A */
8853         wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
8854         tmp = wm_mask;
8855         tmp &= ~LATENCY_WATERMARK_MASK(3);
8856         tmp |= LATENCY_WATERMARK_MASK(1);
8857         WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
8858         WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
8859                (LATENCY_LOW_WATERMARK(latency_watermark_a) |
8860                 LATENCY_HIGH_WATERMARK(line_time)));
8861         /* select wm B */
8862         tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
8863         tmp &= ~LATENCY_WATERMARK_MASK(3);
8864         tmp |= LATENCY_WATERMARK_MASK(2);
8865         WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
8866         WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
8867                (LATENCY_LOW_WATERMARK(latency_watermark_b) |
8868                 LATENCY_HIGH_WATERMARK(line_time)));
8869         /* restore original selection */
8870         WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
8871
8872         /* save values for DPM */
8873         radeon_crtc->line_time = line_time;
8874         radeon_crtc->wm_high = latency_watermark_a;
8875         radeon_crtc->wm_low = latency_watermark_b;
8876 }
8877
8878 /**
8879  * dce8_bandwidth_update - program display watermarks
8880  *
8881  * @rdev: radeon_device pointer
8882  *
8883  * Calculate and program the display watermarks and line
8884  * buffer allocation (CIK).
8885  */
8886 void dce8_bandwidth_update(struct radeon_device *rdev)
8887 {
8888         struct drm_display_mode *mode = NULL;
8889         u32 num_heads = 0, lb_size;
8890         int i;
8891
8892         radeon_update_display_priority(rdev);
8893
8894         for (i = 0; i < rdev->num_crtc; i++) {
8895                 if (rdev->mode_info.crtcs[i]->base.enabled)
8896                         num_heads++;
8897         }
8898         for (i = 0; i < rdev->num_crtc; i++) {
8899                 mode = &rdev->mode_info.crtcs[i]->base.mode;
8900                 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
8901                 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
8902         }
8903 }
8904
8905 /**
8906  * cik_get_gpu_clock_counter - return GPU clock counter snapshot
8907  *
8908  * @rdev: radeon_device pointer
8909  *
8910  * Fetches a GPU clock counter snapshot (SI).
8911  * Returns the 64 bit clock counter snapshot.
8912  */
8913 uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
8914 {
8915         uint64_t clock;
8916
8917         mutex_lock(&rdev->gpu_clock_mutex);
8918         WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
8919         clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
8920                 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
8921         mutex_unlock(&rdev->gpu_clock_mutex);
8922         return clock;
8923 }
8924
8925 static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
8926                               u32 cntl_reg, u32 status_reg)
8927 {
8928         int r, i;
8929         struct atom_clock_dividers dividers;
8930         uint32_t tmp;
8931
8932         r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
8933                                            clock, false, &dividers);
8934         if (r)
8935                 return r;
8936
8937         tmp = RREG32_SMC(cntl_reg);
8938         tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
8939         tmp |= dividers.post_divider;
8940         WREG32_SMC(cntl_reg, tmp);
8941
8942         for (i = 0; i < 100; i++) {
8943                 if (RREG32_SMC(status_reg) & DCLK_STATUS)
8944                         break;
8945                 mdelay(10);
8946         }
8947         if (i == 100)
8948                 return -ETIMEDOUT;
8949
8950         return 0;
8951 }
8952
8953 int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
8954 {
8955         int r = 0;
8956
8957         r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
8958         if (r)
8959                 return r;
8960
8961         r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
8962         return r;
8963 }
8964
8965 int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
8966 {
8967         int r, i;
8968         struct atom_clock_dividers dividers;
8969         u32 tmp;
8970
8971         r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
8972                                            ecclk, false, &dividers);
8973         if (r)
8974                 return r;
8975
8976         for (i = 0; i < 100; i++) {
8977                 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
8978                         break;
8979                 mdelay(10);
8980         }
8981         if (i == 100)
8982                 return -ETIMEDOUT;
8983
8984         tmp = RREG32_SMC(CG_ECLK_CNTL);
8985         tmp &= ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK);
8986         tmp |= dividers.post_divider;
8987         WREG32_SMC(CG_ECLK_CNTL, tmp);
8988
8989         for (i = 0; i < 100; i++) {
8990                 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
8991                         break;
8992                 mdelay(10);
8993         }
8994         if (i == 100)
8995                 return -ETIMEDOUT;
8996
8997         return 0;
8998 }
8999
9000 static void cik_pcie_gen3_enable(struct radeon_device *rdev)
9001 {
9002         struct pci_dev *root = rdev->pdev->bus->self;
9003         int bridge_pos, gpu_pos;
9004         u32 speed_cntl, mask, current_data_rate;
9005         int ret, i;
9006         u16 tmp16;
9007
9008         if (radeon_pcie_gen2 == 0)
9009                 return;
9010
9011         if (rdev->flags & RADEON_IS_IGP)
9012                 return;
9013
9014         if (!(rdev->flags & RADEON_IS_PCIE))
9015                 return;
9016
9017         ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
9018         if (ret != 0)
9019                 return;
9020
9021         if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
9022                 return;
9023
9024         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9025         current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
9026                 LC_CURRENT_DATA_RATE_SHIFT;
9027         if (mask & DRM_PCIE_SPEED_80) {
9028                 if (current_data_rate == 2) {
9029                         DRM_INFO("PCIE gen 3 link speeds already enabled\n");
9030                         return;
9031                 }
9032                 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
9033         } else if (mask & DRM_PCIE_SPEED_50) {
9034                 if (current_data_rate == 1) {
9035                         DRM_INFO("PCIE gen 2 link speeds already enabled\n");
9036                         return;
9037                 }
9038                 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
9039         }
9040
9041         bridge_pos = pci_pcie_cap(root);
9042         if (!bridge_pos)
9043                 return;
9044
9045         gpu_pos = pci_pcie_cap(rdev->pdev);
9046         if (!gpu_pos)
9047                 return;
9048
9049         if (mask & DRM_PCIE_SPEED_80) {
9050                 /* re-try equalization if gen3 is not already enabled */
9051                 if (current_data_rate != 2) {
9052                         u16 bridge_cfg, gpu_cfg;
9053                         u16 bridge_cfg2, gpu_cfg2;
9054                         u32 max_lw, current_lw, tmp;
9055
9056                         pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9057                         pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9058
9059                         tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
9060                         pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9061
9062                         tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
9063                         pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9064
9065                         tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9066                         max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
9067                         current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
9068
9069                         if (current_lw < max_lw) {
9070                                 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9071                                 if (tmp & LC_RENEGOTIATION_SUPPORT) {
9072                                         tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
9073                                         tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
9074                                         tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
9075                                         WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
9076                                 }
9077                         }
9078
9079                         for (i = 0; i < 10; i++) {
9080                                 /* check status */
9081                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
9082                                 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
9083                                         break;
9084
9085                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9086                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9087
9088                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
9089                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
9090
9091                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9092                                 tmp |= LC_SET_QUIESCE;
9093                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9094
9095                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9096                                 tmp |= LC_REDO_EQ;
9097                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9098
9099                                 mdelay(100);
9100
9101                                 /* linkctl */
9102                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
9103                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9104                                 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
9105                                 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9106
9107                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
9108                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9109                                 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
9110                                 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9111
9112                                 /* linkctl2 */
9113                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
9114                                 tmp16 &= ~((1 << 4) | (7 << 9));
9115                                 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
9116                                 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
9117
9118                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9119                                 tmp16 &= ~((1 << 4) | (7 << 9));
9120                                 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
9121                                 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9122
9123                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9124                                 tmp &= ~LC_SET_QUIESCE;
9125                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9126                         }
9127                 }
9128         }
9129
9130         /* set the link speed */
9131         speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
9132         speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
9133         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9134
9135         pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9136         tmp16 &= ~0xf;
9137         if (mask & DRM_PCIE_SPEED_80)
9138                 tmp16 |= 3; /* gen3 */
9139         else if (mask & DRM_PCIE_SPEED_50)
9140                 tmp16 |= 2; /* gen2 */
9141         else
9142                 tmp16 |= 1; /* gen1 */
9143         pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9144
9145         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9146         speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
9147         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9148
9149         for (i = 0; i < rdev->usec_timeout; i++) {
9150                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9151                 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
9152                         break;
9153                 udelay(1);
9154         }
9155 }
9156
9157 static void cik_program_aspm(struct radeon_device *rdev)
9158 {
9159         u32 data, orig;
9160         bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
9161         bool disable_clkreq = false;
9162
9163         if (radeon_aspm == 0)
9164                 return;
9165
9166         /* XXX double check IGPs */
9167         if (rdev->flags & RADEON_IS_IGP)
9168                 return;
9169
9170         if (!(rdev->flags & RADEON_IS_PCIE))
9171                 return;
9172
9173         orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9174         data &= ~LC_XMIT_N_FTS_MASK;
9175         data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
9176         if (orig != data)
9177                 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
9178
9179         orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
9180         data |= LC_GO_TO_RECOVERY;
9181         if (orig != data)
9182                 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
9183
9184         orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
9185         data |= P_IGNORE_EDB_ERR;
9186         if (orig != data)
9187                 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
9188
9189         orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9190         data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
9191         data |= LC_PMI_TO_L1_DIS;
9192         if (!disable_l0s)
9193                 data |= LC_L0S_INACTIVITY(7);
9194
9195         if (!disable_l1) {
9196                 data |= LC_L1_INACTIVITY(7);
9197                 data &= ~LC_PMI_TO_L1_DIS;
9198                 if (orig != data)
9199                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9200
9201                 if (!disable_plloff_in_l1) {
9202                         bool clk_req_support;
9203
9204                         orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
9205                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9206                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9207                         if (orig != data)
9208                                 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
9209
9210                         orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
9211                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9212                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9213                         if (orig != data)
9214                                 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
9215
9216                         orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
9217                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9218                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9219                         if (orig != data)
9220                                 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
9221
9222                         orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
9223                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9224                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9225                         if (orig != data)
9226                                 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
9227
9228                         orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9229                         data &= ~LC_DYN_LANES_PWR_STATE_MASK;
9230                         data |= LC_DYN_LANES_PWR_STATE(3);
9231                         if (orig != data)
9232                                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
9233
9234                         if (!disable_clkreq) {
9235                                 struct pci_dev *root = rdev->pdev->bus->self;
9236                                 u32 lnkcap;
9237
9238                                 clk_req_support = false;
9239                                 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
9240                                 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
9241                                         clk_req_support = true;
9242                         } else {
9243                                 clk_req_support = false;
9244                         }
9245
9246                         if (clk_req_support) {
9247                                 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
9248                                 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
9249                                 if (orig != data)
9250                                         WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
9251
9252                                 orig = data = RREG32_SMC(THM_CLK_CNTL);
9253                                 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
9254                                 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
9255                                 if (orig != data)
9256                                         WREG32_SMC(THM_CLK_CNTL, data);
9257
9258                                 orig = data = RREG32_SMC(MISC_CLK_CTRL);
9259                                 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
9260                                 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
9261                                 if (orig != data)
9262                                         WREG32_SMC(MISC_CLK_CTRL, data);
9263
9264                                 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
9265                                 data &= ~BCLK_AS_XCLK;
9266                                 if (orig != data)
9267                                         WREG32_SMC(CG_CLKPIN_CNTL, data);
9268
9269                                 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
9270                                 data &= ~FORCE_BIF_REFCLK_EN;
9271                                 if (orig != data)
9272                                         WREG32_SMC(CG_CLKPIN_CNTL_2, data);
9273
9274                                 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
9275                                 data &= ~MPLL_CLKOUT_SEL_MASK;
9276                                 data |= MPLL_CLKOUT_SEL(4);
9277                                 if (orig != data)
9278                                         WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
9279                         }
9280                 }
9281         } else {
9282                 if (orig != data)
9283                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9284         }
9285
9286         orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
9287         data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
9288         if (orig != data)
9289                 WREG32_PCIE_PORT(PCIE_CNTL2, data);
9290
9291         if (!disable_l0s) {
9292                 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9293                 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
9294                         data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9295                         if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
9296                                 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9297                                 data &= ~LC_L0S_INACTIVITY_MASK;
9298                                 if (orig != data)
9299                                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9300                         }
9301                 }
9302         }
9303 }