drivers/gpu/drm/radeon/cik.c

   1 /*
   2  * Copyright 2012 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/slab.h>
  26 #include <linux/module.h>
  27 #include "drmP.h"
  28 #include "radeon.h"
  29 #include "radeon_asic.h"
  30 #include "cikd.h"
  31 #include "atom.h"
  32 #include "cik_blit_shaders.h"
  33 #include "radeon_ucode.h"
  34 #include "clearstate_ci.h"
  35
  36 MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
  37 MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
  38 MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
  39 MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
  40 MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
  41 MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
  42 MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
  43 MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
  44 MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
  45 MODULE_FIRMWARE("radeon/KAVERI_me.bin");
  46 MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
  47 MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
  48 MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
  49 MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
  50 MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
  51 MODULE_FIRMWARE("radeon/KABINI_me.bin");
  52 MODULE_FIRMWARE("radeon/KABINI_ce.bin");
  53 MODULE_FIRMWARE("radeon/KABINI_mec.bin");
  54 MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
  55 MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
  56
  57 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
  58 extern void r600_ih_ring_fini(struct radeon_device *rdev);
  59 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
  60 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
  61 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
  62 extern void sumo_rlc_fini(struct radeon_device *rdev);
  63 extern int sumo_rlc_init(struct radeon_device *rdev);
  64 extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
  65 extern void si_rlc_reset(struct radeon_device *rdev);
  66 extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
  67 extern int cik_sdma_resume(struct radeon_device *rdev);
  68 extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
  69 extern void cik_sdma_fini(struct radeon_device *rdev);
  70 extern void cik_sdma_vm_set_page(struct radeon_device *rdev,
  71                                  struct radeon_ib *ib,
  72                                  uint64_t pe,
  73                                  uint64_t addr, unsigned count,
  74                                  uint32_t incr, uint32_t flags);
  75 static void cik_rlc_stop(struct radeon_device *rdev);
  76 static void cik_pcie_gen3_enable(struct radeon_device *rdev);
  77 static void cik_program_aspm(struct radeon_device *rdev);
  78 static void cik_init_pg(struct radeon_device *rdev);
  79 static void cik_init_cg(struct radeon_device *rdev);
  80
  81 /* get temperature in millidegrees */
  82 int ci_get_temp(struct radeon_device *rdev)
  83 {
  84         u32 temp;
  85         int actual_temp = 0;
  86
  87         temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
  88                 CTF_TEMP_SHIFT;
  89
  90         if (temp & 0x200)
  91                 actual_temp = 255;
  92         else
  93                 actual_temp = temp & 0x1ff;
  94
  95         actual_temp = actual_temp * 1000;
  96
  97         return actual_temp;
  98 }
  99
 100 /* get temperature in millidegrees */
 101 int kv_get_temp(struct radeon_device *rdev)
 102 {
 103         u32 temp;
 104         int actual_temp = 0;
 105
 106         temp = RREG32_SMC(0xC0300E0C);
 107
 108         if (temp)
 109                 actual_temp = (temp / 8) - 49;
 110         else
 111                 actual_temp = 0;
 112
 113         actual_temp = actual_temp * 1000;
 114
 115         return actual_temp;
 116 }
 117
 118 /*
 119  * Indirect registers accessor
 120  */
 121 u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
 122 {
 123         u32 r;
 124
 125         WREG32(PCIE_INDEX, reg);
 126         (void)RREG32(PCIE_INDEX);
 127         r = RREG32(PCIE_DATA);
 128         return r;
 129 }
 130
 131 void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
 132 {
 133         WREG32(PCIE_INDEX, reg);
 134         (void)RREG32(PCIE_INDEX);
 135         WREG32(PCIE_DATA, v);
 136         (void)RREG32(PCIE_DATA);
 137 }
 138
 139 static const u32 spectre_rlc_save_restore_register_list[] =
 140 {
 141         (0x0e00 << 16) | (0xc12c >> 2),
 142         0x00000000,
 143         (0x0e00 << 16) | (0xc140 >> 2),
 144         0x00000000,
 145         (0x0e00 << 16) | (0xc150 >> 2),
 146         0x00000000,
 147         (0x0e00 << 16) | (0xc15c >> 2),
 148         0x00000000,
 149         (0x0e00 << 16) | (0xc168 >> 2),
 150         0x00000000,
 151         (0x0e00 << 16) | (0xc170 >> 2),
 152         0x00000000,
 153         (0x0e00 << 16) | (0xc178 >> 2),
 154         0x00000000,
 155         (0x0e00 << 16) | (0xc204 >> 2),
 156         0x00000000,
 157         (0x0e00 << 16) | (0xc2b4 >> 2),
 158         0x00000000,
 159         (0x0e00 << 16) | (0xc2b8 >> 2),
 160         0x00000000,
 161         (0x0e00 << 16) | (0xc2bc >> 2),
 162         0x00000000,
 163         (0x0e00 << 16) | (0xc2c0 >> 2),
 164         0x00000000,
 165         (0x0e00 << 16) | (0x8228 >> 2),
 166         0x00000000,
 167         (0x0e00 << 16) | (0x829c >> 2),
 168         0x00000000,
 169         (0x0e00 << 16) | (0x869c >> 2),
 170         0x00000000,
 171         (0x0600 << 16) | (0x98f4 >> 2),
 172         0x00000000,
 173         (0x0e00 << 16) | (0x98f8 >> 2),
 174         0x00000000,
 175         (0x0e00 << 16) | (0x9900 >> 2),
 176         0x00000000,
 177         (0x0e00 << 16) | (0xc260 >> 2),
 178         0x00000000,
 179         (0x0e00 << 16) | (0x90e8 >> 2),
 180         0x00000000,
 181         (0x0e00 << 16) | (0x3c000 >> 2),
 182         0x00000000,
 183         (0x0e00 << 16) | (0x3c00c >> 2),
 184         0x00000000,
 185         (0x0e00 << 16) | (0x8c1c >> 2),
 186         0x00000000,
 187         (0x0e00 << 16) | (0x9700 >> 2),
 188         0x00000000,
 189         (0x0e00 << 16) | (0xcd20 >> 2),
 190         0x00000000,
 191         (0x4e00 << 16) | (0xcd20 >> 2),
 192         0x00000000,
 193         (0x5e00 << 16) | (0xcd20 >> 2),
 194         0x00000000,
 195         (0x6e00 << 16) | (0xcd20 >> 2),
 196         0x00000000,
 197         (0x7e00 << 16) | (0xcd20 >> 2),
 198         0x00000000,
 199         (0x8e00 << 16) | (0xcd20 >> 2),
 200         0x00000000,
 201         (0x9e00 << 16) | (0xcd20 >> 2),
 202         0x00000000,
 203         (0xae00 << 16) | (0xcd20 >> 2),
 204         0x00000000,
 205         (0xbe00 << 16) | (0xcd20 >> 2),
 206         0x00000000,
 207         (0x0e00 << 16) | (0x89bc >> 2),
 208         0x00000000,
 209         (0x0e00 << 16) | (0x8900 >> 2),
 210         0x00000000,
 211         0x3,
 212         (0x0e00 << 16) | (0xc130 >> 2),
 213         0x00000000,
 214         (0x0e00 << 16) | (0xc134 >> 2),
 215         0x00000000,
 216         (0x0e00 << 16) | (0xc1fc >> 2),
 217         0x00000000,
 218         (0x0e00 << 16) | (0xc208 >> 2),
 219         0x00000000,
 220         (0x0e00 << 16) | (0xc264 >> 2),
 221         0x00000000,
 222         (0x0e00 << 16) | (0xc268 >> 2),
 223         0x00000000,
 224         (0x0e00 << 16) | (0xc26c >> 2),
 225         0x00000000,
 226         (0x0e00 << 16) | (0xc270 >> 2),
 227         0x00000000,
 228         (0x0e00 << 16) | (0xc274 >> 2),
 229         0x00000000,
 230         (0x0e00 << 16) | (0xc278 >> 2),
 231         0x00000000,
 232         (0x0e00 << 16) | (0xc27c >> 2),
 233         0x00000000,
 234         (0x0e00 << 16) | (0xc280 >> 2),
 235         0x00000000,
 236         (0x0e00 << 16) | (0xc284 >> 2),
 237         0x00000000,
 238         (0x0e00 << 16) | (0xc288 >> 2),
 239         0x00000000,
 240         (0x0e00 << 16) | (0xc28c >> 2),
 241         0x00000000,
 242         (0x0e00 << 16) | (0xc290 >> 2),
 243         0x00000000,
 244         (0x0e00 << 16) | (0xc294 >> 2),
 245         0x00000000,
 246         (0x0e00 << 16) | (0xc298 >> 2),
 247         0x00000000,
 248         (0x0e00 << 16) | (0xc29c >> 2),
 249         0x00000000,
 250         (0x0e00 << 16) | (0xc2a0 >> 2),
 251         0x00000000,
 252         (0x0e00 << 16) | (0xc2a4 >> 2),
 253         0x00000000,
 254         (0x0e00 << 16) | (0xc2a8 >> 2),
 255         0x00000000,
 256         (0x0e00 << 16) | (0xc2ac  >> 2),
 257         0x00000000,
 258         (0x0e00 << 16) | (0xc2b0 >> 2),
 259         0x00000000,
 260         (0x0e00 << 16) | (0x301d0 >> 2),
 261         0x00000000,
 262         (0x0e00 << 16) | (0x30238 >> 2),
 263         0x00000000,
 264         (0x0e00 << 16) | (0x30250 >> 2),
 265         0x00000000,
 266         (0x0e00 << 16) | (0x30254 >> 2),
 267         0x00000000,
 268         (0x0e00 << 16) | (0x30258 >> 2),
 269         0x00000000,
 270         (0x0e00 << 16) | (0x3025c >> 2),
 271         0x00000000,
 272         (0x4e00 << 16) | (0xc900 >> 2),
 273         0x00000000,
 274         (0x5e00 << 16) | (0xc900 >> 2),
 275         0x00000000,
 276         (0x6e00 << 16) | (0xc900 >> 2),
 277         0x00000000,
 278         (0x7e00 << 16) | (0xc900 >> 2),
 279         0x00000000,
 280         (0x8e00 << 16) | (0xc900 >> 2),
 281         0x00000000,
 282         (0x9e00 << 16) | (0xc900 >> 2),
 283         0x00000000,
 284         (0xae00 << 16) | (0xc900 >> 2),
 285         0x00000000,
 286         (0xbe00 << 16) | (0xc900 >> 2),
 287         0x00000000,
 288         (0x4e00 << 16) | (0xc904 >> 2),
 289         0x00000000,
 290         (0x5e00 << 16) | (0xc904 >> 2),
 291         0x00000000,
 292         (0x6e00 << 16) | (0xc904 >> 2),
 293         0x00000000,
 294         (0x7e00 << 16) | (0xc904 >> 2),
 295         0x00000000,
 296         (0x8e00 << 16) | (0xc904 >> 2),
 297         0x00000000,
 298         (0x9e00 << 16) | (0xc904 >> 2),
 299         0x00000000,
 300         (0xae00 << 16) | (0xc904 >> 2),
 301         0x00000000,
 302         (0xbe00 << 16) | (0xc904 >> 2),
 303         0x00000000,
 304         (0x4e00 << 16) | (0xc908 >> 2),
 305         0x00000000,
 306         (0x5e00 << 16) | (0xc908 >> 2),
 307         0x00000000,
 308         (0x6e00 << 16) | (0xc908 >> 2),
 309         0x00000000,
 310         (0x7e00 << 16) | (0xc908 >> 2),
 311         0x00000000,
 312         (0x8e00 << 16) | (0xc908 >> 2),
 313         0x00000000,
 314         (0x9e00 << 16) | (0xc908 >> 2),
 315         0x00000000,
 316         (0xae00 << 16) | (0xc908 >> 2),
 317         0x00000000,
 318         (0xbe00 << 16) | (0xc908 >> 2),
 319         0x00000000,
 320         (0x4e00 << 16) | (0xc90c >> 2),
 321         0x00000000,
 322         (0x5e00 << 16) | (0xc90c >> 2),
 323         0x00000000,
 324         (0x6e00 << 16) | (0xc90c >> 2),
 325         0x00000000,
 326         (0x7e00 << 16) | (0xc90c >> 2),
 327         0x00000000,
 328         (0x8e00 << 16) | (0xc90c >> 2),
 329         0x00000000,
 330         (0x9e00 << 16) | (0xc90c >> 2),
 331         0x00000000,
 332         (0xae00 << 16) | (0xc90c >> 2),
 333         0x00000000,
 334         (0xbe00 << 16) | (0xc90c >> 2),
 335         0x00000000,
 336         (0x4e00 << 16) | (0xc910 >> 2),
 337         0x00000000,
 338         (0x5e00 << 16) | (0xc910 >> 2),
 339         0x00000000,
 340         (0x6e00 << 16) | (0xc910 >> 2),
 341         0x00000000,
 342         (0x7e00 << 16) | (0xc910 >> 2),
 343         0x00000000,
 344         (0x8e00 << 16) | (0xc910 >> 2),
 345         0x00000000,
 346         (0x9e00 << 16) | (0xc910 >> 2),
 347         0x00000000,
 348         (0xae00 << 16) | (0xc910 >> 2),
 349         0x00000000,
 350         (0xbe00 << 16) | (0xc910 >> 2),
 351         0x00000000,
 352         (0x0e00 << 16) | (0xc99c >> 2),
 353         0x00000000,
 354         (0x0e00 << 16) | (0x9834 >> 2),
 355         0x00000000,
 356         (0x0000 << 16) | (0x30f00 >> 2),
 357         0x00000000,
 358         (0x0001 << 16) | (0x30f00 >> 2),
 359         0x00000000,
 360         (0x0000 << 16) | (0x30f04 >> 2),
 361         0x00000000,
 362         (0x0001 << 16) | (0x30f04 >> 2),
 363         0x00000000,
 364         (0x0000 << 16) | (0x30f08 >> 2),
 365         0x00000000,
 366         (0x0001 << 16) | (0x30f08 >> 2),
 367         0x00000000,
 368         (0x0000 << 16) | (0x30f0c >> 2),
 369         0x00000000,
 370         (0x0001 << 16) | (0x30f0c >> 2),
 371         0x00000000,
 372         (0x0600 << 16) | (0x9b7c >> 2),
 373         0x00000000,
 374         (0x0e00 << 16) | (0x8a14 >> 2),
 375         0x00000000,
 376         (0x0e00 << 16) | (0x8a18 >> 2),
 377         0x00000000,
 378         (0x0600 << 16) | (0x30a00 >> 2),
 379         0x00000000,
 380         (0x0e00 << 16) | (0x8bf0 >> 2),
 381         0x00000000,
 382         (0x0e00 << 16) | (0x8bcc >> 2),
 383         0x00000000,
 384         (0x0e00 << 16) | (0x8b24 >> 2),
 385         0x00000000,
 386         (0x0e00 << 16) | (0x30a04 >> 2),
 387         0x00000000,
 388         (0x0600 << 16) | (0x30a10 >> 2),
 389         0x00000000,
 390         (0x0600 << 16) | (0x30a14 >> 2),
 391         0x00000000,
 392         (0x0600 << 16) | (0x30a18 >> 2),
 393         0x00000000,
 394         (0x0600 << 16) | (0x30a2c >> 2),
 395         0x00000000,
 396         (0x0e00 << 16) | (0xc700 >> 2),
 397         0x00000000,
 398         (0x0e00 << 16) | (0xc704 >> 2),
 399         0x00000000,
 400         (0x0e00 << 16) | (0xc708 >> 2),
 401         0x00000000,
 402         (0x0e00 << 16) | (0xc768 >> 2),
 403         0x00000000,
 404         (0x0400 << 16) | (0xc770 >> 2),
 405         0x00000000,
 406         (0x0400 << 16) | (0xc774 >> 2),
 407         0x00000000,
 408         (0x0400 << 16) | (0xc778 >> 2),
 409         0x00000000,
 410         (0x0400 << 16) | (0xc77c >> 2),
 411         0x00000000,
 412         (0x0400 << 16) | (0xc780 >> 2),
 413         0x00000000,
 414         (0x0400 << 16) | (0xc784 >> 2),
 415         0x00000000,
 416         (0x0400 << 16) | (0xc788 >> 2),
 417         0x00000000,
 418         (0x0400 << 16) | (0xc78c >> 2),
 419         0x00000000,
 420         (0x0400 << 16) | (0xc798 >> 2),
 421         0x00000000,
 422         (0x0400 << 16) | (0xc79c >> 2),
 423         0x00000000,
 424         (0x0400 << 16) | (0xc7a0 >> 2),
 425         0x00000000,
 426         (0x0400 << 16) | (0xc7a4 >> 2),
 427         0x00000000,
 428         (0x0400 << 16) | (0xc7a8 >> 2),
 429         0x00000000,
 430         (0x0400 << 16) | (0xc7ac >> 2),
 431         0x00000000,
 432         (0x0400 << 16) | (0xc7b0 >> 2),
 433         0x00000000,
 434         (0x0400 << 16) | (0xc7b4 >> 2),
 435         0x00000000,
 436         (0x0e00 << 16) | (0x9100 >> 2),
 437         0x00000000,
 438         (0x0e00 << 16) | (0x3c010 >> 2),
 439         0x00000000,
 440         (0x0e00 << 16) | (0x92a8 >> 2),
 441         0x00000000,
 442         (0x0e00 << 16) | (0x92ac >> 2),
 443         0x00000000,
 444         (0x0e00 << 16) | (0x92b4 >> 2),
 445         0x00000000,
 446         (0x0e00 << 16) | (0x92b8 >> 2),
 447         0x00000000,
 448         (0x0e00 << 16) | (0x92bc >> 2),
 449         0x00000000,
 450         (0x0e00 << 16) | (0x92c0 >> 2),
 451         0x00000000,
 452         (0x0e00 << 16) | (0x92c4 >> 2),
 453         0x00000000,
 454         (0x0e00 << 16) | (0x92c8 >> 2),
 455         0x00000000,
 456         (0x0e00 << 16) | (0x92cc >> 2),
 457         0x00000000,
 458         (0x0e00 << 16) | (0x92d0 >> 2),
 459         0x00000000,
 460         (0x0e00 << 16) | (0x8c00 >> 2),
 461         0x00000000,
 462         (0x0e00 << 16) | (0x8c04 >> 2),
 463         0x00000000,
 464         (0x0e00 << 16) | (0x8c20 >> 2),
 465         0x00000000,
 466         (0x0e00 << 16) | (0x8c38 >> 2),
 467         0x00000000,
 468         (0x0e00 << 16) | (0x8c3c >> 2),
 469         0x00000000,
 470         (0x0e00 << 16) | (0xae00 >> 2),
 471         0x00000000,
 472         (0x0e00 << 16) | (0x9604 >> 2),
 473         0x00000000,
 474         (0x0e00 << 16) | (0xac08 >> 2),
 475         0x00000000,
 476         (0x0e00 << 16) | (0xac0c >> 2),
 477         0x00000000,
 478         (0x0e00 << 16) | (0xac10 >> 2),
 479         0x00000000,
 480         (0x0e00 << 16) | (0xac14 >> 2),
 481         0x00000000,
 482         (0x0e00 << 16) | (0xac58 >> 2),
 483         0x00000000,
 484         (0x0e00 << 16) | (0xac68 >> 2),
 485         0x00000000,
 486         (0x0e00 << 16) | (0xac6c >> 2),
 487         0x00000000,
 488         (0x0e00 << 16) | (0xac70 >> 2),
 489         0x00000000,
 490         (0x0e00 << 16) | (0xac74 >> 2),
 491         0x00000000,
 492         (0x0e00 << 16) | (0xac78 >> 2),
 493         0x00000000,
 494         (0x0e00 << 16) | (0xac7c >> 2),
 495         0x00000000,
 496         (0x0e00 << 16) | (0xac80 >> 2),
 497         0x00000000,
 498         (0x0e00 << 16) | (0xac84 >> 2),
 499         0x00000000,
 500         (0x0e00 << 16) | (0xac88 >> 2),
 501         0x00000000,
 502         (0x0e00 << 16) | (0xac8c >> 2),
 503         0x00000000,
 504         (0x0e00 << 16) | (0x970c >> 2),
 505         0x00000000,
 506         (0x0e00 << 16) | (0x9714 >> 2),
 507         0x00000000,
 508         (0x0e00 << 16) | (0x9718 >> 2),
 509         0x00000000,
 510         (0x0e00 << 16) | (0x971c >> 2),
 511         0x00000000,
 512         (0x0e00 << 16) | (0x31068 >> 2),
 513         0x00000000,
 514         (0x4e00 << 16) | (0x31068 >> 2),
 515         0x00000000,
 516         (0x5e00 << 16) | (0x31068 >> 2),
 517         0x00000000,
 518         (0x6e00 << 16) | (0x31068 >> 2),
 519         0x00000000,
 520         (0x7e00 << 16) | (0x31068 >> 2),
 521         0x00000000,
 522         (0x8e00 << 16) | (0x31068 >> 2),
 523         0x00000000,
 524         (0x9e00 << 16) | (0x31068 >> 2),
 525         0x00000000,
 526         (0xae00 << 16) | (0x31068 >> 2),
 527         0x00000000,
 528         (0xbe00 << 16) | (0x31068 >> 2),
 529         0x00000000,
 530         (0x0e00 << 16) | (0xcd10 >> 2),
 531         0x00000000,
 532         (0x0e00 << 16) | (0xcd14 >> 2),
 533         0x00000000,
 534         (0x0e00 << 16) | (0x88b0 >> 2),
 535         0x00000000,
 536         (0x0e00 << 16) | (0x88b4 >> 2),
 537         0x00000000,
 538         (0x0e00 << 16) | (0x88b8 >> 2),
 539         0x00000000,
 540         (0x0e00 << 16) | (0x88bc >> 2),
 541         0x00000000,
 542         (0x0400 << 16) | (0x89c0 >> 2),
 543         0x00000000,
 544         (0x0e00 << 16) | (0x88c4 >> 2),
 545         0x00000000,
 546         (0x0e00 << 16) | (0x88c8 >> 2),
 547         0x00000000,
 548         (0x0e00 << 16) | (0x88d0 >> 2),
 549         0x00000000,
 550         (0x0e00 << 16) | (0x88d4 >> 2),
 551         0x00000000,
 552         (0x0e00 << 16) | (0x88d8 >> 2),
 553         0x00000000,
 554         (0x0e00 << 16) | (0x8980 >> 2),
 555         0x00000000,
 556         (0x0e00 << 16) | (0x30938 >> 2),
 557         0x00000000,
 558         (0x0e00 << 16) | (0x3093c >> 2),
 559         0x00000000,
 560         (0x0e00 << 16) | (0x30940 >> 2),
 561         0x00000000,
 562         (0x0e00 << 16) | (0x89a0 >> 2),
 563         0x00000000,
 564         (0x0e00 << 16) | (0x30900 >> 2),
 565         0x00000000,
 566         (0x0e00 << 16) | (0x30904 >> 2),
 567         0x00000000,
 568         (0x0e00 << 16) | (0x89b4 >> 2),
 569         0x00000000,
 570         (0x0e00 << 16) | (0x3c210 >> 2),
 571         0x00000000,
 572         (0x0e00 << 16) | (0x3c214 >> 2),
 573         0x00000000,
 574         (0x0e00 << 16) | (0x3c218 >> 2),
 575         0x00000000,
 576         (0x0e00 << 16) | (0x8904 >> 2),
 577         0x00000000,
 578         0x5,
 579         (0x0e00 << 16) | (0x8c28 >> 2),
 580         (0x0e00 << 16) | (0x8c2c >> 2),
 581         (0x0e00 << 16) | (0x8c30 >> 2),
 582         (0x0e00 << 16) | (0x8c34 >> 2),
 583         (0x0e00 << 16) | (0x9600 >> 2),
 584 };
 585
 586 static const u32 kalindi_rlc_save_restore_register_list[] =
 587 {
 588         (0x0e00 << 16) | (0xc12c >> 2),
 589         0x00000000,
 590         (0x0e00 << 16) | (0xc140 >> 2),
 591         0x00000000,
 592         (0x0e00 << 16) | (0xc150 >> 2),
 593         0x00000000,
 594         (0x0e00 << 16) | (0xc15c >> 2),
 595         0x00000000,
 596         (0x0e00 << 16) | (0xc168 >> 2),
 597         0x00000000,
 598         (0x0e00 << 16) | (0xc170 >> 2),
 599         0x00000000,
 600         (0x0e00 << 16) | (0xc204 >> 2),
 601         0x00000000,
 602         (0x0e00 << 16) | (0xc2b4 >> 2),
 603         0x00000000,
 604         (0x0e00 << 16) | (0xc2b8 >> 2),
 605         0x00000000,
 606         (0x0e00 << 16) | (0xc2bc >> 2),
 607         0x00000000,
 608         (0x0e00 << 16) | (0xc2c0 >> 2),
 609         0x00000000,
 610         (0x0e00 << 16) | (0x8228 >> 2),
 611         0x00000000,
 612         (0x0e00 << 16) | (0x829c >> 2),
 613         0x00000000,
 614         (0x0e00 << 16) | (0x869c >> 2),
 615         0x00000000,
 616         (0x0600 << 16) | (0x98f4 >> 2),
 617         0x00000000,
 618         (0x0e00 << 16) | (0x98f8 >> 2),
 619         0x00000000,
 620         (0x0e00 << 16) | (0x9900 >> 2),
 621         0x00000000,
 622         (0x0e00 << 16) | (0xc260 >> 2),
 623         0x00000000,
 624         (0x0e00 << 16) | (0x90e8 >> 2),
 625         0x00000000,
 626         (0x0e00 << 16) | (0x3c000 >> 2),
 627         0x00000000,
 628         (0x0e00 << 16) | (0x3c00c >> 2),
 629         0x00000000,
 630         (0x0e00 << 16) | (0x8c1c >> 2),
 631         0x00000000,
 632         (0x0e00 << 16) | (0x9700 >> 2),
 633         0x00000000,
 634         (0x0e00 << 16) | (0xcd20 >> 2),
 635         0x00000000,
 636         (0x4e00 << 16) | (0xcd20 >> 2),
 637         0x00000000,
 638         (0x5e00 << 16) | (0xcd20 >> 2),
 639         0x00000000,
 640         (0x6e00 << 16) | (0xcd20 >> 2),
 641         0x00000000,
 642         (0x7e00 << 16) | (0xcd20 >> 2),
 643         0x00000000,
 644         (0x0e00 << 16) | (0x89bc >> 2),
 645         0x00000000,
 646         (0x0e00 << 16) | (0x8900 >> 2),
 647         0x00000000,
 648         0x3,
 649         (0x0e00 << 16) | (0xc130 >> 2),
 650         0x00000000,
 651         (0x0e00 << 16) | (0xc134 >> 2),
 652         0x00000000,
 653         (0x0e00 << 16) | (0xc1fc >> 2),
 654         0x00000000,
 655         (0x0e00 << 16) | (0xc208 >> 2),
 656         0x00000000,
 657         (0x0e00 << 16) | (0xc264 >> 2),
 658         0x00000000,
 659         (0x0e00 << 16) | (0xc268 >> 2),
 660         0x00000000,
 661         (0x0e00 << 16) | (0xc26c >> 2),
 662         0x00000000,
 663         (0x0e00 << 16) | (0xc270 >> 2),
 664         0x00000000,
 665         (0x0e00 << 16) | (0xc274 >> 2),
 666         0x00000000,
 667         (0x0e00 << 16) | (0xc28c >> 2),
 668         0x00000000,
 669         (0x0e00 << 16) | (0xc290 >> 2),
 670         0x00000000,
 671         (0x0e00 << 16) | (0xc294 >> 2),
 672         0x00000000,
 673         (0x0e00 << 16) | (0xc298 >> 2),
 674         0x00000000,
 675         (0x0e00 << 16) | (0xc2a0 >> 2),
 676         0x00000000,
 677         (0x0e00 << 16) | (0xc2a4 >> 2),
 678         0x00000000,
 679         (0x0e00 << 16) | (0xc2a8 >> 2),
 680         0x00000000,
 681         (0x0e00 << 16) | (0xc2ac >> 2),
 682         0x00000000,
 683         (0x0e00 << 16) | (0x301d0 >> 2),
 684         0x00000000,
 685         (0x0e00 << 16) | (0x30238 >> 2),
 686         0x00000000,
 687         (0x0e00 << 16) | (0x30250 >> 2),
 688         0x00000000,
 689         (0x0e00 << 16) | (0x30254 >> 2),
 690         0x00000000,
 691         (0x0e00 << 16) | (0x30258 >> 2),
 692         0x00000000,
 693         (0x0e00 << 16) | (0x3025c >> 2),
 694         0x00000000,
 695         (0x4e00 << 16) | (0xc900 >> 2),
 696         0x00000000,
 697         (0x5e00 << 16) | (0xc900 >> 2),
 698         0x00000000,
 699         (0x6e00 << 16) | (0xc900 >> 2),
 700         0x00000000,
 701         (0x7e00 << 16) | (0xc900 >> 2),
 702         0x00000000,
 703         (0x4e00 << 16) | (0xc904 >> 2),
 704         0x00000000,
 705         (0x5e00 << 16) | (0xc904 >> 2),
 706         0x00000000,
 707         (0x6e00 << 16) | (0xc904 >> 2),
 708         0x00000000,
 709         (0x7e00 << 16) | (0xc904 >> 2),
 710         0x00000000,
 711         (0x4e00 << 16) | (0xc908 >> 2),
 712         0x00000000,
 713         (0x5e00 << 16) | (0xc908 >> 2),
 714         0x00000000,
 715         (0x6e00 << 16) | (0xc908 >> 2),
 716         0x00000000,
 717         (0x7e00 << 16) | (0xc908 >> 2),
 718         0x00000000,
 719         (0x4e00 << 16) | (0xc90c >> 2),
 720         0x00000000,
 721         (0x5e00 << 16) | (0xc90c >> 2),
 722         0x00000000,
 723         (0x6e00 << 16) | (0xc90c >> 2),
 724         0x00000000,
 725         (0x7e00 << 16) | (0xc90c >> 2),
 726         0x00000000,
 727         (0x4e00 << 16) | (0xc910 >> 2),
 728         0x00000000,
 729         (0x5e00 << 16) | (0xc910 >> 2),
 730         0x00000000,
 731         (0x6e00 << 16) | (0xc910 >> 2),
 732         0x00000000,
 733         (0x7e00 << 16) | (0xc910 >> 2),
 734         0x00000000,
 735         (0x0e00 << 16) | (0xc99c >> 2),
 736         0x00000000,
 737         (0x0e00 << 16) | (0x9834 >> 2),
 738         0x00000000,
 739         (0x0000 << 16) | (0x30f00 >> 2),
 740         0x00000000,
 741         (0x0000 << 16) | (0x30f04 >> 2),
 742         0x00000000,
 743         (0x0000 << 16) | (0x30f08 >> 2),
 744         0x00000000,
 745         (0x0000 << 16) | (0x30f0c >> 2),
 746         0x00000000,
 747         (0x0600 << 16) | (0x9b7c >> 2),
 748         0x00000000,
 749         (0x0e00 << 16) | (0x8a14 >> 2),
 750         0x00000000,
 751         (0x0e00 << 16) | (0x8a18 >> 2),
 752         0x00000000,
 753         (0x0600 << 16) | (0x30a00 >> 2),
 754         0x00000000,
 755         (0x0e00 << 16) | (0x8bf0 >> 2),
 756         0x00000000,
 757         (0x0e00 << 16) | (0x8bcc >> 2),
 758         0x00000000,
 759         (0x0e00 << 16) | (0x8b24 >> 2),
 760         0x00000000,
 761         (0x0e00 << 16) | (0x30a04 >> 2),
 762         0x00000000,
 763         (0x0600 << 16) | (0x30a10 >> 2),
 764         0x00000000,
 765         (0x0600 << 16) | (0x30a14 >> 2),
 766         0x00000000,
 767         (0x0600 << 16) | (0x30a18 >> 2),
 768         0x00000000,
 769         (0x0600 << 16) | (0x30a2c >> 2),
 770         0x00000000,
 771         (0x0e00 << 16) | (0xc700 >> 2),
 772         0x00000000,
 773         (0x0e00 << 16) | (0xc704 >> 2),
 774         0x00000000,
 775         (0x0e00 << 16) | (0xc708 >> 2),
 776         0x00000000,
 777         (0x0e00 << 16) | (0xc768 >> 2),
 778         0x00000000,
 779         (0x0400 << 16) | (0xc770 >> 2),
 780         0x00000000,
 781         (0x0400 << 16) | (0xc774 >> 2),
 782         0x00000000,
 783         (0x0400 << 16) | (0xc798 >> 2),
 784         0x00000000,
 785         (0x0400 << 16) | (0xc79c >> 2),
 786         0x00000000,
 787         (0x0e00 << 16) | (0x9100 >> 2),
 788         0x00000000,
 789         (0x0e00 << 16) | (0x3c010 >> 2),
 790         0x00000000,
 791         (0x0e00 << 16) | (0x8c00 >> 2),
 792         0x00000000,
 793         (0x0e00 << 16) | (0x8c04 >> 2),
 794         0x00000000,
 795         (0x0e00 << 16) | (0x8c20 >> 2),
 796         0x00000000,
 797         (0x0e00 << 16) | (0x8c38 >> 2),
 798         0x00000000,
 799         (0x0e00 << 16) | (0x8c3c >> 2),
 800         0x00000000,
 801         (0x0e00 << 16) | (0xae00 >> 2),
 802         0x00000000,
 803         (0x0e00 << 16) | (0x9604 >> 2),
 804         0x00000000,
 805         (0x0e00 << 16) | (0xac08 >> 2),
 806         0x00000000,
 807         (0x0e00 << 16) | (0xac0c >> 2),
 808         0x00000000,
 809         (0x0e00 << 16) | (0xac10 >> 2),
 810         0x00000000,
 811         (0x0e00 << 16) | (0xac14 >> 2),
 812         0x00000000,
 813         (0x0e00 << 16) | (0xac58 >> 2),
 814         0x00000000,
 815         (0x0e00 << 16) | (0xac68 >> 2),
 816         0x00000000,
 817         (0x0e00 << 16) | (0xac6c >> 2),
 818         0x00000000,
 819         (0x0e00 << 16) | (0xac70 >> 2),
 820         0x00000000,
 821         (0x0e00 << 16) | (0xac74 >> 2),
 822         0x00000000,
 823         (0x0e00 << 16) | (0xac78 >> 2),
 824         0x00000000,
 825         (0x0e00 << 16) | (0xac7c >> 2),
 826         0x00000000,
 827         (0x0e00 << 16) | (0xac80 >> 2),
 828         0x00000000,
 829         (0x0e00 << 16) | (0xac84 >> 2),
 830         0x00000000,
 831         (0x0e00 << 16) | (0xac88 >> 2),
 832         0x00000000,
 833         (0x0e00 << 16) | (0xac8c >> 2),
 834         0x00000000,
 835         (0x0e00 << 16) | (0x970c >> 2),
 836         0x00000000,
 837         (0x0e00 << 16) | (0x9714 >> 2),
 838         0x00000000,
 839         (0x0e00 << 16) | (0x9718 >> 2),
 840         0x00000000,
 841         (0x0e00 << 16) | (0x971c >> 2),
 842         0x00000000,
 843         (0x0e00 << 16) | (0x31068 >> 2),
 844         0x00000000,
 845         (0x4e00 << 16) | (0x31068 >> 2),
 846         0x00000000,
 847         (0x5e00 << 16) | (0x31068 >> 2),
 848         0x00000000,
 849         (0x6e00 << 16) | (0x31068 >> 2),
 850         0x00000000,
 851         (0x7e00 << 16) | (0x31068 >> 2),
 852         0x00000000,
 853         (0x0e00 << 16) | (0xcd10 >> 2),
 854         0x00000000,
 855         (0x0e00 << 16) | (0xcd14 >> 2),
 856         0x00000000,
 857         (0x0e00 << 16) | (0x88b0 >> 2),
 858         0x00000000,
 859         (0x0e00 << 16) | (0x88b4 >> 2),
 860         0x00000000,
 861         (0x0e00 << 16) | (0x88b8 >> 2),
 862         0x00000000,
 863         (0x0e00 << 16) | (0x88bc >> 2),
 864         0x00000000,
 865         (0x0400 << 16) | (0x89c0 >> 2),
 866         0x00000000,
 867         (0x0e00 << 16) | (0x88c4 >> 2),
 868         0x00000000,
 869         (0x0e00 << 16) | (0x88c8 >> 2),
 870         0x00000000,
 871         (0x0e00 << 16) | (0x88d0 >> 2),
 872         0x00000000,
 873         (0x0e00 << 16) | (0x88d4 >> 2),
 874         0x00000000,
 875         (0x0e00 << 16) | (0x88d8 >> 2),
 876         0x00000000,
 877         (0x0e00 << 16) | (0x8980 >> 2),
 878         0x00000000,
 879         (0x0e00 << 16) | (0x30938 >> 2),
 880         0x00000000,
 881         (0x0e00 << 16) | (0x3093c >> 2),
 882         0x00000000,
 883         (0x0e00 << 16) | (0x30940 >> 2),
 884         0x00000000,
 885         (0x0e00 << 16) | (0x89a0 >> 2),
 886         0x00000000,
 887         (0x0e00 << 16) | (0x30900 >> 2),
 888         0x00000000,
 889         (0x0e00 << 16) | (0x30904 >> 2),
 890         0x00000000,
 891         (0x0e00 << 16) | (0x89b4 >> 2),
 892         0x00000000,
 893         (0x0e00 << 16) | (0x3e1fc >> 2),
 894         0x00000000,
 895         (0x0e00 << 16) | (0x3c210 >> 2),
 896         0x00000000,
 897         (0x0e00 << 16) | (0x3c214 >> 2),
 898         0x00000000,
 899         (0x0e00 << 16) | (0x3c218 >> 2),
 900         0x00000000,
 901         (0x0e00 << 16) | (0x8904 >> 2),
 902         0x00000000,
 903         0x5,
 904         (0x0e00 << 16) | (0x8c28 >> 2),
 905         (0x0e00 << 16) | (0x8c2c >> 2),
 906         (0x0e00 << 16) | (0x8c30 >> 2),
 907         (0x0e00 << 16) | (0x8c34 >> 2),
 908         (0x0e00 << 16) | (0x9600 >> 2),
 909 };
 910
 911 static const u32 bonaire_golden_spm_registers[] =
 912 {
 913         0x30800, 0xe0ffffff, 0xe0000000
 914 };
 915
 916 static const u32 bonaire_golden_common_registers[] =
 917 {
 918         0xc770, 0xffffffff, 0x00000800,
 919         0xc774, 0xffffffff, 0x00000800,
 920         0xc798, 0xffffffff, 0x00007fbf,
 921         0xc79c, 0xffffffff, 0x00007faf
 922 };
 923
 924 static const u32 bonaire_golden_registers[] =
 925 {
 926         0x3354, 0x00000333, 0x00000333,
 927         0x3350, 0x000c0fc0, 0x00040200,
 928         0x9a10, 0x00010000, 0x00058208,
 929         0x3c000, 0xffff1fff, 0x00140000,
 930         0x3c200, 0xfdfc0fff, 0x00000100,
 931         0x3c234, 0x40000000, 0x40000200,
 932         0x9830, 0xffffffff, 0x00000000,
 933         0x9834, 0xf00fffff, 0x00000400,
 934         0x9838, 0x0002021c, 0x00020200,
 935         0xc78, 0x00000080, 0x00000000,
 936         0x5bb0, 0x000000f0, 0x00000070,
 937         0x5bc0, 0xf0311fff, 0x80300000,
 938         0x98f8, 0x73773777, 0x12010001,
 939         0x350c, 0x00810000, 0x408af000,
 940         0x7030, 0x31000111, 0x00000011,
 941         0x2f48, 0x73773777, 0x12010001,
 942         0x220c, 0x00007fb6, 0x0021a1b1,
 943         0x2210, 0x00007fb6, 0x002021b1,
 944         0x2180, 0x00007fb6, 0x00002191,
 945         0x2218, 0x00007fb6, 0x002121b1,
 946         0x221c, 0x00007fb6, 0x002021b1,
 947         0x21dc, 0x00007fb6, 0x00002191,
 948         0x21e0, 0x00007fb6, 0x00002191,
 949         0x3628, 0x0000003f, 0x0000000a,
 950         0x362c, 0x0000003f, 0x0000000a,
 951         0x2ae4, 0x00073ffe, 0x000022a2,
 952         0x240c, 0x000007ff, 0x00000000,
 953         0x8a14, 0xf000003f, 0x00000007,
 954         0x8bf0, 0x00002001, 0x00000001,
 955         0x8b24, 0xffffffff, 0x00ffffff,
 956         0x30a04, 0x0000ff0f, 0x00000000,
 957         0x28a4c, 0x07ffffff, 0x06000000,
 958         0x4d8, 0x00000fff, 0x00000100,
 959         0x3e78, 0x00000001, 0x00000002,
 960         0x9100, 0x03000000, 0x0362c688,
 961         0x8c00, 0x000000ff, 0x00000001,
 962         0xe40, 0x00001fff, 0x00001fff,
 963         0x9060, 0x0000007f, 0x00000020,
 964         0x9508, 0x00010000, 0x00010000,
 965         0xac14, 0x000003ff, 0x000000f3,
 966         0xac0c, 0xffffffff, 0x00001032
 967 };
 968
 969 static const u32 bonaire_mgcg_cgcg_init[] =
 970 {
 971         0xc420, 0xffffffff, 0xfffffffc,
 972         0x30800, 0xffffffff, 0xe0000000,
 973         0x3c2a0, 0xffffffff, 0x00000100,
 974         0x3c208, 0xffffffff, 0x00000100,
 975         0x3c2c0, 0xffffffff, 0xc0000100,
 976         0x3c2c8, 0xffffffff, 0xc0000100,
 977         0x3c2c4, 0xffffffff, 0xc0000100,
 978         0x55e4, 0xffffffff, 0x00600100,
 979         0x3c280, 0xffffffff, 0x00000100,
 980         0x3c214, 0xffffffff, 0x06000100,
 981         0x3c220, 0xffffffff, 0x00000100,
 982         0x3c218, 0xffffffff, 0x06000100,
 983         0x3c204, 0xffffffff, 0x00000100,
 984         0x3c2e0, 0xffffffff, 0x00000100,
 985         0x3c224, 0xffffffff, 0x00000100,
 986         0x3c200, 0xffffffff, 0x00000100,
 987         0x3c230, 0xffffffff, 0x00000100,
 988         0x3c234, 0xffffffff, 0x00000100,
 989         0x3c250, 0xffffffff, 0x00000100,
 990         0x3c254, 0xffffffff, 0x00000100,
 991         0x3c258, 0xffffffff, 0x00000100,
 992         0x3c25c, 0xffffffff, 0x00000100,
 993         0x3c260, 0xffffffff, 0x00000100,
 994         0x3c27c, 0xffffffff, 0x00000100,
 995         0x3c278, 0xffffffff, 0x00000100,
 996         0x3c210, 0xffffffff, 0x06000100,
 997         0x3c290, 0xffffffff, 0x00000100,
 998         0x3c274, 0xffffffff, 0x00000100,
 999         0x3c2b4, 0xffffffff, 0x00000100,
1000         0x3c2b0, 0xffffffff, 0x00000100,
1001         0x3c270, 0xffffffff, 0x00000100,
1002         0x30800, 0xffffffff, 0xe0000000,
1003         0x3c020, 0xffffffff, 0x00010000,
1004         0x3c024, 0xffffffff, 0x00030002,
1005         0x3c028, 0xffffffff, 0x00040007,
1006         0x3c02c, 0xffffffff, 0x00060005,
1007         0x3c030, 0xffffffff, 0x00090008,
1008         0x3c034, 0xffffffff, 0x00010000,
1009         0x3c038, 0xffffffff, 0x00030002,
1010         0x3c03c, 0xffffffff, 0x00040007,
1011         0x3c040, 0xffffffff, 0x00060005,
1012         0x3c044, 0xffffffff, 0x00090008,
1013         0x3c048, 0xffffffff, 0x00010000,
1014         0x3c04c, 0xffffffff, 0x00030002,
1015         0x3c050, 0xffffffff, 0x00040007,
1016         0x3c054, 0xffffffff, 0x00060005,
1017         0x3c058, 0xffffffff, 0x00090008,
1018         0x3c05c, 0xffffffff, 0x00010000,
1019         0x3c060, 0xffffffff, 0x00030002,
1020         0x3c064, 0xffffffff, 0x00040007,
1021         0x3c068, 0xffffffff, 0x00060005,
1022         0x3c06c, 0xffffffff, 0x00090008,
1023         0x3c070, 0xffffffff, 0x00010000,
1024         0x3c074, 0xffffffff, 0x00030002,
1025         0x3c078, 0xffffffff, 0x00040007,
1026         0x3c07c, 0xffffffff, 0x00060005,
1027         0x3c080, 0xffffffff, 0x00090008,
1028         0x3c084, 0xffffffff, 0x00010000,
1029         0x3c088, 0xffffffff, 0x00030002,
1030         0x3c08c, 0xffffffff, 0x00040007,
1031         0x3c090, 0xffffffff, 0x00060005,
1032         0x3c094, 0xffffffff, 0x00090008,
1033         0x3c098, 0xffffffff, 0x00010000,
1034         0x3c09c, 0xffffffff, 0x00030002,
1035         0x3c0a0, 0xffffffff, 0x00040007,
1036         0x3c0a4, 0xffffffff, 0x00060005,
1037         0x3c0a8, 0xffffffff, 0x00090008,
1038         0x3c000, 0xffffffff, 0x96e00200,
1039         0x8708, 0xffffffff, 0x00900100,
1040         0xc424, 0xffffffff, 0x0020003f,
1041         0x38, 0xffffffff, 0x0140001c,
1042         0x3c, 0x000f0000, 0x000f0000,
1043         0x220, 0xffffffff, 0xC060000C,
1044         0x224, 0xc0000fff, 0x00000100,
1045         0xf90, 0xffffffff, 0x00000100,
1046         0xf98, 0x00000101, 0x00000000,
1047         0x20a8, 0xffffffff, 0x00000104,
1048         0x55e4, 0xff000fff, 0x00000100,
1049         0x30cc, 0xc0000fff, 0x00000104,
1050         0xc1e4, 0x00000001, 0x00000001,
1051         0xd00c, 0xff000ff0, 0x00000100,
1052         0xd80c, 0xff000ff0, 0x00000100
1053 };
1054
1055 static const u32 spectre_golden_spm_registers[] =
1056 {
1057         0x30800, 0xe0ffffff, 0xe0000000
1058 };
1059
1060 static const u32 spectre_golden_common_registers[] =
1061 {
1062         0xc770, 0xffffffff, 0x00000800,
1063         0xc774, 0xffffffff, 0x00000800,
1064         0xc798, 0xffffffff, 0x00007fbf,
1065         0xc79c, 0xffffffff, 0x00007faf
1066 };
1067
1068 static const u32 spectre_golden_registers[] =
1069 {
1070         0x3c000, 0xffff1fff, 0x96940200,
1071         0x3c00c, 0xffff0001, 0xff000000,
1072         0x3c200, 0xfffc0fff, 0x00000100,
1073         0x6ed8, 0x00010101, 0x00010000,
1074         0x9834, 0xf00fffff, 0x00000400,
1075         0x9838, 0xfffffffc, 0x00020200,
1076         0x5bb0, 0x000000f0, 0x00000070,
1077         0x5bc0, 0xf0311fff, 0x80300000,
1078         0x98f8, 0x73773777, 0x12010001,
1079         0x9b7c, 0x00ff0000, 0x00fc0000,
1080         0x2f48, 0x73773777, 0x12010001,
1081         0x8a14, 0xf000003f, 0x00000007,
1082         0x8b24, 0xffffffff, 0x00ffffff,
1083         0x28350, 0x3f3f3fff, 0x00000082,
1084         0x28355, 0x0000003f, 0x00000000,
1085         0x3e78, 0x00000001, 0x00000002,
1086         0x913c, 0xffff03df, 0x00000004,
1087         0xc768, 0x00000008, 0x00000008,
1088         0x8c00, 0x000008ff, 0x00000800,
1089         0x9508, 0x00010000, 0x00010000,
1090         0xac0c, 0xffffffff, 0x54763210,
1091         0x214f8, 0x01ff01ff, 0x00000002,
1092         0x21498, 0x007ff800, 0x00200000,
1093         0x2015c, 0xffffffff, 0x00000f40,
1094         0x30934, 0xffffffff, 0x00000001
1095 };
1096
1097 static const u32 spectre_mgcg_cgcg_init[] =
1098 {
1099         0xc420, 0xffffffff, 0xfffffffc,
1100         0x30800, 0xffffffff, 0xe0000000,
1101         0x3c2a0, 0xffffffff, 0x00000100,
1102         0x3c208, 0xffffffff, 0x00000100,
1103         0x3c2c0, 0xffffffff, 0x00000100,
1104         0x3c2c8, 0xffffffff, 0x00000100,
1105         0x3c2c4, 0xffffffff, 0x00000100,
1106         0x55e4, 0xffffffff, 0x00600100,
1107         0x3c280, 0xffffffff, 0x00000100,
1108         0x3c214, 0xffffffff, 0x06000100,
1109         0x3c220, 0xffffffff, 0x00000100,
1110         0x3c218, 0xffffffff, 0x06000100,
1111         0x3c204, 0xffffffff, 0x00000100,
1112         0x3c2e0, 0xffffffff, 0x00000100,
1113         0x3c224, 0xffffffff, 0x00000100,
1114         0x3c200, 0xffffffff, 0x00000100,
1115         0x3c230, 0xffffffff, 0x00000100,
1116         0x3c234, 0xffffffff, 0x00000100,
1117         0x3c250, 0xffffffff, 0x00000100,
1118         0x3c254, 0xffffffff, 0x00000100,
1119         0x3c258, 0xffffffff, 0x00000100,
1120         0x3c25c, 0xffffffff, 0x00000100,
1121         0x3c260, 0xffffffff, 0x00000100,
1122         0x3c27c, 0xffffffff, 0x00000100,
1123         0x3c278, 0xffffffff, 0x00000100,
1124         0x3c210, 0xffffffff, 0x06000100,
1125         0x3c290, 0xffffffff, 0x00000100,
1126         0x3c274, 0xffffffff, 0x00000100,
1127         0x3c2b4, 0xffffffff, 0x00000100,
1128         0x3c2b0, 0xffffffff, 0x00000100,
1129         0x3c270, 0xffffffff, 0x00000100,
1130         0x30800, 0xffffffff, 0xe0000000,
1131         0x3c020, 0xffffffff, 0x00010000,
1132         0x3c024, 0xffffffff, 0x00030002,
1133         0x3c028, 0xffffffff, 0x00040007,
1134         0x3c02c, 0xffffffff, 0x00060005,
1135         0x3c030, 0xffffffff, 0x00090008,
1136         0x3c034, 0xffffffff, 0x00010000,
1137         0x3c038, 0xffffffff, 0x00030002,
1138         0x3c03c, 0xffffffff, 0x00040007,
1139         0x3c040, 0xffffffff, 0x00060005,
1140         0x3c044, 0xffffffff, 0x00090008,
1141         0x3c048, 0xffffffff, 0x00010000,
1142         0x3c04c, 0xffffffff, 0x00030002,
1143         0x3c050, 0xffffffff, 0x00040007,
1144         0x3c054, 0xffffffff, 0x00060005,
1145         0x3c058, 0xffffffff, 0x00090008,
1146         0x3c05c, 0xffffffff, 0x00010000,
1147         0x3c060, 0xffffffff, 0x00030002,
1148         0x3c064, 0xffffffff, 0x00040007,
1149         0x3c068, 0xffffffff, 0x00060005,
1150         0x3c06c, 0xffffffff, 0x00090008,
1151         0x3c070, 0xffffffff, 0x00010000,
1152         0x3c074, 0xffffffff, 0x00030002,
1153         0x3c078, 0xffffffff, 0x00040007,
1154         0x3c07c, 0xffffffff, 0x00060005,
1155         0x3c080, 0xffffffff, 0x00090008,
1156         0x3c084, 0xffffffff, 0x00010000,
1157         0x3c088, 0xffffffff, 0x00030002,
1158         0x3c08c, 0xffffffff, 0x00040007,
1159         0x3c090, 0xffffffff, 0x00060005,
1160         0x3c094, 0xffffffff, 0x00090008,
1161         0x3c098, 0xffffffff, 0x00010000,
1162         0x3c09c, 0xffffffff, 0x00030002,
1163         0x3c0a0, 0xffffffff, 0x00040007,
1164         0x3c0a4, 0xffffffff, 0x00060005,
1165         0x3c0a8, 0xffffffff, 0x00090008,
1166         0x3c0ac, 0xffffffff, 0x00010000,
1167         0x3c0b0, 0xffffffff, 0x00030002,
1168         0x3c0b4, 0xffffffff, 0x00040007,
1169         0x3c0b8, 0xffffffff, 0x00060005,
1170         0x3c0bc, 0xffffffff, 0x00090008,
1171         0x3c000, 0xffffffff, 0x96e00200,
1172         0x8708, 0xffffffff, 0x00900100,
1173         0xc424, 0xffffffff, 0x0020003f,
1174         0x38, 0xffffffff, 0x0140001c,
1175         0x3c, 0x000f0000, 0x000f0000,
1176         0x220, 0xffffffff, 0xC060000C,
1177         0x224, 0xc0000fff, 0x00000100,
1178         0xf90, 0xffffffff, 0x00000100,
1179         0xf98, 0x00000101, 0x00000000,
1180         0x20a8, 0xffffffff, 0x00000104,
1181         0x55e4, 0xff000fff, 0x00000100,
1182         0x30cc, 0xc0000fff, 0x00000104,
1183         0xc1e4, 0x00000001, 0x00000001,
1184         0xd00c, 0xff000ff0, 0x00000100,
1185         0xd80c, 0xff000ff0, 0x00000100
1186 };
1187
1188 static const u32 kalindi_golden_spm_registers[] =
1189 {
1190         0x30800, 0xe0ffffff, 0xe0000000
1191 };
1192
1193 static const u32 kalindi_golden_common_registers[] =
1194 {
1195         0xc770, 0xffffffff, 0x00000800,
1196         0xc774, 0xffffffff, 0x00000800,
1197         0xc798, 0xffffffff, 0x00007fbf,
1198         0xc79c, 0xffffffff, 0x00007faf
1199 };
1200
1201 static const u32 kalindi_golden_registers[] =
1202 {
1203         0x3c000, 0xffffdfff, 0x6e944040,
1204         0x55e4, 0xff607fff, 0xfc000100,
1205         0x3c220, 0xff000fff, 0x00000100,
1206         0x3c224, 0xff000fff, 0x00000100,
1207         0x3c200, 0xfffc0fff, 0x00000100,
1208         0x6ed8, 0x00010101, 0x00010000,
1209         0x9830, 0xffffffff, 0x00000000,
1210         0x9834, 0xf00fffff, 0x00000400,
1211         0x5bb0, 0x000000f0, 0x00000070,
1212         0x5bc0, 0xf0311fff, 0x80300000,
1213         0x98f8, 0x73773777, 0x12010001,
1214         0x98fc, 0xffffffff, 0x00000010,
1215         0x9b7c, 0x00ff0000, 0x00fc0000,
1216         0x8030, 0x00001f0f, 0x0000100a,
1217         0x2f48, 0x73773777, 0x12010001,
1218         0x2408, 0x000fffff, 0x000c007f,
1219         0x8a14, 0xf000003f, 0x00000007,
1220         0x8b24, 0x3fff3fff, 0x00ffcfff,
1221         0x30a04, 0x0000ff0f, 0x00000000,
1222         0x28a4c, 0x07ffffff, 0x06000000,
1223         0x4d8, 0x00000fff, 0x00000100,
1224         0x3e78, 0x00000001, 0x00000002,
1225         0xc768, 0x00000008, 0x00000008,
1226         0x8c00, 0x000000ff, 0x00000003,
1227         0x214f8, 0x01ff01ff, 0x00000002,
1228         0x21498, 0x007ff800, 0x00200000,
1229         0x2015c, 0xffffffff, 0x00000f40,
1230         0x88c4, 0x001f3ae3, 0x00000082,
1231         0x88d4, 0x0000001f, 0x00000010,
1232         0x30934, 0xffffffff, 0x00000000
1233 };
1234
1235 static const u32 kalindi_mgcg_cgcg_init[] =
1236 {
1237         0xc420, 0xffffffff, 0xfffffffc,
1238         0x30800, 0xffffffff, 0xe0000000,
1239         0x3c2a0, 0xffffffff, 0x00000100,
1240         0x3c208, 0xffffffff, 0x00000100,
1241         0x3c2c0, 0xffffffff, 0x00000100,
1242         0x3c2c8, 0xffffffff, 0x00000100,
1243         0x3c2c4, 0xffffffff, 0x00000100,
1244         0x55e4, 0xffffffff, 0x00600100,
1245         0x3c280, 0xffffffff, 0x00000100,
1246         0x3c214, 0xffffffff, 0x06000100,
1247         0x3c220, 0xffffffff, 0x00000100,
1248         0x3c218, 0xffffffff, 0x06000100,
1249         0x3c204, 0xffffffff, 0x00000100,
1250         0x3c2e0, 0xffffffff, 0x00000100,
1251         0x3c224, 0xffffffff, 0x00000100,
1252         0x3c200, 0xffffffff, 0x00000100,
1253         0x3c230, 0xffffffff, 0x00000100,
1254         0x3c234, 0xffffffff, 0x00000100,
1255         0x3c250, 0xffffffff, 0x00000100,
1256         0x3c254, 0xffffffff, 0x00000100,
1257         0x3c258, 0xffffffff, 0x00000100,
1258         0x3c25c, 0xffffffff, 0x00000100,
1259         0x3c260, 0xffffffff, 0x00000100,
1260         0x3c27c, 0xffffffff, 0x00000100,
1261         0x3c278, 0xffffffff, 0x00000100,
1262         0x3c210, 0xffffffff, 0x06000100,
1263         0x3c290, 0xffffffff, 0x00000100,
1264         0x3c274, 0xffffffff, 0x00000100,
1265         0x3c2b4, 0xffffffff, 0x00000100,
1266         0x3c2b0, 0xffffffff, 0x00000100,
1267         0x3c270, 0xffffffff, 0x00000100,
1268         0x30800, 0xffffffff, 0xe0000000,
1269         0x3c020, 0xffffffff, 0x00010000,
1270         0x3c024, 0xffffffff, 0x00030002,
1271         0x3c028, 0xffffffff, 0x00040007,
1272         0x3c02c, 0xffffffff, 0x00060005,
1273         0x3c030, 0xffffffff, 0x00090008,
1274         0x3c034, 0xffffffff, 0x00010000,
1275         0x3c038, 0xffffffff, 0x00030002,
1276         0x3c03c, 0xffffffff, 0x00040007,
1277         0x3c040, 0xffffffff, 0x00060005,
1278         0x3c044, 0xffffffff, 0x00090008,
1279         0x3c000, 0xffffffff, 0x96e00200,
1280         0x8708, 0xffffffff, 0x00900100,
1281         0xc424, 0xffffffff, 0x0020003f,
1282         0x38, 0xffffffff, 0x0140001c,
1283         0x3c, 0x000f0000, 0x000f0000,
1284         0x220, 0xffffffff, 0xC060000C,
1285         0x224, 0xc0000fff, 0x00000100,
1286         0x20a8, 0xffffffff, 0x00000104,
1287         0x55e4, 0xff000fff, 0x00000100,
1288         0x30cc, 0xc0000fff, 0x00000104,
1289         0xc1e4, 0x00000001, 0x00000001,
1290         0xd00c, 0xff000ff0, 0x00000100,
1291         0xd80c, 0xff000ff0, 0x00000100
1292 };
1293
1294 static void cik_init_golden_registers(struct radeon_device *rdev)
1295 {
1296         switch (rdev->family) {
1297         case CHIP_BONAIRE:
1298                 radeon_program_register_sequence(rdev,
1299                                                  bonaire_mgcg_cgcg_init,
1300                                                  (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1301                 radeon_program_register_sequence(rdev,
1302                                                  bonaire_golden_registers,
1303                                                  (const u32)ARRAY_SIZE(bonaire_golden_registers));
1304                 radeon_program_register_sequence(rdev,
1305                                                  bonaire_golden_common_registers,
1306                                                  (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1307                 radeon_program_register_sequence(rdev,
1308                                                  bonaire_golden_spm_registers,
1309                                                  (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1310                 break;
1311         case CHIP_KABINI:
1312                 radeon_program_register_sequence(rdev,
1313                                                  kalindi_mgcg_cgcg_init,
1314                                                  (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1315                 radeon_program_register_sequence(rdev,
1316                                                  kalindi_golden_registers,
1317                                                  (const u32)ARRAY_SIZE(kalindi_golden_registers));
1318                 radeon_program_register_sequence(rdev,
1319                                                  kalindi_golden_common_registers,
1320                                                  (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1321                 radeon_program_register_sequence(rdev,
1322                                                  kalindi_golden_spm_registers,
1323                                                  (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1324                 break;
1325         case CHIP_KAVERI:
1326                 radeon_program_register_sequence(rdev,
1327                                                  spectre_mgcg_cgcg_init,
1328                                                  (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1329                 radeon_program_register_sequence(rdev,
1330                                                  spectre_golden_registers,
1331                                                  (const u32)ARRAY_SIZE(spectre_golden_registers));
1332                 radeon_program_register_sequence(rdev,
1333                                                  spectre_golden_common_registers,
1334                                                  (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1335                 radeon_program_register_sequence(rdev,
1336                                                  spectre_golden_spm_registers,
1337                                                  (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1338                 break;
1339         default:
1340                 break;
1341         }
1342 }
1343
1344 /**
1345  * cik_get_xclk - get the xclk
1346  *
1347  * @rdev: radeon_device pointer
1348  *
1349  * Returns the reference clock used by the gfx engine
1350  * (CIK).
1351  */
1352 u32 cik_get_xclk(struct radeon_device *rdev)
1353 {
1354         u32 reference_clock = rdev->clock.spll.reference_freq;
1355
1356         if (rdev->flags & RADEON_IS_IGP) {
1357                 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1358                         return reference_clock / 2;
1359         } else {
1360                 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1361                         return reference_clock / 4;
1362         }
1363         return reference_clock;
1364 }
1365
1366 /**
1367  * cik_mm_rdoorbell - read a doorbell dword
1368  *
1369  * @rdev: radeon_device pointer
1370  * @offset: byte offset into the aperture
1371  *
1372  * Returns the value in the doorbell aperture at the
1373  * requested offset (CIK).
1374  */
1375 u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 offset)
1376 {
1377         if (offset < rdev->doorbell.size) {
1378                 return readl(((void __iomem *)rdev->doorbell.ptr) + offset);
1379         } else {
1380                 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", offset);
1381                 return 0;
1382         }
1383 }
1384
1385 /**
1386  * cik_mm_wdoorbell - write a doorbell dword
1387  *
1388  * @rdev: radeon_device pointer
1389  * @offset: byte offset into the aperture
1390  * @v: value to write
1391  *
1392  * Writes @v to the doorbell aperture at the
1393  * requested offset (CIK).
1394  */
1395 void cik_mm_wdoorbell(struct radeon_device *rdev, u32 offset, u32 v)
1396 {
1397         if (offset < rdev->doorbell.size) {
1398                 writel(v, ((void __iomem *)rdev->doorbell.ptr) + offset);
1399         } else {
1400                 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", offset);
1401         }
1402 }
1403
1404 #define BONAIRE_IO_MC_REGS_SIZE 36
1405
1406 static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1407 {
1408         {0x00000070, 0x04400000},
1409         {0x00000071, 0x80c01803},
1410         {0x00000072, 0x00004004},
1411         {0x00000073, 0x00000100},
1412         {0x00000074, 0x00ff0000},
1413         {0x00000075, 0x34000000},
1414         {0x00000076, 0x08000014},
1415         {0x00000077, 0x00cc08ec},
1416         {0x00000078, 0x00000400},
1417         {0x00000079, 0x00000000},
1418         {0x0000007a, 0x04090000},
1419         {0x0000007c, 0x00000000},
1420         {0x0000007e, 0x4408a8e8},
1421         {0x0000007f, 0x00000304},
1422         {0x00000080, 0x00000000},
1423         {0x00000082, 0x00000001},
1424         {0x00000083, 0x00000002},
1425         {0x00000084, 0xf3e4f400},
1426         {0x00000085, 0x052024e3},
1427         {0x00000087, 0x00000000},
1428         {0x00000088, 0x01000000},
1429         {0x0000008a, 0x1c0a0000},
1430         {0x0000008b, 0xff010000},
1431         {0x0000008d, 0xffffefff},
1432         {0x0000008e, 0xfff3efff},
1433         {0x0000008f, 0xfff3efbf},
1434         {0x00000092, 0xf7ffffff},
1435         {0x00000093, 0xffffff7f},
1436         {0x00000095, 0x00101101},
1437         {0x00000096, 0x00000fff},
1438         {0x00000097, 0x00116fff},
1439         {0x00000098, 0x60010000},
1440         {0x00000099, 0x10010000},
1441         {0x0000009a, 0x00006000},
1442         {0x0000009b, 0x00001000},
1443         {0x0000009f, 0x00b48000}
1444 };
1445
1446 /**
1447  * cik_srbm_select - select specific register instances
1448  *
1449  * @rdev: radeon_device pointer
1450  * @me: selected ME (micro engine)
1451  * @pipe: pipe
1452  * @queue: queue
1453  * @vmid: VMID
1454  *
1455  * Switches the currently active registers instances.  Some
1456  * registers are instanced per VMID, others are instanced per
1457  * me/pipe/queue combination.
1458  */
1459 static void cik_srbm_select(struct radeon_device *rdev,
1460                             u32 me, u32 pipe, u32 queue, u32 vmid)
1461 {
1462         u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1463                              MEID(me & 0x3) |
1464                              VMID(vmid & 0xf) |
1465                              QUEUEID(queue & 0x7));
1466         WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1467 }
1468
1469 /* ucode loading */
1470 /**
1471  * ci_mc_load_microcode - load MC ucode into the hw
1472  *
1473  * @rdev: radeon_device pointer
1474  *
1475  * Load the GDDR MC ucode into the hw (CIK).
1476  * Returns 0 on success, error on failure.
1477  */
1478 static int ci_mc_load_microcode(struct radeon_device *rdev)
1479 {
1480         const __be32 *fw_data;
1481         u32 running, blackout = 0;
1482         u32 *io_mc_regs;
1483         int i, ucode_size, regs_size;
1484
1485         if (!rdev->mc_fw)
1486                 return -EINVAL;
1487
1488         switch (rdev->family) {
1489         case CHIP_BONAIRE:
1490         default:
1491                 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1492                 ucode_size = CIK_MC_UCODE_SIZE;
1493                 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1494                 break;
1495         }
1496
1497         running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1498
1499         if (running == 0) {
1500                 if (running) {
1501                         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1502                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1503                 }
1504
1505                 /* reset the engine and set to writable */
1506                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1507                 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1508
1509                 /* load mc io regs */
1510                 for (i = 0; i < regs_size; i++) {
1511                         WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1512                         WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1513                 }
1514                 /* load the MC ucode */
1515                 fw_data = (const __be32 *)rdev->mc_fw->data;
1516                 for (i = 0; i < ucode_size; i++)
1517                         WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1518
1519                 /* put the engine back into the active state */
1520                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1521                 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1522                 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1523
1524                 /* wait for training to complete */
1525                 for (i = 0; i < rdev->usec_timeout; i++) {
1526                         if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1527                                 break;
1528                         udelay(1);
1529                 }
1530                 for (i = 0; i < rdev->usec_timeout; i++) {
1531                         if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1532                                 break;
1533                         udelay(1);
1534                 }
1535
1536                 if (running)
1537                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1538         }
1539
1540         return 0;
1541 }
1542
1543 /**
1544  * cik_init_microcode - load ucode images from disk
1545  *
1546  * @rdev: radeon_device pointer
1547  *
1548  * Use the firmware interface to load the ucode images into
1549  * the driver (not loaded into hw).
1550  * Returns 0 on success, error on failure.
1551  */
1552 static int cik_init_microcode(struct radeon_device *rdev)
1553 {
1554         const char *chip_name;
1555         size_t pfp_req_size, me_req_size, ce_req_size,
1556                 mec_req_size, rlc_req_size, mc_req_size,
1557                 sdma_req_size, smc_req_size;
1558         char fw_name[30];
1559         int err;
1560
1561         DRM_DEBUG("\n");
1562
1563         switch (rdev->family) {
1564         case CHIP_BONAIRE:
1565                 chip_name = "BONAIRE";
1566                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1567                 me_req_size = CIK_ME_UCODE_SIZE * 4;
1568                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1569                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1570                 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1571                 mc_req_size = CIK_MC_UCODE_SIZE * 4;
1572                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1573                 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
1574                 break;
1575         case CHIP_KAVERI:
1576                 chip_name = "KAVERI";
1577                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1578                 me_req_size = CIK_ME_UCODE_SIZE * 4;
1579                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1580                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1581                 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
1582                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1583                 break;
1584         case CHIP_KABINI:
1585                 chip_name = "KABINI";
1586                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1587                 me_req_size = CIK_ME_UCODE_SIZE * 4;
1588                 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1589                 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1590                 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
1591                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1592                 break;
1593         default: BUG();
1594         }
1595
1596         DRM_INFO("Loading %s Microcode\n", chip_name);
1597
1598         snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1599         err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1600         if (err)
1601                 goto out;
1602         if (rdev->pfp_fw->size != pfp_req_size) {
1603                 printk(KERN_ERR
1604                        "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1605                        rdev->pfp_fw->size, fw_name);
1606                 err = -EINVAL;
1607                 goto out;
1608         }
1609
1610         snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1611         err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1612         if (err)
1613                 goto out;
1614         if (rdev->me_fw->size != me_req_size) {
1615                 printk(KERN_ERR
1616                        "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1617                        rdev->me_fw->size, fw_name);
1618                 err = -EINVAL;
1619         }
1620
1621         snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1622         err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1623         if (err)
1624                 goto out;
1625         if (rdev->ce_fw->size != ce_req_size) {
1626                 printk(KERN_ERR
1627                        "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1628                        rdev->ce_fw->size, fw_name);
1629                 err = -EINVAL;
1630         }
1631
1632         snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
1633         err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
1634         if (err)
1635                 goto out;
1636         if (rdev->mec_fw->size != mec_req_size) {
1637                 printk(KERN_ERR
1638                        "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1639                        rdev->mec_fw->size, fw_name);
1640                 err = -EINVAL;
1641         }
1642
1643         snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
1644         err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1645         if (err)
1646                 goto out;
1647         if (rdev->rlc_fw->size != rlc_req_size) {
1648                 printk(KERN_ERR
1649                        "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
1650                        rdev->rlc_fw->size, fw_name);
1651                 err = -EINVAL;
1652         }
1653
1654         snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
1655         err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
1656         if (err)
1657                 goto out;
1658         if (rdev->sdma_fw->size != sdma_req_size) {
1659                 printk(KERN_ERR
1660                        "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
1661                        rdev->sdma_fw->size, fw_name);
1662                 err = -EINVAL;
1663         }
1664
1665         /* No SMC, MC ucode on APUs */
1666         if (!(rdev->flags & RADEON_IS_IGP)) {
1667                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1668                 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1669                 if (err)
1670                         goto out;
1671                 if (rdev->mc_fw->size != mc_req_size) {
1672                         printk(KERN_ERR
1673                                "cik_mc: Bogus length %zu in firmware \"%s\"\n",
1674                                rdev->mc_fw->size, fw_name);
1675                         err = -EINVAL;
1676                 }
1677
1678                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1679                 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1680                 if (err) {
1681                         printk(KERN_ERR
1682                                "smc: error loading firmware \"%s\"\n",
1683                                fw_name);
1684                         release_firmware(rdev->smc_fw);
1685                         rdev->smc_fw = NULL;
1686                 } else if (rdev->smc_fw->size != smc_req_size) {
1687                         printk(KERN_ERR
1688                                "cik_smc: Bogus length %zu in firmware \"%s\"\n",
1689                                rdev->smc_fw->size, fw_name);
1690                         err = -EINVAL;
1691                 }
1692         }
1693
1694 out:
1695         if (err) {
1696                 if (err != -EINVAL)
1697                         printk(KERN_ERR
1698                                "cik_cp: Failed to load firmware \"%s\"\n",
1699                                fw_name);
1700                 release_firmware(rdev->pfp_fw);
1701                 rdev->pfp_fw = NULL;
1702                 release_firmware(rdev->me_fw);
1703                 rdev->me_fw = NULL;
1704                 release_firmware(rdev->ce_fw);
1705                 rdev->ce_fw = NULL;
1706                 release_firmware(rdev->rlc_fw);
1707                 rdev->rlc_fw = NULL;
1708                 release_firmware(rdev->mc_fw);
1709                 rdev->mc_fw = NULL;
1710                 release_firmware(rdev->smc_fw);
1711                 rdev->smc_fw = NULL;
1712         }
1713         return err;
1714 }
1715
1716 /*
1717  * Core functions
1718  */
1719 /**
1720  * cik_tiling_mode_table_init - init the hw tiling table
1721  *
1722  * @rdev: radeon_device pointer
1723  *
1724  * Starting with SI, the tiling setup is done globally in a
1725  * set of 32 tiling modes.  Rather than selecting each set of
1726  * parameters per surface as on older asics, we just select
1727  * which index in the tiling table we want to use, and the
1728  * surface uses those parameters (CIK).
1729  */
1730 static void cik_tiling_mode_table_init(struct radeon_device *rdev)
1731 {
1732         const u32 num_tile_mode_states = 32;
1733         const u32 num_secondary_tile_mode_states = 16;
1734         u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
1735         u32 num_pipe_configs;
1736         u32 num_rbs = rdev->config.cik.max_backends_per_se *
1737                 rdev->config.cik.max_shader_engines;
1738
1739         switch (rdev->config.cik.mem_row_size_in_kb) {
1740         case 1:
1741                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
1742                 break;
1743         case 2:
1744         default:
1745                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
1746                 break;
1747         case 4:
1748                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
1749                 break;
1750         }
1751
1752         num_pipe_configs = rdev->config.cik.max_tile_pipes;
1753         if (num_pipe_configs > 8)
1754                 num_pipe_configs = 8; /* ??? */
1755
1756         if (num_pipe_configs == 8) {
1757                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
1758                         switch (reg_offset) {
1759                         case 0:
1760                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1761                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1762                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1763                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
1764                                 break;
1765                         case 1:
1766                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1767                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1768                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1769                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
1770                                 break;
1771                         case 2:
1772                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1773                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1774                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1775                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
1776                                 break;
1777                         case 3:
1778                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1779                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1780                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1781                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
1782                                 break;
1783                         case 4:
1784                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1785                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1786                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1787                                                  TILE_SPLIT(split_equal_to_row_size));
1788                                 break;
1789                         case 5:
1790                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1791                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
1792                                 break;
1793                         case 6:
1794                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1795                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1796                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1797                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
1798                                 break;
1799                         case 7:
1800                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1801                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1802                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1803                                                  TILE_SPLIT(split_equal_to_row_size));
1804                                 break;
1805                         case 8:
1806                                 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
1807                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
1808                                 break;
1809                         case 9:
1810                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1811                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
1812                                 break;
1813                         case 10:
1814                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1815                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1816                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1817                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1818                                 break;
1819                         case 11:
1820                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1821                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1822                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1823                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1824                                 break;
1825                         case 12:
1826                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1827                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
1828                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1829                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1830                                 break;
1831                         case 13:
1832                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1833                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
1834                                 break;
1835                         case 14:
1836                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1837                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1838                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1839                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1840                                 break;
1841                         case 16:
1842                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1843                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1844                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1845                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1846                                 break;
1847                         case 17:
1848                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1849                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
1850                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1851                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1852                                 break;
1853                         case 27:
1854                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1855                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
1856                                 break;
1857                         case 28:
1858                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1859                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1860                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1861                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1862                                 break;
1863                         case 29:
1864                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
1865                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1866                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1867                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1868                                 break;
1869                         case 30:
1870                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
1871                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
1872                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
1873                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
1874                                 break;
1875                         default:
1876                                 gb_tile_moden = 0;
1877                                 break;
1878                         }
1879                         rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
1880                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
1881                 }
1882                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
1883                         switch (reg_offset) {
1884                         case 0:
1885                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1886                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1887                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1888                                                  NUM_BANKS(ADDR_SURF_16_BANK));
1889                                 break;
1890                         case 1:
1891                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1892                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1893                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1894                                                  NUM_BANKS(ADDR_SURF_16_BANK));
1895                                 break;
1896                         case 2:
1897                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1898                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1899                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1900                                                  NUM_BANKS(ADDR_SURF_16_BANK));
1901                                 break;
1902                         case 3:
1903                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1904                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1905                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1906                                                  NUM_BANKS(ADDR_SURF_16_BANK));
1907                                 break;
1908                         case 4:
1909                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1910                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1911                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1912                                                  NUM_BANKS(ADDR_SURF_8_BANK));
1913                                 break;
1914                         case 5:
1915                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1916                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1917                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1918                                                  NUM_BANKS(ADDR_SURF_4_BANK));
1919                                 break;
1920                         case 6:
1921                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1922                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1923                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1924                                                  NUM_BANKS(ADDR_SURF_2_BANK));
1925                                 break;
1926                         case 8:
1927                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1928                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
1929                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1930                                                  NUM_BANKS(ADDR_SURF_16_BANK));
1931                                 break;
1932                         case 9:
1933                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1934                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1935                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
1936                                                  NUM_BANKS(ADDR_SURF_16_BANK));
1937                                 break;
1938                         case 10:
1939                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1940                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1941                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1942                                                  NUM_BANKS(ADDR_SURF_16_BANK));
1943                                 break;
1944                         case 11:
1945                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1946                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1947                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
1948                                                  NUM_BANKS(ADDR_SURF_16_BANK));
1949                                 break;
1950                         case 12:
1951                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1952                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1953                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1954                                                  NUM_BANKS(ADDR_SURF_8_BANK));
1955                                 break;
1956                         case 13:
1957                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1958                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1959                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1960                                                  NUM_BANKS(ADDR_SURF_4_BANK));
1961                                 break;
1962                         case 14:
1963                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1964                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1965                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
1966                                                  NUM_BANKS(ADDR_SURF_2_BANK));
1967                                 break;
1968                         default:
1969                                 gb_tile_moden = 0;
1970                                 break;
1971                         }
1972                         WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
1973                 }
1974         } else if (num_pipe_configs == 4) {
1975                 if (num_rbs == 4) {
1976                         for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
1977                                 switch (reg_offset) {
1978                                 case 0:
1979                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1980                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1981                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1982                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
1983                                         break;
1984                                 case 1:
1985                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1986                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1987                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1988                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
1989                                         break;
1990                                 case 2:
1991                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1992                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1993                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
1994                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
1995                                         break;
1996                                 case 3:
1997                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1998                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
1999                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2000                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2001                                         break;
2002                                 case 4:
2003                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2004                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2005                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2006                                                          TILE_SPLIT(split_equal_to_row_size));
2007                                         break;
2008                                 case 5:
2009                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2010                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2011                                         break;
2012                                 case 6:
2013                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2014                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2015                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2016                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2017                                         break;
2018                                 case 7:
2019                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2020                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2021                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2022                                                          TILE_SPLIT(split_equal_to_row_size));
2023                                         break;
2024                                 case 8:
2025                                         gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2026                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16));
2027                                         break;
2028                                 case 9:
2029                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2030                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2031                                         break;
2032                                 case 10:
2033                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2034                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2035                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2036                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2037                                         break;
2038                                 case 11:
2039                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2040                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2041                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2042                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2043                                         break;
2044                                 case 12:
2045                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2046                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2047                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2048                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2049                                         break;
2050                                 case 13:
2051                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2052                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2053                                         break;
2054                                 case 14:
2055                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2056                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2057                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2058                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2059                                         break;
2060                                 case 16:
2061                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2062                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2063                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2064                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2065                                         break;
2066                                 case 17:
2067                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2068                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2069                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2070                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2071                                         break;
2072                                 case 27:
2073                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2074                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2075                                         break;
2076                                 case 28:
2077                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2078                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2079                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2080                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2081                                         break;
2082                                 case 29:
2083                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2084                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2085                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2086                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2087                                         break;
2088                                 case 30:
2089                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2090                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2091                                                          PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2092                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2093                                         break;
2094                                 default:
2095                                         gb_tile_moden = 0;
2096                                         break;
2097                                 }
2098                                 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2099                                 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2100                         }
2101                 } else if (num_rbs < 4) {
2102                         for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2103                                 switch (reg_offset) {
2104                                 case 0:
2105                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2106                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2107                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2108                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2109                                         break;
2110                                 case 1:
2111                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2112                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2113                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2114                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2115                                         break;
2116                                 case 2:
2117                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2118                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2119                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2120                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2121                                         break;
2122                                 case 3:
2123                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2124                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2125                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2126                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2127                                         break;
2128                                 case 4:
2129                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2130                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2131                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2132                                                          TILE_SPLIT(split_equal_to_row_size));
2133                                         break;
2134                                 case 5:
2135                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2136                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2137                                         break;
2138                                 case 6:
2139                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2140                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2141                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2142                                                          TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2143                                         break;
2144                                 case 7:
2145                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2146                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2147                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2148                                                          TILE_SPLIT(split_equal_to_row_size));
2149                                         break;
2150                                 case 8:
2151                                         gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2152                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16));
2153                                         break;
2154                                 case 9:
2155                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2156                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2157                                         break;
2158                                 case 10:
2159                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2160                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2161                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2162                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2163                                         break;
2164                                 case 11:
2165                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2166                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2167                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2168                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2169                                         break;
2170                                 case 12:
2171                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2172                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2173                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2174                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2175                                         break;
2176                                 case 13:
2177                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2178                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2179                                         break;
2180                                 case 14:
2181                                         gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2182                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2183                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2184                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2185                                         break;
2186                                 case 16:
2187                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2188                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2189                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2190                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2191                                         break;
2192                                 case 17:
2193                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2194                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2195                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2196                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2197                                         break;
2198                                 case 27:
2199                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2200                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2201                                         break;
2202                                 case 28:
2203                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2204                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2205                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2206                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2207                                         break;
2208                                 case 29:
2209                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2210                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2211                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2212                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2213                                         break;
2214                                 case 30:
2215                                         gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2216                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2217                                                          PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2218                                                          SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2219                                         break;
2220                                 default:
2221                                         gb_tile_moden = 0;
2222                                         break;
2223                                 }
2224                                 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2225                                 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2226                         }
2227                 }
2228                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2229                         switch (reg_offset) {
2230                         case 0:
2231                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2232                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2233                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2234                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2235                                 break;
2236                         case 1:
2237                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2238                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2239                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2240                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2241                                 break;
2242                         case 2:
2243                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2244                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2245                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2246                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2247                                 break;
2248                         case 3:
2249                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2250                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2251                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2252                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2253                                 break;
2254                         case 4:
2255                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2256                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2257                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2258                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2259                                 break;
2260                         case 5:
2261                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2262                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2263                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2264                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2265                                 break;
2266                         case 6:
2267                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2268                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2269                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2270                                                  NUM_BANKS(ADDR_SURF_4_BANK));
2271                                 break;
2272                         case 8:
2273                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2274                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2275                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2276                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2277                                 break;
2278                         case 9:
2279                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2280                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2281                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2282                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2283                                 break;
2284                         case 10:
2285                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2286                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2287                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2288                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2289                                 break;
2290                         case 11:
2291                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2292                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2293                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2294                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2295                                 break;
2296                         case 12:
2297                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2298                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2299                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2300                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2301                                 break;
2302                         case 13:
2303                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2304                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2305                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2306                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2307                                 break;
2308                         case 14:
2309                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2310                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2311                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2312                                                  NUM_BANKS(ADDR_SURF_4_BANK));
2313                                 break;
2314                         default:
2315                                 gb_tile_moden = 0;
2316                                 break;
2317                         }
2318                         WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2319                 }
2320         } else if (num_pipe_configs == 2) {
2321                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2322                         switch (reg_offset) {
2323                         case 0:
2324                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2325                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2326                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2327                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2328                                 break;
2329                         case 1:
2330                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2331                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2332                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2333                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2334                                 break;
2335                         case 2:
2336                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2337                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2338                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2339                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2340                                 break;
2341                         case 3:
2342                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2343                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2344                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2345                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2346                                 break;
2347                         case 4:
2348                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2349                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2350                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2351                                                  TILE_SPLIT(split_equal_to_row_size));
2352                                 break;
2353                         case 5:
2354                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2355                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2356                                 break;
2357                         case 6:
2358                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2359                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2360                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2361                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2362                                 break;
2363                         case 7:
2364                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2365                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2366                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2367                                                  TILE_SPLIT(split_equal_to_row_size));
2368                                 break;
2369                         case 8:
2370                                 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED);
2371                                 break;
2372                         case 9:
2373                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2374                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2375                                 break;
2376                         case 10:
2377                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2378                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2379                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2380                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2381                                 break;
2382                         case 11:
2383                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2384                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2385                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2386                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2387                                 break;
2388                         case 12:
2389                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2390                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2391                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2392                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2393                                 break;
2394                         case 13:
2395                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2396                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2397                                 break;
2398                         case 14:
2399                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2400                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2401                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2402                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2403                                 break;
2404                         case 16:
2405                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2406                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2407                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2408                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2409                                 break;
2410                         case 17:
2411                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2412                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2413                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2414                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2415                                 break;
2416                         case 27:
2417                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2418                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2419                                 break;
2420                         case 28:
2421                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2422                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2423                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2424                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2425                                 break;
2426                         case 29:
2427                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2428                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2429                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2430                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2431                                 break;
2432                         case 30:
2433                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2434                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2435                                                  PIPE_CONFIG(ADDR_SURF_P2) |
2436                                                  SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2437                                 break;
2438                         default:
2439                                 gb_tile_moden = 0;
2440                                 break;
2441                         }
2442                         rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2443                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2444                 }
2445                 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2446                         switch (reg_offset) {
2447                         case 0:
2448                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2449                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2450                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2451                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2452                                 break;
2453                         case 1:
2454                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2455                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2456                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2457                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2458                                 break;
2459                         case 2:
2460                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2461                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2462                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2463                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2464                                 break;
2465                         case 3:
2466                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2467                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2468                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2469                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2470                                 break;
2471                         case 4:
2472                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2473                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2474                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2475                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2476                                 break;
2477                         case 5:
2478                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2479                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2480                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2481                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2482                                 break;
2483                         case 6:
2484                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2485                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2486                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2487                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2488                                 break;
2489                         case 8:
2490                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2491                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2492                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2493                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2494                                 break;
2495                         case 9:
2496                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2497                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2498                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2499                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2500                                 break;
2501                         case 10:
2502                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2503                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2504                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2505                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2506                                 break;
2507                         case 11:
2508                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2509                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2510                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2511                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2512                                 break;
2513                         case 12:
2514                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2515                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2516                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2517                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2518                                 break;
2519                         case 13:
2520                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2521                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2522                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2523                                                  NUM_BANKS(ADDR_SURF_16_BANK));
2524                                 break;
2525                         case 14:
2526                                 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2527                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2528                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2529                                                  NUM_BANKS(ADDR_SURF_8_BANK));
2530                                 break;
2531                         default:
2532                                 gb_tile_moden = 0;
2533                                 break;
2534                         }
2535                         WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2536                 }
2537         } else
2538                 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
2539 }
2540
2541 /**
2542  * cik_select_se_sh - select which SE, SH to address
2543  *
2544  * @rdev: radeon_device pointer
2545  * @se_num: shader engine to address
2546  * @sh_num: sh block to address
2547  *
2548  * Select which SE, SH combinations to address. Certain
2549  * registers are instanced per SE or SH.  0xffffffff means
2550  * broadcast to all SEs or SHs (CIK).
2551  */
2552 static void cik_select_se_sh(struct radeon_device *rdev,
2553                              u32 se_num, u32 sh_num)
2554 {
2555         u32 data = INSTANCE_BROADCAST_WRITES;
2556
2557         if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
2558                 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
2559         else if (se_num == 0xffffffff)
2560                 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2561         else if (sh_num == 0xffffffff)
2562                 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2563         else
2564                 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2565         WREG32(GRBM_GFX_INDEX, data);
2566 }
2567
2568 /**
2569  * cik_create_bitmask - create a bitmask
2570  *
2571  * @bit_width: length of the mask
2572  *
2573  * create a variable length bit mask (CIK).
2574  * Returns the bitmask.
2575  */
2576 static u32 cik_create_bitmask(u32 bit_width)
2577 {
2578         u32 i, mask = 0;
2579
2580         for (i = 0; i < bit_width; i++) {
2581                 mask <<= 1;
2582                 mask |= 1;
2583         }
2584         return mask;
2585 }
2586
2587 /**
2588  * cik_select_se_sh - select which SE, SH to address
2589  *
2590  * @rdev: radeon_device pointer
2591  * @max_rb_num: max RBs (render backends) for the asic
2592  * @se_num: number of SEs (shader engines) for the asic
2593  * @sh_per_se: number of SH blocks per SE for the asic
2594  *
2595  * Calculates the bitmask of disabled RBs (CIK).
2596  * Returns the disabled RB bitmask.
2597  */
2598 static u32 cik_get_rb_disabled(struct radeon_device *rdev,
2599                               u32 max_rb_num, u32 se_num,
2600                               u32 sh_per_se)
2601 {
2602         u32 data, mask;
2603
2604         data = RREG32(CC_RB_BACKEND_DISABLE);
2605         if (data & 1)
2606                 data &= BACKEND_DISABLE_MASK;
2607         else
2608                 data = 0;
2609         data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
2610
2611         data >>= BACKEND_DISABLE_SHIFT;
2612
2613         mask = cik_create_bitmask(max_rb_num / se_num / sh_per_se);
2614
2615         return data & mask;
2616 }
2617
2618 /**
2619  * cik_setup_rb - setup the RBs on the asic
2620  *
2621  * @rdev: radeon_device pointer
2622  * @se_num: number of SEs (shader engines) for the asic
2623  * @sh_per_se: number of SH blocks per SE for the asic
2624  * @max_rb_num: max RBs (render backends) for the asic
2625  *
2626  * Configures per-SE/SH RB registers (CIK).
2627  */
2628 static void cik_setup_rb(struct radeon_device *rdev,
2629                          u32 se_num, u32 sh_per_se,
2630                          u32 max_rb_num)
2631 {
2632         int i, j;
2633         u32 data, mask;
2634         u32 disabled_rbs = 0;
2635         u32 enabled_rbs = 0;
2636
2637         for (i = 0; i < se_num; i++) {
2638                 for (j = 0; j < sh_per_se; j++) {
2639                         cik_select_se_sh(rdev, i, j);
2640                         data = cik_get_rb_disabled(rdev, max_rb_num, se_num, sh_per_se);
2641                         disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
2642                 }
2643         }
2644         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2645
2646         mask = 1;
2647         for (i = 0; i < max_rb_num; i++) {
2648                 if (!(disabled_rbs & mask))
2649                         enabled_rbs |= mask;
2650                 mask <<= 1;
2651         }
2652
2653         for (i = 0; i < se_num; i++) {
2654                 cik_select_se_sh(rdev, i, 0xffffffff);
2655                 data = 0;
2656                 for (j = 0; j < sh_per_se; j++) {
2657                         switch (enabled_rbs & 3) {
2658                         case 1:
2659                                 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
2660                                 break;
2661                         case 2:
2662                                 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
2663                                 break;
2664                         case 3:
2665                         default:
2666                                 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
2667                                 break;
2668                         }
2669                         enabled_rbs >>= 2;
2670                 }
2671                 WREG32(PA_SC_RASTER_CONFIG, data);
2672         }
2673         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2674 }
2675
2676 /**
2677  * cik_gpu_init - setup the 3D engine
2678  *
2679  * @rdev: radeon_device pointer
2680  *
2681  * Configures the 3D engine and tiling configuration
2682  * registers so that the 3D engine is usable.
2683  */
2684 static void cik_gpu_init(struct radeon_device *rdev)
2685 {
2686         u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
2687         u32 mc_shared_chmap, mc_arb_ramcfg;
2688         u32 hdp_host_path_cntl;
2689         u32 tmp;
2690         int i, j;
2691
2692         switch (rdev->family) {
2693         case CHIP_BONAIRE:
2694                 rdev->config.cik.max_shader_engines = 2;
2695                 rdev->config.cik.max_tile_pipes = 4;
2696                 rdev->config.cik.max_cu_per_sh = 7;
2697                 rdev->config.cik.max_sh_per_se = 1;
2698                 rdev->config.cik.max_backends_per_se = 2;
2699                 rdev->config.cik.max_texture_channel_caches = 4;
2700                 rdev->config.cik.max_gprs = 256;
2701                 rdev->config.cik.max_gs_threads = 32;
2702                 rdev->config.cik.max_hw_contexts = 8;
2703
2704                 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
2705                 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
2706                 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
2707                 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
2708                 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
2709                 break;
2710         case CHIP_KAVERI:
2711                 rdev->config.cik.max_shader_engines = 1;
2712                 rdev->config.cik.max_tile_pipes = 4;
2713                 if ((rdev->pdev->device == 0x1304) ||
2714                     (rdev->pdev->device == 0x1305) ||
2715                     (rdev->pdev->device == 0x130C) ||
2716                     (rdev->pdev->device == 0x130F) ||
2717                     (rdev->pdev->device == 0x1310) ||
2718                     (rdev->pdev->device == 0x1311) ||
2719                     (rdev->pdev->device == 0x131C)) {
2720                         rdev->config.cik.max_cu_per_sh = 8;
2721                         rdev->config.cik.max_backends_per_se = 2;
2722                 } else if ((rdev->pdev->device == 0x1309) ||
2723                            (rdev->pdev->device == 0x130A) ||
2724                            (rdev->pdev->device == 0x130D) ||
2725                            (rdev->pdev->device == 0x1313)) {
2726                         rdev->config.cik.max_cu_per_sh = 6;
2727                         rdev->config.cik.max_backends_per_se = 2;
2728                 } else if ((rdev->pdev->device == 0x1306) ||
2729                            (rdev->pdev->device == 0x1307) ||
2730                            (rdev->pdev->device == 0x130B) ||
2731                            (rdev->pdev->device == 0x130E) ||
2732                            (rdev->pdev->device == 0x1315) ||
2733                            (rdev->pdev->device == 0x131B)) {
2734                         rdev->config.cik.max_cu_per_sh = 4;
2735                         rdev->config.cik.max_backends_per_se = 1;
2736                 } else {
2737                         rdev->config.cik.max_cu_per_sh = 3;
2738                         rdev->config.cik.max_backends_per_se = 1;
2739                 }
2740                 rdev->config.cik.max_sh_per_se = 1;
2741                 rdev->config.cik.max_texture_channel_caches = 4;
2742                 rdev->config.cik.max_gprs = 256;
2743                 rdev->config.cik.max_gs_threads = 16;
2744                 rdev->config.cik.max_hw_contexts = 8;
2745
2746                 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
2747                 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
2748                 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
2749                 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
2750                 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
2751                 break;
2752         case CHIP_KABINI:
2753         default:
2754                 rdev->config.cik.max_shader_engines = 1;
2755                 rdev->config.cik.max_tile_pipes = 2;
2756                 rdev->config.cik.max_cu_per_sh = 2;
2757                 rdev->config.cik.max_sh_per_se = 1;
2758                 rdev->config.cik.max_backends_per_se = 1;
2759                 rdev->config.cik.max_texture_channel_caches = 2;
2760                 rdev->config.cik.max_gprs = 256;
2761                 rdev->config.cik.max_gs_threads = 16;
2762                 rdev->config.cik.max_hw_contexts = 8;
2763
2764                 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
2765                 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
2766                 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
2767                 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
2768                 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
2769                 break;
2770         }
2771
2772         /* Initialize HDP */
2773         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2774                 WREG32((0x2c14 + j), 0x00000000);
2775                 WREG32((0x2c18 + j), 0x00000000);
2776                 WREG32((0x2c1c + j), 0x00000000);
2777                 WREG32((0x2c20 + j), 0x00000000);
2778                 WREG32((0x2c24 + j), 0x00000000);
2779         }
2780
2781         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
2782
2783         WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2784
2785         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
2786         mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
2787
2788         rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
2789         rdev->config.cik.mem_max_burst_length_bytes = 256;
2790         tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
2791         rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
2792         if (rdev->config.cik.mem_row_size_in_kb > 4)
2793                 rdev->config.cik.mem_row_size_in_kb = 4;
2794         /* XXX use MC settings? */
2795         rdev->config.cik.shader_engine_tile_size = 32;
2796         rdev->config.cik.num_gpus = 1;
2797         rdev->config.cik.multi_gpu_tile_size = 64;
2798
2799         /* fix up row size */
2800         gb_addr_config &= ~ROW_SIZE_MASK;
2801         switch (rdev->config.cik.mem_row_size_in_kb) {
2802         case 1:
2803         default:
2804                 gb_addr_config |= ROW_SIZE(0);
2805                 break;
2806         case 2:
2807                 gb_addr_config |= ROW_SIZE(1);
2808                 break;
2809         case 4:
2810                 gb_addr_config |= ROW_SIZE(2);
2811                 break;
2812         }
2813
2814         /* setup tiling info dword.  gb_addr_config is not adequate since it does
2815          * not have bank info, so create a custom tiling dword.
2816          * bits 3:0   num_pipes
2817          * bits 7:4   num_banks
2818          * bits 11:8  group_size
2819          * bits 15:12 row_size
2820          */
2821         rdev->config.cik.tile_config = 0;
2822         switch (rdev->config.cik.num_tile_pipes) {
2823         case 1:
2824                 rdev->config.cik.tile_config |= (0 << 0);
2825                 break;
2826         case 2:
2827                 rdev->config.cik.tile_config |= (1 << 0);
2828                 break;
2829         case 4:
2830                 rdev->config.cik.tile_config |= (2 << 0);
2831                 break;
2832         case 8:
2833         default:
2834                 /* XXX what about 12? */
2835                 rdev->config.cik.tile_config |= (3 << 0);
2836                 break;
2837         }
2838         if ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT)
2839                 rdev->config.cik.tile_config |= 1 << 4;
2840         else
2841                 rdev->config.cik.tile_config |= 0 << 4;
2842         rdev->config.cik.tile_config |=
2843                 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
2844         rdev->config.cik.tile_config |=
2845                 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
2846
2847         WREG32(GB_ADDR_CONFIG, gb_addr_config);
2848         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
2849         WREG32(DMIF_ADDR_CALC, gb_addr_config);
2850         WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
2851         WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
2852         WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
2853         WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
2854         WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
2855
2856         cik_tiling_mode_table_init(rdev);
2857
2858         cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
2859                      rdev->config.cik.max_sh_per_se,
2860                      rdev->config.cik.max_backends_per_se);
2861
2862         /* set HW defaults for 3D engine */
2863         WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
2864
2865         WREG32(SX_DEBUG_1, 0x20);
2866
2867         WREG32(TA_CNTL_AUX, 0x00010000);
2868
2869         tmp = RREG32(SPI_CONFIG_CNTL);
2870         tmp |= 0x03000000;
2871         WREG32(SPI_CONFIG_CNTL, tmp);
2872
2873         WREG32(SQ_CONFIG, 1);
2874
2875         WREG32(DB_DEBUG, 0);
2876
2877         tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
2878         tmp |= 0x00000400;
2879         WREG32(DB_DEBUG2, tmp);
2880
2881         tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
2882         tmp |= 0x00020200;
2883         WREG32(DB_DEBUG3, tmp);
2884
2885         tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
2886         tmp |= 0x00018208;
2887         WREG32(CB_HW_CONTROL, tmp);
2888
2889         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
2890
2891         WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
2892                                  SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
2893                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
2894                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
2895
2896         WREG32(VGT_NUM_INSTANCES, 1);
2897
2898         WREG32(CP_PERFMON_CNTL, 0);
2899
2900         WREG32(SQ_CONFIG, 0);
2901
2902         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
2903                                           FORCE_EOV_MAX_REZ_CNT(255)));
2904
2905         WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
2906                AUTO_INVLD_EN(ES_AND_GS_AUTO));
2907
2908         WREG32(VGT_GS_VERTEX_REUSE, 16);
2909         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
2910
2911         tmp = RREG32(HDP_MISC_CNTL);
2912         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
2913         WREG32(HDP_MISC_CNTL, tmp);
2914
2915         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
2916         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
2917
2918         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
2919         WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
2920
2921         udelay(50);
2922 }
2923
2924 /*
2925  * GPU scratch registers helpers function.
2926  */
2927 /**
2928  * cik_scratch_init - setup driver info for CP scratch regs
2929  *
2930  * @rdev: radeon_device pointer
2931  *
2932  * Set up the number and offset of the CP scratch registers.
2933  * NOTE: use of CP scratch registers is a legacy inferface and
2934  * is not used by default on newer asics (r6xx+).  On newer asics,
2935  * memory buffers are used for fences rather than scratch regs.
2936  */
2937 static void cik_scratch_init(struct radeon_device *rdev)
2938 {
2939         int i;
2940
2941         rdev->scratch.num_reg = 7;
2942         rdev->scratch.reg_base = SCRATCH_REG0;
2943         for (i = 0; i < rdev->scratch.num_reg; i++) {
2944                 rdev->scratch.free[i] = true;
2945                 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
2946         }
2947 }
2948
2949 /**
2950  * cik_ring_test - basic gfx ring test
2951  *
2952  * @rdev: radeon_device pointer
2953  * @ring: radeon_ring structure holding ring information
2954  *
2955  * Allocate a scratch register and write to it using the gfx ring (CIK).
2956  * Provides a basic gfx ring test to verify that the ring is working.
2957  * Used by cik_cp_gfx_resume();
2958  * Returns 0 on success, error on failure.
2959  */
2960 int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
2961 {
2962         uint32_t scratch;
2963         uint32_t tmp = 0;
2964         unsigned i;
2965         int r;
2966
2967         r = radeon_scratch_get(rdev, &scratch);
2968         if (r) {
2969                 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
2970                 return r;
2971         }
2972         WREG32(scratch, 0xCAFEDEAD);
2973         r = radeon_ring_lock(rdev, ring, 3);
2974         if (r) {
2975                 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
2976                 radeon_scratch_free(rdev, scratch);
2977                 return r;
2978         }
2979         radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
2980         radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
2981         radeon_ring_write(ring, 0xDEADBEEF);
2982         radeon_ring_unlock_commit(rdev, ring);
2983
2984         for (i = 0; i < rdev->usec_timeout; i++) {
2985                 tmp = RREG32(scratch);
2986                 if (tmp == 0xDEADBEEF)
2987                         break;
2988                 DRM_UDELAY(1);
2989         }
2990         if (i < rdev->usec_timeout) {
2991                 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
2992         } else {
2993                 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
2994                           ring->idx, scratch, tmp);
2995                 r = -EINVAL;
2996         }
2997         radeon_scratch_free(rdev, scratch);
2998         return r;
2999 }
3000
3001 /**
3002  * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
3003  *
3004  * @rdev: radeon_device pointer
3005  * @fence: radeon fence object
3006  *
3007  * Emits a fence sequnce number on the gfx ring and flushes
3008  * GPU caches.
3009  */
3010 void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3011                              struct radeon_fence *fence)
3012 {
3013         struct radeon_ring *ring = &rdev->ring[fence->ring];
3014         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3015
3016         /* EVENT_WRITE_EOP - flush caches, send int */
3017         radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3018         radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3019                                  EOP_TC_ACTION_EN |
3020                                  EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3021                                  EVENT_INDEX(5)));
3022         radeon_ring_write(ring, addr & 0xfffffffc);
3023         radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3024         radeon_ring_write(ring, fence->seq);
3025         radeon_ring_write(ring, 0);
3026         /* HDP flush */
3027         /* We should be using the new WAIT_REG_MEM special op packet here
3028          * but it causes the CP to hang
3029          */
3030         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3031         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
3032                                  WRITE_DATA_DST_SEL(0)));
3033         radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
3034         radeon_ring_write(ring, 0);
3035         radeon_ring_write(ring, 0);
3036 }
3037
3038 /**
3039  * cik_fence_compute_ring_emit - emit a fence on the compute ring
3040  *
3041  * @rdev: radeon_device pointer
3042  * @fence: radeon fence object
3043  *
3044  * Emits a fence sequnce number on the compute ring and flushes
3045  * GPU caches.
3046  */
3047 void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3048                                  struct radeon_fence *fence)
3049 {
3050         struct radeon_ring *ring = &rdev->ring[fence->ring];
3051         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3052
3053         /* RELEASE_MEM - flush caches, send int */
3054         radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3055         radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3056                                  EOP_TC_ACTION_EN |
3057                                  EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3058                                  EVENT_INDEX(5)));
3059         radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3060         radeon_ring_write(ring, addr & 0xfffffffc);
3061         radeon_ring_write(ring, upper_32_bits(addr));
3062         radeon_ring_write(ring, fence->seq);
3063         radeon_ring_write(ring, 0);
3064         /* HDP flush */
3065         /* We should be using the new WAIT_REG_MEM special op packet here
3066          * but it causes the CP to hang
3067          */
3068         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3069         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
3070                                  WRITE_DATA_DST_SEL(0)));
3071         radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
3072         radeon_ring_write(ring, 0);
3073         radeon_ring_write(ring, 0);
3074 }
3075
3076 void cik_semaphore_ring_emit(struct radeon_device *rdev,
3077                              struct radeon_ring *ring,
3078                              struct radeon_semaphore *semaphore,
3079                              bool emit_wait)
3080 {
3081         uint64_t addr = semaphore->gpu_addr;
3082         unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3083
3084         radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3085         radeon_ring_write(ring, addr & 0xffffffff);
3086         radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
3087 }
3088
3089 /*
3090  * IB stuff
3091  */
3092 /**
3093  * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
3094  *
3095  * @rdev: radeon_device pointer
3096  * @ib: radeon indirect buffer object
3097  *
3098  * Emits an DE (drawing engine) or CE (constant engine) IB
3099  * on the gfx ring.  IBs are usually generated by userspace
3100  * acceleration drivers and submitted to the kernel for
3101  * sheduling on the ring.  This function schedules the IB
3102  * on the gfx ring for execution by the GPU.
3103  */
3104 void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3105 {
3106         struct radeon_ring *ring = &rdev->ring[ib->ring];
3107         u32 header, control = INDIRECT_BUFFER_VALID;
3108
3109         if (ib->is_const_ib) {
3110                 /* set switch buffer packet before const IB */
3111                 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3112                 radeon_ring_write(ring, 0);
3113
3114                 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3115         } else {
3116                 u32 next_rptr;
3117                 if (ring->rptr_save_reg) {
3118                         next_rptr = ring->wptr + 3 + 4;
3119                         radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3120                         radeon_ring_write(ring, ((ring->rptr_save_reg -
3121                                                   PACKET3_SET_UCONFIG_REG_START) >> 2));
3122                         radeon_ring_write(ring, next_rptr);
3123                 } else if (rdev->wb.enabled) {
3124                         next_rptr = ring->wptr + 5 + 4;
3125                         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3126                         radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
3127                         radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3128                         radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
3129                         radeon_ring_write(ring, next_rptr);
3130                 }
3131
3132                 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3133         }
3134
3135         control |= ib->length_dw |
3136                 (ib->vm ? (ib->vm->id << 24) : 0);
3137
3138         radeon_ring_write(ring, header);
3139         radeon_ring_write(ring,
3140 #ifdef __BIG_ENDIAN
3141                           (2 << 0) |
3142 #endif
3143                           (ib->gpu_addr & 0xFFFFFFFC));
3144         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3145         radeon_ring_write(ring, control);
3146 }
3147
3148 /**
3149  * cik_ib_test - basic gfx ring IB test
3150  *
3151  * @rdev: radeon_device pointer
3152  * @ring: radeon_ring structure holding ring information
3153  *
3154  * Allocate an IB and execute it on the gfx ring (CIK).
3155  * Provides a basic gfx ring test to verify that IBs are working.
3156  * Returns 0 on success, error on failure.
3157  */
3158 int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3159 {
3160         struct radeon_ib ib;
3161         uint32_t scratch;
3162         uint32_t tmp = 0;
3163         unsigned i;
3164         int r;
3165
3166         r = radeon_scratch_get(rdev, &scratch);
3167         if (r) {
3168                 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3169                 return r;
3170         }
3171         WREG32(scratch, 0xCAFEDEAD);
3172         r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3173         if (r) {
3174                 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
3175                 return r;
3176         }
3177         ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
3178         ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
3179         ib.ptr[2] = 0xDEADBEEF;
3180         ib.length_dw = 3;
3181         r = radeon_ib_schedule(rdev, &ib, NULL);
3182         if (r) {
3183                 radeon_scratch_free(rdev, scratch);
3184                 radeon_ib_free(rdev, &ib);
3185                 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3186                 return r;
3187         }
3188         r = radeon_fence_wait(ib.fence, false);
3189         if (r) {
3190                 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
3191                 return r;
3192         }
3193         for (i = 0; i < rdev->usec_timeout; i++) {
3194                 tmp = RREG32(scratch);
3195                 if (tmp == 0xDEADBEEF)
3196                         break;
3197                 DRM_UDELAY(1);
3198         }
3199         if (i < rdev->usec_timeout) {
3200                 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3201         } else {
3202                 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3203                           scratch, tmp);
3204                 r = -EINVAL;
3205         }
3206         radeon_scratch_free(rdev, scratch);
3207         radeon_ib_free(rdev, &ib);
3208         return r;
3209 }
3210
3211 /*
3212  * CP.
3213  * On CIK, gfx and compute now have independant command processors.
3214  *
3215  * GFX
3216  * Gfx consists of a single ring and can process both gfx jobs and
3217  * compute jobs.  The gfx CP consists of three microengines (ME):
3218  * PFP - Pre-Fetch Parser
3219  * ME - Micro Engine
3220  * CE - Constant Engine
3221  * The PFP and ME make up what is considered the Drawing Engine (DE).
3222  * The CE is an asynchronous engine used for updating buffer desciptors
3223  * used by the DE so that they can be loaded into cache in parallel
3224  * while the DE is processing state update packets.
3225  *
3226  * Compute
3227  * The compute CP consists of two microengines (ME):
3228  * MEC1 - Compute MicroEngine 1
3229  * MEC2 - Compute MicroEngine 2
3230  * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
3231  * The queues are exposed to userspace and are programmed directly
3232  * by the compute runtime.
3233  */
3234 /**
3235  * cik_cp_gfx_enable - enable/disable the gfx CP MEs
3236  *
3237  * @rdev: radeon_device pointer
3238  * @enable: enable or disable the MEs
3239  *
3240  * Halts or unhalts the gfx MEs.
3241  */
3242 static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
3243 {
3244         if (enable)
3245                 WREG32(CP_ME_CNTL, 0);
3246         else {
3247                 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3248                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3249         }
3250         udelay(50);
3251 }
3252
3253 /**
3254  * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
3255  *
3256  * @rdev: radeon_device pointer
3257  *
3258  * Loads the gfx PFP, ME, and CE ucode.
3259  * Returns 0 for success, -EINVAL if the ucode is not available.
3260  */
3261 static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
3262 {
3263         const __be32 *fw_data;
3264         int i;
3265
3266         if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3267                 return -EINVAL;
3268
3269         cik_cp_gfx_enable(rdev, false);
3270
3271         /* PFP */
3272         fw_data = (const __be32 *)rdev->pfp_fw->data;
3273         WREG32(CP_PFP_UCODE_ADDR, 0);
3274         for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
3275                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3276         WREG32(CP_PFP_UCODE_ADDR, 0);
3277
3278         /* CE */
3279         fw_data = (const __be32 *)rdev->ce_fw->data;
3280         WREG32(CP_CE_UCODE_ADDR, 0);
3281         for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
3282                 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3283         WREG32(CP_CE_UCODE_ADDR, 0);
3284
3285         /* ME */
3286         fw_data = (const __be32 *)rdev->me_fw->data;
3287         WREG32(CP_ME_RAM_WADDR, 0);
3288         for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
3289                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3290         WREG32(CP_ME_RAM_WADDR, 0);
3291
3292         WREG32(CP_PFP_UCODE_ADDR, 0);
3293         WREG32(CP_CE_UCODE_ADDR, 0);
3294         WREG32(CP_ME_RAM_WADDR, 0);
3295         WREG32(CP_ME_RAM_RADDR, 0);
3296         return 0;
3297 }
3298
3299 /**
3300  * cik_cp_gfx_start - start the gfx ring
3301  *
3302  * @rdev: radeon_device pointer
3303  *
3304  * Enables the ring and loads the clear state context and other
3305  * packets required to init the ring.
3306  * Returns 0 for success, error for failure.
3307  */
3308 static int cik_cp_gfx_start(struct radeon_device *rdev)
3309 {
3310         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3311         int r, i;
3312
3313         /* init the CP */
3314         WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
3315         WREG32(CP_ENDIAN_SWAP, 0);
3316         WREG32(CP_DEVICE_ID, 1);
3317
3318         cik_cp_gfx_enable(rdev, true);
3319
3320         r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
3321         if (r) {
3322                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3323                 return r;
3324         }
3325
3326         /* init the CE partitions.  CE only used for gfx on CIK */
3327         radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3328         radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3329         radeon_ring_write(ring, 0xc000);
3330         radeon_ring_write(ring, 0xc000);
3331
3332         /* setup clear context state */
3333         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3334         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3335
3336         radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3337         radeon_ring_write(ring, 0x80000000);
3338         radeon_ring_write(ring, 0x80000000);
3339
3340         for (i = 0; i < cik_default_size; i++)
3341                 radeon_ring_write(ring, cik_default_state[i]);
3342
3343         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3344         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3345
3346         /* set clear context state */
3347         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3348         radeon_ring_write(ring, 0);
3349
3350         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3351         radeon_ring_write(ring, 0x00000316);
3352         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3353         radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3354
3355         radeon_ring_unlock_commit(rdev, ring);
3356
3357         return 0;
3358 }
3359
3360 /**
3361  * cik_cp_gfx_fini - stop the gfx ring
3362  *
3363  * @rdev: radeon_device pointer
3364  *
3365  * Stop the gfx ring and tear down the driver ring
3366  * info.
3367  */
3368 static void cik_cp_gfx_fini(struct radeon_device *rdev)
3369 {
3370         cik_cp_gfx_enable(rdev, false);
3371         radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3372 }
3373
3374 /**
3375  * cik_cp_gfx_resume - setup the gfx ring buffer registers
3376  *
3377  * @rdev: radeon_device pointer
3378  *
3379  * Program the location and size of the gfx ring buffer
3380  * and test it to make sure it's working.
3381  * Returns 0 for success, error for failure.
3382  */
3383 static int cik_cp_gfx_resume(struct radeon_device *rdev)
3384 {
3385         struct radeon_ring *ring;
3386         u32 tmp;
3387         u32 rb_bufsz;
3388         u64 rb_addr;
3389         int r;
3390
3391         WREG32(CP_SEM_WAIT_TIMER, 0x0);
3392         WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3393
3394         /* Set the write pointer delay */
3395         WREG32(CP_RB_WPTR_DELAY, 0);
3396
3397         /* set the RB to use vmid 0 */
3398         WREG32(CP_RB_VMID, 0);
3399
3400         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3401
3402         /* ring 0 - compute and gfx */
3403         /* Set ring buffer size */
3404         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3405         rb_bufsz = order_base_2(ring->ring_size / 8);
3406         tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3407 #ifdef __BIG_ENDIAN
3408         tmp |= BUF_SWAP_32BIT;
3409 #endif
3410         WREG32(CP_RB0_CNTL, tmp);
3411
3412         /* Initialize the ring buffer's read and write pointers */
3413         WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3414         ring->wptr = 0;
3415         WREG32(CP_RB0_WPTR, ring->wptr);
3416
3417         /* set the wb address wether it's enabled or not */
3418         WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3419         WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3420
3421         /* scratch register shadowing is no longer supported */
3422         WREG32(SCRATCH_UMSK, 0);
3423
3424         if (!rdev->wb.enabled)
3425                 tmp |= RB_NO_UPDATE;
3426
3427         mdelay(1);
3428         WREG32(CP_RB0_CNTL, tmp);
3429
3430         rb_addr = ring->gpu_addr >> 8;
3431         WREG32(CP_RB0_BASE, rb_addr);
3432         WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
3433
3434         ring->rptr = RREG32(CP_RB0_RPTR);
3435
3436         /* start the ring */
3437         cik_cp_gfx_start(rdev);
3438         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3439         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3440         if (r) {
3441                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3442                 return r;
3443         }
3444         return 0;
3445 }
3446
3447 u32 cik_compute_ring_get_rptr(struct radeon_device *rdev,
3448                               struct radeon_ring *ring)
3449 {
3450         u32 rptr;
3451
3452
3453
3454         if (rdev->wb.enabled) {
3455                 rptr = le32_to_cpu(rdev->wb.wb[ring->rptr_offs/4]);
3456         } else {
3457                 mutex_lock(&rdev->srbm_mutex);
3458                 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
3459                 rptr = RREG32(CP_HQD_PQ_RPTR);
3460                 cik_srbm_select(rdev, 0, 0, 0, 0);
3461                 mutex_unlock(&rdev->srbm_mutex);
3462         }
3463
3464         return rptr;
3465 }
3466
3467 u32 cik_compute_ring_get_wptr(struct radeon_device *rdev,
3468                               struct radeon_ring *ring)
3469 {
3470         u32 wptr;
3471
3472         if (rdev->wb.enabled) {
3473                 wptr = le32_to_cpu(rdev->wb.wb[ring->wptr_offs/4]);
3474         } else {
3475                 mutex_lock(&rdev->srbm_mutex);
3476                 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
3477                 wptr = RREG32(CP_HQD_PQ_WPTR);
3478                 cik_srbm_select(rdev, 0, 0, 0, 0);
3479                 mutex_unlock(&rdev->srbm_mutex);
3480         }
3481
3482         return wptr;
3483 }
3484
3485 void cik_compute_ring_set_wptr(struct radeon_device *rdev,
3486                                struct radeon_ring *ring)
3487 {
3488         rdev->wb.wb[ring->wptr_offs/4] = cpu_to_le32(ring->wptr);
3489         WDOORBELL32(ring->doorbell_offset, ring->wptr);
3490 }
3491
3492 /**
3493  * cik_cp_compute_enable - enable/disable the compute CP MEs
3494  *
3495  * @rdev: radeon_device pointer
3496  * @enable: enable or disable the MEs
3497  *
3498  * Halts or unhalts the compute MEs.
3499  */
3500 static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
3501 {
3502         if (enable)
3503                 WREG32(CP_MEC_CNTL, 0);
3504         else
3505                 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
3506         udelay(50);
3507 }
3508
3509 /**
3510  * cik_cp_compute_load_microcode - load the compute CP ME ucode
3511  *
3512  * @rdev: radeon_device pointer
3513  *
3514  * Loads the compute MEC1&2 ucode.
3515  * Returns 0 for success, -EINVAL if the ucode is not available.
3516  */
3517 static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
3518 {
3519         const __be32 *fw_data;
3520         int i;
3521
3522         if (!rdev->mec_fw)
3523                 return -EINVAL;
3524
3525         cik_cp_compute_enable(rdev, false);
3526
3527         /* MEC1 */
3528         fw_data = (const __be32 *)rdev->mec_fw->data;
3529         WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
3530         for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
3531                 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
3532         WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
3533
3534         if (rdev->family == CHIP_KAVERI) {
3535                 /* MEC2 */
3536                 fw_data = (const __be32 *)rdev->mec_fw->data;
3537                 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
3538                 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
3539                         WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
3540                 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
3541         }
3542
3543         return 0;
3544 }
3545
3546 /**
3547  * cik_cp_compute_start - start the compute queues
3548  *
3549  * @rdev: radeon_device pointer
3550  *
3551  * Enable the compute queues.
3552  * Returns 0 for success, error for failure.
3553  */
3554 static int cik_cp_compute_start(struct radeon_device *rdev)
3555 {
3556         cik_cp_compute_enable(rdev, true);
3557
3558         return 0;
3559 }
3560
3561 /**
3562  * cik_cp_compute_fini - stop the compute queues
3563  *
3564  * @rdev: radeon_device pointer
3565  *
3566  * Stop the compute queues and tear down the driver queue
3567  * info.
3568  */
3569 static void cik_cp_compute_fini(struct radeon_device *rdev)
3570 {
3571         int i, idx, r;
3572
3573         cik_cp_compute_enable(rdev, false);
3574
3575         for (i = 0; i < 2; i++) {
3576                 if (i == 0)
3577                         idx = CAYMAN_RING_TYPE_CP1_INDEX;
3578                 else
3579                         idx = CAYMAN_RING_TYPE_CP2_INDEX;
3580
3581                 if (rdev->ring[idx].mqd_obj) {
3582                         r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
3583                         if (unlikely(r != 0))
3584                                 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
3585
3586                         radeon_bo_unpin(rdev->ring[idx].mqd_obj);
3587                         radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
3588
3589                         radeon_bo_unref(&rdev->ring[idx].mqd_obj);
3590                         rdev->ring[idx].mqd_obj = NULL;
3591                 }
3592         }
3593 }
3594
3595 static void cik_mec_fini(struct radeon_device *rdev)
3596 {
3597         int r;
3598
3599         if (rdev->mec.hpd_eop_obj) {
3600                 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
3601                 if (unlikely(r != 0))
3602                         dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
3603                 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
3604                 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
3605
3606                 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
3607                 rdev->mec.hpd_eop_obj = NULL;
3608         }
3609 }
3610
3611 #define MEC_HPD_SIZE 2048
3612
3613 static int cik_mec_init(struct radeon_device *rdev)
3614 {
3615         int r;
3616         u32 *hpd;
3617
3618         /*
3619          * KV:    2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
3620          * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
3621          */
3622         if (rdev->family == CHIP_KAVERI)
3623                 rdev->mec.num_mec = 2;
3624         else
3625                 rdev->mec.num_mec = 1;
3626         rdev->mec.num_pipe = 4;
3627         rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
3628
3629         if (rdev->mec.hpd_eop_obj == NULL) {
3630                 r = radeon_bo_create(rdev,
3631                                      rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
3632                                      PAGE_SIZE, true,
3633                                      RADEON_GEM_DOMAIN_GTT, NULL,
3634                                      &rdev->mec.hpd_eop_obj);
3635                 if (r) {
3636                         dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
3637                         return r;
3638                 }
3639         }
3640
3641         r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
3642         if (unlikely(r != 0)) {
3643                 cik_mec_fini(rdev);
3644                 return r;
3645         }
3646         r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
3647                           &rdev->mec.hpd_eop_gpu_addr);
3648         if (r) {
3649                 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
3650                 cik_mec_fini(rdev);
3651                 return r;
3652         }
3653         r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
3654         if (r) {
3655                 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
3656                 cik_mec_fini(rdev);
3657                 return r;
3658         }
3659
3660         /* clear memory.  Not sure if this is required or not */
3661         memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
3662
3663         radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
3664         radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
3665
3666         return 0;
3667 }
3668
3669 struct hqd_registers
3670 {
3671         u32 cp_mqd_base_addr;
3672         u32 cp_mqd_base_addr_hi;
3673         u32 cp_hqd_active;
3674         u32 cp_hqd_vmid;
3675         u32 cp_hqd_persistent_state;
3676         u32 cp_hqd_pipe_priority;
3677         u32 cp_hqd_queue_priority;
3678         u32 cp_hqd_quantum;
3679         u32 cp_hqd_pq_base;
3680         u32 cp_hqd_pq_base_hi;
3681         u32 cp_hqd_pq_rptr;
3682         u32 cp_hqd_pq_rptr_report_addr;
3683         u32 cp_hqd_pq_rptr_report_addr_hi;
3684         u32 cp_hqd_pq_wptr_poll_addr;
3685         u32 cp_hqd_pq_wptr_poll_addr_hi;
3686         u32 cp_hqd_pq_doorbell_control;
3687         u32 cp_hqd_pq_wptr;
3688         u32 cp_hqd_pq_control;
3689         u32 cp_hqd_ib_base_addr;
3690         u32 cp_hqd_ib_base_addr_hi;
3691         u32 cp_hqd_ib_rptr;
3692         u32 cp_hqd_ib_control;
3693         u32 cp_hqd_iq_timer;
3694         u32 cp_hqd_iq_rptr;
3695         u32 cp_hqd_dequeue_request;
3696         u32 cp_hqd_dma_offload;
3697         u32 cp_hqd_sema_cmd;
3698         u32 cp_hqd_msg_type;
3699         u32 cp_hqd_atomic0_preop_lo;
3700         u32 cp_hqd_atomic0_preop_hi;
3701         u32 cp_hqd_atomic1_preop_lo;
3702         u32 cp_hqd_atomic1_preop_hi;
3703         u32 cp_hqd_hq_scheduler0;
3704         u32 cp_hqd_hq_scheduler1;
3705         u32 cp_mqd_control;
3706 };
3707
3708 struct bonaire_mqd
3709 {
3710         u32 header;
3711         u32 dispatch_initiator;
3712         u32 dimensions[3];
3713         u32 start_idx[3];
3714         u32 num_threads[3];
3715         u32 pipeline_stat_enable;
3716         u32 perf_counter_enable;
3717         u32 pgm[2];
3718         u32 tba[2];
3719         u32 tma[2];
3720         u32 pgm_rsrc[2];
3721         u32 vmid;
3722         u32 resource_limits;
3723         u32 static_thread_mgmt01[2];
3724         u32 tmp_ring_size;
3725         u32 static_thread_mgmt23[2];
3726         u32 restart[3];
3727         u32 thread_trace_enable;
3728         u32 reserved1;
3729         u32 user_data[16];
3730         u32 vgtcs_invoke_count[2];
3731         struct hqd_registers queue_state;
3732         u32 dequeue_cntr;
3733         u32 interrupt_queue[64];
3734 };
3735
3736 /**
3737  * cik_cp_compute_resume - setup the compute queue registers
3738  *
3739  * @rdev: radeon_device pointer
3740  *
3741  * Program the compute queues and test them to make sure they
3742  * are working.
3743  * Returns 0 for success, error for failure.
3744  */
3745 static int cik_cp_compute_resume(struct radeon_device *rdev)
3746 {
3747         int r, i, idx;
3748         u32 tmp;
3749         bool use_doorbell = true;
3750         u64 hqd_gpu_addr;
3751         u64 mqd_gpu_addr;
3752         u64 eop_gpu_addr;
3753         u64 wb_gpu_addr;
3754         u32 *buf;
3755         struct bonaire_mqd *mqd;
3756
3757         r = cik_cp_compute_start(rdev);
3758         if (r)
3759                 return r;
3760
3761         /* fix up chicken bits */
3762         tmp = RREG32(CP_CPF_DEBUG);
3763         tmp |= (1 << 23);
3764         WREG32(CP_CPF_DEBUG, tmp);
3765
3766         /* init the pipes */
3767         mutex_lock(&rdev->srbm_mutex);
3768         for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) {
3769                 int me = (i < 4) ? 1 : 2;
3770                 int pipe = (i < 4) ? i : (i - 4);
3771
3772                 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2);
3773
3774                 cik_srbm_select(rdev, me, pipe, 0, 0);
3775
3776                 /* write the EOP addr */
3777                 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
3778                 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
3779
3780                 /* set the VMID assigned */
3781                 WREG32(CP_HPD_EOP_VMID, 0);
3782
3783                 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3784                 tmp = RREG32(CP_HPD_EOP_CONTROL);
3785                 tmp &= ~EOP_SIZE_MASK;
3786                 tmp |= order_base_2(MEC_HPD_SIZE / 8);
3787                 WREG32(CP_HPD_EOP_CONTROL, tmp);
3788         }
3789         cik_srbm_select(rdev, 0, 0, 0, 0);
3790         mutex_unlock(&rdev->srbm_mutex);
3791
3792         /* init the queues.  Just two for now. */
3793         for (i = 0; i < 2; i++) {
3794                 if (i == 0)
3795                         idx = CAYMAN_RING_TYPE_CP1_INDEX;
3796                 else
3797                         idx = CAYMAN_RING_TYPE_CP2_INDEX;
3798
3799                 if (rdev->ring[idx].mqd_obj == NULL) {
3800                         r = radeon_bo_create(rdev,
3801                                              sizeof(struct bonaire_mqd),
3802                                              PAGE_SIZE, true,
3803                                              RADEON_GEM_DOMAIN_GTT, NULL,
3804                                              &rdev->ring[idx].mqd_obj);
3805                         if (r) {
3806                                 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
3807                                 return r;
3808                         }
3809                 }
3810
3811                 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
3812                 if (unlikely(r != 0)) {
3813                         cik_cp_compute_fini(rdev);
3814                         return r;
3815                 }
3816                 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
3817                                   &mqd_gpu_addr);
3818                 if (r) {
3819                         dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
3820                         cik_cp_compute_fini(rdev);
3821                         return r;
3822                 }
3823                 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
3824                 if (r) {
3825                         dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
3826                         cik_cp_compute_fini(rdev);
3827                         return r;
3828                 }
3829
3830                 /* doorbell offset */
3831                 rdev->ring[idx].doorbell_offset =
3832                         (rdev->ring[idx].doorbell_page_num * PAGE_SIZE) + 0;
3833
3834                 /* init the mqd struct */
3835                 memset(buf, 0, sizeof(struct bonaire_mqd));
3836
3837                 mqd = (struct bonaire_mqd *)buf;
3838                 mqd->header = 0xC0310800;
3839                 mqd->static_thread_mgmt01[0] = 0xffffffff;
3840                 mqd->static_thread_mgmt01[1] = 0xffffffff;
3841                 mqd->static_thread_mgmt23[0] = 0xffffffff;
3842                 mqd->static_thread_mgmt23[1] = 0xffffffff;
3843
3844                 mutex_lock(&rdev->srbm_mutex);
3845                 cik_srbm_select(rdev, rdev->ring[idx].me,
3846                                 rdev->ring[idx].pipe,
3847                                 rdev->ring[idx].queue, 0);
3848
3849                 /* disable wptr polling */
3850                 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
3851                 tmp &= ~WPTR_POLL_EN;
3852                 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
3853
3854                 /* enable doorbell? */
3855                 mqd->queue_state.cp_hqd_pq_doorbell_control =
3856                         RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
3857                 if (use_doorbell)
3858                         mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
3859                 else
3860                         mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
3861                 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
3862                        mqd->queue_state.cp_hqd_pq_doorbell_control);
3863
3864                 /* disable the queue if it's active */
3865                 mqd->queue_state.cp_hqd_dequeue_request = 0;
3866                 mqd->queue_state.cp_hqd_pq_rptr = 0;
3867                 mqd->queue_state.cp_hqd_pq_wptr= 0;
3868                 if (RREG32(CP_HQD_ACTIVE) & 1) {
3869                         WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
3870                         for (i = 0; i < rdev->usec_timeout; i++) {
3871                                 if (!(RREG32(CP_HQD_ACTIVE) & 1))
3872                                         break;
3873                                 udelay(1);
3874                         }
3875                         WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
3876                         WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
3877                         WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
3878                 }
3879
3880                 /* set the pointer to the MQD */
3881                 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
3882                 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
3883                 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
3884                 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
3885                 /* set MQD vmid to 0 */
3886                 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
3887                 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
3888                 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
3889
3890                 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3891                 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
3892                 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
3893                 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
3894                 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
3895                 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
3896
3897                 /* set up the HQD, this is similar to CP_RB0_CNTL */
3898                 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
3899                 mqd->queue_state.cp_hqd_pq_control &=
3900                         ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
3901
3902                 mqd->queue_state.cp_hqd_pq_control |=
3903                         order_base_2(rdev->ring[idx].ring_size / 8);
3904                 mqd->queue_state.cp_hqd_pq_control |=
3905                         (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
3906 #ifdef __BIG_ENDIAN
3907                 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
3908 #endif
3909                 mqd->queue_state.cp_hqd_pq_control &=
3910                         ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
3911                 mqd->queue_state.cp_hqd_pq_control |=
3912                         PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
3913                 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
3914
3915                 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
3916                 if (i == 0)
3917                         wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
3918                 else
3919                         wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
3920                 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
3921                 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
3922                 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
3923                 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
3924                        mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
3925
3926                 /* set the wb address wether it's enabled or not */
3927                 if (i == 0)
3928                         wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
3929                 else
3930                         wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
3931                 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
3932                 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
3933                         upper_32_bits(wb_gpu_addr) & 0xffff;
3934                 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
3935                        mqd->queue_state.cp_hqd_pq_rptr_report_addr);
3936                 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
3937                        mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
3938
3939                 /* enable the doorbell if requested */
3940                 if (use_doorbell) {
3941                         mqd->queue_state.cp_hqd_pq_doorbell_control =
3942                                 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
3943                         mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
3944                         mqd->queue_state.cp_hqd_pq_doorbell_control |=
3945                                 DOORBELL_OFFSET(rdev->ring[idx].doorbell_offset / 4);
3946                         mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
3947                         mqd->queue_state.cp_hqd_pq_doorbell_control &=
3948                                 ~(DOORBELL_SOURCE | DOORBELL_HIT);
3949
3950                 } else {
3951                         mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
3952                 }
3953                 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
3954                        mqd->queue_state.cp_hqd_pq_doorbell_control);
3955
3956                 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3957                 rdev->ring[idx].wptr = 0;
3958                 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
3959                 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
3960                 rdev->ring[idx].rptr = RREG32(CP_HQD_PQ_RPTR);
3961                 mqd->queue_state.cp_hqd_pq_rptr = rdev->ring[idx].rptr;
3962
3963                 /* set the vmid for the queue */
3964                 mqd->queue_state.cp_hqd_vmid = 0;
3965                 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
3966
3967                 /* activate the queue */
3968                 mqd->queue_state.cp_hqd_active = 1;
3969                 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
3970
3971                 cik_srbm_select(rdev, 0, 0, 0, 0);
3972                 mutex_unlock(&rdev->srbm_mutex);
3973
3974                 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
3975                 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
3976
3977                 rdev->ring[idx].ready = true;
3978                 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
3979                 if (r)
3980                         rdev->ring[idx].ready = false;
3981         }
3982
3983         return 0;
3984 }
3985
3986 static void cik_cp_enable(struct radeon_device *rdev, bool enable)
3987 {
3988         cik_cp_gfx_enable(rdev, enable);
3989         cik_cp_compute_enable(rdev, enable);
3990 }
3991
3992 static int cik_cp_load_microcode(struct radeon_device *rdev)
3993 {
3994         int r;
3995
3996         r = cik_cp_gfx_load_microcode(rdev);
3997         if (r)
3998                 return r;
3999         r = cik_cp_compute_load_microcode(rdev);
4000         if (r)
4001                 return r;
4002
4003         return 0;
4004 }
4005
4006 static void cik_cp_fini(struct radeon_device *rdev)
4007 {
4008         cik_cp_gfx_fini(rdev);
4009         cik_cp_compute_fini(rdev);
4010 }
4011
4012 static int cik_cp_resume(struct radeon_device *rdev)
4013 {
4014         int r;
4015
4016         r = cik_cp_load_microcode(rdev);
4017         if (r)
4018                 return r;
4019
4020         r = cik_cp_gfx_resume(rdev);
4021         if (r)
4022                 return r;
4023         r = cik_cp_compute_resume(rdev);
4024         if (r)
4025                 return r;
4026
4027         return 0;
4028 }
4029
4030 static void cik_print_gpu_status_regs(struct radeon_device *rdev)
4031 {
4032         dev_info(rdev->dev, "  GRBM_STATUS=0x%08X\n",
4033                 RREG32(GRBM_STATUS));
4034         dev_info(rdev->dev, "  GRBM_STATUS2=0x%08X\n",
4035                 RREG32(GRBM_STATUS2));
4036         dev_info(rdev->dev, "  GRBM_STATUS_SE0=0x%08X\n",
4037                 RREG32(GRBM_STATUS_SE0));
4038         dev_info(rdev->dev, "  GRBM_STATUS_SE1=0x%08X\n",
4039                 RREG32(GRBM_STATUS_SE1));
4040         dev_info(rdev->dev, "  GRBM_STATUS_SE2=0x%08X\n",
4041                 RREG32(GRBM_STATUS_SE2));
4042         dev_info(rdev->dev, "  GRBM_STATUS_SE3=0x%08X\n",
4043                 RREG32(GRBM_STATUS_SE3));
4044         dev_info(rdev->dev, "  SRBM_STATUS=0x%08X\n",
4045                 RREG32(SRBM_STATUS));
4046         dev_info(rdev->dev, "  SRBM_STATUS2=0x%08X\n",
4047                 RREG32(SRBM_STATUS2));
4048         dev_info(rdev->dev, "  SDMA0_STATUS_REG   = 0x%08X\n",
4049                 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
4050         dev_info(rdev->dev, "  SDMA1_STATUS_REG   = 0x%08X\n",
4051                  RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
4052         dev_info(rdev->dev, "  CP_STAT = 0x%08x\n", RREG32(CP_STAT));
4053         dev_info(rdev->dev, "  CP_STALLED_STAT1 = 0x%08x\n",
4054                  RREG32(CP_STALLED_STAT1));
4055         dev_info(rdev->dev, "  CP_STALLED_STAT2 = 0x%08x\n",
4056                  RREG32(CP_STALLED_STAT2));
4057         dev_info(rdev->dev, "  CP_STALLED_STAT3 = 0x%08x\n",
4058                  RREG32(CP_STALLED_STAT3));
4059         dev_info(rdev->dev, "  CP_CPF_BUSY_STAT = 0x%08x\n",
4060                  RREG32(CP_CPF_BUSY_STAT));
4061         dev_info(rdev->dev, "  CP_CPF_STALLED_STAT1 = 0x%08x\n",
4062                  RREG32(CP_CPF_STALLED_STAT1));
4063         dev_info(rdev->dev, "  CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
4064         dev_info(rdev->dev, "  CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
4065         dev_info(rdev->dev, "  CP_CPC_STALLED_STAT1 = 0x%08x\n",
4066                  RREG32(CP_CPC_STALLED_STAT1));
4067         dev_info(rdev->dev, "  CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
4068 }
4069
4070 /**
4071  * cik_gpu_check_soft_reset - check which blocks are busy
4072  *
4073  * @rdev: radeon_device pointer
4074  *
4075  * Check which blocks are busy and return the relevant reset
4076  * mask to be used by cik_gpu_soft_reset().
4077  * Returns a mask of the blocks to be reset.
4078  */
4079 u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
4080 {
4081         u32 reset_mask = 0;
4082         u32 tmp;
4083
4084         /* GRBM_STATUS */
4085         tmp = RREG32(GRBM_STATUS);
4086         if (tmp & (PA_BUSY | SC_BUSY |
4087                    BCI_BUSY | SX_BUSY |
4088                    TA_BUSY | VGT_BUSY |
4089                    DB_BUSY | CB_BUSY |
4090                    GDS_BUSY | SPI_BUSY |
4091                    IA_BUSY | IA_BUSY_NO_DMA))
4092                 reset_mask |= RADEON_RESET_GFX;
4093
4094         if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
4095                 reset_mask |= RADEON_RESET_CP;
4096
4097         /* GRBM_STATUS2 */
4098         tmp = RREG32(GRBM_STATUS2);
4099         if (tmp & RLC_BUSY)
4100                 reset_mask |= RADEON_RESET_RLC;
4101
4102         /* SDMA0_STATUS_REG */
4103         tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
4104         if (!(tmp & SDMA_IDLE))
4105                 reset_mask |= RADEON_RESET_DMA;
4106
4107         /* SDMA1_STATUS_REG */
4108         tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
4109         if (!(tmp & SDMA_IDLE))
4110                 reset_mask |= RADEON_RESET_DMA1;
4111
4112         /* SRBM_STATUS2 */
4113         tmp = RREG32(SRBM_STATUS2);
4114         if (tmp & SDMA_BUSY)
4115                 reset_mask |= RADEON_RESET_DMA;
4116
4117         if (tmp & SDMA1_BUSY)
4118                 reset_mask |= RADEON_RESET_DMA1;
4119
4120         /* SRBM_STATUS */
4121         tmp = RREG32(SRBM_STATUS);
4122
4123         if (tmp & IH_BUSY)
4124                 reset_mask |= RADEON_RESET_IH;
4125
4126         if (tmp & SEM_BUSY)
4127                 reset_mask |= RADEON_RESET_SEM;
4128
4129         if (tmp & GRBM_RQ_PENDING)
4130                 reset_mask |= RADEON_RESET_GRBM;
4131
4132         if (tmp & VMC_BUSY)
4133                 reset_mask |= RADEON_RESET_VMC;
4134
4135         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
4136                    MCC_BUSY | MCD_BUSY))
4137                 reset_mask |= RADEON_RESET_MC;
4138
4139         if (evergreen_is_display_hung(rdev))
4140                 reset_mask |= RADEON_RESET_DISPLAY;
4141
4142         /* Skip MC reset as it's mostly likely not hung, just busy */
4143         if (reset_mask & RADEON_RESET_MC) {
4144                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
4145                 reset_mask &= ~RADEON_RESET_MC;
4146         }
4147
4148         return reset_mask;
4149 }
4150
4151 /**
4152  * cik_gpu_soft_reset - soft reset GPU
4153  *
4154  * @rdev: radeon_device pointer
4155  * @reset_mask: mask of which blocks to reset
4156  *
4157  * Soft reset the blocks specified in @reset_mask.
4158  */
4159 static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
4160 {
4161         struct evergreen_mc_save save;
4162         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
4163         u32 tmp;
4164
4165         if (reset_mask == 0)
4166                 return;
4167
4168         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
4169
4170         cik_print_gpu_status_regs(rdev);
4171         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
4172                  RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4173         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4174                  RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
4175
4176         /* stop the rlc */
4177         cik_rlc_stop(rdev);
4178
4179         /* Disable GFX parsing/prefetching */
4180         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
4181
4182         /* Disable MEC parsing/prefetching */
4183         WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
4184
4185         if (reset_mask & RADEON_RESET_DMA) {
4186                 /* sdma0 */
4187                 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
4188                 tmp |= SDMA_HALT;
4189                 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
4190         }
4191         if (reset_mask & RADEON_RESET_DMA1) {
4192                 /* sdma1 */
4193                 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
4194                 tmp |= SDMA_HALT;
4195                 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
4196         }
4197
4198         evergreen_mc_stop(rdev, &save);
4199         if (evergreen_mc_wait_for_idle(rdev)) {
4200                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4201         }
4202
4203         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
4204                 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
4205
4206         if (reset_mask & RADEON_RESET_CP) {
4207                 grbm_soft_reset |= SOFT_RESET_CP;
4208
4209                 srbm_soft_reset |= SOFT_RESET_GRBM;
4210         }
4211
4212         if (reset_mask & RADEON_RESET_DMA)
4213                 srbm_soft_reset |= SOFT_RESET_SDMA;
4214
4215         if (reset_mask & RADEON_RESET_DMA1)
4216                 srbm_soft_reset |= SOFT_RESET_SDMA1;
4217
4218         if (reset_mask & RADEON_RESET_DISPLAY)
4219                 srbm_soft_reset |= SOFT_RESET_DC;
4220
4221         if (reset_mask & RADEON_RESET_RLC)
4222                 grbm_soft_reset |= SOFT_RESET_RLC;
4223
4224         if (reset_mask & RADEON_RESET_SEM)
4225                 srbm_soft_reset |= SOFT_RESET_SEM;
4226
4227         if (reset_mask & RADEON_RESET_IH)
4228                 srbm_soft_reset |= SOFT_RESET_IH;
4229
4230         if (reset_mask & RADEON_RESET_GRBM)
4231                 srbm_soft_reset |= SOFT_RESET_GRBM;
4232
4233         if (reset_mask & RADEON_RESET_VMC)
4234                 srbm_soft_reset |= SOFT_RESET_VMC;
4235
4236         if (!(rdev->flags & RADEON_IS_IGP)) {
4237                 if (reset_mask & RADEON_RESET_MC)
4238                         srbm_soft_reset |= SOFT_RESET_MC;
4239         }
4240
4241         if (grbm_soft_reset) {
4242                 tmp = RREG32(GRBM_SOFT_RESET);
4243                 tmp |= grbm_soft_reset;
4244                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4245                 WREG32(GRBM_SOFT_RESET, tmp);
4246                 tmp = RREG32(GRBM_SOFT_RESET);
4247
4248                 udelay(50);
4249
4250                 tmp &= ~grbm_soft_reset;
4251                 WREG32(GRBM_SOFT_RESET, tmp);
4252                 tmp = RREG32(GRBM_SOFT_RESET);
4253         }
4254
4255         if (srbm_soft_reset) {
4256                 tmp = RREG32(SRBM_SOFT_RESET);
4257                 tmp |= srbm_soft_reset;
4258                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
4259                 WREG32(SRBM_SOFT_RESET, tmp);
4260                 tmp = RREG32(SRBM_SOFT_RESET);
4261
4262                 udelay(50);
4263
4264                 tmp &= ~srbm_soft_reset;
4265                 WREG32(SRBM_SOFT_RESET, tmp);
4266                 tmp = RREG32(SRBM_SOFT_RESET);
4267         }
4268
4269         /* Wait a little for things to settle down */
4270         udelay(50);
4271
4272         evergreen_mc_resume(rdev, &save);
4273         udelay(50);
4274
4275         cik_print_gpu_status_regs(rdev);
4276 }
4277
4278 /**
4279  * cik_asic_reset - soft reset GPU
4280  *
4281  * @rdev: radeon_device pointer
4282  *
4283  * Look up which blocks are hung and attempt
4284  * to reset them.
4285  * Returns 0 for success.
4286  */
4287 int cik_asic_reset(struct radeon_device *rdev)
4288 {
4289         u32 reset_mask;
4290
4291         reset_mask = cik_gpu_check_soft_reset(rdev);
4292
4293         if (reset_mask)
4294                 r600_set_bios_scratch_engine_hung(rdev, true);
4295
4296         cik_gpu_soft_reset(rdev, reset_mask);
4297
4298         reset_mask = cik_gpu_check_soft_reset(rdev);
4299
4300         if (!reset_mask)
4301                 r600_set_bios_scratch_engine_hung(rdev, false);
4302
4303         return 0;
4304 }
4305
4306 /**
4307  * cik_gfx_is_lockup - check if the 3D engine is locked up
4308  *
4309  * @rdev: radeon_device pointer
4310  * @ring: radeon_ring structure holding ring information
4311  *
4312  * Check if the 3D engine is locked up (CIK).
4313  * Returns true if the engine is locked, false if not.
4314  */
4315 bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4316 {
4317         u32 reset_mask = cik_gpu_check_soft_reset(rdev);
4318
4319         if (!(reset_mask & (RADEON_RESET_GFX |
4320                             RADEON_RESET_COMPUTE |
4321                             RADEON_RESET_CP))) {
4322                 radeon_ring_lockup_update(ring);
4323                 return false;
4324         }
4325         /* force CP activities */
4326         radeon_ring_force_activity(rdev, ring);
4327         return radeon_ring_test_lockup(rdev, ring);
4328 }
4329
4330 /* MC */
4331 /**
4332  * cik_mc_program - program the GPU memory controller
4333  *
4334  * @rdev: radeon_device pointer
4335  *
4336  * Set the location of vram, gart, and AGP in the GPU's
4337  * physical address space (CIK).
4338  */
4339 static void cik_mc_program(struct radeon_device *rdev)
4340 {
4341         struct evergreen_mc_save save;
4342         u32 tmp;
4343         int i, j;
4344
4345         /* Initialize HDP */
4346         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
4347                 WREG32((0x2c14 + j), 0x00000000);
4348                 WREG32((0x2c18 + j), 0x00000000);
4349                 WREG32((0x2c1c + j), 0x00000000);
4350                 WREG32((0x2c20 + j), 0x00000000);
4351                 WREG32((0x2c24 + j), 0x00000000);
4352         }
4353         WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
4354
4355         evergreen_mc_stop(rdev, &save);
4356         if (radeon_mc_wait_for_idle(rdev)) {
4357                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4358         }
4359         /* Lockout access through VGA aperture*/
4360         WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
4361         /* Update configuration */
4362         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
4363                rdev->mc.vram_start >> 12);
4364         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
4365                rdev->mc.vram_end >> 12);
4366         WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
4367                rdev->vram_scratch.gpu_addr >> 12);
4368         tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
4369         tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
4370         WREG32(MC_VM_FB_LOCATION, tmp);
4371         /* XXX double check these! */
4372         WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
4373         WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
4374         WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
4375         WREG32(MC_VM_AGP_BASE, 0);
4376         WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
4377         WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
4378         if (radeon_mc_wait_for_idle(rdev)) {
4379                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4380         }
4381         evergreen_mc_resume(rdev, &save);
4382         /* we need to own VRAM, so turn off the VGA renderer here
4383          * to stop it overwriting our objects */
4384         rv515_vga_render_disable(rdev);
4385 }
4386
4387 /**
4388  * cik_mc_init - initialize the memory controller driver params
4389  *
4390  * @rdev: radeon_device pointer
4391  *
4392  * Look up the amount of vram, vram width, and decide how to place
4393  * vram and gart within the GPU's physical address space (CIK).
4394  * Returns 0 for success.
4395  */
4396 static int cik_mc_init(struct radeon_device *rdev)
4397 {
4398         u32 tmp;
4399         int chansize, numchan;
4400
4401         /* Get VRAM informations */
4402         rdev->mc.vram_is_ddr = true;
4403         tmp = RREG32(MC_ARB_RAMCFG);
4404         if (tmp & CHANSIZE_MASK) {
4405                 chansize = 64;
4406         } else {
4407                 chansize = 32;
4408         }
4409         tmp = RREG32(MC_SHARED_CHMAP);
4410         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
4411         case 0:
4412         default:
4413                 numchan = 1;
4414                 break;
4415         case 1:
4416                 numchan = 2;
4417                 break;
4418         case 2:
4419                 numchan = 4;
4420                 break;
4421         case 3:
4422                 numchan = 8;
4423                 break;
4424         case 4:
4425                 numchan = 3;
4426                 break;
4427         case 5:
4428                 numchan = 6;
4429                 break;
4430         case 6:
4431                 numchan = 10;
4432                 break;
4433         case 7:
4434                 numchan = 12;
4435                 break;
4436         case 8:
4437                 numchan = 16;
4438                 break;
4439         }
4440         rdev->mc.vram_width = numchan * chansize;
4441         /* Could aper size report 0 ? */
4442         rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
4443         rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
4444         /* size in MB on si */
4445         rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
4446         rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
4447         rdev->mc.visible_vram_size = rdev->mc.aper_size;
4448         si_vram_gtt_location(rdev, &rdev->mc);
4449         radeon_update_bandwidth_info(rdev);
4450
4451         return 0;
4452 }
4453
4454 /*
4455  * GART
4456  * VMID 0 is the physical GPU addresses as used by the kernel.
4457  * VMIDs 1-15 are used for userspace clients and are handled
4458  * by the radeon vm/hsa code.
4459  */
4460 /**
4461  * cik_pcie_gart_tlb_flush - gart tlb flush callback
4462  *
4463  * @rdev: radeon_device pointer
4464  *
4465  * Flush the TLB for the VMID 0 page table (CIK).
4466  */
4467 void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
4468 {
4469         /* flush hdp cache */
4470         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
4471
4472         /* bits 0-15 are the VM contexts0-15 */
4473         WREG32(VM_INVALIDATE_REQUEST, 0x1);
4474 }
4475
4476 /**
4477  * cik_pcie_gart_enable - gart enable
4478  *
4479  * @rdev: radeon_device pointer
4480  *
4481  * This sets up the TLBs, programs the page tables for VMID0,
4482  * sets up the hw for VMIDs 1-15 which are allocated on
4483  * demand, and sets up the global locations for the LDS, GDS,
4484  * and GPUVM for FSA64 clients (CIK).
4485  * Returns 0 for success, errors for failure.
4486  */
4487 static int cik_pcie_gart_enable(struct radeon_device *rdev)
4488 {
4489         int r, i;
4490
4491         if (rdev->gart.robj == NULL) {
4492                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
4493                 return -EINVAL;
4494         }
4495         r = radeon_gart_table_vram_pin(rdev);
4496         if (r)
4497                 return r;
4498         radeon_gart_restore(rdev);
4499         /* Setup TLB control */
4500         WREG32(MC_VM_MX_L1_TLB_CNTL,
4501                (0xA << 7) |
4502                ENABLE_L1_TLB |
4503                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4504                ENABLE_ADVANCED_DRIVER_MODEL |
4505                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4506         /* Setup L2 cache */
4507         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
4508                ENABLE_L2_FRAGMENT_PROCESSING |
4509                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4510                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4511                EFFECTIVE_L2_QUEUE_SIZE(7) |
4512                CONTEXT1_IDENTITY_ACCESS_MODE(1));
4513         WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
4514         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4515                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
4516         /* setup context0 */
4517         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
4518         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
4519         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
4520         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
4521                         (u32)(rdev->dummy_page.addr >> 12));
4522         WREG32(VM_CONTEXT0_CNTL2, 0);
4523         WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
4524                                   RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
4525
4526         WREG32(0x15D4, 0);
4527         WREG32(0x15D8, 0);
4528         WREG32(0x15DC, 0);
4529
4530         /* empty context1-15 */
4531         /* FIXME start with 4G, once using 2 level pt switch to full
4532          * vm size space
4533          */
4534         /* set vm size, must be a multiple of 4 */
4535         WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
4536         WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
4537         for (i = 1; i < 16; i++) {
4538                 if (i < 8)
4539                         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
4540                                rdev->gart.table_addr >> 12);
4541                 else
4542                         WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
4543                                rdev->gart.table_addr >> 12);
4544         }
4545
4546         /* enable context1-15 */
4547         WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
4548                (u32)(rdev->dummy_page.addr >> 12));
4549         WREG32(VM_CONTEXT1_CNTL2, 4);
4550         WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
4551                                 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4552                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4553                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4554                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4555                                 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
4556                                 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
4557                                 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
4558                                 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
4559                                 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
4560                                 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
4561                                 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4562                                 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
4563
4564         /* TC cache setup ??? */
4565         WREG32(TC_CFG_L1_LOAD_POLICY0, 0);
4566         WREG32(TC_CFG_L1_LOAD_POLICY1, 0);
4567         WREG32(TC_CFG_L1_STORE_POLICY, 0);
4568
4569         WREG32(TC_CFG_L2_LOAD_POLICY0, 0);
4570         WREG32(TC_CFG_L2_LOAD_POLICY1, 0);
4571         WREG32(TC_CFG_L2_STORE_POLICY0, 0);
4572         WREG32(TC_CFG_L2_STORE_POLICY1, 0);
4573         WREG32(TC_CFG_L2_ATOMIC_POLICY, 0);
4574
4575         WREG32(TC_CFG_L1_VOLATILE, 0);
4576         WREG32(TC_CFG_L2_VOLATILE, 0);
4577
4578         if (rdev->family == CHIP_KAVERI) {
4579                 u32 tmp = RREG32(CHUB_CONTROL);
4580                 tmp &= ~BYPASS_VM;
4581                 WREG32(CHUB_CONTROL, tmp);
4582         }
4583
4584         /* XXX SH_MEM regs */
4585         /* where to put LDS, scratch, GPUVM in FSA64 space */
4586         mutex_lock(&rdev->srbm_mutex);
4587         for (i = 0; i < 16; i++) {
4588                 cik_srbm_select(rdev, 0, 0, 0, i);
4589                 /* CP and shaders */
4590                 WREG32(SH_MEM_CONFIG, 0);
4591                 WREG32(SH_MEM_APE1_BASE, 1);
4592                 WREG32(SH_MEM_APE1_LIMIT, 0);
4593                 WREG32(SH_MEM_BASES, 0);
4594                 /* SDMA GFX */
4595                 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
4596                 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
4597                 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
4598                 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
4599                 /* XXX SDMA RLC - todo */
4600         }
4601         cik_srbm_select(rdev, 0, 0, 0, 0);
4602         mutex_unlock(&rdev->srbm_mutex);
4603
4604         cik_pcie_gart_tlb_flush(rdev);
4605         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
4606                  (unsigned)(rdev->mc.gtt_size >> 20),
4607                  (unsigned long long)rdev->gart.table_addr);
4608         rdev->gart.ready = true;
4609         return 0;
4610 }
4611
4612 /**
4613  * cik_pcie_gart_disable - gart disable
4614  *
4615  * @rdev: radeon_device pointer
4616  *
4617  * This disables all VM page table (CIK).
4618  */
4619 static void cik_pcie_gart_disable(struct radeon_device *rdev)
4620 {
4621         /* Disable all tables */
4622         WREG32(VM_CONTEXT0_CNTL, 0);
4623         WREG32(VM_CONTEXT1_CNTL, 0);
4624         /* Setup TLB control */
4625         WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4626                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4627         /* Setup L2 cache */
4628         WREG32(VM_L2_CNTL,
4629                ENABLE_L2_FRAGMENT_PROCESSING |
4630                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4631                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4632                EFFECTIVE_L2_QUEUE_SIZE(7) |
4633                CONTEXT1_IDENTITY_ACCESS_MODE(1));
4634         WREG32(VM_L2_CNTL2, 0);
4635         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4636                L2_CACHE_BIGK_FRAGMENT_SIZE(6));
4637         radeon_gart_table_vram_unpin(rdev);
4638 }
4639
4640 /**
4641  * cik_pcie_gart_fini - vm fini callback
4642  *
4643  * @rdev: radeon_device pointer
4644  *
4645  * Tears down the driver GART/VM setup (CIK).
4646  */
4647 static void cik_pcie_gart_fini(struct radeon_device *rdev)
4648 {
4649         cik_pcie_gart_disable(rdev);
4650         radeon_gart_table_vram_free(rdev);
4651         radeon_gart_fini(rdev);
4652 }
4653
4654 /* vm parser */
4655 /**
4656  * cik_ib_parse - vm ib_parse callback
4657  *
4658  * @rdev: radeon_device pointer
4659  * @ib: indirect buffer pointer
4660  *
4661  * CIK uses hw IB checking so this is a nop (CIK).
4662  */
4663 int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4664 {
4665         return 0;
4666 }
4667
4668 /*
4669  * vm
4670  * VMID 0 is the physical GPU addresses as used by the kernel.
4671  * VMIDs 1-15 are used for userspace clients and are handled
4672  * by the radeon vm/hsa code.
4673  */
4674 /**
4675  * cik_vm_init - cik vm init callback
4676  *
4677  * @rdev: radeon_device pointer
4678  *
4679  * Inits cik specific vm parameters (number of VMs, base of vram for
4680  * VMIDs 1-15) (CIK).
4681  * Returns 0 for success.
4682  */
4683 int cik_vm_init(struct radeon_device *rdev)
4684 {
4685         /* number of VMs */
4686         rdev->vm_manager.nvm = 16;
4687         /* base offset of vram pages */
4688         if (rdev->flags & RADEON_IS_IGP) {
4689                 u64 tmp = RREG32(MC_VM_FB_OFFSET);
4690                 tmp <<= 22;
4691                 rdev->vm_manager.vram_base_offset = tmp;
4692         } else
4693                 rdev->vm_manager.vram_base_offset = 0;
4694
4695         return 0;
4696 }
4697
4698 /**
4699  * cik_vm_fini - cik vm fini callback
4700  *
4701  * @rdev: radeon_device pointer
4702  *
4703  * Tear down any asic specific VM setup (CIK).
4704  */
4705 void cik_vm_fini(struct radeon_device *rdev)
4706 {
4707 }
4708
4709 /**
4710  * cik_vm_decode_fault - print human readable fault info
4711  *
4712  * @rdev: radeon_device pointer
4713  * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
4714  * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
4715  *
4716  * Print human readable fault information (CIK).
4717  */
4718 static void cik_vm_decode_fault(struct radeon_device *rdev,
4719                                 u32 status, u32 addr, u32 mc_client)
4720 {
4721         u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
4722         u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
4723         u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
4724         char *block = (char *)&mc_client;
4725
4726         printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
4727                protections, vmid, addr,
4728                (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
4729                block, mc_id);
4730 }
4731
4732 /**
4733  * cik_vm_flush - cik vm flush using the CP
4734  *
4735  * @rdev: radeon_device pointer
4736  *
4737  * Update the page table base and flush the VM TLB
4738  * using the CP (CIK).
4739  */
4740 void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
4741 {
4742         struct radeon_ring *ring = &rdev->ring[ridx];
4743
4744         if (vm == NULL)
4745                 return;
4746
4747         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4748         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4749                                  WRITE_DATA_DST_SEL(0)));
4750         if (vm->id < 8) {
4751                 radeon_ring_write(ring,
4752                                   (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
4753         } else {
4754                 radeon_ring_write(ring,
4755                                   (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
4756         }
4757         radeon_ring_write(ring, 0);
4758         radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
4759
4760         /* update SH_MEM_* regs */
4761         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4762         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4763                                  WRITE_DATA_DST_SEL(0)));
4764         radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
4765         radeon_ring_write(ring, 0);
4766         radeon_ring_write(ring, VMID(vm->id));
4767
4768         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
4769         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4770                                  WRITE_DATA_DST_SEL(0)));
4771         radeon_ring_write(ring, SH_MEM_BASES >> 2);
4772         radeon_ring_write(ring, 0);
4773
4774         radeon_ring_write(ring, 0); /* SH_MEM_BASES */
4775         radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
4776         radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
4777         radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
4778
4779         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4780         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4781                                  WRITE_DATA_DST_SEL(0)));
4782         radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
4783         radeon_ring_write(ring, 0);
4784         radeon_ring_write(ring, VMID(0));
4785
4786         /* HDP flush */
4787         /* We should be using the WAIT_REG_MEM packet here like in
4788          * cik_fence_ring_emit(), but it causes the CP to hang in this
4789          * context...
4790          */
4791         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4792         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4793                                  WRITE_DATA_DST_SEL(0)));
4794         radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
4795         radeon_ring_write(ring, 0);
4796         radeon_ring_write(ring, 0);
4797
4798         /* bits 0-15 are the VM contexts0-15 */
4799         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4800         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4801                                  WRITE_DATA_DST_SEL(0)));
4802         radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
4803         radeon_ring_write(ring, 0);
4804         radeon_ring_write(ring, 1 << vm->id);
4805
4806         /* compute doesn't have PFP */
4807         if (ridx == RADEON_RING_TYPE_GFX_INDEX) {
4808                 /* sync PFP to ME, otherwise we might get invalid PFP reads */
4809                 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
4810                 radeon_ring_write(ring, 0x0);
4811         }
4812 }
4813
4814 /**
4815  * cik_vm_set_page - update the page tables using sDMA
4816  *
4817  * @rdev: radeon_device pointer
4818  * @ib: indirect buffer to fill with commands
4819  * @pe: addr of the page entry
4820  * @addr: dst addr to write into pe
4821  * @count: number of page entries to update
4822  * @incr: increase next addr by incr bytes
4823  * @flags: access flags
4824  *
4825  * Update the page tables using CP or sDMA (CIK).
4826  */
4827 void cik_vm_set_page(struct radeon_device *rdev,
4828                      struct radeon_ib *ib,
4829                      uint64_t pe,
4830                      uint64_t addr, unsigned count,
4831                      uint32_t incr, uint32_t flags)
4832 {
4833         uint32_t r600_flags = cayman_vm_page_flags(rdev, flags);
4834         uint64_t value;
4835         unsigned ndw;
4836
4837         if (rdev->asic->vm.pt_ring_index == RADEON_RING_TYPE_GFX_INDEX) {
4838                 /* CP */
4839                 while (count) {
4840                         ndw = 2 + count * 2;
4841                         if (ndw > 0x3FFE)
4842                                 ndw = 0x3FFE;
4843
4844                         ib->ptr[ib->length_dw++] = PACKET3(PACKET3_WRITE_DATA, ndw);
4845                         ib->ptr[ib->length_dw++] = (WRITE_DATA_ENGINE_SEL(0) |
4846                                                     WRITE_DATA_DST_SEL(1));
4847                         ib->ptr[ib->length_dw++] = pe;
4848                         ib->ptr[ib->length_dw++] = upper_32_bits(pe);
4849                         for (; ndw > 2; ndw -= 2, --count, pe += 8) {
4850                                 if (flags & RADEON_VM_PAGE_SYSTEM) {
4851                                         value = radeon_vm_map_gart(rdev, addr);
4852                                         value &= 0xFFFFFFFFFFFFF000ULL;
4853                                 } else if (flags & RADEON_VM_PAGE_VALID) {
4854                                         value = addr;
4855                                 } else {
4856                                         value = 0;
4857                                 }
4858                                 addr += incr;
4859                                 value |= r600_flags;
4860                                 ib->ptr[ib->length_dw++] = value;
4861                                 ib->ptr[ib->length_dw++] = upper_32_bits(value);
4862                         }
4863                 }
4864         } else {
4865                 /* DMA */
4866                 cik_sdma_vm_set_page(rdev, ib, pe, addr, count, incr, flags);
4867         }
4868 }
4869
4870 /*
4871  * RLC
4872  * The RLC is a multi-purpose microengine that handles a
4873  * variety of functions, the most important of which is
4874  * the interrupt controller.
4875  */
4876 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
4877                                           bool enable)
4878 {
4879         u32 tmp = RREG32(CP_INT_CNTL_RING0);
4880
4881         if (enable)
4882                 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4883         else
4884                 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4885         WREG32(CP_INT_CNTL_RING0, tmp);
4886 }
4887
4888 static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
4889 {
4890         u32 tmp;
4891
4892         tmp = RREG32(RLC_LB_CNTL);
4893         if (enable)
4894                 tmp |= LOAD_BALANCE_ENABLE;
4895         else
4896                 tmp &= ~LOAD_BALANCE_ENABLE;
4897         WREG32(RLC_LB_CNTL, tmp);
4898 }
4899
4900 static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
4901 {
4902         u32 i, j, k;
4903         u32 mask;
4904
4905         for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
4906                 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
4907                         cik_select_se_sh(rdev, i, j);
4908                         for (k = 0; k < rdev->usec_timeout; k++) {
4909                                 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
4910                                         break;
4911                                 udelay(1);
4912                         }
4913                 }
4914         }
4915         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
4916
4917         mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
4918         for (k = 0; k < rdev->usec_timeout; k++) {
4919                 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
4920                         break;
4921                 udelay(1);
4922         }
4923 }
4924
4925 static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
4926 {
4927         u32 tmp;
4928
4929         tmp = RREG32(RLC_CNTL);
4930         if (tmp != rlc)
4931                 WREG32(RLC_CNTL, rlc);
4932 }
4933
4934 static u32 cik_halt_rlc(struct radeon_device *rdev)
4935 {
4936         u32 data, orig;
4937
4938         orig = data = RREG32(RLC_CNTL);
4939
4940         if (data & RLC_ENABLE) {
4941                 u32 i;
4942
4943                 data &= ~RLC_ENABLE;
4944                 WREG32(RLC_CNTL, data);
4945
4946                 for (i = 0; i < rdev->usec_timeout; i++) {
4947                         if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
4948                                 break;
4949                         udelay(1);
4950                 }
4951
4952                 cik_wait_for_rlc_serdes(rdev);
4953         }
4954
4955         return orig;
4956 }
4957
4958 void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
4959 {
4960         u32 tmp, i, mask;
4961
4962         tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
4963         WREG32(RLC_GPR_REG2, tmp);
4964
4965         mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
4966         for (i = 0; i < rdev->usec_timeout; i++) {
4967                 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
4968                         break;
4969                 udelay(1);
4970         }
4971
4972         for (i = 0; i < rdev->usec_timeout; i++) {
4973                 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
4974                         break;
4975                 udelay(1);
4976         }
4977 }
4978
4979 void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
4980 {
4981         u32 tmp;
4982
4983         tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
4984         WREG32(RLC_GPR_REG2, tmp);
4985 }
4986
4987 /**
4988  * cik_rlc_stop - stop the RLC ME
4989  *
4990  * @rdev: radeon_device pointer
4991  *
4992  * Halt the RLC ME (MicroEngine) (CIK).
4993  */
4994 static void cik_rlc_stop(struct radeon_device *rdev)
4995 {
4996         WREG32(RLC_CNTL, 0);
4997
4998         cik_enable_gui_idle_interrupt(rdev, false);
4999
5000         cik_wait_for_rlc_serdes(rdev);
5001 }
5002
5003 /**
5004  * cik_rlc_start - start the RLC ME
5005  *
5006  * @rdev: radeon_device pointer
5007  *
5008  * Unhalt the RLC ME (MicroEngine) (CIK).
5009  */
5010 static void cik_rlc_start(struct radeon_device *rdev)
5011 {
5012         WREG32(RLC_CNTL, RLC_ENABLE);
5013
5014         cik_enable_gui_idle_interrupt(rdev, true);
5015
5016         udelay(50);
5017 }
5018
5019 /**
5020  * cik_rlc_resume - setup the RLC hw
5021  *
5022  * @rdev: radeon_device pointer
5023  *
5024  * Initialize the RLC registers, load the ucode,
5025  * and start the RLC (CIK).
5026  * Returns 0 for success, -EINVAL if the ucode is not available.
5027  */
5028 static int cik_rlc_resume(struct radeon_device *rdev)
5029 {
5030         u32 i, size, tmp;
5031         const __be32 *fw_data;
5032
5033         if (!rdev->rlc_fw)
5034                 return -EINVAL;
5035
5036         switch (rdev->family) {
5037         case CHIP_BONAIRE:
5038         default:
5039                 size = BONAIRE_RLC_UCODE_SIZE;
5040                 break;
5041         case CHIP_KAVERI:
5042                 size = KV_RLC_UCODE_SIZE;
5043                 break;
5044         case CHIP_KABINI:
5045                 size = KB_RLC_UCODE_SIZE;
5046                 break;
5047         }
5048
5049         cik_rlc_stop(rdev);
5050
5051         /* disable CG */
5052         tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
5053         WREG32(RLC_CGCG_CGLS_CTRL, tmp);
5054
5055         si_rlc_reset(rdev);
5056
5057         cik_init_pg(rdev);
5058
5059         cik_init_cg(rdev);
5060
5061         WREG32(RLC_LB_CNTR_INIT, 0);
5062         WREG32(RLC_LB_CNTR_MAX, 0x00008000);
5063
5064         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5065         WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5066         WREG32(RLC_LB_PARAMS, 0x00600408);
5067         WREG32(RLC_LB_CNTL, 0x80000004);
5068
5069         WREG32(RLC_MC_CNTL, 0);
5070         WREG32(RLC_UCODE_CNTL, 0);
5071
5072         fw_data = (const __be32 *)rdev->rlc_fw->data;
5073                 WREG32(RLC_GPM_UCODE_ADDR, 0);
5074         for (i = 0; i < size; i++)
5075                 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
5076         WREG32(RLC_GPM_UCODE_ADDR, 0);
5077
5078         /* XXX - find out what chips support lbpw */
5079         cik_enable_lbpw(rdev, false);
5080
5081         if (rdev->family == CHIP_BONAIRE)
5082                 WREG32(RLC_DRIVER_DMA_STATUS, 0);
5083
5084         cik_rlc_start(rdev);
5085
5086         return 0;
5087 }
5088
5089 static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
5090 {
5091         u32 data, orig, tmp, tmp2;
5092
5093         orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5094
5095         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
5096                 cik_enable_gui_idle_interrupt(rdev, true);
5097
5098                 tmp = cik_halt_rlc(rdev);
5099
5100                 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5101                 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5102                 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5103                 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
5104                 WREG32(RLC_SERDES_WR_CTRL, tmp2);
5105
5106                 cik_update_rlc(rdev, tmp);
5107
5108                 data |= CGCG_EN | CGLS_EN;
5109         } else {
5110                 cik_enable_gui_idle_interrupt(rdev, false);
5111
5112                 RREG32(CB_CGTT_SCLK_CTRL);
5113                 RREG32(CB_CGTT_SCLK_CTRL);
5114                 RREG32(CB_CGTT_SCLK_CTRL);
5115                 RREG32(CB_CGTT_SCLK_CTRL);
5116
5117                 data &= ~(CGCG_EN | CGLS_EN);
5118         }
5119
5120         if (orig != data)
5121                 WREG32(RLC_CGCG_CGLS_CTRL, data);
5122
5123 }
5124
5125 static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
5126 {
5127         u32 data, orig, tmp = 0;
5128
5129         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5130                 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
5131                         if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5132                                 orig = data = RREG32(CP_MEM_SLP_CNTL);
5133                                 data |= CP_MEM_LS_EN;
5134                                 if (orig != data)
5135                                         WREG32(CP_MEM_SLP_CNTL, data);
5136                         }
5137                 }
5138
5139                 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5140                 data &= 0xfffffffd;
5141                 if (orig != data)
5142                         WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5143
5144                 tmp = cik_halt_rlc(rdev);
5145
5146                 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5147                 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5148                 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5149                 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
5150                 WREG32(RLC_SERDES_WR_CTRL, data);
5151
5152                 cik_update_rlc(rdev, tmp);
5153
5154                 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
5155                         orig = data = RREG32(CGTS_SM_CTRL_REG);
5156                         data &= ~SM_MODE_MASK;
5157                         data |= SM_MODE(0x2);
5158                         data |= SM_MODE_ENABLE;
5159                         data &= ~CGTS_OVERRIDE;
5160                         if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
5161                             (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
5162                                 data &= ~CGTS_LS_OVERRIDE;
5163                         data &= ~ON_MONITOR_ADD_MASK;
5164                         data |= ON_MONITOR_ADD_EN;
5165                         data |= ON_MONITOR_ADD(0x96);
5166                         if (orig != data)
5167                                 WREG32(CGTS_SM_CTRL_REG, data);
5168                 }
5169         } else {
5170                 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5171                 data |= 0x00000002;
5172                 if (orig != data)
5173                         WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5174
5175                 data = RREG32(RLC_MEM_SLP_CNTL);
5176                 if (data & RLC_MEM_LS_EN) {
5177                         data &= ~RLC_MEM_LS_EN;
5178                         WREG32(RLC_MEM_SLP_CNTL, data);
5179                 }
5180
5181                 data = RREG32(CP_MEM_SLP_CNTL);
5182                 if (data & CP_MEM_LS_EN) {
5183                         data &= ~CP_MEM_LS_EN;
5184                         WREG32(CP_MEM_SLP_CNTL, data);
5185                 }
5186
5187                 orig = data = RREG32(CGTS_SM_CTRL_REG);
5188                 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
5189                 if (orig != data)
5190                         WREG32(CGTS_SM_CTRL_REG, data);
5191
5192                 tmp = cik_halt_rlc(rdev);
5193
5194                 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5195                 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5196                 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5197                 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
5198                 WREG32(RLC_SERDES_WR_CTRL, data);
5199
5200                 cik_update_rlc(rdev, tmp);
5201         }
5202 }
5203
5204 static const u32 mc_cg_registers[] =
5205 {
5206         MC_HUB_MISC_HUB_CG,
5207         MC_HUB_MISC_SIP_CG,
5208         MC_HUB_MISC_VM_CG,
5209         MC_XPB_CLK_GAT,
5210         ATC_MISC_CG,
5211         MC_CITF_MISC_WR_CG,
5212         MC_CITF_MISC_RD_CG,
5213         MC_CITF_MISC_VM_CG,
5214         VM_L2_CG,
5215 };
5216
5217 static void cik_enable_mc_ls(struct radeon_device *rdev,
5218                              bool enable)
5219 {
5220         int i;
5221         u32 orig, data;
5222
5223         for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5224                 orig = data = RREG32(mc_cg_registers[i]);
5225                 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
5226                         data |= MC_LS_ENABLE;
5227                 else
5228                         data &= ~MC_LS_ENABLE;
5229                 if (data != orig)
5230                         WREG32(mc_cg_registers[i], data);
5231         }
5232 }
5233
5234 static void cik_enable_mc_mgcg(struct radeon_device *rdev,
5235                                bool enable)
5236 {
5237         int i;
5238         u32 orig, data;
5239
5240         for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5241                 orig = data = RREG32(mc_cg_registers[i]);
5242                 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
5243                         data |= MC_CG_ENABLE;
5244                 else
5245                         data &= ~MC_CG_ENABLE;
5246                 if (data != orig)
5247                         WREG32(mc_cg_registers[i], data);
5248         }
5249 }
5250
5251 static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
5252                                  bool enable)
5253 {
5254         u32 orig, data;
5255
5256         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
5257                 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
5258                 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
5259         } else {
5260                 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
5261                 data |= 0xff000000;
5262                 if (data != orig)
5263                         WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
5264
5265                 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
5266                 data |= 0xff000000;
5267                 if (data != orig)
5268                         WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
5269         }
5270 }
5271
5272 static void cik_enable_sdma_mgls(struct radeon_device *rdev,
5273                                  bool enable)
5274 {
5275         u32 orig, data;
5276
5277         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
5278                 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
5279                 data |= 0x100;
5280                 if (orig != data)
5281                         WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
5282
5283                 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
5284                 data |= 0x100;
5285                 if (orig != data)
5286                         WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
5287         } else {
5288                 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
5289                 data &= ~0x100;
5290                 if (orig != data)
5291                         WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
5292
5293                 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
5294                 data &= ~0x100;
5295                 if (orig != data)
5296                         WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
5297         }
5298 }
5299
5300 static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
5301                                 bool enable)
5302 {
5303         u32 orig, data;
5304
5305         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
5306                 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5307                 data = 0xfff;
5308                 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
5309
5310                 orig = data = RREG32(UVD_CGC_CTRL);
5311                 data |= DCM;
5312                 if (orig != data)
5313                         WREG32(UVD_CGC_CTRL, data);
5314         } else {
5315                 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5316                 data &= ~0xfff;
5317                 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
5318
5319                 orig = data = RREG32(UVD_CGC_CTRL);
5320                 data &= ~DCM;
5321                 if (orig != data)
5322                         WREG32(UVD_CGC_CTRL, data);
5323         }
5324 }
5325
5326 static void cik_enable_bif_mgls(struct radeon_device *rdev,
5327                                bool enable)
5328 {
5329         u32 orig, data;
5330
5331         orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
5332
5333         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5334                 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
5335                         REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
5336         else
5337                 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
5338                           REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
5339
5340         if (orig != data)
5341                 WREG32_PCIE_PORT(PCIE_CNTL2, data);
5342 }
5343
5344 static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
5345                                 bool enable)
5346 {
5347         u32 orig, data;
5348
5349         orig = data = RREG32(HDP_HOST_PATH_CNTL);
5350
5351         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
5352                 data &= ~CLOCK_GATING_DIS;
5353         else
5354                 data |= CLOCK_GATING_DIS;
5355
5356         if (orig != data)
5357                 WREG32(HDP_HOST_PATH_CNTL, data);
5358 }
5359
5360 static void cik_enable_hdp_ls(struct radeon_device *rdev,
5361                               bool enable)
5362 {
5363         u32 orig, data;
5364
5365         orig = data = RREG32(HDP_MEM_POWER_LS);
5366
5367         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
5368                 data |= HDP_LS_ENABLE;
5369         else
5370                 data &= ~HDP_LS_ENABLE;
5371
5372         if (orig != data)
5373                 WREG32(HDP_MEM_POWER_LS, data);
5374 }
5375
5376 void cik_update_cg(struct radeon_device *rdev,
5377                    u32 block, bool enable)
5378 {
5379         if (block & RADEON_CG_BLOCK_GFX) {
5380                 /* order matters! */
5381                 if (enable) {
5382                         cik_enable_mgcg(rdev, true);
5383                         cik_enable_cgcg(rdev, true);
5384                 } else {
5385                         cik_enable_cgcg(rdev, false);
5386                         cik_enable_mgcg(rdev, false);
5387                 }
5388         }
5389
5390         if (block & RADEON_CG_BLOCK_MC) {
5391                 if (!(rdev->flags & RADEON_IS_IGP)) {
5392                         cik_enable_mc_mgcg(rdev, enable);
5393                         cik_enable_mc_ls(rdev, enable);
5394                 }
5395         }
5396
5397         if (block & RADEON_CG_BLOCK_SDMA) {
5398                 cik_enable_sdma_mgcg(rdev, enable);
5399                 cik_enable_sdma_mgls(rdev, enable);
5400         }
5401
5402         if (block & RADEON_CG_BLOCK_BIF) {
5403                 cik_enable_bif_mgls(rdev, enable);
5404         }
5405
5406         if (block & RADEON_CG_BLOCK_UVD) {
5407                 if (rdev->has_uvd)
5408                         cik_enable_uvd_mgcg(rdev, enable);
5409         }
5410
5411         if (block & RADEON_CG_BLOCK_HDP) {
5412                 cik_enable_hdp_mgcg(rdev, enable);
5413                 cik_enable_hdp_ls(rdev, enable);
5414         }
5415 }
5416
5417 static void cik_init_cg(struct radeon_device *rdev)
5418 {
5419
5420         cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
5421
5422         if (rdev->has_uvd)
5423                 si_init_uvd_internal_cg(rdev);
5424
5425         cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
5426                              RADEON_CG_BLOCK_SDMA |
5427                              RADEON_CG_BLOCK_BIF |
5428                              RADEON_CG_BLOCK_UVD |
5429                              RADEON_CG_BLOCK_HDP), true);
5430 }
5431
5432 static void cik_fini_cg(struct radeon_device *rdev)
5433 {
5434         cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
5435                              RADEON_CG_BLOCK_SDMA |
5436                              RADEON_CG_BLOCK_BIF |
5437                              RADEON_CG_BLOCK_UVD |
5438                              RADEON_CG_BLOCK_HDP), false);
5439
5440         cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
5441 }
5442
5443 static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
5444                                           bool enable)
5445 {
5446         u32 data, orig;
5447
5448         orig = data = RREG32(RLC_PG_CNTL);
5449         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
5450                 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
5451         else
5452                 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
5453         if (orig != data)
5454                 WREG32(RLC_PG_CNTL, data);
5455 }
5456
5457 static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
5458                                           bool enable)
5459 {
5460         u32 data, orig;
5461
5462         orig = data = RREG32(RLC_PG_CNTL);
5463         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
5464                 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
5465         else
5466                 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
5467         if (orig != data)
5468                 WREG32(RLC_PG_CNTL, data);
5469 }
5470
5471 static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
5472 {
5473         u32 data, orig;
5474
5475         orig = data = RREG32(RLC_PG_CNTL);
5476         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
5477                 data &= ~DISABLE_CP_PG;
5478         else
5479                 data |= DISABLE_CP_PG;
5480         if (orig != data)
5481                 WREG32(RLC_PG_CNTL, data);
5482 }
5483
5484 static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
5485 {
5486         u32 data, orig;
5487
5488         orig = data = RREG32(RLC_PG_CNTL);
5489         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
5490                 data &= ~DISABLE_GDS_PG;
5491         else
5492                 data |= DISABLE_GDS_PG;
5493         if (orig != data)
5494                 WREG32(RLC_PG_CNTL, data);
5495 }
5496
5497 #define CP_ME_TABLE_SIZE    96
5498 #define CP_ME_TABLE_OFFSET  2048
5499 #define CP_MEC_TABLE_OFFSET 4096
5500
5501 void cik_init_cp_pg_table(struct radeon_device *rdev)
5502 {
5503         const __be32 *fw_data;
5504         volatile u32 *dst_ptr;
5505         int me, i, max_me = 4;
5506         u32 bo_offset = 0;
5507         u32 table_offset;
5508
5509         if (rdev->family == CHIP_KAVERI)
5510                 max_me = 5;
5511
5512         if (rdev->rlc.cp_table_ptr == NULL)
5513                 return;
5514
5515         /* write the cp table buffer */
5516         dst_ptr = rdev->rlc.cp_table_ptr;
5517         for (me = 0; me < max_me; me++) {
5518                 if (me == 0) {
5519                         fw_data = (const __be32 *)rdev->ce_fw->data;
5520                         table_offset = CP_ME_TABLE_OFFSET;
5521                 } else if (me == 1) {
5522                         fw_data = (const __be32 *)rdev->pfp_fw->data;
5523                         table_offset = CP_ME_TABLE_OFFSET;
5524                 } else if (me == 2) {
5525                         fw_data = (const __be32 *)rdev->me_fw->data;
5526                         table_offset = CP_ME_TABLE_OFFSET;
5527                 } else {
5528                         fw_data = (const __be32 *)rdev->mec_fw->data;
5529                         table_offset = CP_MEC_TABLE_OFFSET;
5530                 }
5531
5532                 for (i = 0; i < CP_ME_TABLE_SIZE; i ++) {
5533                         dst_ptr[bo_offset + i] = be32_to_cpu(fw_data[table_offset + i]);
5534                 }
5535                 bo_offset += CP_ME_TABLE_SIZE;
5536         }
5537 }
5538
5539 static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
5540                                 bool enable)
5541 {
5542         u32 data, orig;
5543
5544         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_CG)) {
5545                 orig = data = RREG32(RLC_PG_CNTL);
5546                 data |= GFX_PG_ENABLE;
5547                 if (orig != data)
5548                         WREG32(RLC_PG_CNTL, data);
5549
5550                 orig = data = RREG32(RLC_AUTO_PG_CTRL);
5551                 data |= AUTO_PG_EN;
5552                 if (orig != data)
5553                         WREG32(RLC_AUTO_PG_CTRL, data);
5554         } else {
5555                 orig = data = RREG32(RLC_PG_CNTL);
5556                 data &= ~GFX_PG_ENABLE;
5557                 if (orig != data)
5558                         WREG32(RLC_PG_CNTL, data);
5559
5560                 orig = data = RREG32(RLC_AUTO_PG_CTRL);
5561                 data &= ~AUTO_PG_EN;
5562                 if (orig != data)
5563                         WREG32(RLC_AUTO_PG_CTRL, data);
5564
5565                 data = RREG32(DB_RENDER_CONTROL);
5566         }
5567 }
5568
5569 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
5570 {
5571         u32 mask = 0, tmp, tmp1;
5572         int i;
5573
5574         cik_select_se_sh(rdev, se, sh);
5575         tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
5576         tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
5577         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5578
5579         tmp &= 0xffff0000;
5580
5581         tmp |= tmp1;
5582         tmp >>= 16;
5583
5584         for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
5585                 mask <<= 1;
5586                 mask |= 1;
5587         }
5588
5589         return (~tmp) & mask;
5590 }
5591
5592 static void cik_init_ao_cu_mask(struct radeon_device *rdev)
5593 {
5594         u32 i, j, k, active_cu_number = 0;
5595         u32 mask, counter, cu_bitmap;
5596         u32 tmp = 0;
5597
5598         for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
5599                 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
5600                         mask = 1;
5601                         cu_bitmap = 0;
5602                         counter = 0;
5603                         for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
5604                                 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
5605                                         if (counter < 2)
5606                                                 cu_bitmap |= mask;
5607                                         counter ++;
5608                                 }
5609                                 mask <<= 1;
5610                         }
5611
5612                         active_cu_number += counter;
5613                         tmp |= (cu_bitmap << (i * 16 + j * 8));
5614                 }
5615         }
5616
5617         WREG32(RLC_PG_AO_CU_MASK, tmp);
5618
5619         tmp = RREG32(RLC_MAX_PG_CU);
5620         tmp &= ~MAX_PU_CU_MASK;
5621         tmp |= MAX_PU_CU(active_cu_number);
5622         WREG32(RLC_MAX_PG_CU, tmp);
5623 }
5624
5625 static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
5626                                        bool enable)
5627 {
5628         u32 data, orig;
5629
5630         orig = data = RREG32(RLC_PG_CNTL);
5631         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
5632                 data |= STATIC_PER_CU_PG_ENABLE;
5633         else
5634                 data &= ~STATIC_PER_CU_PG_ENABLE;
5635         if (orig != data)
5636                 WREG32(RLC_PG_CNTL, data);
5637 }
5638
5639 static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
5640                                         bool enable)
5641 {
5642         u32 data, orig;
5643
5644         orig = data = RREG32(RLC_PG_CNTL);
5645         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
5646                 data |= DYN_PER_CU_PG_ENABLE;
5647         else
5648                 data &= ~DYN_PER_CU_PG_ENABLE;
5649         if (orig != data)
5650                 WREG32(RLC_PG_CNTL, data);
5651 }
5652
5653 #define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
5654 #define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET    0x3D
5655
5656 static void cik_init_gfx_cgpg(struct radeon_device *rdev)
5657 {
5658         u32 data, orig;
5659         u32 i;
5660
5661         if (rdev->rlc.cs_data) {
5662                 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
5663                 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
5664                 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
5665                 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
5666         } else {
5667                 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
5668                 for (i = 0; i < 3; i++)
5669                         WREG32(RLC_GPM_SCRATCH_DATA, 0);
5670         }
5671         if (rdev->rlc.reg_list) {
5672                 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
5673                 for (i = 0; i < rdev->rlc.reg_list_size; i++)
5674                         WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
5675         }
5676
5677         orig = data = RREG32(RLC_PG_CNTL);
5678         data |= GFX_PG_SRC;
5679         if (orig != data)
5680                 WREG32(RLC_PG_CNTL, data);
5681
5682         WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5683         WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
5684
5685         data = RREG32(CP_RB_WPTR_POLL_CNTL);
5686         data &= ~IDLE_POLL_COUNT_MASK;
5687         data |= IDLE_POLL_COUNT(0x60);
5688         WREG32(CP_RB_WPTR_POLL_CNTL, data);
5689
5690         data = 0x10101010;
5691         WREG32(RLC_PG_DELAY, data);
5692
5693         data = RREG32(RLC_PG_DELAY_2);
5694         data &= ~0xff;
5695         data |= 0x3;
5696         WREG32(RLC_PG_DELAY_2, data);
5697
5698         data = RREG32(RLC_AUTO_PG_CTRL);
5699         data &= ~GRBM_REG_SGIT_MASK;
5700         data |= GRBM_REG_SGIT(0x700);
5701         WREG32(RLC_AUTO_PG_CTRL, data);
5702
5703 }
5704
5705 static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
5706 {
5707         cik_enable_gfx_cgpg(rdev, enable);
5708         cik_enable_gfx_static_mgpg(rdev, enable);
5709         cik_enable_gfx_dynamic_mgpg(rdev, enable);
5710 }
5711
5712 u32 cik_get_csb_size(struct radeon_device *rdev)
5713 {
5714         u32 count = 0;
5715         const struct cs_section_def *sect = NULL;
5716         const struct cs_extent_def *ext = NULL;
5717
5718         if (rdev->rlc.cs_data == NULL)
5719                 return 0;
5720
5721         /* begin clear state */
5722         count += 2;
5723         /* context control state */
5724         count += 3;
5725
5726         for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5727                 for (ext = sect->section; ext->extent != NULL; ++ext) {
5728                         if (sect->id == SECT_CONTEXT)
5729                                 count += 2 + ext->reg_count;
5730                         else
5731                                 return 0;
5732                 }
5733         }
5734         /* pa_sc_raster_config/pa_sc_raster_config1 */
5735         count += 4;
5736         /* end clear state */
5737         count += 2;
5738         /* clear state */
5739         count += 2;
5740
5741         return count;
5742 }
5743
5744 void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
5745 {
5746         u32 count = 0, i;
5747         const struct cs_section_def *sect = NULL;
5748         const struct cs_extent_def *ext = NULL;
5749
5750         if (rdev->rlc.cs_data == NULL)
5751                 return;
5752         if (buffer == NULL)
5753                 return;
5754
5755         buffer[count++] = PACKET3(PACKET3_PREAMBLE_CNTL, 0);
5756         buffer[count++] = PACKET3_PREAMBLE_BEGIN_CLEAR_STATE;
5757
5758         buffer[count++] = PACKET3(PACKET3_CONTEXT_CONTROL, 1);
5759         buffer[count++] = 0x80000000;
5760         buffer[count++] = 0x80000000;
5761
5762         for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5763                 for (ext = sect->section; ext->extent != NULL; ++ext) {
5764                         if (sect->id == SECT_CONTEXT) {
5765                                 buffer[count++] = PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count);
5766                                 buffer[count++] = ext->reg_index - 0xa000;
5767                                 for (i = 0; i < ext->reg_count; i++)
5768                                         buffer[count++] = ext->extent[i];
5769                         } else {
5770                                 return;
5771                         }
5772                 }
5773         }
5774
5775         buffer[count++] = PACKET3(PACKET3_SET_CONTEXT_REG, 2);
5776         buffer[count++] = PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START;
5777         switch (rdev->family) {
5778         case CHIP_BONAIRE:
5779                 buffer[count++] = 0x16000012;
5780                 buffer[count++] = 0x00000000;
5781                 break;
5782         case CHIP_KAVERI:
5783                 buffer[count++] = 0x00000000; /* XXX */
5784                 buffer[count++] = 0x00000000;
5785                 break;
5786         case CHIP_KABINI:
5787                 buffer[count++] = 0x00000000; /* XXX */
5788                 buffer[count++] = 0x00000000;
5789                 break;
5790         default:
5791                 buffer[count++] = 0x00000000;
5792                 buffer[count++] = 0x00000000;
5793                 break;
5794         }
5795
5796         buffer[count++] = PACKET3(PACKET3_PREAMBLE_CNTL, 0);
5797         buffer[count++] = PACKET3_PREAMBLE_END_CLEAR_STATE;
5798
5799         buffer[count++] = PACKET3(PACKET3_CLEAR_STATE, 0);
5800         buffer[count++] = 0;
5801 }
5802
5803 static void cik_init_pg(struct radeon_device *rdev)
5804 {
5805         if (rdev->pg_flags) {
5806                 cik_enable_sck_slowdown_on_pu(rdev, true);
5807                 cik_enable_sck_slowdown_on_pd(rdev, true);
5808                 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_CG) {
5809                         cik_init_gfx_cgpg(rdev);
5810                         cik_enable_cp_pg(rdev, true);
5811                         cik_enable_gds_pg(rdev, true);
5812                 }
5813                 cik_init_ao_cu_mask(rdev);
5814                 cik_update_gfx_pg(rdev, true);
5815         }
5816 }
5817
5818 static void cik_fini_pg(struct radeon_device *rdev)
5819 {
5820         if (rdev->pg_flags) {
5821                 cik_update_gfx_pg(rdev, false);
5822                 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_CG) {
5823                         cik_enable_cp_pg(rdev, false);
5824                         cik_enable_gds_pg(rdev, false);
5825                 }
5826         }
5827 }
5828
5829 /*
5830  * Interrupts
5831  * Starting with r6xx, interrupts are handled via a ring buffer.
5832  * Ring buffers are areas of GPU accessible memory that the GPU
5833  * writes interrupt vectors into and the host reads vectors out of.
5834  * There is a rptr (read pointer) that determines where the
5835  * host is currently reading, and a wptr (write pointer)
5836  * which determines where the GPU has written.  When the
5837  * pointers are equal, the ring is idle.  When the GPU
5838  * writes vectors to the ring buffer, it increments the
5839  * wptr.  When there is an interrupt, the host then starts
5840  * fetching commands and processing them until the pointers are
5841  * equal again at which point it updates the rptr.
5842  */
5843
5844 /**
5845  * cik_enable_interrupts - Enable the interrupt ring buffer
5846  *
5847  * @rdev: radeon_device pointer
5848  *
5849  * Enable the interrupt ring buffer (CIK).
5850  */
5851 static void cik_enable_interrupts(struct radeon_device *rdev)
5852 {
5853         u32 ih_cntl = RREG32(IH_CNTL);
5854         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5855
5856         ih_cntl |= ENABLE_INTR;
5857         ih_rb_cntl |= IH_RB_ENABLE;
5858         WREG32(IH_CNTL, ih_cntl);
5859         WREG32(IH_RB_CNTL, ih_rb_cntl);
5860         rdev->ih.enabled = true;
5861 }
5862
5863 /**
5864  * cik_disable_interrupts - Disable the interrupt ring buffer
5865  *
5866  * @rdev: radeon_device pointer
5867  *
5868  * Disable the interrupt ring buffer (CIK).
5869  */
5870 static void cik_disable_interrupts(struct radeon_device *rdev)
5871 {
5872         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5873         u32 ih_cntl = RREG32(IH_CNTL);
5874
5875         ih_rb_cntl &= ~IH_RB_ENABLE;
5876         ih_cntl &= ~ENABLE_INTR;
5877         WREG32(IH_RB_CNTL, ih_rb_cntl);
5878         WREG32(IH_CNTL, ih_cntl);
5879         /* set rptr, wptr to 0 */
5880         WREG32(IH_RB_RPTR, 0);
5881         WREG32(IH_RB_WPTR, 0);
5882         rdev->ih.enabled = false;
5883         rdev->ih.rptr = 0;
5884 }
5885
5886 /**
5887  * cik_disable_interrupt_state - Disable all interrupt sources
5888  *
5889  * @rdev: radeon_device pointer
5890  *
5891  * Clear all interrupt enable bits used by the driver (CIK).
5892  */
5893 static void cik_disable_interrupt_state(struct radeon_device *rdev)
5894 {
5895         u32 tmp;
5896
5897         /* gfx ring */
5898         WREG32(CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5899         /* sdma */
5900         tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5901         WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5902         tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5903         WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5904         /* compute queues */
5905         WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
5906         WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
5907         WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
5908         WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
5909         WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
5910         WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
5911         WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
5912         WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
5913         /* grbm */
5914         WREG32(GRBM_INT_CNTL, 0);
5915         /* vline/vblank, etc. */
5916         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5917         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5918         if (rdev->num_crtc >= 4) {
5919                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5920                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5921         }
5922         if (rdev->num_crtc >= 6) {
5923                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5924                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5925         }
5926
5927         /* dac hotplug */
5928         WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
5929
5930         /* digital hotplug */
5931         tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5932         WREG32(DC_HPD1_INT_CONTROL, tmp);
5933         tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5934         WREG32(DC_HPD2_INT_CONTROL, tmp);
5935         tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5936         WREG32(DC_HPD3_INT_CONTROL, tmp);
5937         tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5938         WREG32(DC_HPD4_INT_CONTROL, tmp);
5939         tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5940         WREG32(DC_HPD5_INT_CONTROL, tmp);
5941         tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5942         WREG32(DC_HPD6_INT_CONTROL, tmp);
5943
5944 }
5945
5946 /**
5947  * cik_irq_init - init and enable the interrupt ring
5948  *
5949  * @rdev: radeon_device pointer
5950  *
5951  * Allocate a ring buffer for the interrupt controller,
5952  * enable the RLC, disable interrupts, enable the IH
5953  * ring buffer and enable it (CIK).
5954  * Called at device load and reume.
5955  * Returns 0 for success, errors for failure.
5956  */
5957 static int cik_irq_init(struct radeon_device *rdev)
5958 {
5959         int ret = 0;
5960         int rb_bufsz;
5961         u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
5962
5963         /* allocate ring */
5964         ret = r600_ih_ring_alloc(rdev);
5965         if (ret)
5966                 return ret;
5967
5968         /* disable irqs */
5969         cik_disable_interrupts(rdev);
5970
5971         /* init rlc */
5972         ret = cik_rlc_resume(rdev);
5973         if (ret) {
5974                 r600_ih_ring_fini(rdev);
5975                 return ret;
5976         }
5977
5978         /* setup interrupt control */
5979         /* XXX this should actually be a bus address, not an MC address. same on older asics */
5980         WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
5981         interrupt_cntl = RREG32(INTERRUPT_CNTL);
5982         /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
5983          * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
5984          */
5985         interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
5986         /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
5987         interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
5988         WREG32(INTERRUPT_CNTL, interrupt_cntl);
5989
5990         WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
5991         rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
5992
5993         ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
5994                       IH_WPTR_OVERFLOW_CLEAR |
5995                       (rb_bufsz << 1));
5996
5997         if (rdev->wb.enabled)
5998                 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
5999
6000         /* set the writeback address whether it's enabled or not */
6001         WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
6002         WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
6003
6004         WREG32(IH_RB_CNTL, ih_rb_cntl);
6005
6006         /* set rptr, wptr to 0 */
6007         WREG32(IH_RB_RPTR, 0);
6008         WREG32(IH_RB_WPTR, 0);
6009
6010         /* Default settings for IH_CNTL (disabled at first) */
6011         ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
6012         /* RPTR_REARM only works if msi's are enabled */
6013         if (rdev->msi_enabled)
6014                 ih_cntl |= RPTR_REARM;
6015         WREG32(IH_CNTL, ih_cntl);
6016
6017         /* force the active interrupt state to all disabled */
6018         cik_disable_interrupt_state(rdev);
6019
6020         pci_set_master(rdev->pdev);
6021
6022         /* enable irqs */
6023         cik_enable_interrupts(rdev);
6024
6025         return ret;
6026 }
6027
6028 /**
6029  * cik_irq_set - enable/disable interrupt sources
6030  *
6031  * @rdev: radeon_device pointer
6032  *
6033  * Enable interrupt sources on the GPU (vblanks, hpd,
6034  * etc.) (CIK).
6035  * Returns 0 for success, errors for failure.
6036  */
6037 int cik_irq_set(struct radeon_device *rdev)
6038 {
6039         u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE |
6040                 PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
6041         u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;
6042         u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
6043         u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
6044         u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
6045         u32 grbm_int_cntl = 0;
6046         u32 dma_cntl, dma_cntl1;
6047         u32 thermal_int;
6048
6049         if (!rdev->irq.installed) {
6050                 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6051                 return -EINVAL;
6052         }
6053         /* don't enable anything if the ih is disabled */
6054         if (!rdev->ih.enabled) {
6055                 cik_disable_interrupts(rdev);
6056                 /* force the active interrupt state to all disabled */
6057                 cik_disable_interrupt_state(rdev);
6058                 return 0;
6059         }
6060
6061         hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
6062         hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
6063         hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
6064         hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
6065         hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
6066         hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
6067
6068         dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6069         dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6070
6071         cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6072         cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6073         cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6074         cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6075         cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6076         cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6077         cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6078         cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6079
6080         if (rdev->flags & RADEON_IS_IGP)
6081                 thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) &
6082                         ~(THERM_INTH_MASK | THERM_INTL_MASK);
6083         else
6084                 thermal_int = RREG32_SMC(CG_THERMAL_INT) &
6085                         ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
6086
6087         /* enable CP interrupts on all rings */
6088         if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
6089                 DRM_DEBUG("cik_irq_set: sw int gfx\n");
6090                 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
6091         }
6092         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
6093                 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6094                 DRM_DEBUG("si_irq_set: sw int cp1\n");
6095                 if (ring->me == 1) {
6096                         switch (ring->pipe) {
6097                         case 0:
6098                                 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6099                                 break;
6100                         case 1:
6101                                 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6102                                 break;
6103                         case 2:
6104                                 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6105                                 break;
6106                         case 3:
6107                                 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6108                                 break;
6109                         default:
6110                                 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6111                                 break;
6112                         }
6113                 } else if (ring->me == 2) {
6114                         switch (ring->pipe) {
6115                         case 0:
6116                                 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6117                                 break;
6118                         case 1:
6119                                 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6120                                 break;
6121                         case 2:
6122                                 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6123                                 break;
6124                         case 3:
6125                                 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6126                                 break;
6127                         default:
6128                                 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6129                                 break;
6130                         }
6131                 } else {
6132                         DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
6133                 }
6134         }
6135         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
6136                 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6137                 DRM_DEBUG("si_irq_set: sw int cp2\n");
6138                 if (ring->me == 1) {
6139                         switch (ring->pipe) {
6140                         case 0:
6141                                 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6142                                 break;
6143                         case 1:
6144                                 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6145                                 break;
6146                         case 2:
6147                                 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6148                                 break;
6149                         case 3:
6150                                 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6151                                 break;
6152                         default:
6153                                 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6154                                 break;
6155                         }
6156                 } else if (ring->me == 2) {
6157                         switch (ring->pipe) {
6158                         case 0:
6159                                 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6160                                 break;
6161                         case 1:
6162                                 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6163                                 break;
6164                         case 2:
6165                                 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6166                                 break;
6167                         case 3:
6168                                 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6169                                 break;
6170                         default:
6171                                 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6172                                 break;
6173                         }
6174                 } else {
6175                         DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
6176                 }
6177         }
6178
6179         if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
6180                 DRM_DEBUG("cik_irq_set: sw int dma\n");
6181                 dma_cntl |= TRAP_ENABLE;
6182         }
6183
6184         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
6185                 DRM_DEBUG("cik_irq_set: sw int dma1\n");
6186                 dma_cntl1 |= TRAP_ENABLE;
6187         }
6188
6189         if (rdev->irq.crtc_vblank_int[0] ||
6190             atomic_read(&rdev->irq.pflip[0])) {
6191                 DRM_DEBUG("cik_irq_set: vblank 0\n");
6192                 crtc1 |= VBLANK_INTERRUPT_MASK;
6193         }
6194         if (rdev->irq.crtc_vblank_int[1] ||
6195             atomic_read(&rdev->irq.pflip[1])) {
6196                 DRM_DEBUG("cik_irq_set: vblank 1\n");
6197                 crtc2 |= VBLANK_INTERRUPT_MASK;
6198         }
6199         if (rdev->irq.crtc_vblank_int[2] ||
6200             atomic_read(&rdev->irq.pflip[2])) {
6201                 DRM_DEBUG("cik_irq_set: vblank 2\n");
6202                 crtc3 |= VBLANK_INTERRUPT_MASK;
6203         }
6204         if (rdev->irq.crtc_vblank_int[3] ||
6205             atomic_read(&rdev->irq.pflip[3])) {
6206                 DRM_DEBUG("cik_irq_set: vblank 3\n");
6207                 crtc4 |= VBLANK_INTERRUPT_MASK;
6208         }
6209         if (rdev->irq.crtc_vblank_int[4] ||
6210             atomic_read(&rdev->irq.pflip[4])) {
6211                 DRM_DEBUG("cik_irq_set: vblank 4\n");
6212                 crtc5 |= VBLANK_INTERRUPT_MASK;
6213         }
6214         if (rdev->irq.crtc_vblank_int[5] ||
6215             atomic_read(&rdev->irq.pflip[5])) {
6216                 DRM_DEBUG("cik_irq_set: vblank 5\n");
6217                 crtc6 |= VBLANK_INTERRUPT_MASK;
6218         }
6219         if (rdev->irq.hpd[0]) {
6220                 DRM_DEBUG("cik_irq_set: hpd 1\n");
6221                 hpd1 |= DC_HPDx_INT_EN;
6222         }
6223         if (rdev->irq.hpd[1]) {
6224                 DRM_DEBUG("cik_irq_set: hpd 2\n");
6225                 hpd2 |= DC_HPDx_INT_EN;
6226         }
6227         if (rdev->irq.hpd[2]) {
6228                 DRM_DEBUG("cik_irq_set: hpd 3\n");
6229                 hpd3 |= DC_HPDx_INT_EN;
6230         }
6231         if (rdev->irq.hpd[3]) {
6232                 DRM_DEBUG("cik_irq_set: hpd 4\n");
6233                 hpd4 |= DC_HPDx_INT_EN;
6234         }
6235         if (rdev->irq.hpd[4]) {
6236                 DRM_DEBUG("cik_irq_set: hpd 5\n");
6237                 hpd5 |= DC_HPDx_INT_EN;
6238         }
6239         if (rdev->irq.hpd[5]) {
6240                 DRM_DEBUG("cik_irq_set: hpd 6\n");
6241                 hpd6 |= DC_HPDx_INT_EN;
6242         }
6243
6244         if (rdev->irq.dpm_thermal) {
6245                 DRM_DEBUG("dpm thermal\n");
6246                 if (rdev->flags & RADEON_IS_IGP)
6247                         thermal_int |= THERM_INTH_MASK | THERM_INTL_MASK;
6248                 else
6249                         thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
6250         }
6251
6252         WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
6253
6254         WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
6255         WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
6256
6257         WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
6258         WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
6259         WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
6260         WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
6261         WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
6262         WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
6263         WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
6264         WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
6265
6266         WREG32(GRBM_INT_CNTL, grbm_int_cntl);
6267
6268         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
6269         WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
6270         if (rdev->num_crtc >= 4) {
6271                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
6272                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
6273         }
6274         if (rdev->num_crtc >= 6) {
6275                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
6276                 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
6277         }
6278
6279         WREG32(DC_HPD1_INT_CONTROL, hpd1);
6280         WREG32(DC_HPD2_INT_CONTROL, hpd2);
6281         WREG32(DC_HPD3_INT_CONTROL, hpd3);
6282         WREG32(DC_HPD4_INT_CONTROL, hpd4);
6283         WREG32(DC_HPD5_INT_CONTROL, hpd5);
6284         WREG32(DC_HPD6_INT_CONTROL, hpd6);
6285
6286         if (rdev->flags & RADEON_IS_IGP)
6287                 WREG32_SMC(CG_THERMAL_INT_CTRL, thermal_int);
6288         else
6289                 WREG32_SMC(CG_THERMAL_INT, thermal_int);
6290
6291         return 0;
6292 }
6293
6294 /**
6295  * cik_irq_ack - ack interrupt sources
6296  *
6297  * @rdev: radeon_device pointer
6298  *
6299  * Ack interrupt sources on the GPU (vblanks, hpd,
6300  * etc.) (CIK).  Certain interrupts sources are sw
6301  * generated and do not require an explicit ack.
6302  */
6303 static inline void cik_irq_ack(struct radeon_device *rdev)
6304 {
6305         u32 tmp;
6306
6307         rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
6308         rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
6309         rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
6310         rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
6311         rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
6312         rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
6313         rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
6314
6315         if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
6316                 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
6317         if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
6318                 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
6319         if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
6320                 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
6321         if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
6322                 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
6323
6324         if (rdev->num_crtc >= 4) {
6325                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
6326                         WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
6327                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
6328                         WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
6329                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
6330                         WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
6331                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
6332                         WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
6333         }
6334
6335         if (rdev->num_crtc >= 6) {
6336                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
6337                         WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
6338                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
6339                         WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
6340                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
6341                         WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
6342                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
6343                         WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
6344         }
6345
6346         if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
6347                 tmp = RREG32(DC_HPD1_INT_CONTROL);
6348                 tmp |= DC_HPDx_INT_ACK;
6349                 WREG32(DC_HPD1_INT_CONTROL, tmp);
6350         }
6351         if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
6352                 tmp = RREG32(DC_HPD2_INT_CONTROL);
6353                 tmp |= DC_HPDx_INT_ACK;
6354                 WREG32(DC_HPD2_INT_CONTROL, tmp);
6355         }
6356         if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6357                 tmp = RREG32(DC_HPD3_INT_CONTROL);
6358                 tmp |= DC_HPDx_INT_ACK;
6359                 WREG32(DC_HPD3_INT_CONTROL, tmp);
6360         }
6361         if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6362                 tmp = RREG32(DC_HPD4_INT_CONTROL);
6363                 tmp |= DC_HPDx_INT_ACK;
6364                 WREG32(DC_HPD4_INT_CONTROL, tmp);
6365         }
6366         if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6367                 tmp = RREG32(DC_HPD5_INT_CONTROL);
6368                 tmp |= DC_HPDx_INT_ACK;
6369                 WREG32(DC_HPD5_INT_CONTROL, tmp);
6370         }
6371         if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6372                 tmp = RREG32(DC_HPD5_INT_CONTROL);
6373                 tmp |= DC_HPDx_INT_ACK;
6374                 WREG32(DC_HPD6_INT_CONTROL, tmp);
6375         }
6376 }
6377
6378 /**
6379  * cik_irq_disable - disable interrupts
6380  *
6381  * @rdev: radeon_device pointer
6382  *
6383  * Disable interrupts on the hw (CIK).
6384  */
6385 static void cik_irq_disable(struct radeon_device *rdev)
6386 {
6387         cik_disable_interrupts(rdev);
6388         /* Wait and acknowledge irq */
6389         mdelay(1);
6390         cik_irq_ack(rdev);
6391         cik_disable_interrupt_state(rdev);
6392 }
6393
6394 /**
6395  * cik_irq_disable - disable interrupts for suspend
6396  *
6397  * @rdev: radeon_device pointer
6398  *
6399  * Disable interrupts and stop the RLC (CIK).
6400  * Used for suspend.
6401  */
6402 static void cik_irq_suspend(struct radeon_device *rdev)
6403 {
6404         cik_irq_disable(rdev);
6405         cik_rlc_stop(rdev);
6406 }
6407
6408 /**
6409  * cik_irq_fini - tear down interrupt support
6410  *
6411  * @rdev: radeon_device pointer
6412  *
6413  * Disable interrupts on the hw and free the IH ring
6414  * buffer (CIK).
6415  * Used for driver unload.
6416  */
6417 static void cik_irq_fini(struct radeon_device *rdev)
6418 {
6419         cik_irq_suspend(rdev);
6420         r600_ih_ring_fini(rdev);
6421 }
6422
6423 /**
6424  * cik_get_ih_wptr - get the IH ring buffer wptr
6425  *
6426  * @rdev: radeon_device pointer
6427  *
6428  * Get the IH ring buffer wptr from either the register
6429  * or the writeback memory buffer (CIK).  Also check for
6430  * ring buffer overflow and deal with it.
6431  * Used by cik_irq_process().
6432  * Returns the value of the wptr.
6433  */
6434 static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
6435 {
6436         u32 wptr, tmp;
6437
6438         if (rdev->wb.enabled)
6439                 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
6440         else
6441                 wptr = RREG32(IH_RB_WPTR);
6442
6443         if (wptr & RB_OVERFLOW) {
6444                 /* When a ring buffer overflow happen start parsing interrupt
6445                  * from the last not overwritten vector (wptr + 16). Hopefully
6446                  * this should allow us to catchup.
6447                  */
6448                 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
6449                         wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
6450                 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
6451                 tmp = RREG32(IH_RB_CNTL);
6452                 tmp |= IH_WPTR_OVERFLOW_CLEAR;
6453                 WREG32(IH_RB_CNTL, tmp);
6454         }
6455         return (wptr & rdev->ih.ptr_mask);
6456 }
6457
6458 /*        CIK IV Ring
6459  * Each IV ring entry is 128 bits:
6460  * [7:0]    - interrupt source id
6461  * [31:8]   - reserved
6462  * [59:32]  - interrupt source data
6463  * [63:60]  - reserved
6464  * [71:64]  - RINGID
6465  *            CP:
6466  *            ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
6467  *            QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
6468  *                     - for gfx, hw shader state (0=PS...5=LS, 6=CS)
6469  *            ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
6470  *            PIPE_ID - ME0 0=3D
6471  *                    - ME1&2 compute dispatcher (4 pipes each)
6472  *            SDMA:
6473  *            INSTANCE_ID [1:0], QUEUE_ID[1:0]
6474  *            INSTANCE_ID - 0 = sdma0, 1 = sdma1
6475  *            QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
6476  * [79:72]  - VMID
6477  * [95:80]  - PASID
6478  * [127:96] - reserved
6479  */
6480 /**
6481  * cik_irq_process - interrupt handler
6482  *
6483  * @rdev: radeon_device pointer
6484  *
6485  * Interrupt hander (CIK).  Walk the IH ring,
6486  * ack interrupts and schedule work to handle
6487  * interrupt events.
6488  * Returns irq process return code.
6489  */
6490 int cik_irq_process(struct radeon_device *rdev)
6491 {
6492         struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6493         struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6494         u32 wptr;
6495         u32 rptr;
6496         u32 src_id, src_data, ring_id;
6497         u8 me_id, pipe_id, queue_id;
6498         u32 ring_index;
6499         bool queue_hotplug = false;
6500         bool queue_reset = false;
6501         u32 addr, status, mc_client;
6502         bool queue_thermal = false;
6503
6504         if (!rdev->ih.enabled || rdev->shutdown)
6505                 return IRQ_NONE;
6506
6507         wptr = cik_get_ih_wptr(rdev);
6508
6509 restart_ih:
6510         /* is somebody else already processing irqs? */
6511         if (atomic_xchg(&rdev->ih.lock, 1))
6512                 return IRQ_NONE;
6513
6514         rptr = rdev->ih.rptr;
6515         DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
6516
6517         /* Order reading of wptr vs. reading of IH ring data */
6518         rmb();
6519
6520         /* display interrupts */
6521         cik_irq_ack(rdev);
6522
6523         while (rptr != wptr) {
6524                 /* wptr/rptr are in bytes! */
6525                 ring_index = rptr / 4;
6526                 src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
6527                 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
6528                 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
6529
6530                 switch (src_id) {
6531                 case 1: /* D1 vblank/vline */
6532                         switch (src_data) {
6533                         case 0: /* D1 vblank */
6534                                 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
6535                                         if (rdev->irq.crtc_vblank_int[0]) {
6536                                                 drm_handle_vblank(rdev->ddev, 0);
6537                                                 rdev->pm.vblank_sync = true;
6538                                                 wake_up(&rdev->irq.vblank_queue);
6539                                         }
6540                                         if (atomic_read(&rdev->irq.pflip[0]))
6541                                                 radeon_crtc_handle_flip(rdev, 0);
6542                                         rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
6543                                         DRM_DEBUG("IH: D1 vblank\n");
6544                                 }
6545                                 break;
6546                         case 1: /* D1 vline */
6547                                 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
6548                                         rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
6549                                         DRM_DEBUG("IH: D1 vline\n");
6550                                 }
6551                                 break;
6552                         default:
6553                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6554                                 break;
6555                         }
6556                         break;
6557                 case 2: /* D2 vblank/vline */
6558                         switch (src_data) {
6559                         case 0: /* D2 vblank */
6560                                 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
6561                                         if (rdev->irq.crtc_vblank_int[1]) {
6562                                                 drm_handle_vblank(rdev->ddev, 1);
6563                                                 rdev->pm.vblank_sync = true;
6564                                                 wake_up(&rdev->irq.vblank_queue);
6565                                         }
6566                                         if (atomic_read(&rdev->irq.pflip[1]))
6567                                                 radeon_crtc_handle_flip(rdev, 1);
6568                                         rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
6569                                         DRM_DEBUG("IH: D2 vblank\n");
6570                                 }
6571                                 break;
6572                         case 1: /* D2 vline */
6573                                 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
6574                                         rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
6575                                         DRM_DEBUG("IH: D2 vline\n");
6576                                 }
6577                                 break;
6578                         default:
6579                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6580                                 break;
6581                         }
6582                         break;
6583                 case 3: /* D3 vblank/vline */
6584                         switch (src_data) {
6585                         case 0: /* D3 vblank */
6586                                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
6587                                         if (rdev->irq.crtc_vblank_int[2]) {
6588                                                 drm_handle_vblank(rdev->ddev, 2);
6589                                                 rdev->pm.vblank_sync = true;
6590                                                 wake_up(&rdev->irq.vblank_queue);
6591                                         }
6592                                         if (atomic_read(&rdev->irq.pflip[2]))
6593                                                 radeon_crtc_handle_flip(rdev, 2);
6594                                         rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
6595                                         DRM_DEBUG("IH: D3 vblank\n");
6596                                 }
6597                                 break;
6598                         case 1: /* D3 vline */
6599                                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
6600                                         rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
6601                                         DRM_DEBUG("IH: D3 vline\n");
6602                                 }
6603                                 break;
6604                         default:
6605                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6606                                 break;
6607                         }
6608                         break;
6609                 case 4: /* D4 vblank/vline */
6610                         switch (src_data) {
6611                         case 0: /* D4 vblank */
6612                                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
6613                                         if (rdev->irq.crtc_vblank_int[3]) {
6614                                                 drm_handle_vblank(rdev->ddev, 3);
6615                                                 rdev->pm.vblank_sync = true;
6616                                                 wake_up(&rdev->irq.vblank_queue);
6617                                         }
6618                                         if (atomic_read(&rdev->irq.pflip[3]))
6619                                                 radeon_crtc_handle_flip(rdev, 3);
6620                                         rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
6621                                         DRM_DEBUG("IH: D4 vblank\n");
6622                                 }
6623                                 break;
6624                         case 1: /* D4 vline */
6625                                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
6626                                         rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
6627                                         DRM_DEBUG("IH: D4 vline\n");
6628                                 }
6629                                 break;
6630                         default:
6631                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6632                                 break;
6633                         }
6634                         break;
6635                 case 5: /* D5 vblank/vline */
6636                         switch (src_data) {
6637                         case 0: /* D5 vblank */
6638                                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
6639                                         if (rdev->irq.crtc_vblank_int[4]) {
6640                                                 drm_handle_vblank(rdev->ddev, 4);
6641                                                 rdev->pm.vblank_sync = true;
6642                                                 wake_up(&rdev->irq.vblank_queue);
6643                                         }
6644                                         if (atomic_read(&rdev->irq.pflip[4]))
6645                                                 radeon_crtc_handle_flip(rdev, 4);
6646                                         rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
6647                                         DRM_DEBUG("IH: D5 vblank\n");
6648                                 }
6649                                 break;
6650                         case 1: /* D5 vline */
6651                                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
6652                                         rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
6653                                         DRM_DEBUG("IH: D5 vline\n");
6654                                 }
6655                                 break;
6656                         default:
6657                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6658                                 break;
6659                         }
6660                         break;
6661                 case 6: /* D6 vblank/vline */
6662                         switch (src_data) {
6663                         case 0: /* D6 vblank */
6664                                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
6665                                         if (rdev->irq.crtc_vblank_int[5]) {
6666                                                 drm_handle_vblank(rdev->ddev, 5);
6667                                                 rdev->pm.vblank_sync = true;
6668                                                 wake_up(&rdev->irq.vblank_queue);
6669                                         }
6670                                         if (atomic_read(&rdev->irq.pflip[5]))
6671                                                 radeon_crtc_handle_flip(rdev, 5);
6672                                         rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
6673                                         DRM_DEBUG("IH: D6 vblank\n");
6674                                 }
6675                                 break;
6676                         case 1: /* D6 vline */
6677                                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
6678                                         rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
6679                                         DRM_DEBUG("IH: D6 vline\n");
6680                                 }
6681                                 break;
6682                         default:
6683                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6684                                 break;
6685                         }
6686                         break;
6687                 case 42: /* HPD hotplug */
6688                         switch (src_data) {
6689                         case 0:
6690                                 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
6691                                         rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
6692                                         queue_hotplug = true;
6693                                         DRM_DEBUG("IH: HPD1\n");
6694                                 }
6695                                 break;
6696                         case 1:
6697                                 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
6698                                         rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
6699                                         queue_hotplug = true;
6700                                         DRM_DEBUG("IH: HPD2\n");
6701                                 }
6702                                 break;
6703                         case 2:
6704                                 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6705                                         rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
6706                                         queue_hotplug = true;
6707                                         DRM_DEBUG("IH: HPD3\n");
6708                                 }
6709                                 break;
6710                         case 3:
6711                                 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6712                                         rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
6713                                         queue_hotplug = true;
6714                                         DRM_DEBUG("IH: HPD4\n");
6715                                 }
6716                                 break;
6717                         case 4:
6718                                 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6719                                         rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
6720                                         queue_hotplug = true;
6721                                         DRM_DEBUG("IH: HPD5\n");
6722                                 }
6723                                 break;
6724                         case 5:
6725                                 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6726                                         rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
6727                                         queue_hotplug = true;
6728                                         DRM_DEBUG("IH: HPD6\n");
6729                                 }
6730                                 break;
6731                         default:
6732                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6733                                 break;
6734                         }
6735                         break;
6736                 case 124: /* UVD */
6737                         DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
6738                         radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
6739                         break;
6740                 case 146:
6741                 case 147:
6742                         addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
6743                         status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
6744                         mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
6745                         dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
6746                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
6747                                 addr);
6748                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
6749                                 status);
6750                         cik_vm_decode_fault(rdev, status, addr, mc_client);
6751                         /* reset addr and status */
6752                         WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
6753                         break;
6754                 case 176: /* GFX RB CP_INT */
6755                 case 177: /* GFX IB CP_INT */
6756                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6757                         break;
6758                 case 181: /* CP EOP event */
6759                         DRM_DEBUG("IH: CP EOP\n");
6760                         /* XXX check the bitfield order! */
6761                         me_id = (ring_id & 0x60) >> 5;
6762                         pipe_id = (ring_id & 0x18) >> 3;
6763                         queue_id = (ring_id & 0x7) >> 0;
6764                         switch (me_id) {
6765                         case 0:
6766                                 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6767                                 break;
6768                         case 1:
6769                         case 2:
6770                                 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
6771                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6772                                 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
6773                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6774                                 break;
6775                         }
6776                         break;
6777                 case 184: /* CP Privileged reg access */
6778                         DRM_ERROR("Illegal register access in command stream\n");
6779                         /* XXX check the bitfield order! */
6780                         me_id = (ring_id & 0x60) >> 5;
6781                         pipe_id = (ring_id & 0x18) >> 3;
6782                         queue_id = (ring_id & 0x7) >> 0;
6783                         switch (me_id) {
6784                         case 0:
6785                                 /* This results in a full GPU reset, but all we need to do is soft
6786                                  * reset the CP for gfx
6787                                  */
6788                                 queue_reset = true;
6789                                 break;
6790                         case 1:
6791                                 /* XXX compute */
6792                                 queue_reset = true;
6793                                 break;
6794                         case 2:
6795                                 /* XXX compute */
6796                                 queue_reset = true;
6797                                 break;
6798                         }
6799                         break;
6800                 case 185: /* CP Privileged inst */
6801                         DRM_ERROR("Illegal instruction in command stream\n");
6802                         /* XXX check the bitfield order! */
6803                         me_id = (ring_id & 0x60) >> 5;
6804                         pipe_id = (ring_id & 0x18) >> 3;
6805                         queue_id = (ring_id & 0x7) >> 0;
6806                         switch (me_id) {
6807                         case 0:
6808                                 /* This results in a full GPU reset, but all we need to do is soft
6809                                  * reset the CP for gfx
6810                                  */
6811                                 queue_reset = true;
6812                                 break;
6813                         case 1:
6814                                 /* XXX compute */
6815                                 queue_reset = true;
6816                                 break;
6817                         case 2:
6818                                 /* XXX compute */
6819                                 queue_reset = true;
6820                                 break;
6821                         }
6822                         break;
6823                 case 224: /* SDMA trap event */
6824                         /* XXX check the bitfield order! */
6825                         me_id = (ring_id & 0x3) >> 0;
6826                         queue_id = (ring_id & 0xc) >> 2;
6827                         DRM_DEBUG("IH: SDMA trap\n");
6828                         switch (me_id) {
6829                         case 0:
6830                                 switch (queue_id) {
6831                                 case 0:
6832                                         radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
6833                                         break;
6834                                 case 1:
6835                                         /* XXX compute */
6836                                         break;
6837                                 case 2:
6838                                         /* XXX compute */
6839                                         break;
6840                                 }
6841                                 break;
6842                         case 1:
6843                                 switch (queue_id) {
6844                                 case 0:
6845                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6846                                         break;
6847                                 case 1:
6848                                         /* XXX compute */
6849                                         break;
6850                                 case 2:
6851                                         /* XXX compute */
6852                                         break;
6853                                 }
6854                                 break;
6855                         }
6856                         break;
6857                 case 230: /* thermal low to high */
6858                         DRM_DEBUG("IH: thermal low to high\n");
6859                         rdev->pm.dpm.thermal.high_to_low = false;
6860                         queue_thermal = true;
6861                         break;
6862                 case 231: /* thermal high to low */
6863                         DRM_DEBUG("IH: thermal high to low\n");
6864                         rdev->pm.dpm.thermal.high_to_low = true;
6865                         queue_thermal = true;
6866                         break;
6867                 case 233: /* GUI IDLE */
6868                         DRM_DEBUG("IH: GUI idle\n");
6869                         break;
6870                 case 241: /* SDMA Privileged inst */
6871                 case 247: /* SDMA Privileged inst */
6872                         DRM_ERROR("Illegal instruction in SDMA command stream\n");
6873                         /* XXX check the bitfield order! */
6874                         me_id = (ring_id & 0x3) >> 0;
6875                         queue_id = (ring_id & 0xc) >> 2;
6876                         switch (me_id) {
6877                         case 0:
6878                                 switch (queue_id) {
6879                                 case 0:
6880                                         queue_reset = true;
6881                                         break;
6882                                 case 1:
6883                                         /* XXX compute */
6884                                         queue_reset = true;
6885                                         break;
6886                                 case 2:
6887                                         /* XXX compute */
6888                                         queue_reset = true;
6889                                         break;
6890                                 }
6891                                 break;
6892                         case 1:
6893                                 switch (queue_id) {
6894                                 case 0:
6895                                         queue_reset = true;
6896                                         break;
6897                                 case 1:
6898                                         /* XXX compute */
6899                                         queue_reset = true;
6900                                         break;
6901                                 case 2:
6902                                         /* XXX compute */
6903                                         queue_reset = true;
6904                                         break;
6905                                 }
6906                                 break;
6907                         }
6908                         break;
6909                 default:
6910                         DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6911                         break;
6912                 }
6913
6914                 /* wptr/rptr are in bytes! */
6915                 rptr += 16;
6916                 rptr &= rdev->ih.ptr_mask;
6917         }
6918         if (queue_hotplug)
6919                 schedule_work(&rdev->hotplug_work);
6920         if (queue_reset)
6921                 schedule_work(&rdev->reset_work);
6922         if (queue_thermal)
6923                 schedule_work(&rdev->pm.dpm.thermal.work);
6924         rdev->ih.rptr = rptr;
6925         WREG32(IH_RB_RPTR, rdev->ih.rptr);
6926         atomic_set(&rdev->ih.lock, 0);
6927
6928         /* make sure wptr hasn't changed while processing */
6929         wptr = cik_get_ih_wptr(rdev);
6930         if (wptr != rptr)
6931                 goto restart_ih;
6932
6933         return IRQ_HANDLED;
6934 }
6935
6936 /*
6937  * startup/shutdown callbacks
6938  */
6939 /**
6940  * cik_startup - program the asic to a functional state
6941  *
6942  * @rdev: radeon_device pointer
6943  *
6944  * Programs the asic to a functional state (CIK).
6945  * Called by cik_init() and cik_resume().
6946  * Returns 0 for success, error for failure.
6947  */
6948 static int cik_startup(struct radeon_device *rdev)
6949 {
6950         struct radeon_ring *ring;
6951         int r;
6952
6953         /* enable pcie gen2/3 link */
6954         cik_pcie_gen3_enable(rdev);
6955         /* enable aspm */
6956         cik_program_aspm(rdev);
6957
6958         /* scratch needs to be initialized before MC */
6959         r = r600_vram_scratch_init(rdev);
6960         if (r)
6961                 return r;
6962
6963         cik_mc_program(rdev);
6964
6965         if (rdev->flags & RADEON_IS_IGP) {
6966                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
6967                     !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
6968                         r = cik_init_microcode(rdev);
6969                         if (r) {
6970                                 DRM_ERROR("Failed to load firmware!\n");
6971                                 return r;
6972                         }
6973                 }
6974         } else {
6975                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
6976                     !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
6977                     !rdev->mc_fw) {
6978                         r = cik_init_microcode(rdev);
6979                         if (r) {
6980                                 DRM_ERROR("Failed to load firmware!\n");
6981                                 return r;
6982                         }
6983                 }
6984
6985                 r = ci_mc_load_microcode(rdev);
6986                 if (r) {
6987                         DRM_ERROR("Failed to load MC firmware!\n");
6988                         return r;
6989                 }
6990         }
6991
6992         r = cik_pcie_gart_enable(rdev);
6993         if (r)
6994                 return r;
6995         cik_gpu_init(rdev);
6996
6997         /* allocate rlc buffers */
6998         if (rdev->flags & RADEON_IS_IGP) {
6999                 if (rdev->family == CHIP_KAVERI) {
7000                         rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
7001                         rdev->rlc.reg_list_size =
7002                                 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
7003                 } else {
7004                         rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
7005                         rdev->rlc.reg_list_size =
7006                                 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
7007                 }
7008         }
7009         rdev->rlc.cs_data = ci_cs_data;
7010         rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
7011         r = sumo_rlc_init(rdev);
7012         if (r) {
7013                 DRM_ERROR("Failed to init rlc BOs!\n");
7014                 return r;
7015         }
7016
7017         /* allocate wb buffer */
7018         r = radeon_wb_init(rdev);
7019         if (r)
7020                 return r;
7021
7022         /* allocate mec buffers */
7023         r = cik_mec_init(rdev);
7024         if (r) {
7025                 DRM_ERROR("Failed to init MEC BOs!\n");
7026                 return r;
7027         }
7028
7029         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
7030         if (r) {
7031                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7032                 return r;
7033         }
7034
7035         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7036         if (r) {
7037                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7038                 return r;
7039         }
7040
7041         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7042         if (r) {
7043                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7044                 return r;
7045         }
7046
7047         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
7048         if (r) {
7049                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7050                 return r;
7051         }
7052
7053         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7054         if (r) {
7055                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7056                 return r;
7057         }
7058
7059         r = radeon_uvd_resume(rdev);
7060         if (!r) {
7061                 r = uvd_v4_2_resume(rdev);
7062                 if (!r) {
7063                         r = radeon_fence_driver_start_ring(rdev,
7064                                                            R600_RING_TYPE_UVD_INDEX);
7065                         if (r)
7066                                 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
7067                 }
7068         }
7069         if (r)
7070                 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
7071
7072         /* Enable IRQ */
7073         if (!rdev->irq.installed) {
7074                 r = radeon_irq_kms_init(rdev);
7075                 if (r)
7076                         return r;
7077         }
7078
7079         r = cik_irq_init(rdev);
7080         if (r) {
7081                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
7082                 radeon_irq_kms_fini(rdev);
7083                 return r;
7084         }
7085         cik_irq_set(rdev);
7086
7087         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7088         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
7089                              CP_RB0_RPTR, CP_RB0_WPTR,
7090                              RADEON_CP_PACKET2);
7091         if (r)
7092                 return r;
7093
7094         /* set up the compute queues */
7095         /* type-2 packets are deprecated on MEC, use type-3 instead */
7096         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7097         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
7098                              CP_HQD_PQ_RPTR, CP_HQD_PQ_WPTR,
7099                              PACKET3(PACKET3_NOP, 0x3FFF));
7100         if (r)
7101                 return r;
7102         ring->me = 1; /* first MEC */
7103         ring->pipe = 0; /* first pipe */
7104         ring->queue = 0; /* first queue */
7105         ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
7106
7107         /* type-2 packets are deprecated on MEC, use type-3 instead */
7108         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7109         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
7110                              CP_HQD_PQ_RPTR, CP_HQD_PQ_WPTR,
7111                              PACKET3(PACKET3_NOP, 0x3FFF));
7112         if (r)
7113                 return r;
7114         /* dGPU only have 1 MEC */
7115         ring->me = 1; /* first MEC */
7116         ring->pipe = 0; /* first pipe */
7117         ring->queue = 1; /* second queue */
7118         ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
7119
7120         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7121         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
7122                              SDMA0_GFX_RB_RPTR + SDMA0_REGISTER_OFFSET,
7123                              SDMA0_GFX_RB_WPTR + SDMA0_REGISTER_OFFSET,
7124                              SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7125         if (r)
7126                 return r;
7127
7128         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7129         r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
7130                              SDMA0_GFX_RB_RPTR + SDMA1_REGISTER_OFFSET,
7131                              SDMA0_GFX_RB_WPTR + SDMA1_REGISTER_OFFSET,
7132                              SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7133         if (r)
7134                 return r;
7135
7136         r = cik_cp_resume(rdev);
7137         if (r)
7138                 return r;
7139
7140         r = cik_sdma_resume(rdev);
7141         if (r)
7142                 return r;
7143
7144         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7145         if (ring->ring_size) {
7146                 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
7147                                      UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
7148                                      RADEON_CP_PACKET2);
7149                 if (!r)
7150                         r = uvd_v1_0_init(rdev);
7151                 if (r)
7152                         DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
7153         }
7154
7155         r = radeon_ib_pool_init(rdev);
7156         if (r) {
7157                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
7158                 return r;
7159         }
7160
7161         r = radeon_vm_manager_init(rdev);
7162         if (r) {
7163                 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
7164                 return r;
7165         }
7166
7167         r = dce6_audio_init(rdev);
7168         if (r)
7169                 return r;
7170
7171         return 0;
7172 }
7173
7174 /**
7175  * cik_resume - resume the asic to a functional state
7176  *
7177  * @rdev: radeon_device pointer
7178  *
7179  * Programs the asic to a functional state (CIK).
7180  * Called at resume.
7181  * Returns 0 for success, error for failure.
7182  */
7183 int cik_resume(struct radeon_device *rdev)
7184 {
7185         int r;
7186
7187         /* post card */
7188         atom_asic_init(rdev->mode_info.atom_context);
7189
7190         /* init golden registers */
7191         cik_init_golden_registers(rdev);
7192
7193         rdev->accel_working = true;
7194         r = cik_startup(rdev);
7195         if (r) {
7196                 DRM_ERROR("cik startup failed on resume\n");
7197                 rdev->accel_working = false;
7198                 return r;
7199         }
7200
7201         return r;
7202
7203 }
7204
7205 /**
7206  * cik_suspend - suspend the asic
7207  *
7208  * @rdev: radeon_device pointer
7209  *
7210  * Bring the chip into a state suitable for suspend (CIK).
7211  * Called at suspend.
7212  * Returns 0 for success.
7213  */
7214 int cik_suspend(struct radeon_device *rdev)
7215 {
7216         dce6_audio_fini(rdev);
7217         radeon_vm_manager_fini(rdev);
7218         cik_cp_enable(rdev, false);
7219         cik_sdma_enable(rdev, false);
7220         uvd_v1_0_fini(rdev);
7221         radeon_uvd_suspend(rdev);
7222         cik_fini_pg(rdev);
7223         cik_fini_cg(rdev);
7224         cik_irq_suspend(rdev);
7225         radeon_wb_disable(rdev);
7226         cik_pcie_gart_disable(rdev);
7227         return 0;
7228 }
7229
7230 /* Plan is to move initialization in that function and use
7231  * helper function so that radeon_device_init pretty much
7232  * do nothing more than calling asic specific function. This
7233  * should also allow to remove a bunch of callback function
7234  * like vram_info.
7235  */
7236 /**
7237  * cik_init - asic specific driver and hw init
7238  *
7239  * @rdev: radeon_device pointer
7240  *
7241  * Setup asic specific driver variables and program the hw
7242  * to a functional state (CIK).
7243  * Called at driver startup.
7244  * Returns 0 for success, errors for failure.
7245  */
7246 int cik_init(struct radeon_device *rdev)
7247 {
7248         struct radeon_ring *ring;
7249         int r;
7250
7251         /* Read BIOS */
7252         if (!radeon_get_bios(rdev)) {
7253                 if (ASIC_IS_AVIVO(rdev))
7254                         return -EINVAL;
7255         }
7256         /* Must be an ATOMBIOS */
7257         if (!rdev->is_atom_bios) {
7258                 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
7259                 return -EINVAL;
7260         }
7261         r = radeon_atombios_init(rdev);
7262         if (r)
7263                 return r;
7264
7265         /* Post card if necessary */
7266         if (!radeon_card_posted(rdev)) {
7267                 if (!rdev->bios) {
7268                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
7269                         return -EINVAL;
7270                 }
7271                 DRM_INFO("GPU not posted. posting now...\n");
7272                 atom_asic_init(rdev->mode_info.atom_context);
7273         }
7274         /* init golden registers */
7275         cik_init_golden_registers(rdev);
7276         /* Initialize scratch registers */
7277         cik_scratch_init(rdev);
7278         /* Initialize surface registers */
7279         radeon_surface_init(rdev);
7280         /* Initialize clocks */
7281         radeon_get_clock_info(rdev->ddev);
7282
7283         /* Fence driver */
7284         r = radeon_fence_driver_init(rdev);
7285         if (r)
7286                 return r;
7287
7288         /* initialize memory controller */
7289         r = cik_mc_init(rdev);
7290         if (r)
7291                 return r;
7292         /* Memory manager */
7293         r = radeon_bo_init(rdev);
7294         if (r)
7295                 return r;
7296
7297         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7298         ring->ring_obj = NULL;
7299         r600_ring_init(rdev, ring, 1024 * 1024);
7300
7301         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7302         ring->ring_obj = NULL;
7303         r600_ring_init(rdev, ring, 1024 * 1024);
7304         r = radeon_doorbell_get(rdev, &ring->doorbell_page_num);
7305         if (r)
7306                 return r;
7307
7308         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7309         ring->ring_obj = NULL;
7310         r600_ring_init(rdev, ring, 1024 * 1024);
7311         r = radeon_doorbell_get(rdev, &ring->doorbell_page_num);
7312         if (r)
7313                 return r;
7314
7315         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7316         ring->ring_obj = NULL;
7317         r600_ring_init(rdev, ring, 256 * 1024);
7318
7319         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7320         ring->ring_obj = NULL;
7321         r600_ring_init(rdev, ring, 256 * 1024);
7322
7323         r = radeon_uvd_init(rdev);
7324         if (!r) {
7325                 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7326                 ring->ring_obj = NULL;
7327                 r600_ring_init(rdev, ring, 4096);
7328         }
7329
7330         rdev->ih.ring_obj = NULL;
7331         r600_ih_ring_init(rdev, 64 * 1024);
7332
7333         r = r600_pcie_gart_init(rdev);
7334         if (r)
7335                 return r;
7336
7337         rdev->accel_working = true;
7338         r = cik_startup(rdev);
7339         if (r) {
7340                 dev_err(rdev->dev, "disabling GPU acceleration\n");
7341                 cik_cp_fini(rdev);
7342                 cik_sdma_fini(rdev);
7343                 cik_irq_fini(rdev);
7344                 sumo_rlc_fini(rdev);
7345                 cik_mec_fini(rdev);
7346                 radeon_wb_fini(rdev);
7347                 radeon_ib_pool_fini(rdev);
7348                 radeon_vm_manager_fini(rdev);
7349                 radeon_irq_kms_fini(rdev);
7350                 cik_pcie_gart_fini(rdev);
7351                 rdev->accel_working = false;
7352         }
7353
7354         /* Don't start up if the MC ucode is missing.
7355          * The default clocks and voltages before the MC ucode
7356          * is loaded are not suffient for advanced operations.
7357          */
7358         if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
7359                 DRM_ERROR("radeon: MC ucode required for NI+.\n");
7360                 return -EINVAL;
7361         }
7362
7363         return 0;
7364 }
7365
7366 /**
7367  * cik_fini - asic specific driver and hw fini
7368  *
7369  * @rdev: radeon_device pointer
7370  *
7371  * Tear down the asic specific driver variables and program the hw
7372  * to an idle state (CIK).
7373  * Called at driver unload.
7374  */
7375 void cik_fini(struct radeon_device *rdev)
7376 {
7377         cik_cp_fini(rdev);
7378         cik_sdma_fini(rdev);
7379         cik_fini_pg(rdev);
7380         cik_fini_cg(rdev);
7381         cik_irq_fini(rdev);
7382         sumo_rlc_fini(rdev);
7383         cik_mec_fini(rdev);
7384         radeon_wb_fini(rdev);
7385         radeon_vm_manager_fini(rdev);
7386         radeon_ib_pool_fini(rdev);
7387         radeon_irq_kms_fini(rdev);
7388         uvd_v1_0_fini(rdev);
7389         radeon_uvd_fini(rdev);
7390         cik_pcie_gart_fini(rdev);
7391         r600_vram_scratch_fini(rdev);
7392         radeon_gem_fini(rdev);
7393         radeon_fence_driver_fini(rdev);
7394         radeon_bo_fini(rdev);
7395         radeon_atombios_fini(rdev);
7396         kfree(rdev->bios);
7397         rdev->bios = NULL;
7398 }
7399
7400 /* display watermark setup */
7401 /**
7402  * dce8_line_buffer_adjust - Set up the line buffer
7403  *
7404  * @rdev: radeon_device pointer
7405  * @radeon_crtc: the selected display controller
7406  * @mode: the current display mode on the selected display
7407  * controller
7408  *
7409  * Setup up the line buffer allocation for
7410  * the selected display controller (CIK).
7411  * Returns the line buffer size in pixels.
7412  */
7413 static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
7414                                    struct radeon_crtc *radeon_crtc,
7415                                    struct drm_display_mode *mode)
7416 {
7417         u32 tmp, buffer_alloc, i;
7418         u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
7419         /*
7420          * Line Buffer Setup
7421          * There are 6 line buffers, one for each display controllers.
7422          * There are 3 partitions per LB. Select the number of partitions
7423          * to enable based on the display width.  For display widths larger
7424          * than 4096, you need use to use 2 display controllers and combine
7425          * them using the stereo blender.
7426          */
7427         if (radeon_crtc->base.enabled && mode) {
7428                 if (mode->crtc_hdisplay < 1920) {
7429                         tmp = 1;
7430                         buffer_alloc = 2;
7431                 } else if (mode->crtc_hdisplay < 2560) {
7432                         tmp = 2;
7433                         buffer_alloc = 2;
7434                 } else if (mode->crtc_hdisplay < 4096) {
7435                         tmp = 0;
7436                         buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
7437                 } else {
7438                         DRM_DEBUG_KMS("Mode too big for LB!\n");
7439                         tmp = 0;
7440                         buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
7441                 }
7442         } else {
7443                 tmp = 1;
7444                 buffer_alloc = 0;
7445         }
7446
7447         WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
7448                LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
7449
7450         WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
7451                DMIF_BUFFERS_ALLOCATED(buffer_alloc));
7452         for (i = 0; i < rdev->usec_timeout; i++) {
7453                 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
7454                     DMIF_BUFFERS_ALLOCATED_COMPLETED)
7455                         break;
7456                 udelay(1);
7457         }
7458
7459         if (radeon_crtc->base.enabled && mode) {
7460                 switch (tmp) {
7461                 case 0:
7462                 default:
7463                         return 4096 * 2;
7464                 case 1:
7465                         return 1920 * 2;
7466                 case 2:
7467                         return 2560 * 2;
7468                 }
7469         }
7470
7471         /* controller not enabled, so no lb used */
7472         return 0;
7473 }
7474
7475 /**
7476  * cik_get_number_of_dram_channels - get the number of dram channels
7477  *
7478  * @rdev: radeon_device pointer
7479  *
7480  * Look up the number of video ram channels (CIK).
7481  * Used for display watermark bandwidth calculations
7482  * Returns the number of dram channels
7483  */
7484 static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
7485 {
7486         u32 tmp = RREG32(MC_SHARED_CHMAP);
7487
7488         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
7489         case 0:
7490         default:
7491                 return 1;
7492         case 1:
7493                 return 2;
7494         case 2:
7495                 return 4;
7496         case 3:
7497                 return 8;
7498         case 4:
7499                 return 3;
7500         case 5:
7501                 return 6;
7502         case 6:
7503                 return 10;
7504         case 7:
7505                 return 12;
7506         case 8:
7507                 return 16;
7508         }
7509 }
7510
7511 struct dce8_wm_params {
7512         u32 dram_channels; /* number of dram channels */
7513         u32 yclk;          /* bandwidth per dram data pin in kHz */
7514         u32 sclk;          /* engine clock in kHz */
7515         u32 disp_clk;      /* display clock in kHz */
7516         u32 src_width;     /* viewport width */
7517         u32 active_time;   /* active display time in ns */
7518         u32 blank_time;    /* blank time in ns */
7519         bool interlaced;    /* mode is interlaced */
7520         fixed20_12 vsc;    /* vertical scale ratio */
7521         u32 num_heads;     /* number of active crtcs */
7522         u32 bytes_per_pixel; /* bytes per pixel display + overlay */
7523         u32 lb_size;       /* line buffer allocated to pipe */
7524         u32 vtaps;         /* vertical scaler taps */
7525 };
7526
7527 /**
7528  * dce8_dram_bandwidth - get the dram bandwidth
7529  *
7530  * @wm: watermark calculation data
7531  *
7532  * Calculate the raw dram bandwidth (CIK).
7533  * Used for display watermark bandwidth calculations
7534  * Returns the dram bandwidth in MBytes/s
7535  */
7536 static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
7537 {
7538         /* Calculate raw DRAM Bandwidth */
7539         fixed20_12 dram_efficiency; /* 0.7 */
7540         fixed20_12 yclk, dram_channels, bandwidth;
7541         fixed20_12 a;
7542
7543         a.full = dfixed_const(1000);
7544         yclk.full = dfixed_const(wm->yclk);
7545         yclk.full = dfixed_div(yclk, a);
7546         dram_channels.full = dfixed_const(wm->dram_channels * 4);
7547         a.full = dfixed_const(10);
7548         dram_efficiency.full = dfixed_const(7);
7549         dram_efficiency.full = dfixed_div(dram_efficiency, a);
7550         bandwidth.full = dfixed_mul(dram_channels, yclk);
7551         bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
7552
7553         return dfixed_trunc(bandwidth);
7554 }
7555
7556 /**
7557  * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
7558  *
7559  * @wm: watermark calculation data
7560  *
7561  * Calculate the dram bandwidth used for display (CIK).
7562  * Used for display watermark bandwidth calculations
7563  * Returns the dram bandwidth for display in MBytes/s
7564  */
7565 static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
7566 {
7567         /* Calculate DRAM Bandwidth and the part allocated to display. */
7568         fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
7569         fixed20_12 yclk, dram_channels, bandwidth;
7570         fixed20_12 a;
7571
7572         a.full = dfixed_const(1000);
7573         yclk.full = dfixed_const(wm->yclk);
7574         yclk.full = dfixed_div(yclk, a);
7575         dram_channels.full = dfixed_const(wm->dram_channels * 4);
7576         a.full = dfixed_const(10);
7577         disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
7578         disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
7579         bandwidth.full = dfixed_mul(dram_channels, yclk);
7580         bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
7581
7582         return dfixed_trunc(bandwidth);
7583 }
7584
7585 /**
7586  * dce8_data_return_bandwidth - get the data return bandwidth
7587  *
7588  * @wm: watermark calculation data
7589  *
7590  * Calculate the data return bandwidth used for display (CIK).
7591  * Used for display watermark bandwidth calculations
7592  * Returns the data return bandwidth in MBytes/s
7593  */
7594 static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
7595 {
7596         /* Calculate the display Data return Bandwidth */
7597         fixed20_12 return_efficiency; /* 0.8 */
7598         fixed20_12 sclk, bandwidth;
7599         fixed20_12 a;
7600
7601         a.full = dfixed_const(1000);
7602         sclk.full = dfixed_const(wm->sclk);
7603         sclk.full = dfixed_div(sclk, a);
7604         a.full = dfixed_const(10);
7605         return_efficiency.full = dfixed_const(8);
7606         return_efficiency.full = dfixed_div(return_efficiency, a);
7607         a.full = dfixed_const(32);
7608         bandwidth.full = dfixed_mul(a, sclk);
7609         bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
7610
7611         return dfixed_trunc(bandwidth);
7612 }
7613
7614 /**
7615  * dce8_dmif_request_bandwidth - get the dmif bandwidth
7616  *
7617  * @wm: watermark calculation data
7618  *
7619  * Calculate the dmif bandwidth used for display (CIK).
7620  * Used for display watermark bandwidth calculations
7621  * Returns the dmif bandwidth in MBytes/s
7622  */
7623 static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
7624 {
7625         /* Calculate the DMIF Request Bandwidth */
7626         fixed20_12 disp_clk_request_efficiency; /* 0.8 */
7627         fixed20_12 disp_clk, bandwidth;
7628         fixed20_12 a, b;
7629
7630         a.full = dfixed_const(1000);
7631         disp_clk.full = dfixed_const(wm->disp_clk);
7632         disp_clk.full = dfixed_div(disp_clk, a);
7633         a.full = dfixed_const(32);
7634         b.full = dfixed_mul(a, disp_clk);
7635
7636         a.full = dfixed_const(10);
7637         disp_clk_request_efficiency.full = dfixed_const(8);
7638         disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
7639
7640         bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
7641
7642         return dfixed_trunc(bandwidth);
7643 }
7644
7645 /**
7646  * dce8_available_bandwidth - get the min available bandwidth
7647  *
7648  * @wm: watermark calculation data
7649  *
7650  * Calculate the min available bandwidth used for display (CIK).
7651  * Used for display watermark bandwidth calculations
7652  * Returns the min available bandwidth in MBytes/s
7653  */
7654 static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
7655 {
7656         /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
7657         u32 dram_bandwidth = dce8_dram_bandwidth(wm);
7658         u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
7659         u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
7660
7661         return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
7662 }
7663
7664 /**
7665  * dce8_average_bandwidth - get the average available bandwidth
7666  *
7667  * @wm: watermark calculation data
7668  *
7669  * Calculate the average available bandwidth used for display (CIK).
7670  * Used for display watermark bandwidth calculations
7671  * Returns the average available bandwidth in MBytes/s
7672  */
7673 static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
7674 {
7675         /* Calculate the display mode Average Bandwidth
7676          * DisplayMode should contain the source and destination dimensions,
7677          * timing, etc.
7678          */
7679         fixed20_12 bpp;
7680         fixed20_12 line_time;
7681         fixed20_12 src_width;
7682         fixed20_12 bandwidth;
7683         fixed20_12 a;
7684
7685         a.full = dfixed_const(1000);
7686         line_time.full = dfixed_const(wm->active_time + wm->blank_time);
7687         line_time.full = dfixed_div(line_time, a);
7688         bpp.full = dfixed_const(wm->bytes_per_pixel);
7689         src_width.full = dfixed_const(wm->src_width);
7690         bandwidth.full = dfixed_mul(src_width, bpp);
7691         bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
7692         bandwidth.full = dfixed_div(bandwidth, line_time);
7693
7694         return dfixed_trunc(bandwidth);
7695 }
7696
7697 /**
7698  * dce8_latency_watermark - get the latency watermark
7699  *
7700  * @wm: watermark calculation data
7701  *
7702  * Calculate the latency watermark (CIK).
7703  * Used for display watermark bandwidth calculations
7704  * Returns the latency watermark in ns
7705  */
7706 static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
7707 {
7708         /* First calculate the latency in ns */
7709         u32 mc_latency = 2000; /* 2000 ns. */
7710         u32 available_bandwidth = dce8_available_bandwidth(wm);
7711         u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
7712         u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
7713         u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
7714         u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
7715                 (wm->num_heads * cursor_line_pair_return_time);
7716         u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
7717         u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
7718         u32 tmp, dmif_size = 12288;
7719         fixed20_12 a, b, c;
7720
7721         if (wm->num_heads == 0)
7722                 return 0;
7723
7724         a.full = dfixed_const(2);
7725         b.full = dfixed_const(1);
7726         if ((wm->vsc.full > a.full) ||
7727             ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
7728             (wm->vtaps >= 5) ||
7729             ((wm->vsc.full >= a.full) && wm->interlaced))
7730                 max_src_lines_per_dst_line = 4;
7731         else
7732                 max_src_lines_per_dst_line = 2;
7733
7734         a.full = dfixed_const(available_bandwidth);
7735         b.full = dfixed_const(wm->num_heads);
7736         a.full = dfixed_div(a, b);
7737
7738         b.full = dfixed_const(mc_latency + 512);
7739         c.full = dfixed_const(wm->disp_clk);
7740         b.full = dfixed_div(b, c);
7741
7742         c.full = dfixed_const(dmif_size);
7743         b.full = dfixed_div(c, b);
7744
7745         tmp = min(dfixed_trunc(a), dfixed_trunc(b));
7746
7747         b.full = dfixed_const(1000);
7748         c.full = dfixed_const(wm->disp_clk);
7749         b.full = dfixed_div(c, b);
7750         c.full = dfixed_const(wm->bytes_per_pixel);
7751         b.full = dfixed_mul(b, c);
7752
7753         lb_fill_bw = min(tmp, dfixed_trunc(b));
7754
7755         a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
7756         b.full = dfixed_const(1000);
7757         c.full = dfixed_const(lb_fill_bw);
7758         b.full = dfixed_div(c, b);
7759         a.full = dfixed_div(a, b);
7760         line_fill_time = dfixed_trunc(a);
7761
7762         if (line_fill_time < wm->active_time)
7763                 return latency;
7764         else
7765                 return latency + (line_fill_time - wm->active_time);
7766
7767 }
7768
7769 /**
7770  * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
7771  * average and available dram bandwidth
7772  *
7773  * @wm: watermark calculation data
7774  *
7775  * Check if the display average bandwidth fits in the display
7776  * dram bandwidth (CIK).
7777  * Used for display watermark bandwidth calculations
7778  * Returns true if the display fits, false if not.
7779  */
7780 static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
7781 {
7782         if (dce8_average_bandwidth(wm) <=
7783             (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
7784                 return true;
7785         else
7786                 return false;
7787 }
7788
7789 /**
7790  * dce8_average_bandwidth_vs_available_bandwidth - check
7791  * average and available bandwidth
7792  *
7793  * @wm: watermark calculation data
7794  *
7795  * Check if the display average bandwidth fits in the display
7796  * available bandwidth (CIK).
7797  * Used for display watermark bandwidth calculations
7798  * Returns true if the display fits, false if not.
7799  */
7800 static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
7801 {
7802         if (dce8_average_bandwidth(wm) <=
7803             (dce8_available_bandwidth(wm) / wm->num_heads))
7804                 return true;
7805         else
7806                 return false;
7807 }
7808
7809 /**
7810  * dce8_check_latency_hiding - check latency hiding
7811  *
7812  * @wm: watermark calculation data
7813  *
7814  * Check latency hiding (CIK).
7815  * Used for display watermark bandwidth calculations
7816  * Returns true if the display fits, false if not.
7817  */
7818 static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
7819 {
7820         u32 lb_partitions = wm->lb_size / wm->src_width;
7821         u32 line_time = wm->active_time + wm->blank_time;
7822         u32 latency_tolerant_lines;
7823         u32 latency_hiding;
7824         fixed20_12 a;
7825
7826         a.full = dfixed_const(1);
7827         if (wm->vsc.full > a.full)
7828                 latency_tolerant_lines = 1;
7829         else {
7830                 if (lb_partitions <= (wm->vtaps + 1))
7831                         latency_tolerant_lines = 1;
7832                 else
7833                         latency_tolerant_lines = 2;
7834         }
7835
7836         latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
7837
7838         if (dce8_latency_watermark(wm) <= latency_hiding)
7839                 return true;
7840         else
7841                 return false;
7842 }
7843
7844 /**
7845  * dce8_program_watermarks - program display watermarks
7846  *
7847  * @rdev: radeon_device pointer
7848  * @radeon_crtc: the selected display controller
7849  * @lb_size: line buffer size
7850  * @num_heads: number of display controllers in use
7851  *
7852  * Calculate and program the display watermarks for the
7853  * selected display controller (CIK).
7854  */
7855 static void dce8_program_watermarks(struct radeon_device *rdev,
7856                                     struct radeon_crtc *radeon_crtc,
7857                                     u32 lb_size, u32 num_heads)
7858 {
7859         struct drm_display_mode *mode = &radeon_crtc->base.mode;
7860         struct dce8_wm_params wm_low, wm_high;
7861         u32 pixel_period;
7862         u32 line_time = 0;
7863         u32 latency_watermark_a = 0, latency_watermark_b = 0;
7864         u32 tmp, wm_mask;
7865
7866         if (radeon_crtc->base.enabled && num_heads && mode) {
7867                 pixel_period = 1000000 / (u32)mode->clock;
7868                 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
7869
7870                 /* watermark for high clocks */
7871                 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
7872                     rdev->pm.dpm_enabled) {
7873                         wm_high.yclk =
7874                                 radeon_dpm_get_mclk(rdev, false) * 10;
7875                         wm_high.sclk =
7876                                 radeon_dpm_get_sclk(rdev, false) * 10;
7877                 } else {
7878                         wm_high.yclk = rdev->pm.current_mclk * 10;
7879                         wm_high.sclk = rdev->pm.current_sclk * 10;
7880                 }
7881
7882                 wm_high.disp_clk = mode->clock;
7883                 wm_high.src_width = mode->crtc_hdisplay;
7884                 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
7885                 wm_high.blank_time = line_time - wm_high.active_time;
7886                 wm_high.interlaced = false;
7887                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
7888                         wm_high.interlaced = true;
7889                 wm_high.vsc = radeon_crtc->vsc;
7890                 wm_high.vtaps = 1;
7891                 if (radeon_crtc->rmx_type != RMX_OFF)
7892                         wm_high.vtaps = 2;
7893                 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
7894                 wm_high.lb_size = lb_size;
7895                 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
7896                 wm_high.num_heads = num_heads;
7897
7898                 /* set for high clocks */
7899                 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
7900
7901                 /* possibly force display priority to high */
7902                 /* should really do this at mode validation time... */
7903                 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
7904                     !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
7905                     !dce8_check_latency_hiding(&wm_high) ||
7906                     (rdev->disp_priority == 2)) {
7907                         DRM_DEBUG_KMS("force priority to high\n");
7908                 }
7909
7910                 /* watermark for low clocks */
7911                 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
7912                     rdev->pm.dpm_enabled) {
7913                         wm_low.yclk =
7914                                 radeon_dpm_get_mclk(rdev, true) * 10;
7915                         wm_low.sclk =
7916                                 radeon_dpm_get_sclk(rdev, true) * 10;
7917                 } else {
7918                         wm_low.yclk = rdev->pm.current_mclk * 10;
7919                         wm_low.sclk = rdev->pm.current_sclk * 10;
7920                 }
7921
7922                 wm_low.disp_clk = mode->clock;
7923                 wm_low.src_width = mode->crtc_hdisplay;
7924                 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
7925                 wm_low.blank_time = line_time - wm_low.active_time;
7926                 wm_low.interlaced = false;
7927                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
7928                         wm_low.interlaced = true;
7929                 wm_low.vsc = radeon_crtc->vsc;
7930                 wm_low.vtaps = 1;
7931                 if (radeon_crtc->rmx_type != RMX_OFF)
7932                         wm_low.vtaps = 2;
7933                 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
7934                 wm_low.lb_size = lb_size;
7935                 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
7936                 wm_low.num_heads = num_heads;
7937
7938                 /* set for low clocks */
7939                 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
7940
7941                 /* possibly force display priority to high */
7942                 /* should really do this at mode validation time... */
7943                 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
7944                     !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
7945                     !dce8_check_latency_hiding(&wm_low) ||
7946                     (rdev->disp_priority == 2)) {
7947                         DRM_DEBUG_KMS("force priority to high\n");
7948                 }
7949         }
7950
7951         /* select wm A */
7952         wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
7953         tmp = wm_mask;
7954         tmp &= ~LATENCY_WATERMARK_MASK(3);
7955         tmp |= LATENCY_WATERMARK_MASK(1);
7956         WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
7957         WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
7958                (LATENCY_LOW_WATERMARK(latency_watermark_a) |
7959                 LATENCY_HIGH_WATERMARK(line_time)));
7960         /* select wm B */
7961         tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
7962         tmp &= ~LATENCY_WATERMARK_MASK(3);
7963         tmp |= LATENCY_WATERMARK_MASK(2);
7964         WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
7965         WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
7966                (LATENCY_LOW_WATERMARK(latency_watermark_b) |
7967                 LATENCY_HIGH_WATERMARK(line_time)));
7968         /* restore original selection */
7969         WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
7970
7971         /* save values for DPM */
7972         radeon_crtc->line_time = line_time;
7973         radeon_crtc->wm_high = latency_watermark_a;
7974         radeon_crtc->wm_low = latency_watermark_b;
7975 }
7976
7977 /**
7978  * dce8_bandwidth_update - program display watermarks
7979  *
7980  * @rdev: radeon_device pointer
7981  *
7982  * Calculate and program the display watermarks and line
7983  * buffer allocation (CIK).
7984  */
7985 void dce8_bandwidth_update(struct radeon_device *rdev)
7986 {
7987         struct drm_display_mode *mode = NULL;
7988         u32 num_heads = 0, lb_size;
7989         int i;
7990
7991         radeon_update_display_priority(rdev);
7992
7993         for (i = 0; i < rdev->num_crtc; i++) {
7994                 if (rdev->mode_info.crtcs[i]->base.enabled)
7995                         num_heads++;
7996         }
7997         for (i = 0; i < rdev->num_crtc; i++) {
7998                 mode = &rdev->mode_info.crtcs[i]->base.mode;
7999                 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
8000                 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
8001         }
8002 }
8003
8004 /**
8005  * cik_get_gpu_clock_counter - return GPU clock counter snapshot
8006  *
8007  * @rdev: radeon_device pointer
8008  *
8009  * Fetches a GPU clock counter snapshot (SI).
8010  * Returns the 64 bit clock counter snapshot.
8011  */
8012 uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
8013 {
8014         uint64_t clock;
8015
8016         mutex_lock(&rdev->gpu_clock_mutex);
8017         WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
8018         clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
8019                 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
8020         mutex_unlock(&rdev->gpu_clock_mutex);
8021         return clock;
8022 }
8023
8024 static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
8025                               u32 cntl_reg, u32 status_reg)
8026 {
8027         int r, i;
8028         struct atom_clock_dividers dividers;
8029         uint32_t tmp;
8030
8031         r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
8032                                            clock, false, &dividers);
8033         if (r)
8034                 return r;
8035
8036         tmp = RREG32_SMC(cntl_reg);
8037         tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
8038         tmp |= dividers.post_divider;
8039         WREG32_SMC(cntl_reg, tmp);
8040
8041         for (i = 0; i < 100; i++) {
8042                 if (RREG32_SMC(status_reg) & DCLK_STATUS)
8043                         break;
8044                 mdelay(10);
8045         }
8046         if (i == 100)
8047                 return -ETIMEDOUT;
8048
8049         return 0;
8050 }
8051
8052 int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
8053 {
8054         int r = 0;
8055
8056         r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
8057         if (r)
8058                 return r;
8059
8060         r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
8061         return r;
8062 }
8063
8064 static void cik_pcie_gen3_enable(struct radeon_device *rdev)
8065 {
8066         struct pci_dev *root = rdev->pdev->bus->self;
8067         int bridge_pos, gpu_pos;
8068         u32 speed_cntl, mask, current_data_rate;
8069         int ret, i;
8070         u16 tmp16;
8071
8072         if (radeon_pcie_gen2 == 0)
8073                 return;
8074
8075         if (rdev->flags & RADEON_IS_IGP)
8076                 return;
8077
8078         if (!(rdev->flags & RADEON_IS_PCIE))
8079                 return;
8080
8081         ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
8082         if (ret != 0)
8083                 return;
8084
8085         if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
8086                 return;
8087
8088         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8089         current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
8090                 LC_CURRENT_DATA_RATE_SHIFT;
8091         if (mask & DRM_PCIE_SPEED_80) {
8092                 if (current_data_rate == 2) {
8093                         DRM_INFO("PCIE gen 3 link speeds already enabled\n");
8094                         return;
8095                 }
8096                 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
8097         } else if (mask & DRM_PCIE_SPEED_50) {
8098                 if (current_data_rate == 1) {
8099                         DRM_INFO("PCIE gen 2 link speeds already enabled\n");
8100                         return;
8101                 }
8102                 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
8103         }
8104
8105         bridge_pos = pci_pcie_cap(root);
8106         if (!bridge_pos)
8107                 return;
8108
8109         gpu_pos = pci_pcie_cap(rdev->pdev);
8110         if (!gpu_pos)
8111                 return;
8112
8113         if (mask & DRM_PCIE_SPEED_80) {
8114                 /* re-try equalization if gen3 is not already enabled */
8115                 if (current_data_rate != 2) {
8116                         u16 bridge_cfg, gpu_cfg;
8117                         u16 bridge_cfg2, gpu_cfg2;
8118                         u32 max_lw, current_lw, tmp;
8119
8120                         pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
8121                         pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
8122
8123                         tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
8124                         pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
8125
8126                         tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
8127                         pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
8128
8129                         tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
8130                         max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
8131                         current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
8132
8133                         if (current_lw < max_lw) {
8134                                 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
8135                                 if (tmp & LC_RENEGOTIATION_SUPPORT) {
8136                                         tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
8137                                         tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
8138                                         tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
8139                                         WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
8140                                 }
8141                         }
8142
8143                         for (i = 0; i < 10; i++) {
8144                                 /* check status */
8145                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
8146                                 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
8147                                         break;
8148
8149                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
8150                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
8151
8152                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
8153                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
8154
8155                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8156                                 tmp |= LC_SET_QUIESCE;
8157                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8158
8159                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8160                                 tmp |= LC_REDO_EQ;
8161                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8162
8163                                 mdelay(100);
8164
8165                                 /* linkctl */
8166                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
8167                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
8168                                 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
8169                                 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
8170
8171                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
8172                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
8173                                 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
8174                                 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
8175
8176                                 /* linkctl2 */
8177                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
8178                                 tmp16 &= ~((1 << 4) | (7 << 9));
8179                                 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
8180                                 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
8181
8182                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
8183                                 tmp16 &= ~((1 << 4) | (7 << 9));
8184                                 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
8185                                 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
8186
8187                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
8188                                 tmp &= ~LC_SET_QUIESCE;
8189                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
8190                         }
8191                 }
8192         }
8193
8194         /* set the link speed */
8195         speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
8196         speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
8197         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
8198
8199         pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
8200         tmp16 &= ~0xf;
8201         if (mask & DRM_PCIE_SPEED_80)
8202                 tmp16 |= 3; /* gen3 */
8203         else if (mask & DRM_PCIE_SPEED_50)
8204                 tmp16 |= 2; /* gen2 */
8205         else
8206                 tmp16 |= 1; /* gen1 */
8207         pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
8208
8209         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8210         speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
8211         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
8212
8213         for (i = 0; i < rdev->usec_timeout; i++) {
8214                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8215                 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
8216                         break;
8217                 udelay(1);
8218         }
8219 }
8220
8221 static void cik_program_aspm(struct radeon_device *rdev)
8222 {
8223         u32 data, orig;
8224         bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
8225         bool disable_clkreq = false;
8226
8227         if (radeon_aspm == 0)
8228                 return;
8229
8230         /* XXX double check IGPs */
8231         if (rdev->flags & RADEON_IS_IGP)
8232                 return;
8233
8234         if (!(rdev->flags & RADEON_IS_PCIE))
8235                 return;
8236
8237         orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
8238         data &= ~LC_XMIT_N_FTS_MASK;
8239         data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
8240         if (orig != data)
8241                 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
8242
8243         orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
8244         data |= LC_GO_TO_RECOVERY;
8245         if (orig != data)
8246                 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
8247
8248         orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
8249         data |= P_IGNORE_EDB_ERR;
8250         if (orig != data)
8251                 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
8252
8253         orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
8254         data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
8255         data |= LC_PMI_TO_L1_DIS;
8256         if (!disable_l0s)
8257                 data |= LC_L0S_INACTIVITY(7);
8258
8259         if (!disable_l1) {
8260                 data |= LC_L1_INACTIVITY(7);
8261                 data &= ~LC_PMI_TO_L1_DIS;
8262                 if (orig != data)
8263                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
8264
8265                 if (!disable_plloff_in_l1) {
8266                         bool clk_req_support;
8267
8268                         orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
8269                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
8270                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
8271                         if (orig != data)
8272                                 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
8273
8274                         orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
8275                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
8276                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
8277                         if (orig != data)
8278                                 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
8279
8280                         orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
8281                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
8282                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
8283                         if (orig != data)
8284                                 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
8285
8286                         orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
8287                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
8288                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
8289                         if (orig != data)
8290                                 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
8291
8292                         orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
8293                         data &= ~LC_DYN_LANES_PWR_STATE_MASK;
8294                         data |= LC_DYN_LANES_PWR_STATE(3);
8295                         if (orig != data)
8296                                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
8297
8298                         if (!disable_clkreq) {
8299                                 struct pci_dev *root = rdev->pdev->bus->self;
8300                                 u32 lnkcap;
8301
8302                                 clk_req_support = false;
8303                                 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
8304                                 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
8305                                         clk_req_support = true;
8306                         } else {
8307                                 clk_req_support = false;
8308                         }
8309
8310                         if (clk_req_support) {
8311                                 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
8312                                 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
8313                                 if (orig != data)
8314                                         WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
8315
8316                                 orig = data = RREG32_SMC(THM_CLK_CNTL);
8317                                 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
8318                                 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
8319                                 if (orig != data)
8320                                         WREG32_SMC(THM_CLK_CNTL, data);
8321
8322                                 orig = data = RREG32_SMC(MISC_CLK_CTRL);
8323                                 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
8324                                 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
8325                                 if (orig != data)
8326                                         WREG32_SMC(MISC_CLK_CTRL, data);
8327
8328                                 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
8329                                 data &= ~BCLK_AS_XCLK;
8330                                 if (orig != data)
8331                                         WREG32_SMC(CG_CLKPIN_CNTL, data);
8332
8333                                 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
8334                                 data &= ~FORCE_BIF_REFCLK_EN;
8335                                 if (orig != data)
8336                                         WREG32_SMC(CG_CLKPIN_CNTL_2, data);
8337
8338                                 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
8339                                 data &= ~MPLL_CLKOUT_SEL_MASK;
8340                                 data |= MPLL_CLKOUT_SEL(4);
8341                                 if (orig != data)
8342                                         WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
8343                         }
8344                 }
8345         } else {
8346                 if (orig != data)
8347                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
8348         }
8349
8350         orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
8351         data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
8352         if (orig != data)
8353                 WREG32_PCIE_PORT(PCIE_CNTL2, data);
8354
8355         if (!disable_l0s) {
8356                 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
8357                 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
8358                         data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
8359                         if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
8360                                 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
8361                                 data &= ~LC_L0S_INACTIVITY_MASK;
8362                                 if (orig != data)
8363                                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
8364                         }
8365                 }
8366         }
8367 }