drivers/gpu/drm/amd/amdgpu/cik.c

   1 /*
   2  * Copyright 2012 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/slab.h>
  26 #include <linux/module.h>
  27 #include "drmP.h"
  28 #include "amdgpu.h"
  29 #include "amdgpu_atombios.h"
  30 #include "amdgpu_ih.h"
  31 #include "amdgpu_uvd.h"
  32 #include "amdgpu_vce.h"
  33 #include "cikd.h"
  34 #include "atom.h"
  35
  36 #include "cik.h"
  37 #include "gmc_v7_0.h"
  38 #include "cik_ih.h"
  39 #include "dce_v8_0.h"
  40 #include "gfx_v7_0.h"
  41 #include "cik_sdma.h"
  42 #include "uvd_v4_2.h"
  43 #include "vce_v2_0.h"
  44 #include "cik_dpm.h"
  45
  46 #include "uvd/uvd_4_2_d.h"
  47
  48 #include "smu/smu_7_0_1_d.h"
  49 #include "smu/smu_7_0_1_sh_mask.h"
  50
  51 #include "dce/dce_8_0_d.h"
  52 #include "dce/dce_8_0_sh_mask.h"
  53
  54 #include "bif/bif_4_1_d.h"
  55 #include "bif/bif_4_1_sh_mask.h"
  56
  57 #include "gca/gfx_7_2_d.h"
  58 #include "gca/gfx_7_2_enum.h"
  59 #include "gca/gfx_7_2_sh_mask.h"
  60
  61 #include "gmc/gmc_7_1_d.h"
  62 #include "gmc/gmc_7_1_sh_mask.h"
  63
  64 #include "oss/oss_2_0_d.h"
  65 #include "oss/oss_2_0_sh_mask.h"
  66
  67 #include "amdgpu_amdkfd.h"
  68
  69 /*
  70  * Indirect registers accessor
  71  */
  72 static u32 cik_pcie_rreg(struct amdgpu_device *adev, u32 reg)
  73 {
  74         unsigned long flags;
  75         u32 r;
  76
  77         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
  78         WREG32(mmPCIE_INDEX, reg);
  79         (void)RREG32(mmPCIE_INDEX);
  80         r = RREG32(mmPCIE_DATA);
  81         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
  82         return r;
  83 }
  84
  85 static void cik_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
  86 {
  87         unsigned long flags;
  88
  89         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
  90         WREG32(mmPCIE_INDEX, reg);
  91         (void)RREG32(mmPCIE_INDEX);
  92         WREG32(mmPCIE_DATA, v);
  93         (void)RREG32(mmPCIE_DATA);
  94         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
  95 }
  96
  97 static u32 cik_smc_rreg(struct amdgpu_device *adev, u32 reg)
  98 {
  99         unsigned long flags;
 100         u32 r;
 101
 102         spin_lock_irqsave(&adev->smc_idx_lock, flags);
 103         WREG32(mmSMC_IND_INDEX_0, (reg));
 104         r = RREG32(mmSMC_IND_DATA_0);
 105         spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 106         return r;
 107 }
 108
 109 static void cik_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 110 {
 111         unsigned long flags;
 112
 113         spin_lock_irqsave(&adev->smc_idx_lock, flags);
 114         WREG32(mmSMC_IND_INDEX_0, (reg));
 115         WREG32(mmSMC_IND_DATA_0, (v));
 116         spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
 117 }
 118
 119 static u32 cik_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg)
 120 {
 121         unsigned long flags;
 122         u32 r;
 123
 124         spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
 125         WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
 126         r = RREG32(mmUVD_CTX_DATA);
 127         spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
 128         return r;
 129 }
 130
 131 static void cik_uvd_ctx_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 132 {
 133         unsigned long flags;
 134
 135         spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
 136         WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
 137         WREG32(mmUVD_CTX_DATA, (v));
 138         spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
 139 }
 140
 141 static u32 cik_didt_rreg(struct amdgpu_device *adev, u32 reg)
 142 {
 143         unsigned long flags;
 144         u32 r;
 145
 146         spin_lock_irqsave(&adev->didt_idx_lock, flags);
 147         WREG32(mmDIDT_IND_INDEX, (reg));
 148         r = RREG32(mmDIDT_IND_DATA);
 149         spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
 150         return r;
 151 }
 152
 153 static void cik_didt_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 154 {
 155         unsigned long flags;
 156
 157         spin_lock_irqsave(&adev->didt_idx_lock, flags);
 158         WREG32(mmDIDT_IND_INDEX, (reg));
 159         WREG32(mmDIDT_IND_DATA, (v));
 160         spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
 161 }
 162
 163 static const u32 bonaire_golden_spm_registers[] =
 164 {
 165         0xc200, 0xe0ffffff, 0xe0000000
 166 };
 167
 168 static const u32 bonaire_golden_common_registers[] =
 169 {
 170         0x31dc, 0xffffffff, 0x00000800,
 171         0x31dd, 0xffffffff, 0x00000800,
 172         0x31e6, 0xffffffff, 0x00007fbf,
 173         0x31e7, 0xffffffff, 0x00007faf
 174 };
 175
 176 static const u32 bonaire_golden_registers[] =
 177 {
 178         0xcd5, 0x00000333, 0x00000333,
 179         0xcd4, 0x000c0fc0, 0x00040200,
 180         0x2684, 0x00010000, 0x00058208,
 181         0xf000, 0xffff1fff, 0x00140000,
 182         0xf080, 0xfdfc0fff, 0x00000100,
 183         0xf08d, 0x40000000, 0x40000200,
 184         0x260c, 0xffffffff, 0x00000000,
 185         0x260d, 0xf00fffff, 0x00000400,
 186         0x260e, 0x0002021c, 0x00020200,
 187         0x31e, 0x00000080, 0x00000000,
 188         0x16ec, 0x000000f0, 0x00000070,
 189         0x16f0, 0xf0311fff, 0x80300000,
 190         0x263e, 0x73773777, 0x12010001,
 191         0xd43, 0x00810000, 0x408af000,
 192         0x1c0c, 0x31000111, 0x00000011,
 193         0xbd2, 0x73773777, 0x12010001,
 194         0x883, 0x00007fb6, 0x0021a1b1,
 195         0x884, 0x00007fb6, 0x002021b1,
 196         0x860, 0x00007fb6, 0x00002191,
 197         0x886, 0x00007fb6, 0x002121b1,
 198         0x887, 0x00007fb6, 0x002021b1,
 199         0x877, 0x00007fb6, 0x00002191,
 200         0x878, 0x00007fb6, 0x00002191,
 201         0xd8a, 0x0000003f, 0x0000000a,
 202         0xd8b, 0x0000003f, 0x0000000a,
 203         0xab9, 0x00073ffe, 0x000022a2,
 204         0x903, 0x000007ff, 0x00000000,
 205         0x2285, 0xf000003f, 0x00000007,
 206         0x22fc, 0x00002001, 0x00000001,
 207         0x22c9, 0xffffffff, 0x00ffffff,
 208         0xc281, 0x0000ff0f, 0x00000000,
 209         0xa293, 0x07ffffff, 0x06000000,
 210         0x136, 0x00000fff, 0x00000100,
 211         0xf9e, 0x00000001, 0x00000002,
 212         0x2440, 0x03000000, 0x0362c688,
 213         0x2300, 0x000000ff, 0x00000001,
 214         0x390, 0x00001fff, 0x00001fff,
 215         0x2418, 0x0000007f, 0x00000020,
 216         0x2542, 0x00010000, 0x00010000,
 217         0x2b05, 0x000003ff, 0x000000f3,
 218         0x2b03, 0xffffffff, 0x00001032
 219 };
 220
 221 static const u32 bonaire_mgcg_cgcg_init[] =
 222 {
 223         0x3108, 0xffffffff, 0xfffffffc,
 224         0xc200, 0xffffffff, 0xe0000000,
 225         0xf0a8, 0xffffffff, 0x00000100,
 226         0xf082, 0xffffffff, 0x00000100,
 227         0xf0b0, 0xffffffff, 0xc0000100,
 228         0xf0b2, 0xffffffff, 0xc0000100,
 229         0xf0b1, 0xffffffff, 0xc0000100,
 230         0x1579, 0xffffffff, 0x00600100,
 231         0xf0a0, 0xffffffff, 0x00000100,
 232         0xf085, 0xffffffff, 0x06000100,
 233         0xf088, 0xffffffff, 0x00000100,
 234         0xf086, 0xffffffff, 0x06000100,
 235         0xf081, 0xffffffff, 0x00000100,
 236         0xf0b8, 0xffffffff, 0x00000100,
 237         0xf089, 0xffffffff, 0x00000100,
 238         0xf080, 0xffffffff, 0x00000100,
 239         0xf08c, 0xffffffff, 0x00000100,
 240         0xf08d, 0xffffffff, 0x00000100,
 241         0xf094, 0xffffffff, 0x00000100,
 242         0xf095, 0xffffffff, 0x00000100,
 243         0xf096, 0xffffffff, 0x00000100,
 244         0xf097, 0xffffffff, 0x00000100,
 245         0xf098, 0xffffffff, 0x00000100,
 246         0xf09f, 0xffffffff, 0x00000100,
 247         0xf09e, 0xffffffff, 0x00000100,
 248         0xf084, 0xffffffff, 0x06000100,
 249         0xf0a4, 0xffffffff, 0x00000100,
 250         0xf09d, 0xffffffff, 0x00000100,
 251         0xf0ad, 0xffffffff, 0x00000100,
 252         0xf0ac, 0xffffffff, 0x00000100,
 253         0xf09c, 0xffffffff, 0x00000100,
 254         0xc200, 0xffffffff, 0xe0000000,
 255         0xf008, 0xffffffff, 0x00010000,
 256         0xf009, 0xffffffff, 0x00030002,
 257         0xf00a, 0xffffffff, 0x00040007,
 258         0xf00b, 0xffffffff, 0x00060005,
 259         0xf00c, 0xffffffff, 0x00090008,
 260         0xf00d, 0xffffffff, 0x00010000,
 261         0xf00e, 0xffffffff, 0x00030002,
 262         0xf00f, 0xffffffff, 0x00040007,
 263         0xf010, 0xffffffff, 0x00060005,
 264         0xf011, 0xffffffff, 0x00090008,
 265         0xf012, 0xffffffff, 0x00010000,
 266         0xf013, 0xffffffff, 0x00030002,
 267         0xf014, 0xffffffff, 0x00040007,
 268         0xf015, 0xffffffff, 0x00060005,
 269         0xf016, 0xffffffff, 0x00090008,
 270         0xf017, 0xffffffff, 0x00010000,
 271         0xf018, 0xffffffff, 0x00030002,
 272         0xf019, 0xffffffff, 0x00040007,
 273         0xf01a, 0xffffffff, 0x00060005,
 274         0xf01b, 0xffffffff, 0x00090008,
 275         0xf01c, 0xffffffff, 0x00010000,
 276         0xf01d, 0xffffffff, 0x00030002,
 277         0xf01e, 0xffffffff, 0x00040007,
 278         0xf01f, 0xffffffff, 0x00060005,
 279         0xf020, 0xffffffff, 0x00090008,
 280         0xf021, 0xffffffff, 0x00010000,
 281         0xf022, 0xffffffff, 0x00030002,
 282         0xf023, 0xffffffff, 0x00040007,
 283         0xf024, 0xffffffff, 0x00060005,
 284         0xf025, 0xffffffff, 0x00090008,
 285         0xf026, 0xffffffff, 0x00010000,
 286         0xf027, 0xffffffff, 0x00030002,
 287         0xf028, 0xffffffff, 0x00040007,
 288         0xf029, 0xffffffff, 0x00060005,
 289         0xf02a, 0xffffffff, 0x00090008,
 290         0xf000, 0xffffffff, 0x96e00200,
 291         0x21c2, 0xffffffff, 0x00900100,
 292         0x3109, 0xffffffff, 0x0020003f,
 293         0xe, 0xffffffff, 0x0140001c,
 294         0xf, 0x000f0000, 0x000f0000,
 295         0x88, 0xffffffff, 0xc060000c,
 296         0x89, 0xc0000fff, 0x00000100,
 297         0x3e4, 0xffffffff, 0x00000100,
 298         0x3e6, 0x00000101, 0x00000000,
 299         0x82a, 0xffffffff, 0x00000104,
 300         0x1579, 0xff000fff, 0x00000100,
 301         0xc33, 0xc0000fff, 0x00000104,
 302         0x3079, 0x00000001, 0x00000001,
 303         0x3403, 0xff000ff0, 0x00000100,
 304         0x3603, 0xff000ff0, 0x00000100
 305 };
 306
 307 static const u32 spectre_golden_spm_registers[] =
 308 {
 309         0xc200, 0xe0ffffff, 0xe0000000
 310 };
 311
 312 static const u32 spectre_golden_common_registers[] =
 313 {
 314         0x31dc, 0xffffffff, 0x00000800,
 315         0x31dd, 0xffffffff, 0x00000800,
 316         0x31e6, 0xffffffff, 0x00007fbf,
 317         0x31e7, 0xffffffff, 0x00007faf
 318 };
 319
 320 static const u32 spectre_golden_registers[] =
 321 {
 322         0xf000, 0xffff1fff, 0x96940200,
 323         0xf003, 0xffff0001, 0xff000000,
 324         0xf080, 0xfffc0fff, 0x00000100,
 325         0x1bb6, 0x00010101, 0x00010000,
 326         0x260d, 0xf00fffff, 0x00000400,
 327         0x260e, 0xfffffffc, 0x00020200,
 328         0x16ec, 0x000000f0, 0x00000070,
 329         0x16f0, 0xf0311fff, 0x80300000,
 330         0x263e, 0x73773777, 0x12010001,
 331         0x26df, 0x00ff0000, 0x00fc0000,
 332         0xbd2, 0x73773777, 0x12010001,
 333         0x2285, 0xf000003f, 0x00000007,
 334         0x22c9, 0xffffffff, 0x00ffffff,
 335         0xa0d4, 0x3f3f3fff, 0x00000082,
 336         0xa0d5, 0x0000003f, 0x00000000,
 337         0xf9e, 0x00000001, 0x00000002,
 338         0x244f, 0xffff03df, 0x00000004,
 339         0x31da, 0x00000008, 0x00000008,
 340         0x2300, 0x000008ff, 0x00000800,
 341         0x2542, 0x00010000, 0x00010000,
 342         0x2b03, 0xffffffff, 0x54763210,
 343         0x853e, 0x01ff01ff, 0x00000002,
 344         0x8526, 0x007ff800, 0x00200000,
 345         0x8057, 0xffffffff, 0x00000f40,
 346         0xc24d, 0xffffffff, 0x00000001
 347 };
 348
 349 static const u32 spectre_mgcg_cgcg_init[] =
 350 {
 351         0x3108, 0xffffffff, 0xfffffffc,
 352         0xc200, 0xffffffff, 0xe0000000,
 353         0xf0a8, 0xffffffff, 0x00000100,
 354         0xf082, 0xffffffff, 0x00000100,
 355         0xf0b0, 0xffffffff, 0x00000100,
 356         0xf0b2, 0xffffffff, 0x00000100,
 357         0xf0b1, 0xffffffff, 0x00000100,
 358         0x1579, 0xffffffff, 0x00600100,
 359         0xf0a0, 0xffffffff, 0x00000100,
 360         0xf085, 0xffffffff, 0x06000100,
 361         0xf088, 0xffffffff, 0x00000100,
 362         0xf086, 0xffffffff, 0x06000100,
 363         0xf081, 0xffffffff, 0x00000100,
 364         0xf0b8, 0xffffffff, 0x00000100,
 365         0xf089, 0xffffffff, 0x00000100,
 366         0xf080, 0xffffffff, 0x00000100,
 367         0xf08c, 0xffffffff, 0x00000100,
 368         0xf08d, 0xffffffff, 0x00000100,
 369         0xf094, 0xffffffff, 0x00000100,
 370         0xf095, 0xffffffff, 0x00000100,
 371         0xf096, 0xffffffff, 0x00000100,
 372         0xf097, 0xffffffff, 0x00000100,
 373         0xf098, 0xffffffff, 0x00000100,
 374         0xf09f, 0xffffffff, 0x00000100,
 375         0xf09e, 0xffffffff, 0x00000100,
 376         0xf084, 0xffffffff, 0x06000100,
 377         0xf0a4, 0xffffffff, 0x00000100,
 378         0xf09d, 0xffffffff, 0x00000100,
 379         0xf0ad, 0xffffffff, 0x00000100,
 380         0xf0ac, 0xffffffff, 0x00000100,
 381         0xf09c, 0xffffffff, 0x00000100,
 382         0xc200, 0xffffffff, 0xe0000000,
 383         0xf008, 0xffffffff, 0x00010000,
 384         0xf009, 0xffffffff, 0x00030002,
 385         0xf00a, 0xffffffff, 0x00040007,
 386         0xf00b, 0xffffffff, 0x00060005,
 387         0xf00c, 0xffffffff, 0x00090008,
 388         0xf00d, 0xffffffff, 0x00010000,
 389         0xf00e, 0xffffffff, 0x00030002,
 390         0xf00f, 0xffffffff, 0x00040007,
 391         0xf010, 0xffffffff, 0x00060005,
 392         0xf011, 0xffffffff, 0x00090008,
 393         0xf012, 0xffffffff, 0x00010000,
 394         0xf013, 0xffffffff, 0x00030002,
 395         0xf014, 0xffffffff, 0x00040007,
 396         0xf015, 0xffffffff, 0x00060005,
 397         0xf016, 0xffffffff, 0x00090008,
 398         0xf017, 0xffffffff, 0x00010000,
 399         0xf018, 0xffffffff, 0x00030002,
 400         0xf019, 0xffffffff, 0x00040007,
 401         0xf01a, 0xffffffff, 0x00060005,
 402         0xf01b, 0xffffffff, 0x00090008,
 403         0xf01c, 0xffffffff, 0x00010000,
 404         0xf01d, 0xffffffff, 0x00030002,
 405         0xf01e, 0xffffffff, 0x00040007,
 406         0xf01f, 0xffffffff, 0x00060005,
 407         0xf020, 0xffffffff, 0x00090008,
 408         0xf021, 0xffffffff, 0x00010000,
 409         0xf022, 0xffffffff, 0x00030002,
 410         0xf023, 0xffffffff, 0x00040007,
 411         0xf024, 0xffffffff, 0x00060005,
 412         0xf025, 0xffffffff, 0x00090008,
 413         0xf026, 0xffffffff, 0x00010000,
 414         0xf027, 0xffffffff, 0x00030002,
 415         0xf028, 0xffffffff, 0x00040007,
 416         0xf029, 0xffffffff, 0x00060005,
 417         0xf02a, 0xffffffff, 0x00090008,
 418         0xf02b, 0xffffffff, 0x00010000,
 419         0xf02c, 0xffffffff, 0x00030002,
 420         0xf02d, 0xffffffff, 0x00040007,
 421         0xf02e, 0xffffffff, 0x00060005,
 422         0xf02f, 0xffffffff, 0x00090008,
 423         0xf000, 0xffffffff, 0x96e00200,
 424         0x21c2, 0xffffffff, 0x00900100,
 425         0x3109, 0xffffffff, 0x0020003f,
 426         0xe, 0xffffffff, 0x0140001c,
 427         0xf, 0x000f0000, 0x000f0000,
 428         0x88, 0xffffffff, 0xc060000c,
 429         0x89, 0xc0000fff, 0x00000100,
 430         0x3e4, 0xffffffff, 0x00000100,
 431         0x3e6, 0x00000101, 0x00000000,
 432         0x82a, 0xffffffff, 0x00000104,
 433         0x1579, 0xff000fff, 0x00000100,
 434         0xc33, 0xc0000fff, 0x00000104,
 435         0x3079, 0x00000001, 0x00000001,
 436         0x3403, 0xff000ff0, 0x00000100,
 437         0x3603, 0xff000ff0, 0x00000100
 438 };
 439
 440 static const u32 kalindi_golden_spm_registers[] =
 441 {
 442         0xc200, 0xe0ffffff, 0xe0000000
 443 };
 444
 445 static const u32 kalindi_golden_common_registers[] =
 446 {
 447         0x31dc, 0xffffffff, 0x00000800,
 448         0x31dd, 0xffffffff, 0x00000800,
 449         0x31e6, 0xffffffff, 0x00007fbf,
 450         0x31e7, 0xffffffff, 0x00007faf
 451 };
 452
 453 static const u32 kalindi_golden_registers[] =
 454 {
 455         0xf000, 0xffffdfff, 0x6e944040,
 456         0x1579, 0xff607fff, 0xfc000100,
 457         0xf088, 0xff000fff, 0x00000100,
 458         0xf089, 0xff000fff, 0x00000100,
 459         0xf080, 0xfffc0fff, 0x00000100,
 460         0x1bb6, 0x00010101, 0x00010000,
 461         0x260c, 0xffffffff, 0x00000000,
 462         0x260d, 0xf00fffff, 0x00000400,
 463         0x16ec, 0x000000f0, 0x00000070,
 464         0x16f0, 0xf0311fff, 0x80300000,
 465         0x263e, 0x73773777, 0x12010001,
 466         0x263f, 0xffffffff, 0x00000010,
 467         0x26df, 0x00ff0000, 0x00fc0000,
 468         0x200c, 0x00001f0f, 0x0000100a,
 469         0xbd2, 0x73773777, 0x12010001,
 470         0x902, 0x000fffff, 0x000c007f,
 471         0x2285, 0xf000003f, 0x00000007,
 472         0x22c9, 0x3fff3fff, 0x00ffcfff,
 473         0xc281, 0x0000ff0f, 0x00000000,
 474         0xa293, 0x07ffffff, 0x06000000,
 475         0x136, 0x00000fff, 0x00000100,
 476         0xf9e, 0x00000001, 0x00000002,
 477         0x31da, 0x00000008, 0x00000008,
 478         0x2300, 0x000000ff, 0x00000003,
 479         0x853e, 0x01ff01ff, 0x00000002,
 480         0x8526, 0x007ff800, 0x00200000,
 481         0x8057, 0xffffffff, 0x00000f40,
 482         0x2231, 0x001f3ae3, 0x00000082,
 483         0x2235, 0x0000001f, 0x00000010,
 484         0xc24d, 0xffffffff, 0x00000000
 485 };
 486
 487 static const u32 kalindi_mgcg_cgcg_init[] =
 488 {
 489         0x3108, 0xffffffff, 0xfffffffc,
 490         0xc200, 0xffffffff, 0xe0000000,
 491         0xf0a8, 0xffffffff, 0x00000100,
 492         0xf082, 0xffffffff, 0x00000100,
 493         0xf0b0, 0xffffffff, 0x00000100,
 494         0xf0b2, 0xffffffff, 0x00000100,
 495         0xf0b1, 0xffffffff, 0x00000100,
 496         0x1579, 0xffffffff, 0x00600100,
 497         0xf0a0, 0xffffffff, 0x00000100,
 498         0xf085, 0xffffffff, 0x06000100,
 499         0xf088, 0xffffffff, 0x00000100,
 500         0xf086, 0xffffffff, 0x06000100,
 501         0xf081, 0xffffffff, 0x00000100,
 502         0xf0b8, 0xffffffff, 0x00000100,
 503         0xf089, 0xffffffff, 0x00000100,
 504         0xf080, 0xffffffff, 0x00000100,
 505         0xf08c, 0xffffffff, 0x00000100,
 506         0xf08d, 0xffffffff, 0x00000100,
 507         0xf094, 0xffffffff, 0x00000100,
 508         0xf095, 0xffffffff, 0x00000100,
 509         0xf096, 0xffffffff, 0x00000100,
 510         0xf097, 0xffffffff, 0x00000100,
 511         0xf098, 0xffffffff, 0x00000100,
 512         0xf09f, 0xffffffff, 0x00000100,
 513         0xf09e, 0xffffffff, 0x00000100,
 514         0xf084, 0xffffffff, 0x06000100,
 515         0xf0a4, 0xffffffff, 0x00000100,
 516         0xf09d, 0xffffffff, 0x00000100,
 517         0xf0ad, 0xffffffff, 0x00000100,
 518         0xf0ac, 0xffffffff, 0x00000100,
 519         0xf09c, 0xffffffff, 0x00000100,
 520         0xc200, 0xffffffff, 0xe0000000,
 521         0xf008, 0xffffffff, 0x00010000,
 522         0xf009, 0xffffffff, 0x00030002,
 523         0xf00a, 0xffffffff, 0x00040007,
 524         0xf00b, 0xffffffff, 0x00060005,
 525         0xf00c, 0xffffffff, 0x00090008,
 526         0xf00d, 0xffffffff, 0x00010000,
 527         0xf00e, 0xffffffff, 0x00030002,
 528         0xf00f, 0xffffffff, 0x00040007,
 529         0xf010, 0xffffffff, 0x00060005,
 530         0xf011, 0xffffffff, 0x00090008,
 531         0xf000, 0xffffffff, 0x96e00200,
 532         0x21c2, 0xffffffff, 0x00900100,
 533         0x3109, 0xffffffff, 0x0020003f,
 534         0xe, 0xffffffff, 0x0140001c,
 535         0xf, 0x000f0000, 0x000f0000,
 536         0x88, 0xffffffff, 0xc060000c,
 537         0x89, 0xc0000fff, 0x00000100,
 538         0x82a, 0xffffffff, 0x00000104,
 539         0x1579, 0xff000fff, 0x00000100,
 540         0xc33, 0xc0000fff, 0x00000104,
 541         0x3079, 0x00000001, 0x00000001,
 542         0x3403, 0xff000ff0, 0x00000100,
 543         0x3603, 0xff000ff0, 0x00000100
 544 };
 545
 546 static const u32 hawaii_golden_spm_registers[] =
 547 {
 548         0xc200, 0xe0ffffff, 0xe0000000
 549 };
 550
 551 static const u32 hawaii_golden_common_registers[] =
 552 {
 553         0xc200, 0xffffffff, 0xe0000000,
 554         0xa0d4, 0xffffffff, 0x3a00161a,
 555         0xa0d5, 0xffffffff, 0x0000002e,
 556         0x2684, 0xffffffff, 0x00018208,
 557         0x263e, 0xffffffff, 0x12011003
 558 };
 559
 560 static const u32 hawaii_golden_registers[] =
 561 {
 562         0xcd5, 0x00000333, 0x00000333,
 563         0x2684, 0x00010000, 0x00058208,
 564         0x260c, 0xffffffff, 0x00000000,
 565         0x260d, 0xf00fffff, 0x00000400,
 566         0x260e, 0x0002021c, 0x00020200,
 567         0x31e, 0x00000080, 0x00000000,
 568         0x16ec, 0x000000f0, 0x00000070,
 569         0x16f0, 0xf0311fff, 0x80300000,
 570         0xd43, 0x00810000, 0x408af000,
 571         0x1c0c, 0x31000111, 0x00000011,
 572         0xbd2, 0x73773777, 0x12010001,
 573         0x848, 0x0000007f, 0x0000001b,
 574         0x877, 0x00007fb6, 0x00002191,
 575         0xd8a, 0x0000003f, 0x0000000a,
 576         0xd8b, 0x0000003f, 0x0000000a,
 577         0xab9, 0x00073ffe, 0x000022a2,
 578         0x903, 0x000007ff, 0x00000000,
 579         0x22fc, 0x00002001, 0x00000001,
 580         0x22c9, 0xffffffff, 0x00ffffff,
 581         0xc281, 0x0000ff0f, 0x00000000,
 582         0xa293, 0x07ffffff, 0x06000000,
 583         0xf9e, 0x00000001, 0x00000002,
 584         0x31da, 0x00000008, 0x00000008,
 585         0x31dc, 0x00000f00, 0x00000800,
 586         0x31dd, 0x00000f00, 0x00000800,
 587         0x31e6, 0x00ffffff, 0x00ff7fbf,
 588         0x31e7, 0x00ffffff, 0x00ff7faf,
 589         0x2300, 0x000000ff, 0x00000800,
 590         0x390, 0x00001fff, 0x00001fff,
 591         0x2418, 0x0000007f, 0x00000020,
 592         0x2542, 0x00010000, 0x00010000,
 593         0x2b80, 0x00100000, 0x000ff07c,
 594         0x2b05, 0x000003ff, 0x0000000f,
 595         0x2b04, 0xffffffff, 0x7564fdec,
 596         0x2b03, 0xffffffff, 0x3120b9a8,
 597         0x2b02, 0x20000000, 0x0f9c0000
 598 };
 599
 600 static const u32 hawaii_mgcg_cgcg_init[] =
 601 {
 602         0x3108, 0xffffffff, 0xfffffffd,
 603         0xc200, 0xffffffff, 0xe0000000,
 604         0xf0a8, 0xffffffff, 0x00000100,
 605         0xf082, 0xffffffff, 0x00000100,
 606         0xf0b0, 0xffffffff, 0x00000100,
 607         0xf0b2, 0xffffffff, 0x00000100,
 608         0xf0b1, 0xffffffff, 0x00000100,
 609         0x1579, 0xffffffff, 0x00200100,
 610         0xf0a0, 0xffffffff, 0x00000100,
 611         0xf085, 0xffffffff, 0x06000100,
 612         0xf088, 0xffffffff, 0x00000100,
 613         0xf086, 0xffffffff, 0x06000100,
 614         0xf081, 0xffffffff, 0x00000100,
 615         0xf0b8, 0xffffffff, 0x00000100,
 616         0xf089, 0xffffffff, 0x00000100,
 617         0xf080, 0xffffffff, 0x00000100,
 618         0xf08c, 0xffffffff, 0x00000100,
 619         0xf08d, 0xffffffff, 0x00000100,
 620         0xf094, 0xffffffff, 0x00000100,
 621         0xf095, 0xffffffff, 0x00000100,
 622         0xf096, 0xffffffff, 0x00000100,
 623         0xf097, 0xffffffff, 0x00000100,
 624         0xf098, 0xffffffff, 0x00000100,
 625         0xf09f, 0xffffffff, 0x00000100,
 626         0xf09e, 0xffffffff, 0x00000100,
 627         0xf084, 0xffffffff, 0x06000100,
 628         0xf0a4, 0xffffffff, 0x00000100,
 629         0xf09d, 0xffffffff, 0x00000100,
 630         0xf0ad, 0xffffffff, 0x00000100,
 631         0xf0ac, 0xffffffff, 0x00000100,
 632         0xf09c, 0xffffffff, 0x00000100,
 633         0xc200, 0xffffffff, 0xe0000000,
 634         0xf008, 0xffffffff, 0x00010000,
 635         0xf009, 0xffffffff, 0x00030002,
 636         0xf00a, 0xffffffff, 0x00040007,
 637         0xf00b, 0xffffffff, 0x00060005,
 638         0xf00c, 0xffffffff, 0x00090008,
 639         0xf00d, 0xffffffff, 0x00010000,
 640         0xf00e, 0xffffffff, 0x00030002,
 641         0xf00f, 0xffffffff, 0x00040007,
 642         0xf010, 0xffffffff, 0x00060005,
 643         0xf011, 0xffffffff, 0x00090008,
 644         0xf012, 0xffffffff, 0x00010000,
 645         0xf013, 0xffffffff, 0x00030002,
 646         0xf014, 0xffffffff, 0x00040007,
 647         0xf015, 0xffffffff, 0x00060005,
 648         0xf016, 0xffffffff, 0x00090008,
 649         0xf017, 0xffffffff, 0x00010000,
 650         0xf018, 0xffffffff, 0x00030002,
 651         0xf019, 0xffffffff, 0x00040007,
 652         0xf01a, 0xffffffff, 0x00060005,
 653         0xf01b, 0xffffffff, 0x00090008,
 654         0xf01c, 0xffffffff, 0x00010000,
 655         0xf01d, 0xffffffff, 0x00030002,
 656         0xf01e, 0xffffffff, 0x00040007,
 657         0xf01f, 0xffffffff, 0x00060005,
 658         0xf020, 0xffffffff, 0x00090008,
 659         0xf021, 0xffffffff, 0x00010000,
 660         0xf022, 0xffffffff, 0x00030002,
 661         0xf023, 0xffffffff, 0x00040007,
 662         0xf024, 0xffffffff, 0x00060005,
 663         0xf025, 0xffffffff, 0x00090008,
 664         0xf026, 0xffffffff, 0x00010000,
 665         0xf027, 0xffffffff, 0x00030002,
 666         0xf028, 0xffffffff, 0x00040007,
 667         0xf029, 0xffffffff, 0x00060005,
 668         0xf02a, 0xffffffff, 0x00090008,
 669         0xf02b, 0xffffffff, 0x00010000,
 670         0xf02c, 0xffffffff, 0x00030002,
 671         0xf02d, 0xffffffff, 0x00040007,
 672         0xf02e, 0xffffffff, 0x00060005,
 673         0xf02f, 0xffffffff, 0x00090008,
 674         0xf030, 0xffffffff, 0x00010000,
 675         0xf031, 0xffffffff, 0x00030002,
 676         0xf032, 0xffffffff, 0x00040007,
 677         0xf033, 0xffffffff, 0x00060005,
 678         0xf034, 0xffffffff, 0x00090008,
 679         0xf035, 0xffffffff, 0x00010000,
 680         0xf036, 0xffffffff, 0x00030002,
 681         0xf037, 0xffffffff, 0x00040007,
 682         0xf038, 0xffffffff, 0x00060005,
 683         0xf039, 0xffffffff, 0x00090008,
 684         0xf03a, 0xffffffff, 0x00010000,
 685         0xf03b, 0xffffffff, 0x00030002,
 686         0xf03c, 0xffffffff, 0x00040007,
 687         0xf03d, 0xffffffff, 0x00060005,
 688         0xf03e, 0xffffffff, 0x00090008,
 689         0x30c6, 0xffffffff, 0x00020200,
 690         0xcd4, 0xffffffff, 0x00000200,
 691         0x570, 0xffffffff, 0x00000400,
 692         0x157a, 0xffffffff, 0x00000000,
 693         0xbd4, 0xffffffff, 0x00000902,
 694         0xf000, 0xffffffff, 0x96940200,
 695         0x21c2, 0xffffffff, 0x00900100,
 696         0x3109, 0xffffffff, 0x0020003f,
 697         0xe, 0xffffffff, 0x0140001c,
 698         0xf, 0x000f0000, 0x000f0000,
 699         0x88, 0xffffffff, 0xc060000c,
 700         0x89, 0xc0000fff, 0x00000100,
 701         0x3e4, 0xffffffff, 0x00000100,
 702         0x3e6, 0x00000101, 0x00000000,
 703         0x82a, 0xffffffff, 0x00000104,
 704         0x1579, 0xff000fff, 0x00000100,
 705         0xc33, 0xc0000fff, 0x00000104,
 706         0x3079, 0x00000001, 0x00000001,
 707         0x3403, 0xff000ff0, 0x00000100,
 708         0x3603, 0xff000ff0, 0x00000100
 709 };
 710
 711 static const u32 godavari_golden_registers[] =
 712 {
 713         0x1579, 0xff607fff, 0xfc000100,
 714         0x1bb6, 0x00010101, 0x00010000,
 715         0x260c, 0xffffffff, 0x00000000,
 716         0x260c0, 0xf00fffff, 0x00000400,
 717         0x184c, 0xffffffff, 0x00010000,
 718         0x16ec, 0x000000f0, 0x00000070,
 719         0x16f0, 0xf0311fff, 0x80300000,
 720         0x263e, 0x73773777, 0x12010001,
 721         0x263f, 0xffffffff, 0x00000010,
 722         0x200c, 0x00001f0f, 0x0000100a,
 723         0xbd2, 0x73773777, 0x12010001,
 724         0x902, 0x000fffff, 0x000c007f,
 725         0x2285, 0xf000003f, 0x00000007,
 726         0x22c9, 0xffffffff, 0x00ff0fff,
 727         0xc281, 0x0000ff0f, 0x00000000,
 728         0xa293, 0x07ffffff, 0x06000000,
 729         0x136, 0x00000fff, 0x00000100,
 730         0x3405, 0x00010000, 0x00810001,
 731         0x3605, 0x00010000, 0x00810001,
 732         0xf9e, 0x00000001, 0x00000002,
 733         0x31da, 0x00000008, 0x00000008,
 734         0x31dc, 0x00000f00, 0x00000800,
 735         0x31dd, 0x00000f00, 0x00000800,
 736         0x31e6, 0x00ffffff, 0x00ff7fbf,
 737         0x31e7, 0x00ffffff, 0x00ff7faf,
 738         0x2300, 0x000000ff, 0x00000001,
 739         0x853e, 0x01ff01ff, 0x00000002,
 740         0x8526, 0x007ff800, 0x00200000,
 741         0x8057, 0xffffffff, 0x00000f40,
 742         0x2231, 0x001f3ae3, 0x00000082,
 743         0x2235, 0x0000001f, 0x00000010,
 744         0xc24d, 0xffffffff, 0x00000000
 745 };
 746
 747 static void cik_init_golden_registers(struct amdgpu_device *adev)
 748 {
 749         /* Some of the registers might be dependent on GRBM_GFX_INDEX */
 750         mutex_lock(&adev->grbm_idx_mutex);
 751
 752         switch (adev->asic_type) {
 753         case CHIP_BONAIRE:
 754                 amdgpu_program_register_sequence(adev,
 755                                                  bonaire_mgcg_cgcg_init,
 756                                                  (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
 757                 amdgpu_program_register_sequence(adev,
 758                                                  bonaire_golden_registers,
 759                                                  (const u32)ARRAY_SIZE(bonaire_golden_registers));
 760                 amdgpu_program_register_sequence(adev,
 761                                                  bonaire_golden_common_registers,
 762                                                  (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
 763                 amdgpu_program_register_sequence(adev,
 764                                                  bonaire_golden_spm_registers,
 765                                                  (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
 766                 break;
 767         case CHIP_KABINI:
 768                 amdgpu_program_register_sequence(adev,
 769                                                  kalindi_mgcg_cgcg_init,
 770                                                  (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
 771                 amdgpu_program_register_sequence(adev,
 772                                                  kalindi_golden_registers,
 773                                                  (const u32)ARRAY_SIZE(kalindi_golden_registers));
 774                 amdgpu_program_register_sequence(adev,
 775                                                  kalindi_golden_common_registers,
 776                                                  (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
 777                 amdgpu_program_register_sequence(adev,
 778                                                  kalindi_golden_spm_registers,
 779                                                  (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
 780                 break;
 781         case CHIP_MULLINS:
 782                 amdgpu_program_register_sequence(adev,
 783                                                  kalindi_mgcg_cgcg_init,
 784                                                  (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
 785                 amdgpu_program_register_sequence(adev,
 786                                                  godavari_golden_registers,
 787                                                  (const u32)ARRAY_SIZE(godavari_golden_registers));
 788                 amdgpu_program_register_sequence(adev,
 789                                                  kalindi_golden_common_registers,
 790                                                  (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
 791                 amdgpu_program_register_sequence(adev,
 792                                                  kalindi_golden_spm_registers,
 793                                                  (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
 794                 break;
 795         case CHIP_KAVERI:
 796                 amdgpu_program_register_sequence(adev,
 797                                                  spectre_mgcg_cgcg_init,
 798                                                  (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
 799                 amdgpu_program_register_sequence(adev,
 800                                                  spectre_golden_registers,
 801                                                  (const u32)ARRAY_SIZE(spectre_golden_registers));
 802                 amdgpu_program_register_sequence(adev,
 803                                                  spectre_golden_common_registers,
 804                                                  (const u32)ARRAY_SIZE(spectre_golden_common_registers));
 805                 amdgpu_program_register_sequence(adev,
 806                                                  spectre_golden_spm_registers,
 807                                                  (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
 808                 break;
 809         case CHIP_HAWAII:
 810                 amdgpu_program_register_sequence(adev,
 811                                                  hawaii_mgcg_cgcg_init,
 812                                                  (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
 813                 amdgpu_program_register_sequence(adev,
 814                                                  hawaii_golden_registers,
 815                                                  (const u32)ARRAY_SIZE(hawaii_golden_registers));
 816                 amdgpu_program_register_sequence(adev,
 817                                                  hawaii_golden_common_registers,
 818                                                  (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
 819                 amdgpu_program_register_sequence(adev,
 820                                                  hawaii_golden_spm_registers,
 821                                                  (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
 822                 break;
 823         default:
 824                 break;
 825         }
 826         mutex_unlock(&adev->grbm_idx_mutex);
 827 }
 828
 829 /**
 830  * cik_get_xclk - get the xclk
 831  *
 832  * @adev: amdgpu_device pointer
 833  *
 834  * Returns the reference clock used by the gfx engine
 835  * (CIK).
 836  */
 837 static u32 cik_get_xclk(struct amdgpu_device *adev)
 838 {
 839         u32 reference_clock = adev->clock.spll.reference_freq;
 840
 841         if (adev->flags & AMD_IS_APU) {
 842                 if (RREG32_SMC(ixGENERAL_PWRMGT) & GENERAL_PWRMGT__GPU_COUNTER_CLK_MASK)
 843                         return reference_clock / 2;
 844         } else {
 845                 if (RREG32_SMC(ixCG_CLKPIN_CNTL) & CG_CLKPIN_CNTL__XTALIN_DIVIDE_MASK)
 846                         return reference_clock / 4;
 847         }
 848         return reference_clock;
 849 }
 850
 851 /**
 852  * cik_srbm_select - select specific register instances
 853  *
 854  * @adev: amdgpu_device pointer
 855  * @me: selected ME (micro engine)
 856  * @pipe: pipe
 857  * @queue: queue
 858  * @vmid: VMID
 859  *
 860  * Switches the currently active registers instances.  Some
 861  * registers are instanced per VMID, others are instanced per
 862  * me/pipe/queue combination.
 863  */
 864 void cik_srbm_select(struct amdgpu_device *adev,
 865                      u32 me, u32 pipe, u32 queue, u32 vmid)
 866 {
 867         u32 srbm_gfx_cntl =
 868                 (((pipe << SRBM_GFX_CNTL__PIPEID__SHIFT) & SRBM_GFX_CNTL__PIPEID_MASK)|
 869                 ((me << SRBM_GFX_CNTL__MEID__SHIFT) & SRBM_GFX_CNTL__MEID_MASK)|
 870                 ((vmid << SRBM_GFX_CNTL__VMID__SHIFT) & SRBM_GFX_CNTL__VMID_MASK)|
 871                 ((queue << SRBM_GFX_CNTL__QUEUEID__SHIFT) & SRBM_GFX_CNTL__QUEUEID_MASK));
 872         WREG32(mmSRBM_GFX_CNTL, srbm_gfx_cntl);
 873 }
 874
 875 static void cik_vga_set_state(struct amdgpu_device *adev, bool state)
 876 {
 877         uint32_t tmp;
 878
 879         tmp = RREG32(mmCONFIG_CNTL);
 880         if (state == false)
 881                 tmp |= CONFIG_CNTL__VGA_DIS_MASK;
 882         else
 883                 tmp &= ~CONFIG_CNTL__VGA_DIS_MASK;
 884         WREG32(mmCONFIG_CNTL, tmp);
 885 }
 886
 887 static bool cik_read_disabled_bios(struct amdgpu_device *adev)
 888 {
 889         u32 bus_cntl;
 890         u32 d1vga_control = 0;
 891         u32 d2vga_control = 0;
 892         u32 vga_render_control = 0;
 893         u32 rom_cntl;
 894         bool r;
 895
 896         bus_cntl = RREG32(mmBUS_CNTL);
 897         if (adev->mode_info.num_crtc) {
 898                 d1vga_control = RREG32(mmD1VGA_CONTROL);
 899                 d2vga_control = RREG32(mmD2VGA_CONTROL);
 900                 vga_render_control = RREG32(mmVGA_RENDER_CONTROL);
 901         }
 902         rom_cntl = RREG32_SMC(ixROM_CNTL);
 903
 904         /* enable the rom */
 905         WREG32(mmBUS_CNTL, (bus_cntl & ~BUS_CNTL__BIOS_ROM_DIS_MASK));
 906         if (adev->mode_info.num_crtc) {
 907                 /* Disable VGA mode */
 908                 WREG32(mmD1VGA_CONTROL,
 909                        (d1vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
 910                                           D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
 911                 WREG32(mmD2VGA_CONTROL,
 912                        (d2vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
 913                                           D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
 914                 WREG32(mmVGA_RENDER_CONTROL,
 915                        (vga_render_control & ~VGA_RENDER_CONTROL__VGA_VSTATUS_CNTL_MASK));
 916         }
 917         WREG32_SMC(ixROM_CNTL, rom_cntl | ROM_CNTL__SCK_OVERWRITE_MASK);
 918
 919         r = amdgpu_read_bios(adev);
 920
 921         /* restore regs */
 922         WREG32(mmBUS_CNTL, bus_cntl);
 923         if (adev->mode_info.num_crtc) {
 924                 WREG32(mmD1VGA_CONTROL, d1vga_control);
 925                 WREG32(mmD2VGA_CONTROL, d2vga_control);
 926                 WREG32(mmVGA_RENDER_CONTROL, vga_render_control);
 927         }
 928         WREG32_SMC(ixROM_CNTL, rom_cntl);
 929         return r;
 930 }
 931
 932 static struct amdgpu_allowed_register_entry cik_allowed_read_registers[] = {
 933         {mmGRBM_STATUS, false},
 934         {mmGB_ADDR_CONFIG, false},
 935         {mmMC_ARB_RAMCFG, false},
 936         {mmGB_TILE_MODE0, false},
 937         {mmGB_TILE_MODE1, false},
 938         {mmGB_TILE_MODE2, false},
 939         {mmGB_TILE_MODE3, false},
 940         {mmGB_TILE_MODE4, false},
 941         {mmGB_TILE_MODE5, false},
 942         {mmGB_TILE_MODE6, false},
 943         {mmGB_TILE_MODE7, false},
 944         {mmGB_TILE_MODE8, false},
 945         {mmGB_TILE_MODE9, false},
 946         {mmGB_TILE_MODE10, false},
 947         {mmGB_TILE_MODE11, false},
 948         {mmGB_TILE_MODE12, false},
 949         {mmGB_TILE_MODE13, false},
 950         {mmGB_TILE_MODE14, false},
 951         {mmGB_TILE_MODE15, false},
 952         {mmGB_TILE_MODE16, false},
 953         {mmGB_TILE_MODE17, false},
 954         {mmGB_TILE_MODE18, false},
 955         {mmGB_TILE_MODE19, false},
 956         {mmGB_TILE_MODE20, false},
 957         {mmGB_TILE_MODE21, false},
 958         {mmGB_TILE_MODE22, false},
 959         {mmGB_TILE_MODE23, false},
 960         {mmGB_TILE_MODE24, false},
 961         {mmGB_TILE_MODE25, false},
 962         {mmGB_TILE_MODE26, false},
 963         {mmGB_TILE_MODE27, false},
 964         {mmGB_TILE_MODE28, false},
 965         {mmGB_TILE_MODE29, false},
 966         {mmGB_TILE_MODE30, false},
 967         {mmGB_TILE_MODE31, false},
 968         {mmGB_MACROTILE_MODE0, false},
 969         {mmGB_MACROTILE_MODE1, false},
 970         {mmGB_MACROTILE_MODE2, false},
 971         {mmGB_MACROTILE_MODE3, false},
 972         {mmGB_MACROTILE_MODE4, false},
 973         {mmGB_MACROTILE_MODE5, false},
 974         {mmGB_MACROTILE_MODE6, false},
 975         {mmGB_MACROTILE_MODE7, false},
 976         {mmGB_MACROTILE_MODE8, false},
 977         {mmGB_MACROTILE_MODE9, false},
 978         {mmGB_MACROTILE_MODE10, false},
 979         {mmGB_MACROTILE_MODE11, false},
 980         {mmGB_MACROTILE_MODE12, false},
 981         {mmGB_MACROTILE_MODE13, false},
 982         {mmGB_MACROTILE_MODE14, false},
 983         {mmGB_MACROTILE_MODE15, false},
 984         {mmCC_RB_BACKEND_DISABLE, false, true},
 985         {mmGC_USER_RB_BACKEND_DISABLE, false, true},
 986         {mmGB_BACKEND_MAP, false, false},
 987         {mmPA_SC_RASTER_CONFIG, false, true},
 988         {mmPA_SC_RASTER_CONFIG_1, false, true},
 989 };
 990
 991 static uint32_t cik_read_indexed_register(struct amdgpu_device *adev,
 992                                           u32 se_num, u32 sh_num,
 993                                           u32 reg_offset)
 994 {
 995         uint32_t val;
 996
 997         mutex_lock(&adev->grbm_idx_mutex);
 998         if (se_num != 0xffffffff || sh_num != 0xffffffff)
 999                 gfx_v7_0_select_se_sh(adev, se_num, sh_num);
1000
1001         val = RREG32(reg_offset);
1002
1003         if (se_num != 0xffffffff || sh_num != 0xffffffff)
1004                 gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff);
1005         mutex_unlock(&adev->grbm_idx_mutex);
1006         return val;
1007 }
1008
1009 static int cik_read_register(struct amdgpu_device *adev, u32 se_num,
1010                              u32 sh_num, u32 reg_offset, u32 *value)
1011 {
1012         uint32_t i;
1013
1014         *value = 0;
1015         for (i = 0; i < ARRAY_SIZE(cik_allowed_read_registers); i++) {
1016                 if (reg_offset != cik_allowed_read_registers[i].reg_offset)
1017                         continue;
1018
1019                 if (!cik_allowed_read_registers[i].untouched)
1020                         *value = cik_allowed_read_registers[i].grbm_indexed ?
1021                                  cik_read_indexed_register(adev, se_num,
1022                                                            sh_num, reg_offset) :
1023                                  RREG32(reg_offset);
1024                 return 0;
1025         }
1026         return -EINVAL;
1027 }
1028
1029 static void cik_print_gpu_status_regs(struct amdgpu_device *adev)
1030 {
1031         dev_info(adev->dev, "  GRBM_STATUS=0x%08X\n",
1032                 RREG32(mmGRBM_STATUS));
1033         dev_info(adev->dev, "  GRBM_STATUS2=0x%08X\n",
1034                 RREG32(mmGRBM_STATUS2));
1035         dev_info(adev->dev, "  GRBM_STATUS_SE0=0x%08X\n",
1036                 RREG32(mmGRBM_STATUS_SE0));
1037         dev_info(adev->dev, "  GRBM_STATUS_SE1=0x%08X\n",
1038                 RREG32(mmGRBM_STATUS_SE1));
1039         dev_info(adev->dev, "  GRBM_STATUS_SE2=0x%08X\n",
1040                 RREG32(mmGRBM_STATUS_SE2));
1041         dev_info(adev->dev, "  GRBM_STATUS_SE3=0x%08X\n",
1042                 RREG32(mmGRBM_STATUS_SE3));
1043         dev_info(adev->dev, "  SRBM_STATUS=0x%08X\n",
1044                 RREG32(mmSRBM_STATUS));
1045         dev_info(adev->dev, "  SRBM_STATUS2=0x%08X\n",
1046                 RREG32(mmSRBM_STATUS2));
1047         dev_info(adev->dev, "  SDMA0_STATUS_REG   = 0x%08X\n",
1048                 RREG32(mmSDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
1049         dev_info(adev->dev, "  SDMA1_STATUS_REG   = 0x%08X\n",
1050                  RREG32(mmSDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
1051         dev_info(adev->dev, "  CP_STAT = 0x%08x\n", RREG32(mmCP_STAT));
1052         dev_info(adev->dev, "  CP_STALLED_STAT1 = 0x%08x\n",
1053                  RREG32(mmCP_STALLED_STAT1));
1054         dev_info(adev->dev, "  CP_STALLED_STAT2 = 0x%08x\n",
1055                  RREG32(mmCP_STALLED_STAT2));
1056         dev_info(adev->dev, "  CP_STALLED_STAT3 = 0x%08x\n",
1057                  RREG32(mmCP_STALLED_STAT3));
1058         dev_info(adev->dev, "  CP_CPF_BUSY_STAT = 0x%08x\n",
1059                  RREG32(mmCP_CPF_BUSY_STAT));
1060         dev_info(adev->dev, "  CP_CPF_STALLED_STAT1 = 0x%08x\n",
1061                  RREG32(mmCP_CPF_STALLED_STAT1));
1062         dev_info(adev->dev, "  CP_CPF_STATUS = 0x%08x\n", RREG32(mmCP_CPF_STATUS));
1063         dev_info(adev->dev, "  CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(mmCP_CPC_BUSY_STAT));
1064         dev_info(adev->dev, "  CP_CPC_STALLED_STAT1 = 0x%08x\n",
1065                  RREG32(mmCP_CPC_STALLED_STAT1));
1066         dev_info(adev->dev, "  CP_CPC_STATUS = 0x%08x\n", RREG32(mmCP_CPC_STATUS));
1067 }
1068
1069 /**
1070  * cik_gpu_check_soft_reset - check which blocks are busy
1071  *
1072  * @adev: amdgpu_device pointer
1073  *
1074  * Check which blocks are busy and return the relevant reset
1075  * mask to be used by cik_gpu_soft_reset().
1076  * Returns a mask of the blocks to be reset.
1077  */
1078 u32 amdgpu_cik_gpu_check_soft_reset(struct amdgpu_device *adev)
1079 {
1080         u32 reset_mask = 0;
1081         u32 tmp;
1082
1083         /* GRBM_STATUS */
1084         tmp = RREG32(mmGRBM_STATUS);
1085         if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK |
1086                    GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK |
1087                    GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK |
1088                    GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK |
1089                    GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK |
1090                    GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK))
1091                 reset_mask |= AMDGPU_RESET_GFX;
1092
1093         if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK))
1094                 reset_mask |= AMDGPU_RESET_CP;
1095
1096         /* GRBM_STATUS2 */
1097         tmp = RREG32(mmGRBM_STATUS2);
1098         if (tmp & GRBM_STATUS2__RLC_BUSY_MASK)
1099                 reset_mask |= AMDGPU_RESET_RLC;
1100
1101         /* SDMA0_STATUS_REG */
1102         tmp = RREG32(mmSDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
1103         if (!(tmp & SDMA0_STATUS_REG__IDLE_MASK))
1104                 reset_mask |= AMDGPU_RESET_DMA;
1105
1106         /* SDMA1_STATUS_REG */
1107         tmp = RREG32(mmSDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
1108         if (!(tmp & SDMA0_STATUS_REG__IDLE_MASK))
1109                 reset_mask |= AMDGPU_RESET_DMA1;
1110
1111         /* SRBM_STATUS2 */
1112         tmp = RREG32(mmSRBM_STATUS2);
1113         if (tmp & SRBM_STATUS2__SDMA_BUSY_MASK)
1114                 reset_mask |= AMDGPU_RESET_DMA;
1115
1116         if (tmp & SRBM_STATUS2__SDMA1_BUSY_MASK)
1117                 reset_mask |= AMDGPU_RESET_DMA1;
1118
1119         /* SRBM_STATUS */
1120         tmp = RREG32(mmSRBM_STATUS);
1121
1122         if (tmp & SRBM_STATUS__IH_BUSY_MASK)
1123                 reset_mask |= AMDGPU_RESET_IH;
1124
1125         if (tmp & SRBM_STATUS__SEM_BUSY_MASK)
1126                 reset_mask |= AMDGPU_RESET_SEM;
1127
1128         if (tmp & SRBM_STATUS__GRBM_RQ_PENDING_MASK)
1129                 reset_mask |= AMDGPU_RESET_GRBM;
1130
1131         if (tmp & SRBM_STATUS__VMC_BUSY_MASK)
1132                 reset_mask |= AMDGPU_RESET_VMC;
1133
1134         if (tmp & (SRBM_STATUS__MCB_BUSY_MASK | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
1135                    SRBM_STATUS__MCC_BUSY_MASK | SRBM_STATUS__MCD_BUSY_MASK))
1136                 reset_mask |= AMDGPU_RESET_MC;
1137
1138         if (amdgpu_display_is_display_hung(adev))
1139                 reset_mask |= AMDGPU_RESET_DISPLAY;
1140
1141         /* Skip MC reset as it's mostly likely not hung, just busy */
1142         if (reset_mask & AMDGPU_RESET_MC) {
1143                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
1144                 reset_mask &= ~AMDGPU_RESET_MC;
1145         }
1146
1147         return reset_mask;
1148 }
1149
1150 /**
1151  * cik_gpu_soft_reset - soft reset GPU
1152  *
1153  * @adev: amdgpu_device pointer
1154  * @reset_mask: mask of which blocks to reset
1155  *
1156  * Soft reset the blocks specified in @reset_mask.
1157  */
1158 static void cik_gpu_soft_reset(struct amdgpu_device *adev, u32 reset_mask)
1159 {
1160         struct amdgpu_mode_mc_save save;
1161         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
1162         u32 tmp;
1163
1164         if (reset_mask == 0)
1165                 return;
1166
1167         dev_info(adev->dev, "GPU softreset: 0x%08X\n", reset_mask);
1168
1169         cik_print_gpu_status_regs(adev);
1170         dev_info(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
1171                  RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_ADDR));
1172         dev_info(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
1173                  RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_STATUS));
1174
1175         /* disable CG/PG */
1176
1177         /* stop the rlc */
1178         gfx_v7_0_rlc_stop(adev);
1179
1180         /* Disable GFX parsing/prefetching */
1181         WREG32(mmCP_ME_CNTL, CP_ME_CNTL__ME_HALT_MASK | CP_ME_CNTL__PFP_HALT_MASK | CP_ME_CNTL__CE_HALT_MASK);
1182
1183         /* Disable MEC parsing/prefetching */
1184         WREG32(mmCP_MEC_CNTL, CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK);
1185
1186         if (reset_mask & AMDGPU_RESET_DMA) {
1187                 /* sdma0 */
1188                 tmp = RREG32(mmSDMA0_F32_CNTL + SDMA0_REGISTER_OFFSET);
1189                 tmp |= SDMA0_F32_CNTL__HALT_MASK;
1190                 WREG32(mmSDMA0_F32_CNTL + SDMA0_REGISTER_OFFSET, tmp);
1191         }
1192         if (reset_mask & AMDGPU_RESET_DMA1) {
1193                 /* sdma1 */
1194                 tmp = RREG32(mmSDMA0_F32_CNTL + SDMA1_REGISTER_OFFSET);
1195                 tmp |= SDMA0_F32_CNTL__HALT_MASK;
1196                 WREG32(mmSDMA0_F32_CNTL + SDMA1_REGISTER_OFFSET, tmp);
1197         }
1198
1199         gmc_v7_0_mc_stop(adev, &save);
1200         if (amdgpu_asic_wait_for_mc_idle(adev)) {
1201                 dev_warn(adev->dev, "Wait for MC idle timedout !\n");
1202         }
1203
1204         if (reset_mask & (AMDGPU_RESET_GFX | AMDGPU_RESET_COMPUTE | AMDGPU_RESET_CP))
1205                 grbm_soft_reset = GRBM_SOFT_RESET__SOFT_RESET_CP_MASK |
1206                         GRBM_SOFT_RESET__SOFT_RESET_GFX_MASK;
1207
1208         if (reset_mask & AMDGPU_RESET_CP) {
1209                 grbm_soft_reset |= GRBM_SOFT_RESET__SOFT_RESET_CP_MASK;
1210
1211                 srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_GRBM_MASK;
1212         }
1213
1214         if (reset_mask & AMDGPU_RESET_DMA)
1215                 srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_SDMA_MASK;
1216
1217         if (reset_mask & AMDGPU_RESET_DMA1)
1218                 srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_SDMA1_MASK;
1219
1220         if (reset_mask & AMDGPU_RESET_DISPLAY)
1221                 srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_DC_MASK;
1222
1223         if (reset_mask & AMDGPU_RESET_RLC)
1224                 grbm_soft_reset |= GRBM_SOFT_RESET__SOFT_RESET_RLC_MASK;
1225
1226         if (reset_mask & AMDGPU_RESET_SEM)
1227                 srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_SEM_MASK;
1228
1229         if (reset_mask & AMDGPU_RESET_IH)
1230                 srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_IH_MASK;
1231
1232         if (reset_mask & AMDGPU_RESET_GRBM)
1233                 srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_GRBM_MASK;
1234
1235         if (reset_mask & AMDGPU_RESET_VMC)
1236                 srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_VMC_MASK;
1237
1238         if (!(adev->flags & AMD_IS_APU)) {
1239                 if (reset_mask & AMDGPU_RESET_MC)
1240                         srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_MC_MASK;
1241         }
1242
1243         if (grbm_soft_reset) {
1244                 tmp = RREG32(mmGRBM_SOFT_RESET);
1245                 tmp |= grbm_soft_reset;
1246                 dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
1247                 WREG32(mmGRBM_SOFT_RESET, tmp);
1248                 tmp = RREG32(mmGRBM_SOFT_RESET);
1249
1250                 udelay(50);
1251
1252                 tmp &= ~grbm_soft_reset;
1253                 WREG32(mmGRBM_SOFT_RESET, tmp);
1254                 tmp = RREG32(mmGRBM_SOFT_RESET);
1255         }
1256
1257         if (srbm_soft_reset) {
1258                 tmp = RREG32(mmSRBM_SOFT_RESET);
1259                 tmp |= srbm_soft_reset;
1260                 dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1261                 WREG32(mmSRBM_SOFT_RESET, tmp);
1262                 tmp = RREG32(mmSRBM_SOFT_RESET);
1263
1264                 udelay(50);
1265
1266                 tmp &= ~srbm_soft_reset;
1267                 WREG32(mmSRBM_SOFT_RESET, tmp);
1268                 tmp = RREG32(mmSRBM_SOFT_RESET);
1269         }
1270
1271         /* Wait a little for things to settle down */
1272         udelay(50);
1273
1274         gmc_v7_0_mc_resume(adev, &save);
1275         udelay(50);
1276
1277         cik_print_gpu_status_regs(adev);
1278 }
1279
1280 struct kv_reset_save_regs {
1281         u32 gmcon_reng_execute;
1282         u32 gmcon_misc;
1283         u32 gmcon_misc3;
1284 };
1285
1286 static void kv_save_regs_for_reset(struct amdgpu_device *adev,
1287                                    struct kv_reset_save_regs *save)
1288 {
1289         save->gmcon_reng_execute = RREG32(mmGMCON_RENG_EXECUTE);
1290         save->gmcon_misc = RREG32(mmGMCON_MISC);
1291         save->gmcon_misc3 = RREG32(mmGMCON_MISC3);
1292
1293         WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute &
1294                 ~GMCON_RENG_EXECUTE__RENG_EXECUTE_ON_PWR_UP_MASK);
1295         WREG32(mmGMCON_MISC, save->gmcon_misc &
1296                 ~(GMCON_MISC__RENG_EXECUTE_ON_REG_UPDATE_MASK |
1297                         GMCON_MISC__STCTRL_STUTTER_EN_MASK));
1298 }
1299
1300 static void kv_restore_regs_for_reset(struct amdgpu_device *adev,
1301                                       struct kv_reset_save_regs *save)
1302 {
1303         int i;
1304
1305         WREG32(mmGMCON_PGFSM_WRITE, 0);
1306         WREG32(mmGMCON_PGFSM_CONFIG, 0x200010ff);
1307
1308         for (i = 0; i < 5; i++)
1309                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1310
1311         WREG32(mmGMCON_PGFSM_WRITE, 0);
1312         WREG32(mmGMCON_PGFSM_CONFIG, 0x300010ff);
1313
1314         for (i = 0; i < 5; i++)
1315                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1316
1317         WREG32(mmGMCON_PGFSM_WRITE, 0x210000);
1318         WREG32(mmGMCON_PGFSM_CONFIG, 0xa00010ff);
1319
1320         for (i = 0; i < 5; i++)
1321                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1322
1323         WREG32(mmGMCON_PGFSM_WRITE, 0x21003);
1324         WREG32(mmGMCON_PGFSM_CONFIG, 0xb00010ff);
1325
1326         for (i = 0; i < 5; i++)
1327                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1328
1329         WREG32(mmGMCON_PGFSM_WRITE, 0x2b00);
1330         WREG32(mmGMCON_PGFSM_CONFIG, 0xc00010ff);
1331
1332         for (i = 0; i < 5; i++)
1333                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1334
1335         WREG32(mmGMCON_PGFSM_WRITE, 0);
1336         WREG32(mmGMCON_PGFSM_CONFIG, 0xd00010ff);
1337
1338         for (i = 0; i < 5; i++)
1339                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1340
1341         WREG32(mmGMCON_PGFSM_WRITE, 0x420000);
1342         WREG32(mmGMCON_PGFSM_CONFIG, 0x100010ff);
1343
1344         for (i = 0; i < 5; i++)
1345                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1346
1347         WREG32(mmGMCON_PGFSM_WRITE, 0x120202);
1348         WREG32(mmGMCON_PGFSM_CONFIG, 0x500010ff);
1349
1350         for (i = 0; i < 5; i++)
1351                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1352
1353         WREG32(mmGMCON_PGFSM_WRITE, 0x3e3e36);
1354         WREG32(mmGMCON_PGFSM_CONFIG, 0x600010ff);
1355
1356         for (i = 0; i < 5; i++)
1357                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1358
1359         WREG32(mmGMCON_PGFSM_WRITE, 0x373f3e);
1360         WREG32(mmGMCON_PGFSM_CONFIG, 0x700010ff);
1361
1362         for (i = 0; i < 5; i++)
1363                 WREG32(mmGMCON_PGFSM_WRITE, 0);
1364
1365         WREG32(mmGMCON_PGFSM_WRITE, 0x3e1332);
1366         WREG32(mmGMCON_PGFSM_CONFIG, 0xe00010ff);
1367
1368         WREG32(mmGMCON_MISC3, save->gmcon_misc3);
1369         WREG32(mmGMCON_MISC, save->gmcon_misc);
1370         WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute);
1371 }
1372
1373 static void cik_gpu_pci_config_reset(struct amdgpu_device *adev)
1374 {
1375         struct amdgpu_mode_mc_save save;
1376         struct kv_reset_save_regs kv_save = { 0 };
1377         u32 tmp, i;
1378
1379         dev_info(adev->dev, "GPU pci config reset\n");
1380
1381         /* disable dpm? */
1382
1383         /* disable cg/pg */
1384
1385         /* Disable GFX parsing/prefetching */
1386         WREG32(mmCP_ME_CNTL, CP_ME_CNTL__ME_HALT_MASK |
1387                 CP_ME_CNTL__PFP_HALT_MASK | CP_ME_CNTL__CE_HALT_MASK);
1388
1389         /* Disable MEC parsing/prefetching */
1390         WREG32(mmCP_MEC_CNTL,
1391                         CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK);
1392
1393         /* sdma0 */
1394         tmp = RREG32(mmSDMA0_F32_CNTL + SDMA0_REGISTER_OFFSET);
1395         tmp |= SDMA0_F32_CNTL__HALT_MASK;
1396         WREG32(mmSDMA0_F32_CNTL + SDMA0_REGISTER_OFFSET, tmp);
1397         /* sdma1 */
1398         tmp = RREG32(mmSDMA0_F32_CNTL + SDMA1_REGISTER_OFFSET);
1399         tmp |= SDMA0_F32_CNTL__HALT_MASK;
1400         WREG32(mmSDMA0_F32_CNTL + SDMA1_REGISTER_OFFSET, tmp);
1401         /* XXX other engines? */
1402
1403         /* halt the rlc, disable cp internal ints */
1404         gfx_v7_0_rlc_stop(adev);
1405
1406         udelay(50);
1407
1408         /* disable mem access */
1409         gmc_v7_0_mc_stop(adev, &save);
1410         if (amdgpu_asic_wait_for_mc_idle(adev)) {
1411                 dev_warn(adev->dev, "Wait for MC idle timed out !\n");
1412         }
1413
1414         if (adev->flags & AMD_IS_APU)
1415                 kv_save_regs_for_reset(adev, &kv_save);
1416
1417         /* disable BM */
1418         pci_clear_master(adev->pdev);
1419         /* reset */
1420         amdgpu_pci_config_reset(adev);
1421
1422         udelay(100);
1423
1424         /* wait for asic to come out of reset */
1425         for (i = 0; i < adev->usec_timeout; i++) {
1426                 if (RREG32(mmCONFIG_MEMSIZE) != 0xffffffff)
1427                         break;
1428                 udelay(1);
1429         }
1430
1431         /* does asic init need to be run first??? */
1432         if (adev->flags & AMD_IS_APU)
1433                 kv_restore_regs_for_reset(adev, &kv_save);
1434 }
1435
1436 static void cik_set_bios_scratch_engine_hung(struct amdgpu_device *adev, bool hung)
1437 {
1438         u32 tmp = RREG32(mmBIOS_SCRATCH_3);
1439
1440         if (hung)
1441                 tmp |= ATOM_S3_ASIC_GUI_ENGINE_HUNG;
1442         else
1443                 tmp &= ~ATOM_S3_ASIC_GUI_ENGINE_HUNG;
1444
1445         WREG32(mmBIOS_SCRATCH_3, tmp);
1446 }
1447
1448 /**
1449  * cik_asic_reset - soft reset GPU
1450  *
1451  * @adev: amdgpu_device pointer
1452  *
1453  * Look up which blocks are hung and attempt
1454  * to reset them.
1455  * Returns 0 for success.
1456  */
1457 static int cik_asic_reset(struct amdgpu_device *adev)
1458 {
1459         u32 reset_mask;
1460
1461         reset_mask = amdgpu_cik_gpu_check_soft_reset(adev);
1462
1463         if (reset_mask)
1464                 cik_set_bios_scratch_engine_hung(adev, true);
1465
1466         /* try soft reset */
1467         cik_gpu_soft_reset(adev, reset_mask);
1468
1469         reset_mask = amdgpu_cik_gpu_check_soft_reset(adev);
1470
1471         /* try pci config reset */
1472         if (reset_mask && amdgpu_hard_reset)
1473                 cik_gpu_pci_config_reset(adev);
1474
1475         reset_mask = amdgpu_cik_gpu_check_soft_reset(adev);
1476
1477         if (!reset_mask)
1478                 cik_set_bios_scratch_engine_hung(adev, false);
1479
1480         return 0;
1481 }
1482
1483 static int cik_set_uvd_clock(struct amdgpu_device *adev, u32 clock,
1484                               u32 cntl_reg, u32 status_reg)
1485 {
1486         int r, i;
1487         struct atom_clock_dividers dividers;
1488         uint32_t tmp;
1489
1490         r = amdgpu_atombios_get_clock_dividers(adev,
1491                                                COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1492                                                clock, false, &dividers);
1493         if (r)
1494                 return r;
1495
1496         tmp = RREG32_SMC(cntl_reg);
1497         tmp &= ~(CG_DCLK_CNTL__DCLK_DIR_CNTL_EN_MASK |
1498                 CG_DCLK_CNTL__DCLK_DIVIDER_MASK);
1499         tmp |= dividers.post_divider;
1500         WREG32_SMC(cntl_reg, tmp);
1501
1502         for (i = 0; i < 100; i++) {
1503                 if (RREG32_SMC(status_reg) & CG_DCLK_STATUS__DCLK_STATUS_MASK)
1504                         break;
1505                 mdelay(10);
1506         }
1507         if (i == 100)
1508                 return -ETIMEDOUT;
1509
1510         return 0;
1511 }
1512
1513 static int cik_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
1514 {
1515         int r = 0;
1516
1517         r = cik_set_uvd_clock(adev, vclk, ixCG_VCLK_CNTL, ixCG_VCLK_STATUS);
1518         if (r)
1519                 return r;
1520
1521         r = cik_set_uvd_clock(adev, dclk, ixCG_DCLK_CNTL, ixCG_DCLK_STATUS);
1522         return r;
1523 }
1524
1525 static int cik_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
1526 {
1527         int r, i;
1528         struct atom_clock_dividers dividers;
1529         u32 tmp;
1530
1531         r = amdgpu_atombios_get_clock_dividers(adev,
1532                                                COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1533                                                ecclk, false, &dividers);
1534         if (r)
1535                 return r;
1536
1537         for (i = 0; i < 100; i++) {
1538                 if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1539                         break;
1540                 mdelay(10);
1541         }
1542         if (i == 100)
1543                 return -ETIMEDOUT;
1544
1545         tmp = RREG32_SMC(ixCG_ECLK_CNTL);
1546         tmp &= ~(CG_ECLK_CNTL__ECLK_DIR_CNTL_EN_MASK |
1547                 CG_ECLK_CNTL__ECLK_DIVIDER_MASK);
1548         tmp |= dividers.post_divider;
1549         WREG32_SMC(ixCG_ECLK_CNTL, tmp);
1550
1551         for (i = 0; i < 100; i++) {
1552                 if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1553                         break;
1554                 mdelay(10);
1555         }
1556         if (i == 100)
1557                 return -ETIMEDOUT;
1558
1559         return 0;
1560 }
1561
1562 static void cik_pcie_gen3_enable(struct amdgpu_device *adev)
1563 {
1564         struct pci_dev *root = adev->pdev->bus->self;
1565         int bridge_pos, gpu_pos;
1566         u32 speed_cntl, mask, current_data_rate;
1567         int ret, i;
1568         u16 tmp16;
1569
1570         if (amdgpu_pcie_gen2 == 0)
1571                 return;
1572
1573         if (adev->flags & AMD_IS_APU)
1574                 return;
1575
1576         ret = drm_pcie_get_speed_cap_mask(adev->ddev, &mask);
1577         if (ret != 0)
1578                 return;
1579
1580         if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
1581                 return;
1582
1583         speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1584         current_data_rate = (speed_cntl & PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK) >>
1585                 PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT;
1586         if (mask & DRM_PCIE_SPEED_80) {
1587                 if (current_data_rate == 2) {
1588                         DRM_INFO("PCIE gen 3 link speeds already enabled\n");
1589                         return;
1590                 }
1591                 DRM_INFO("enabling PCIE gen 3 link speeds, disable with amdgpu.pcie_gen2=0\n");
1592         } else if (mask & DRM_PCIE_SPEED_50) {
1593                 if (current_data_rate == 1) {
1594                         DRM_INFO("PCIE gen 2 link speeds already enabled\n");
1595                         return;
1596                 }
1597                 DRM_INFO("enabling PCIE gen 2 link speeds, disable with amdgpu.pcie_gen2=0\n");
1598         }
1599
1600         bridge_pos = pci_pcie_cap(root);
1601         if (!bridge_pos)
1602                 return;
1603
1604         gpu_pos = pci_pcie_cap(adev->pdev);
1605         if (!gpu_pos)
1606                 return;
1607
1608         if (mask & DRM_PCIE_SPEED_80) {
1609                 /* re-try equalization if gen3 is not already enabled */
1610                 if (current_data_rate != 2) {
1611                         u16 bridge_cfg, gpu_cfg;
1612                         u16 bridge_cfg2, gpu_cfg2;
1613                         u32 max_lw, current_lw, tmp;
1614
1615                         pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
1616                         pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
1617
1618                         tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
1619                         pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
1620
1621                         tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
1622                         pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
1623
1624                         tmp = RREG32_PCIE(ixPCIE_LC_STATUS1);
1625                         max_lw = (tmp & PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH_MASK) >>
1626                                 PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH__SHIFT;
1627                         current_lw = (tmp & PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH_MASK)
1628                                 >> PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH__SHIFT;
1629
1630                         if (current_lw < max_lw) {
1631                                 tmp = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1632                                 if (tmp & PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATION_SUPPORT_MASK) {
1633                                         tmp &= ~(PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_MASK |
1634                                                 PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_DIS_MASK);
1635                                         tmp |= (max_lw <<
1636                                                 PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH__SHIFT);
1637                                         tmp |= PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_SUPPORT_MASK |
1638                                         PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATE_EN_MASK |
1639                                         PCIE_LC_LINK_WIDTH_CNTL__LC_RECONFIG_NOW_MASK;
1640                                         WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, tmp);
1641                                 }
1642                         }
1643
1644                         for (i = 0; i < 10; i++) {
1645                                 /* check status */
1646                                 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
1647                                 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
1648                                         break;
1649
1650                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
1651                                 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
1652
1653                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
1654                                 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
1655
1656                                 tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1657                                 tmp |= PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1658                                 WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1659
1660                                 tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1661                                 tmp |= PCIE_LC_CNTL4__LC_REDO_EQ_MASK;
1662                                 WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1663
1664                                 mdelay(100);
1665
1666                                 /* linkctl */
1667                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
1668                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1669                                 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
1670                                 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
1671
1672                                 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
1673                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1674                                 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
1675                                 pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
1676
1677                                 /* linkctl2 */
1678                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
1679                                 tmp16 &= ~((1 << 4) | (7 << 9));
1680                                 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
1681                                 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
1682
1683                                 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
1684                                 tmp16 &= ~((1 << 4) | (7 << 9));
1685                                 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
1686                                 pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
1687
1688                                 tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1689                                 tmp &= ~PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1690                                 WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1691                         }
1692                 }
1693         }
1694
1695         /* set the link speed */
1696         speed_cntl |= PCIE_LC_SPEED_CNTL__LC_FORCE_EN_SW_SPEED_CHANGE_MASK |
1697                 PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_HW_SPEED_CHANGE_MASK;
1698         speed_cntl &= ~PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_SW_SPEED_CHANGE_MASK;
1699         WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1700
1701         pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
1702         tmp16 &= ~0xf;
1703         if (mask & DRM_PCIE_SPEED_80)
1704                 tmp16 |= 3; /* gen3 */
1705         else if (mask & DRM_PCIE_SPEED_50)
1706                 tmp16 |= 2; /* gen2 */
1707         else
1708                 tmp16 |= 1; /* gen1 */
1709         pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
1710
1711         speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1712         speed_cntl |= PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK;
1713         WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1714
1715         for (i = 0; i < adev->usec_timeout; i++) {
1716                 speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1717                 if ((speed_cntl & PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK) == 0)
1718                         break;
1719                 udelay(1);
1720         }
1721 }
1722
1723 static void cik_program_aspm(struct amdgpu_device *adev)
1724 {
1725         u32 data, orig;
1726         bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
1727         bool disable_clkreq = false;
1728
1729         if (amdgpu_aspm == 0)
1730                 return;
1731
1732         /* XXX double check APUs */
1733         if (adev->flags & AMD_IS_APU)
1734                 return;
1735
1736         orig = data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1737         data &= ~PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_MASK;
1738         data |= (0x24 << PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS__SHIFT) |
1739                 PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_OVERRIDE_EN_MASK;
1740         if (orig != data)
1741                 WREG32_PCIE(ixPCIE_LC_N_FTS_CNTL, data);
1742
1743         orig = data = RREG32_PCIE(ixPCIE_LC_CNTL3);
1744         data |= PCIE_LC_CNTL3__LC_GO_TO_RECOVERY_MASK;
1745         if (orig != data)
1746                 WREG32_PCIE(ixPCIE_LC_CNTL3, data);
1747
1748         orig = data = RREG32_PCIE(ixPCIE_P_CNTL);
1749         data |= PCIE_P_CNTL__P_IGNORE_EDB_ERR_MASK;
1750         if (orig != data)
1751                 WREG32_PCIE(ixPCIE_P_CNTL, data);
1752
1753         orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1754         data &= ~(PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK |
1755                 PCIE_LC_CNTL__LC_L1_INACTIVITY_MASK);
1756         data |= PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1757         if (!disable_l0s)
1758                 data |= (7 << PCIE_LC_CNTL__LC_L0S_INACTIVITY__SHIFT);
1759
1760         if (!disable_l1) {
1761                 data |= (7 << PCIE_LC_CNTL__LC_L1_INACTIVITY__SHIFT);
1762                 data &= ~PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1763                 if (orig != data)
1764                         WREG32_PCIE(ixPCIE_LC_CNTL, data);
1765
1766                 if (!disable_plloff_in_l1) {
1767                         bool clk_req_support;
1768
1769                         orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_0);
1770                         data &= ~(PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1771                                 PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1772                         data |= (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1773                                 (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1774                         if (orig != data)
1775                                 WREG32_PCIE(ixPB0_PIF_PWRDOWN_0, data);
1776
1777                         orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_1);
1778                         data &= ~(PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1779                                 PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1780                         data |= (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1781                                 (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1782                         if (orig != data)
1783                                 WREG32_PCIE(ixPB0_PIF_PWRDOWN_1, data);
1784
1785                         orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_0);
1786                         data &= ~(PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1787                                 PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1788                         data |= (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1789                                 (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1790                         if (orig != data)
1791                                 WREG32_PCIE(ixPB1_PIF_PWRDOWN_0, data);
1792
1793                         orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_1);
1794                         data &= ~(PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1795                                 PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1796                         data |= (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1797                                 (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1798                         if (orig != data)
1799                                 WREG32_PCIE(ixPB1_PIF_PWRDOWN_1, data);
1800
1801                         orig = data = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1802                         data &= ~PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE_MASK;
1803                         data |= ~(3 << PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE__SHIFT);
1804                         if (orig != data)
1805                                 WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, data);
1806
1807                         if (!disable_clkreq) {
1808                                 struct pci_dev *root = adev->pdev->bus->self;
1809                                 u32 lnkcap;
1810
1811                                 clk_req_support = false;
1812                                 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
1813                                 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
1814                                         clk_req_support = true;
1815                         } else {
1816                                 clk_req_support = false;
1817                         }
1818
1819                         if (clk_req_support) {
1820                                 orig = data = RREG32_PCIE(ixPCIE_LC_CNTL2);
1821                                 data |= PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L1_MASK |
1822                                         PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L23_MASK;
1823                                 if (orig != data)
1824                                         WREG32_PCIE(ixPCIE_LC_CNTL2, data);
1825
1826                                 orig = data = RREG32_SMC(ixTHM_CLK_CNTL);
1827                                 data &= ~(THM_CLK_CNTL__CMON_CLK_SEL_MASK |
1828                                         THM_CLK_CNTL__TMON_CLK_SEL_MASK);
1829                                 data |= (1 << THM_CLK_CNTL__CMON_CLK_SEL__SHIFT) |
1830                                         (1 << THM_CLK_CNTL__TMON_CLK_SEL__SHIFT);
1831                                 if (orig != data)
1832                                         WREG32_SMC(ixTHM_CLK_CNTL, data);
1833
1834                                 orig = data = RREG32_SMC(ixMISC_CLK_CTRL);
1835                                 data &= ~(MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL_MASK |
1836                                         MISC_CLK_CTRL__ZCLK_SEL_MASK);
1837                                 data |= (1 << MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL__SHIFT) |
1838                                         (1 << MISC_CLK_CTRL__ZCLK_SEL__SHIFT);
1839                                 if (orig != data)
1840                                         WREG32_SMC(ixMISC_CLK_CTRL, data);
1841
1842                                 orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL);
1843                                 data &= ~CG_CLKPIN_CNTL__BCLK_AS_XCLK_MASK;
1844                                 if (orig != data)
1845                                         WREG32_SMC(ixCG_CLKPIN_CNTL, data);
1846
1847                                 orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL_2);
1848                                 data &= ~CG_CLKPIN_CNTL_2__FORCE_BIF_REFCLK_EN_MASK;
1849                                 if (orig != data)
1850                                         WREG32_SMC(ixCG_CLKPIN_CNTL_2, data);
1851
1852                                 orig = data = RREG32_SMC(ixMPLL_BYPASSCLK_SEL);
1853                                 data &= ~MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL_MASK;
1854                                 data |= (4 << MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL__SHIFT);
1855                                 if (orig != data)
1856                                         WREG32_SMC(ixMPLL_BYPASSCLK_SEL, data);
1857                         }
1858                 }
1859         } else {
1860                 if (orig != data)
1861                         WREG32_PCIE(ixPCIE_LC_CNTL, data);
1862         }
1863
1864         orig = data = RREG32_PCIE(ixPCIE_CNTL2);
1865         data |= PCIE_CNTL2__SLV_MEM_LS_EN_MASK |
1866                 PCIE_CNTL2__MST_MEM_LS_EN_MASK |
1867                 PCIE_CNTL2__REPLAY_MEM_LS_EN_MASK;
1868         if (orig != data)
1869                 WREG32_PCIE(ixPCIE_CNTL2, data);
1870
1871         if (!disable_l0s) {
1872                 data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1873                 if ((data & PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) ==
1874                                 PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) {
1875                         data = RREG32_PCIE(ixPCIE_LC_STATUS1);
1876                         if ((data & PCIE_LC_STATUS1__LC_REVERSE_XMIT_MASK) &&
1877                         (data & PCIE_LC_STATUS1__LC_REVERSE_RCVR_MASK)) {
1878                                 orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1879                                 data &= ~PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK;
1880                                 if (orig != data)
1881                                         WREG32_PCIE(ixPCIE_LC_CNTL, data);
1882                         }
1883                 }
1884         }
1885 }
1886
1887 static uint32_t cik_get_rev_id(struct amdgpu_device *adev)
1888 {
1889         return (RREG32(mmCC_DRM_ID_STRAPS) & CC_DRM_ID_STRAPS__ATI_REV_ID_MASK)
1890                 >> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT;
1891 }
1892
1893 static const struct amdgpu_ip_block_version bonaire_ip_blocks[] =
1894 {
1895         /* ORDER MATTERS! */
1896         {
1897                 .type = AMD_IP_BLOCK_TYPE_COMMON,
1898                 .major = 1,
1899                 .minor = 0,
1900                 .rev = 0,
1901                 .funcs = &cik_common_ip_funcs,
1902         },
1903         {
1904                 .type = AMD_IP_BLOCK_TYPE_GMC,
1905                 .major = 7,
1906                 .minor = 0,
1907                 .rev = 0,
1908                 .funcs = &gmc_v7_0_ip_funcs,
1909         },
1910         {
1911                 .type = AMD_IP_BLOCK_TYPE_IH,
1912                 .major = 2,
1913                 .minor = 0,
1914                 .rev = 0,
1915                 .funcs = &cik_ih_ip_funcs,
1916         },
1917         {
1918                 .type = AMD_IP_BLOCK_TYPE_SMC,
1919                 .major = 7,
1920                 .minor = 0,
1921                 .rev = 0,
1922                 .funcs = &ci_dpm_ip_funcs,
1923         },
1924         {
1925                 .type = AMD_IP_BLOCK_TYPE_DCE,
1926                 .major = 8,
1927                 .minor = 2,
1928                 .rev = 0,
1929                 .funcs = &dce_v8_0_ip_funcs,
1930         },
1931         {
1932                 .type = AMD_IP_BLOCK_TYPE_GFX,
1933                 .major = 7,
1934                 .minor = 2,
1935                 .rev = 0,
1936                 .funcs = &gfx_v7_0_ip_funcs,
1937         },
1938         {
1939                 .type = AMD_IP_BLOCK_TYPE_SDMA,
1940                 .major = 2,
1941                 .minor = 0,
1942                 .rev = 0,
1943                 .funcs = &cik_sdma_ip_funcs,
1944         },
1945         {
1946                 .type = AMD_IP_BLOCK_TYPE_UVD,
1947                 .major = 4,
1948                 .minor = 2,
1949                 .rev = 0,
1950                 .funcs = &uvd_v4_2_ip_funcs,
1951         },
1952         {
1953                 .type = AMD_IP_BLOCK_TYPE_VCE,
1954                 .major = 2,
1955                 .minor = 0,
1956                 .rev = 0,
1957                 .funcs = &vce_v2_0_ip_funcs,
1958         },
1959 };
1960
1961 static const struct amdgpu_ip_block_version hawaii_ip_blocks[] =
1962 {
1963         /* ORDER MATTERS! */
1964         {
1965                 .type = AMD_IP_BLOCK_TYPE_COMMON,
1966                 .major = 1,
1967                 .minor = 0,
1968                 .rev = 0,
1969                 .funcs = &cik_common_ip_funcs,
1970         },
1971         {
1972                 .type = AMD_IP_BLOCK_TYPE_GMC,
1973                 .major = 7,
1974                 .minor = 0,
1975                 .rev = 0,
1976                 .funcs = &gmc_v7_0_ip_funcs,
1977         },
1978         {
1979                 .type = AMD_IP_BLOCK_TYPE_IH,
1980                 .major = 2,
1981                 .minor = 0,
1982                 .rev = 0,
1983                 .funcs = &cik_ih_ip_funcs,
1984         },
1985         {
1986                 .type = AMD_IP_BLOCK_TYPE_SMC,
1987                 .major = 7,
1988                 .minor = 0,
1989                 .rev = 0,
1990                 .funcs = &ci_dpm_ip_funcs,
1991         },
1992         {
1993                 .type = AMD_IP_BLOCK_TYPE_DCE,
1994                 .major = 8,
1995                 .minor = 5,
1996                 .rev = 0,
1997                 .funcs = &dce_v8_0_ip_funcs,
1998         },
1999         {
2000                 .type = AMD_IP_BLOCK_TYPE_GFX,
2001                 .major = 7,
2002                 .minor = 3,
2003                 .rev = 0,
2004                 .funcs = &gfx_v7_0_ip_funcs,
2005         },
2006         {
2007                 .type = AMD_IP_BLOCK_TYPE_SDMA,
2008                 .major = 2,
2009                 .minor = 0,
2010                 .rev = 0,
2011                 .funcs = &cik_sdma_ip_funcs,
2012         },
2013         {
2014                 .type = AMD_IP_BLOCK_TYPE_UVD,
2015                 .major = 4,
2016                 .minor = 2,
2017                 .rev = 0,
2018                 .funcs = &uvd_v4_2_ip_funcs,
2019         },
2020         {
2021                 .type = AMD_IP_BLOCK_TYPE_VCE,
2022                 .major = 2,
2023                 .minor = 0,
2024                 .rev = 0,
2025                 .funcs = &vce_v2_0_ip_funcs,
2026         },
2027 };
2028
2029 static const struct amdgpu_ip_block_version kabini_ip_blocks[] =
2030 {
2031         /* ORDER MATTERS! */
2032         {
2033                 .type = AMD_IP_BLOCK_TYPE_COMMON,
2034                 .major = 1,
2035                 .minor = 0,
2036                 .rev = 0,
2037                 .funcs = &cik_common_ip_funcs,
2038         },
2039         {
2040                 .type = AMD_IP_BLOCK_TYPE_GMC,
2041                 .major = 7,
2042                 .minor = 0,
2043                 .rev = 0,
2044                 .funcs = &gmc_v7_0_ip_funcs,
2045         },
2046         {
2047                 .type = AMD_IP_BLOCK_TYPE_IH,
2048                 .major = 2,
2049                 .minor = 0,
2050                 .rev = 0,
2051                 .funcs = &cik_ih_ip_funcs,
2052         },
2053         {
2054                 .type = AMD_IP_BLOCK_TYPE_SMC,
2055                 .major = 7,
2056                 .minor = 0,
2057                 .rev = 0,
2058                 .funcs = &kv_dpm_ip_funcs,
2059         },
2060         {
2061                 .type = AMD_IP_BLOCK_TYPE_DCE,
2062                 .major = 8,
2063                 .minor = 3,
2064                 .rev = 0,
2065                 .funcs = &dce_v8_0_ip_funcs,
2066         },
2067         {
2068                 .type = AMD_IP_BLOCK_TYPE_GFX,
2069                 .major = 7,
2070                 .minor = 2,
2071                 .rev = 0,
2072                 .funcs = &gfx_v7_0_ip_funcs,
2073         },
2074         {
2075                 .type = AMD_IP_BLOCK_TYPE_SDMA,
2076                 .major = 2,
2077                 .minor = 0,
2078                 .rev = 0,
2079                 .funcs = &cik_sdma_ip_funcs,
2080         },
2081         {
2082                 .type = AMD_IP_BLOCK_TYPE_UVD,
2083                 .major = 4,
2084                 .minor = 2,
2085                 .rev = 0,
2086                 .funcs = &uvd_v4_2_ip_funcs,
2087         },
2088         {
2089                 .type = AMD_IP_BLOCK_TYPE_VCE,
2090                 .major = 2,
2091                 .minor = 0,
2092                 .rev = 0,
2093                 .funcs = &vce_v2_0_ip_funcs,
2094         },
2095 };
2096
2097 static const struct amdgpu_ip_block_version mullins_ip_blocks[] =
2098 {
2099         /* ORDER MATTERS! */
2100         {
2101                 .type = AMD_IP_BLOCK_TYPE_COMMON,
2102                 .major = 1,
2103                 .minor = 0,
2104                 .rev = 0,
2105                 .funcs = &cik_common_ip_funcs,
2106         },
2107         {
2108                 .type = AMD_IP_BLOCK_TYPE_GMC,
2109                 .major = 7,
2110                 .minor = 0,
2111                 .rev = 0,
2112                 .funcs = &gmc_v7_0_ip_funcs,
2113         },
2114         {
2115                 .type = AMD_IP_BLOCK_TYPE_IH,
2116                 .major = 2,
2117                 .minor = 0,
2118                 .rev = 0,
2119                 .funcs = &cik_ih_ip_funcs,
2120         },
2121         {
2122                 .type = AMD_IP_BLOCK_TYPE_SMC,
2123                 .major = 7,
2124                 .minor = 0,
2125                 .rev = 0,
2126                 .funcs = &kv_dpm_ip_funcs,
2127         },
2128         {
2129                 .type = AMD_IP_BLOCK_TYPE_DCE,
2130                 .major = 8,
2131                 .minor = 3,
2132                 .rev = 0,
2133                 .funcs = &dce_v8_0_ip_funcs,
2134         },
2135         {
2136                 .type = AMD_IP_BLOCK_TYPE_GFX,
2137                 .major = 7,
2138                 .minor = 2,
2139                 .rev = 0,
2140                 .funcs = &gfx_v7_0_ip_funcs,
2141         },
2142         {
2143                 .type = AMD_IP_BLOCK_TYPE_SDMA,
2144                 .major = 2,
2145                 .minor = 0,
2146                 .rev = 0,
2147                 .funcs = &cik_sdma_ip_funcs,
2148         },
2149         {
2150                 .type = AMD_IP_BLOCK_TYPE_UVD,
2151                 .major = 4,
2152                 .minor = 2,
2153                 .rev = 0,
2154                 .funcs = &uvd_v4_2_ip_funcs,
2155         },
2156         {
2157                 .type = AMD_IP_BLOCK_TYPE_VCE,
2158                 .major = 2,
2159                 .minor = 0,
2160                 .rev = 0,
2161                 .funcs = &vce_v2_0_ip_funcs,
2162         },
2163 };
2164
2165 static const struct amdgpu_ip_block_version kaveri_ip_blocks[] =
2166 {
2167         /* ORDER MATTERS! */
2168         {
2169                 .type = AMD_IP_BLOCK_TYPE_COMMON,
2170                 .major = 1,
2171                 .minor = 0,
2172                 .rev = 0,
2173                 .funcs = &cik_common_ip_funcs,
2174         },
2175         {
2176                 .type = AMD_IP_BLOCK_TYPE_GMC,
2177                 .major = 7,
2178                 .minor = 0,
2179                 .rev = 0,
2180                 .funcs = &gmc_v7_0_ip_funcs,
2181         },
2182         {
2183                 .type = AMD_IP_BLOCK_TYPE_IH,
2184                 .major = 2,
2185                 .minor = 0,
2186                 .rev = 0,
2187                 .funcs = &cik_ih_ip_funcs,
2188         },
2189         {
2190                 .type = AMD_IP_BLOCK_TYPE_SMC,
2191                 .major = 7,
2192                 .minor = 0,
2193                 .rev = 0,
2194                 .funcs = &kv_dpm_ip_funcs,
2195         },
2196         {
2197                 .type = AMD_IP_BLOCK_TYPE_DCE,
2198                 .major = 8,
2199                 .minor = 1,
2200                 .rev = 0,
2201                 .funcs = &dce_v8_0_ip_funcs,
2202         },
2203         {
2204                 .type = AMD_IP_BLOCK_TYPE_GFX,
2205                 .major = 7,
2206                 .minor = 1,
2207                 .rev = 0,
2208                 .funcs = &gfx_v7_0_ip_funcs,
2209         },
2210         {
2211                 .type = AMD_IP_BLOCK_TYPE_SDMA,
2212                 .major = 2,
2213                 .minor = 0,
2214                 .rev = 0,
2215                 .funcs = &cik_sdma_ip_funcs,
2216         },
2217         {
2218                 .type = AMD_IP_BLOCK_TYPE_UVD,
2219                 .major = 4,
2220                 .minor = 2,
2221                 .rev = 0,
2222                 .funcs = &uvd_v4_2_ip_funcs,
2223         },
2224         {
2225                 .type = AMD_IP_BLOCK_TYPE_VCE,
2226                 .major = 2,
2227                 .minor = 0,
2228                 .rev = 0,
2229                 .funcs = &vce_v2_0_ip_funcs,
2230         },
2231 };
2232
2233 int cik_set_ip_blocks(struct amdgpu_device *adev)
2234 {
2235         switch (adev->asic_type) {
2236         case CHIP_BONAIRE:
2237                 adev->ip_blocks = bonaire_ip_blocks;
2238                 adev->num_ip_blocks = ARRAY_SIZE(bonaire_ip_blocks);
2239                 break;
2240         case CHIP_HAWAII:
2241                 adev->ip_blocks = hawaii_ip_blocks;
2242                 adev->num_ip_blocks = ARRAY_SIZE(hawaii_ip_blocks);
2243                 break;
2244         case CHIP_KAVERI:
2245                 adev->ip_blocks = kaveri_ip_blocks;
2246                 adev->num_ip_blocks = ARRAY_SIZE(kaveri_ip_blocks);
2247                 break;
2248         case CHIP_KABINI:
2249                 adev->ip_blocks = kabini_ip_blocks;
2250                 adev->num_ip_blocks = ARRAY_SIZE(kabini_ip_blocks);
2251                 break;
2252         case CHIP_MULLINS:
2253                 adev->ip_blocks = mullins_ip_blocks;
2254                 adev->num_ip_blocks = ARRAY_SIZE(mullins_ip_blocks);
2255                 break;
2256         default:
2257                 /* FIXME: not supported yet */
2258                 return -EINVAL;
2259         }
2260
2261         return 0;
2262 }
2263
2264 static const struct amdgpu_asic_funcs cik_asic_funcs =
2265 {
2266         .read_disabled_bios = &cik_read_disabled_bios,
2267         .read_register = &cik_read_register,
2268         .reset = &cik_asic_reset,
2269         .set_vga_state = &cik_vga_set_state,
2270         .get_xclk = &cik_get_xclk,
2271         .set_uvd_clocks = &cik_set_uvd_clocks,
2272         .set_vce_clocks = &cik_set_vce_clocks,
2273         .get_cu_info = &gfx_v7_0_get_cu_info,
2274         /* these should be moved to their own ip modules */
2275         .get_gpu_clock_counter = &gfx_v7_0_get_gpu_clock_counter,
2276         .wait_for_mc_idle = &gmc_v7_0_mc_wait_for_idle,
2277 };
2278
2279 static int cik_common_early_init(void *handle)
2280 {
2281         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2282
2283         adev->smc_rreg = &cik_smc_rreg;
2284         adev->smc_wreg = &cik_smc_wreg;
2285         adev->pcie_rreg = &cik_pcie_rreg;
2286         adev->pcie_wreg = &cik_pcie_wreg;
2287         adev->uvd_ctx_rreg = &cik_uvd_ctx_rreg;
2288         adev->uvd_ctx_wreg = &cik_uvd_ctx_wreg;
2289         adev->didt_rreg = &cik_didt_rreg;
2290         adev->didt_wreg = &cik_didt_wreg;
2291
2292         adev->asic_funcs = &cik_asic_funcs;
2293
2294         adev->has_uvd = true;
2295
2296         adev->rev_id = cik_get_rev_id(adev);
2297         adev->external_rev_id = 0xFF;
2298         switch (adev->asic_type) {
2299         case CHIP_BONAIRE:
2300                 adev->cg_flags =
2301                         AMDGPU_CG_SUPPORT_GFX_MGCG |
2302                         AMDGPU_CG_SUPPORT_GFX_MGLS |
2303                         /*AMDGPU_CG_SUPPORT_GFX_CGCG |*/
2304                         AMDGPU_CG_SUPPORT_GFX_CGLS |
2305                         AMDGPU_CG_SUPPORT_GFX_CGTS |
2306                         AMDGPU_CG_SUPPORT_GFX_CGTS_LS |
2307                         AMDGPU_CG_SUPPORT_GFX_CP_LS |
2308                         AMDGPU_CG_SUPPORT_MC_LS |
2309                         AMDGPU_CG_SUPPORT_MC_MGCG |
2310                         AMDGPU_CG_SUPPORT_SDMA_MGCG |
2311                         AMDGPU_CG_SUPPORT_SDMA_LS |
2312                         AMDGPU_CG_SUPPORT_BIF_LS |
2313                         AMDGPU_CG_SUPPORT_VCE_MGCG |
2314                         AMDGPU_CG_SUPPORT_UVD_MGCG |
2315                         AMDGPU_CG_SUPPORT_HDP_LS |
2316                         AMDGPU_CG_SUPPORT_HDP_MGCG;
2317                 adev->pg_flags = 0;
2318                 adev->external_rev_id = adev->rev_id + 0x14;
2319                 break;
2320         case CHIP_HAWAII:
2321                 adev->cg_flags =
2322                         AMDGPU_CG_SUPPORT_GFX_MGCG |
2323                         AMDGPU_CG_SUPPORT_GFX_MGLS |
2324                         /*AMDGPU_CG_SUPPORT_GFX_CGCG |*/
2325                         AMDGPU_CG_SUPPORT_GFX_CGLS |
2326                         AMDGPU_CG_SUPPORT_GFX_CGTS |
2327                         AMDGPU_CG_SUPPORT_GFX_CP_LS |
2328                         AMDGPU_CG_SUPPORT_MC_LS |
2329                         AMDGPU_CG_SUPPORT_MC_MGCG |
2330                         AMDGPU_CG_SUPPORT_SDMA_MGCG |
2331                         AMDGPU_CG_SUPPORT_SDMA_LS |
2332                         AMDGPU_CG_SUPPORT_BIF_LS |
2333                         AMDGPU_CG_SUPPORT_VCE_MGCG |
2334                         AMDGPU_CG_SUPPORT_UVD_MGCG |
2335                         AMDGPU_CG_SUPPORT_HDP_LS |
2336                         AMDGPU_CG_SUPPORT_HDP_MGCG;
2337                 adev->pg_flags = 0;
2338                 adev->external_rev_id = 0x28;
2339                 break;
2340         case CHIP_KAVERI:
2341                 adev->cg_flags =
2342                         AMDGPU_CG_SUPPORT_GFX_MGCG |
2343                         AMDGPU_CG_SUPPORT_GFX_MGLS |
2344                         /*AMDGPU_CG_SUPPORT_GFX_CGCG |*/
2345                         AMDGPU_CG_SUPPORT_GFX_CGLS |
2346                         AMDGPU_CG_SUPPORT_GFX_CGTS |
2347                         AMDGPU_CG_SUPPORT_GFX_CGTS_LS |
2348                         AMDGPU_CG_SUPPORT_GFX_CP_LS |
2349                         AMDGPU_CG_SUPPORT_SDMA_MGCG |
2350                         AMDGPU_CG_SUPPORT_SDMA_LS |
2351                         AMDGPU_CG_SUPPORT_BIF_LS |
2352                         AMDGPU_CG_SUPPORT_VCE_MGCG |
2353                         AMDGPU_CG_SUPPORT_UVD_MGCG |
2354                         AMDGPU_CG_SUPPORT_HDP_LS |
2355                         AMDGPU_CG_SUPPORT_HDP_MGCG;
2356                 adev->pg_flags =
2357                         /*AMDGPU_PG_SUPPORT_GFX_PG |
2358                           AMDGPU_PG_SUPPORT_GFX_SMG |
2359                           AMDGPU_PG_SUPPORT_GFX_DMG |*/
2360                         AMDGPU_PG_SUPPORT_UVD |
2361                         /*AMDGPU_PG_SUPPORT_VCE |
2362                           AMDGPU_PG_SUPPORT_CP |
2363                           AMDGPU_PG_SUPPORT_GDS |
2364                           AMDGPU_PG_SUPPORT_RLC_SMU_HS |
2365                           AMDGPU_PG_SUPPORT_ACP |
2366                           AMDGPU_PG_SUPPORT_SAMU |*/
2367                         0;
2368                 if (adev->pdev->device == 0x1312 ||
2369                         adev->pdev->device == 0x1316 ||
2370                         adev->pdev->device == 0x1317)
2371                         adev->external_rev_id = 0x41;
2372                 else
2373                         adev->external_rev_id = 0x1;
2374                 break;
2375         case CHIP_KABINI:
2376         case CHIP_MULLINS:
2377                 adev->cg_flags =
2378                         AMDGPU_CG_SUPPORT_GFX_MGCG |
2379                         AMDGPU_CG_SUPPORT_GFX_MGLS |
2380                         /*AMDGPU_CG_SUPPORT_GFX_CGCG |*/
2381                         AMDGPU_CG_SUPPORT_GFX_CGLS |
2382                         AMDGPU_CG_SUPPORT_GFX_CGTS |
2383                         AMDGPU_CG_SUPPORT_GFX_CGTS_LS |
2384                         AMDGPU_CG_SUPPORT_GFX_CP_LS |
2385                         AMDGPU_CG_SUPPORT_SDMA_MGCG |
2386                         AMDGPU_CG_SUPPORT_SDMA_LS |
2387                         AMDGPU_CG_SUPPORT_BIF_LS |
2388                         AMDGPU_CG_SUPPORT_VCE_MGCG |
2389                         AMDGPU_CG_SUPPORT_UVD_MGCG |
2390                         AMDGPU_CG_SUPPORT_HDP_LS |
2391                         AMDGPU_CG_SUPPORT_HDP_MGCG;
2392                 adev->pg_flags =
2393                         /*AMDGPU_PG_SUPPORT_GFX_PG |
2394                           AMDGPU_PG_SUPPORT_GFX_SMG | */
2395                         AMDGPU_PG_SUPPORT_UVD |
2396                         /*AMDGPU_PG_SUPPORT_VCE |
2397                           AMDGPU_PG_SUPPORT_CP |
2398                           AMDGPU_PG_SUPPORT_GDS |
2399                           AMDGPU_PG_SUPPORT_RLC_SMU_HS |
2400                           AMDGPU_PG_SUPPORT_SAMU |*/
2401                         0;
2402                 if (adev->asic_type == CHIP_KABINI) {
2403                         if (adev->rev_id == 0)
2404                                 adev->external_rev_id = 0x81;
2405                         else if (adev->rev_id == 1)
2406                                 adev->external_rev_id = 0x82;
2407                         else if (adev->rev_id == 2)
2408                                 adev->external_rev_id = 0x85;
2409                 } else
2410                         adev->external_rev_id = adev->rev_id + 0xa1;
2411                 break;
2412         default:
2413                 /* FIXME: not supported yet */
2414                 return -EINVAL;
2415         }
2416
2417         return 0;
2418 }
2419
2420 static int cik_common_sw_init(void *handle)
2421 {
2422         return 0;
2423 }
2424
2425 static int cik_common_sw_fini(void *handle)
2426 {
2427         return 0;
2428 }
2429
2430 static int cik_common_hw_init(void *handle)
2431 {
2432         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2433
2434         /* move the golden regs per IP block */
2435         cik_init_golden_registers(adev);
2436         /* enable pcie gen2/3 link */
2437         cik_pcie_gen3_enable(adev);
2438         /* enable aspm */
2439         cik_program_aspm(adev);
2440
2441         return 0;
2442 }
2443
2444 static int cik_common_hw_fini(void *handle)
2445 {
2446         return 0;
2447 }
2448
2449 static int cik_common_suspend(void *handle)
2450 {
2451         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2452
2453         amdgpu_amdkfd_suspend(adev);
2454
2455         return cik_common_hw_fini(adev);
2456 }
2457
2458 static int cik_common_resume(void *handle)
2459 {
2460         int r;
2461         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2462
2463         r = cik_common_hw_init(adev);
2464         if (r)
2465                 return r;
2466
2467         return amdgpu_amdkfd_resume(adev);
2468 }
2469
2470 static bool cik_common_is_idle(void *handle)
2471 {
2472         return true;
2473 }
2474
2475 static int cik_common_wait_for_idle(void *handle)
2476 {
2477         return 0;
2478 }
2479
2480 static void cik_common_print_status(void *handle)
2481 {
2482
2483 }
2484
2485 static int cik_common_soft_reset(void *handle)
2486 {
2487         /* XXX hard reset?? */
2488         return 0;
2489 }
2490
2491 static int cik_common_set_clockgating_state(void *handle,
2492                                             enum amd_clockgating_state state)
2493 {
2494         return 0;
2495 }
2496
2497 static int cik_common_set_powergating_state(void *handle,
2498                                             enum amd_powergating_state state)
2499 {
2500         return 0;
2501 }
2502
2503 const struct amd_ip_funcs cik_common_ip_funcs = {
2504         .early_init = cik_common_early_init,
2505         .late_init = NULL,
2506         .sw_init = cik_common_sw_init,
2507         .sw_fini = cik_common_sw_fini,
2508         .hw_init = cik_common_hw_init,
2509         .hw_fini = cik_common_hw_fini,
2510         .suspend = cik_common_suspend,
2511         .resume = cik_common_resume,
2512         .is_idle = cik_common_is_idle,
2513         .wait_for_idle = cik_common_wait_for_idle,
2514         .soft_reset = cik_common_soft_reset,
2515         .print_status = cik_common_print_status,
2516         .set_clockgating_state = cik_common_set_clockgating_state,
2517         .set_powergating_state = cik_common_set_powergating_state,
2518 };