drivers/gpu/drm/radeon/evergreen.c

   1 /*
   2  * Copyright 2010 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/platform_device.h>
  26 #include <linux/slab.h>
  27 #include <drm/drmP.h>
  28 #include "radeon.h"
  29 #include "radeon_asic.h"
  30 #include <drm/radeon_drm.h>
  31 #include "evergreend.h"
  32 #include "atom.h"
  33 #include "avivod.h"
  34 #include "evergreen_reg.h"
  35 #include "evergreen_blit_shaders.h"
  36 #include "radeon_ucode.h"
  37
  38 static const u32 crtc_offsets[6] =
  39 {
  40         EVERGREEN_CRTC0_REGISTER_OFFSET,
  41         EVERGREEN_CRTC1_REGISTER_OFFSET,
  42         EVERGREEN_CRTC2_REGISTER_OFFSET,
  43         EVERGREEN_CRTC3_REGISTER_OFFSET,
  44         EVERGREEN_CRTC4_REGISTER_OFFSET,
  45         EVERGREEN_CRTC5_REGISTER_OFFSET
  46 };
  47
  48 #include "clearstate_evergreen.h"
  49
  50 static u32 sumo_rlc_save_restore_register_list[] =
  51 {
  52         0x98fc,
  53         0x9830,
  54         0x9834,
  55         0x9838,
  56         0x9870,
  57         0x9874,
  58         0x8a14,
  59         0x8b24,
  60         0x8bcc,
  61         0x8b10,
  62         0x8d00,
  63         0x8d04,
  64         0x8c00,
  65         0x8c04,
  66         0x8c08,
  67         0x8c0c,
  68         0x8d8c,
  69         0x8c20,
  70         0x8c24,
  71         0x8c28,
  72         0x8c18,
  73         0x8c1c,
  74         0x8cf0,
  75         0x8e2c,
  76         0x8e38,
  77         0x8c30,
  78         0x9508,
  79         0x9688,
  80         0x9608,
  81         0x960c,
  82         0x9610,
  83         0x9614,
  84         0x88c4,
  85         0x88d4,
  86         0xa008,
  87         0x900c,
  88         0x9100,
  89         0x913c,
  90         0x98f8,
  91         0x98f4,
  92         0x9b7c,
  93         0x3f8c,
  94         0x8950,
  95         0x8954,
  96         0x8a18,
  97         0x8b28,
  98         0x9144,
  99         0x9148,
 100         0x914c,
 101         0x3f90,
 102         0x3f94,
 103         0x915c,
 104         0x9160,
 105         0x9178,
 106         0x917c,
 107         0x9180,
 108         0x918c,
 109         0x9190,
 110         0x9194,
 111         0x9198,
 112         0x919c,
 113         0x91a8,
 114         0x91ac,
 115         0x91b0,
 116         0x91b4,
 117         0x91b8,
 118         0x91c4,
 119         0x91c8,
 120         0x91cc,
 121         0x91d0,
 122         0x91d4,
 123         0x91e0,
 124         0x91e4,
 125         0x91ec,
 126         0x91f0,
 127         0x91f4,
 128         0x9200,
 129         0x9204,
 130         0x929c,
 131         0x9150,
 132         0x802c,
 133 };
 134 static u32 sumo_rlc_save_restore_register_list_size = ARRAY_SIZE(sumo_rlc_save_restore_register_list);
 135
 136 static void evergreen_gpu_init(struct radeon_device *rdev);
 137 void evergreen_fini(struct radeon_device *rdev);
 138 void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
 139 void evergreen_program_aspm(struct radeon_device *rdev);
 140 extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
 141                                      int ring, u32 cp_int_cntl);
 142 extern void cayman_vm_decode_fault(struct radeon_device *rdev,
 143                                    u32 status, u32 addr);
 144
 145 static const u32 evergreen_golden_registers[] =
 146 {
 147         0x3f90, 0xffff0000, 0xff000000,
 148         0x9148, 0xffff0000, 0xff000000,
 149         0x3f94, 0xffff0000, 0xff000000,
 150         0x914c, 0xffff0000, 0xff000000,
 151         0x9b7c, 0xffffffff, 0x00000000,
 152         0x8a14, 0xffffffff, 0x00000007,
 153         0x8b10, 0xffffffff, 0x00000000,
 154         0x960c, 0xffffffff, 0x54763210,
 155         0x88c4, 0xffffffff, 0x000000c2,
 156         0x88d4, 0xffffffff, 0x00000010,
 157         0x8974, 0xffffffff, 0x00000000,
 158         0xc78, 0x00000080, 0x00000080,
 159         0x5eb4, 0xffffffff, 0x00000002,
 160         0x5e78, 0xffffffff, 0x001000f0,
 161         0x6104, 0x01000300, 0x00000000,
 162         0x5bc0, 0x00300000, 0x00000000,
 163         0x7030, 0xffffffff, 0x00000011,
 164         0x7c30, 0xffffffff, 0x00000011,
 165         0x10830, 0xffffffff, 0x00000011,
 166         0x11430, 0xffffffff, 0x00000011,
 167         0x12030, 0xffffffff, 0x00000011,
 168         0x12c30, 0xffffffff, 0x00000011,
 169         0xd02c, 0xffffffff, 0x08421000,
 170         0x240c, 0xffffffff, 0x00000380,
 171         0x8b24, 0xffffffff, 0x00ff0fff,
 172         0x28a4c, 0x06000000, 0x06000000,
 173         0x10c, 0x00000001, 0x00000001,
 174         0x8d00, 0xffffffff, 0x100e4848,
 175         0x8d04, 0xffffffff, 0x00164745,
 176         0x8c00, 0xffffffff, 0xe4000003,
 177         0x8c04, 0xffffffff, 0x40600060,
 178         0x8c08, 0xffffffff, 0x001c001c,
 179         0x8cf0, 0xffffffff, 0x08e00620,
 180         0x8c20, 0xffffffff, 0x00800080,
 181         0x8c24, 0xffffffff, 0x00800080,
 182         0x8c18, 0xffffffff, 0x20202078,
 183         0x8c1c, 0xffffffff, 0x00001010,
 184         0x28350, 0xffffffff, 0x00000000,
 185         0xa008, 0xffffffff, 0x00010000,
 186         0x5cc, 0xffffffff, 0x00000001,
 187         0x9508, 0xffffffff, 0x00000002,
 188         0x913c, 0x0000000f, 0x0000000a
 189 };
 190
 191 static const u32 evergreen_golden_registers2[] =
 192 {
 193         0x2f4c, 0xffffffff, 0x00000000,
 194         0x54f4, 0xffffffff, 0x00000000,
 195         0x54f0, 0xffffffff, 0x00000000,
 196         0x5498, 0xffffffff, 0x00000000,
 197         0x549c, 0xffffffff, 0x00000000,
 198         0x5494, 0xffffffff, 0x00000000,
 199         0x53cc, 0xffffffff, 0x00000000,
 200         0x53c8, 0xffffffff, 0x00000000,
 201         0x53c4, 0xffffffff, 0x00000000,
 202         0x53c0, 0xffffffff, 0x00000000,
 203         0x53bc, 0xffffffff, 0x00000000,
 204         0x53b8, 0xffffffff, 0x00000000,
 205         0x53b4, 0xffffffff, 0x00000000,
 206         0x53b0, 0xffffffff, 0x00000000
 207 };
 208
 209 static const u32 cypress_mgcg_init[] =
 210 {
 211         0x802c, 0xffffffff, 0xc0000000,
 212         0x5448, 0xffffffff, 0x00000100,
 213         0x55e4, 0xffffffff, 0x00000100,
 214         0x160c, 0xffffffff, 0x00000100,
 215         0x5644, 0xffffffff, 0x00000100,
 216         0xc164, 0xffffffff, 0x00000100,
 217         0x8a18, 0xffffffff, 0x00000100,
 218         0x897c, 0xffffffff, 0x06000100,
 219         0x8b28, 0xffffffff, 0x00000100,
 220         0x9144, 0xffffffff, 0x00000100,
 221         0x9a60, 0xffffffff, 0x00000100,
 222         0x9868, 0xffffffff, 0x00000100,
 223         0x8d58, 0xffffffff, 0x00000100,
 224         0x9510, 0xffffffff, 0x00000100,
 225         0x949c, 0xffffffff, 0x00000100,
 226         0x9654, 0xffffffff, 0x00000100,
 227         0x9030, 0xffffffff, 0x00000100,
 228         0x9034, 0xffffffff, 0x00000100,
 229         0x9038, 0xffffffff, 0x00000100,
 230         0x903c, 0xffffffff, 0x00000100,
 231         0x9040, 0xffffffff, 0x00000100,
 232         0xa200, 0xffffffff, 0x00000100,
 233         0xa204, 0xffffffff, 0x00000100,
 234         0xa208, 0xffffffff, 0x00000100,
 235         0xa20c, 0xffffffff, 0x00000100,
 236         0x971c, 0xffffffff, 0x00000100,
 237         0x977c, 0xffffffff, 0x00000100,
 238         0x3f80, 0xffffffff, 0x00000100,
 239         0xa210, 0xffffffff, 0x00000100,
 240         0xa214, 0xffffffff, 0x00000100,
 241         0x4d8, 0xffffffff, 0x00000100,
 242         0x9784, 0xffffffff, 0x00000100,
 243         0x9698, 0xffffffff, 0x00000100,
 244         0x4d4, 0xffffffff, 0x00000200,
 245         0x30cc, 0xffffffff, 0x00000100,
 246         0xd0c0, 0xffffffff, 0xff000100,
 247         0x802c, 0xffffffff, 0x40000000,
 248         0x915c, 0xffffffff, 0x00010000,
 249         0x9160, 0xffffffff, 0x00030002,
 250         0x9178, 0xffffffff, 0x00070000,
 251         0x917c, 0xffffffff, 0x00030002,
 252         0x9180, 0xffffffff, 0x00050004,
 253         0x918c, 0xffffffff, 0x00010006,
 254         0x9190, 0xffffffff, 0x00090008,
 255         0x9194, 0xffffffff, 0x00070000,
 256         0x9198, 0xffffffff, 0x00030002,
 257         0x919c, 0xffffffff, 0x00050004,
 258         0x91a8, 0xffffffff, 0x00010006,
 259         0x91ac, 0xffffffff, 0x00090008,
 260         0x91b0, 0xffffffff, 0x00070000,
 261         0x91b4, 0xffffffff, 0x00030002,
 262         0x91b8, 0xffffffff, 0x00050004,
 263         0x91c4, 0xffffffff, 0x00010006,
 264         0x91c8, 0xffffffff, 0x00090008,
 265         0x91cc, 0xffffffff, 0x00070000,
 266         0x91d0, 0xffffffff, 0x00030002,
 267         0x91d4, 0xffffffff, 0x00050004,
 268         0x91e0, 0xffffffff, 0x00010006,
 269         0x91e4, 0xffffffff, 0x00090008,
 270         0x91e8, 0xffffffff, 0x00000000,
 271         0x91ec, 0xffffffff, 0x00070000,
 272         0x91f0, 0xffffffff, 0x00030002,
 273         0x91f4, 0xffffffff, 0x00050004,
 274         0x9200, 0xffffffff, 0x00010006,
 275         0x9204, 0xffffffff, 0x00090008,
 276         0x9208, 0xffffffff, 0x00070000,
 277         0x920c, 0xffffffff, 0x00030002,
 278         0x9210, 0xffffffff, 0x00050004,
 279         0x921c, 0xffffffff, 0x00010006,
 280         0x9220, 0xffffffff, 0x00090008,
 281         0x9224, 0xffffffff, 0x00070000,
 282         0x9228, 0xffffffff, 0x00030002,
 283         0x922c, 0xffffffff, 0x00050004,
 284         0x9238, 0xffffffff, 0x00010006,
 285         0x923c, 0xffffffff, 0x00090008,
 286         0x9240, 0xffffffff, 0x00070000,
 287         0x9244, 0xffffffff, 0x00030002,
 288         0x9248, 0xffffffff, 0x00050004,
 289         0x9254, 0xffffffff, 0x00010006,
 290         0x9258, 0xffffffff, 0x00090008,
 291         0x925c, 0xffffffff, 0x00070000,
 292         0x9260, 0xffffffff, 0x00030002,
 293         0x9264, 0xffffffff, 0x00050004,
 294         0x9270, 0xffffffff, 0x00010006,
 295         0x9274, 0xffffffff, 0x00090008,
 296         0x9278, 0xffffffff, 0x00070000,
 297         0x927c, 0xffffffff, 0x00030002,
 298         0x9280, 0xffffffff, 0x00050004,
 299         0x928c, 0xffffffff, 0x00010006,
 300         0x9290, 0xffffffff, 0x00090008,
 301         0x9294, 0xffffffff, 0x00000000,
 302         0x929c, 0xffffffff, 0x00000001,
 303         0x802c, 0xffffffff, 0x40010000,
 304         0x915c, 0xffffffff, 0x00010000,
 305         0x9160, 0xffffffff, 0x00030002,
 306         0x9178, 0xffffffff, 0x00070000,
 307         0x917c, 0xffffffff, 0x00030002,
 308         0x9180, 0xffffffff, 0x00050004,
 309         0x918c, 0xffffffff, 0x00010006,
 310         0x9190, 0xffffffff, 0x00090008,
 311         0x9194, 0xffffffff, 0x00070000,
 312         0x9198, 0xffffffff, 0x00030002,
 313         0x919c, 0xffffffff, 0x00050004,
 314         0x91a8, 0xffffffff, 0x00010006,
 315         0x91ac, 0xffffffff, 0x00090008,
 316         0x91b0, 0xffffffff, 0x00070000,
 317         0x91b4, 0xffffffff, 0x00030002,
 318         0x91b8, 0xffffffff, 0x00050004,
 319         0x91c4, 0xffffffff, 0x00010006,
 320         0x91c8, 0xffffffff, 0x00090008,
 321         0x91cc, 0xffffffff, 0x00070000,
 322         0x91d0, 0xffffffff, 0x00030002,
 323         0x91d4, 0xffffffff, 0x00050004,
 324         0x91e0, 0xffffffff, 0x00010006,
 325         0x91e4, 0xffffffff, 0x00090008,
 326         0x91e8, 0xffffffff, 0x00000000,
 327         0x91ec, 0xffffffff, 0x00070000,
 328         0x91f0, 0xffffffff, 0x00030002,
 329         0x91f4, 0xffffffff, 0x00050004,
 330         0x9200, 0xffffffff, 0x00010006,
 331         0x9204, 0xffffffff, 0x00090008,
 332         0x9208, 0xffffffff, 0x00070000,
 333         0x920c, 0xffffffff, 0x00030002,
 334         0x9210, 0xffffffff, 0x00050004,
 335         0x921c, 0xffffffff, 0x00010006,
 336         0x9220, 0xffffffff, 0x00090008,
 337         0x9224, 0xffffffff, 0x00070000,
 338         0x9228, 0xffffffff, 0x00030002,
 339         0x922c, 0xffffffff, 0x00050004,
 340         0x9238, 0xffffffff, 0x00010006,
 341         0x923c, 0xffffffff, 0x00090008,
 342         0x9240, 0xffffffff, 0x00070000,
 343         0x9244, 0xffffffff, 0x00030002,
 344         0x9248, 0xffffffff, 0x00050004,
 345         0x9254, 0xffffffff, 0x00010006,
 346         0x9258, 0xffffffff, 0x00090008,
 347         0x925c, 0xffffffff, 0x00070000,
 348         0x9260, 0xffffffff, 0x00030002,
 349         0x9264, 0xffffffff, 0x00050004,
 350         0x9270, 0xffffffff, 0x00010006,
 351         0x9274, 0xffffffff, 0x00090008,
 352         0x9278, 0xffffffff, 0x00070000,
 353         0x927c, 0xffffffff, 0x00030002,
 354         0x9280, 0xffffffff, 0x00050004,
 355         0x928c, 0xffffffff, 0x00010006,
 356         0x9290, 0xffffffff, 0x00090008,
 357         0x9294, 0xffffffff, 0x00000000,
 358         0x929c, 0xffffffff, 0x00000001,
 359         0x802c, 0xffffffff, 0xc0000000
 360 };
 361
 362 static const u32 redwood_mgcg_init[] =
 363 {
 364         0x802c, 0xffffffff, 0xc0000000,
 365         0x5448, 0xffffffff, 0x00000100,
 366         0x55e4, 0xffffffff, 0x00000100,
 367         0x160c, 0xffffffff, 0x00000100,
 368         0x5644, 0xffffffff, 0x00000100,
 369         0xc164, 0xffffffff, 0x00000100,
 370         0x8a18, 0xffffffff, 0x00000100,
 371         0x897c, 0xffffffff, 0x06000100,
 372         0x8b28, 0xffffffff, 0x00000100,
 373         0x9144, 0xffffffff, 0x00000100,
 374         0x9a60, 0xffffffff, 0x00000100,
 375         0x9868, 0xffffffff, 0x00000100,
 376         0x8d58, 0xffffffff, 0x00000100,
 377         0x9510, 0xffffffff, 0x00000100,
 378         0x949c, 0xffffffff, 0x00000100,
 379         0x9654, 0xffffffff, 0x00000100,
 380         0x9030, 0xffffffff, 0x00000100,
 381         0x9034, 0xffffffff, 0x00000100,
 382         0x9038, 0xffffffff, 0x00000100,
 383         0x903c, 0xffffffff, 0x00000100,
 384         0x9040, 0xffffffff, 0x00000100,
 385         0xa200, 0xffffffff, 0x00000100,
 386         0xa204, 0xffffffff, 0x00000100,
 387         0xa208, 0xffffffff, 0x00000100,
 388         0xa20c, 0xffffffff, 0x00000100,
 389         0x971c, 0xffffffff, 0x00000100,
 390         0x977c, 0xffffffff, 0x00000100,
 391         0x3f80, 0xffffffff, 0x00000100,
 392         0xa210, 0xffffffff, 0x00000100,
 393         0xa214, 0xffffffff, 0x00000100,
 394         0x4d8, 0xffffffff, 0x00000100,
 395         0x9784, 0xffffffff, 0x00000100,
 396         0x9698, 0xffffffff, 0x00000100,
 397         0x4d4, 0xffffffff, 0x00000200,
 398         0x30cc, 0xffffffff, 0x00000100,
 399         0xd0c0, 0xffffffff, 0xff000100,
 400         0x802c, 0xffffffff, 0x40000000,
 401         0x915c, 0xffffffff, 0x00010000,
 402         0x9160, 0xffffffff, 0x00030002,
 403         0x9178, 0xffffffff, 0x00070000,
 404         0x917c, 0xffffffff, 0x00030002,
 405         0x9180, 0xffffffff, 0x00050004,
 406         0x918c, 0xffffffff, 0x00010006,
 407         0x9190, 0xffffffff, 0x00090008,
 408         0x9194, 0xffffffff, 0x00070000,
 409         0x9198, 0xffffffff, 0x00030002,
 410         0x919c, 0xffffffff, 0x00050004,
 411         0x91a8, 0xffffffff, 0x00010006,
 412         0x91ac, 0xffffffff, 0x00090008,
 413         0x91b0, 0xffffffff, 0x00070000,
 414         0x91b4, 0xffffffff, 0x00030002,
 415         0x91b8, 0xffffffff, 0x00050004,
 416         0x91c4, 0xffffffff, 0x00010006,
 417         0x91c8, 0xffffffff, 0x00090008,
 418         0x91cc, 0xffffffff, 0x00070000,
 419         0x91d0, 0xffffffff, 0x00030002,
 420         0x91d4, 0xffffffff, 0x00050004,
 421         0x91e0, 0xffffffff, 0x00010006,
 422         0x91e4, 0xffffffff, 0x00090008,
 423         0x91e8, 0xffffffff, 0x00000000,
 424         0x91ec, 0xffffffff, 0x00070000,
 425         0x91f0, 0xffffffff, 0x00030002,
 426         0x91f4, 0xffffffff, 0x00050004,
 427         0x9200, 0xffffffff, 0x00010006,
 428         0x9204, 0xffffffff, 0x00090008,
 429         0x9294, 0xffffffff, 0x00000000,
 430         0x929c, 0xffffffff, 0x00000001,
 431         0x802c, 0xffffffff, 0xc0000000
 432 };
 433
 434 static const u32 cedar_golden_registers[] =
 435 {
 436         0x3f90, 0xffff0000, 0xff000000,
 437         0x9148, 0xffff0000, 0xff000000,
 438         0x3f94, 0xffff0000, 0xff000000,
 439         0x914c, 0xffff0000, 0xff000000,
 440         0x9b7c, 0xffffffff, 0x00000000,
 441         0x8a14, 0xffffffff, 0x00000007,
 442         0x8b10, 0xffffffff, 0x00000000,
 443         0x960c, 0xffffffff, 0x54763210,
 444         0x88c4, 0xffffffff, 0x000000c2,
 445         0x88d4, 0xffffffff, 0x00000000,
 446         0x8974, 0xffffffff, 0x00000000,
 447         0xc78, 0x00000080, 0x00000080,
 448         0x5eb4, 0xffffffff, 0x00000002,
 449         0x5e78, 0xffffffff, 0x001000f0,
 450         0x6104, 0x01000300, 0x00000000,
 451         0x5bc0, 0x00300000, 0x00000000,
 452         0x7030, 0xffffffff, 0x00000011,
 453         0x7c30, 0xffffffff, 0x00000011,
 454         0x10830, 0xffffffff, 0x00000011,
 455         0x11430, 0xffffffff, 0x00000011,
 456         0xd02c, 0xffffffff, 0x08421000,
 457         0x240c, 0xffffffff, 0x00000380,
 458         0x8b24, 0xffffffff, 0x00ff0fff,
 459         0x28a4c, 0x06000000, 0x06000000,
 460         0x10c, 0x00000001, 0x00000001,
 461         0x8d00, 0xffffffff, 0x100e4848,
 462         0x8d04, 0xffffffff, 0x00164745,
 463         0x8c00, 0xffffffff, 0xe4000003,
 464         0x8c04, 0xffffffff, 0x40600060,
 465         0x8c08, 0xffffffff, 0x001c001c,
 466         0x8cf0, 0xffffffff, 0x08e00410,
 467         0x8c20, 0xffffffff, 0x00800080,
 468         0x8c24, 0xffffffff, 0x00800080,
 469         0x8c18, 0xffffffff, 0x20202078,
 470         0x8c1c, 0xffffffff, 0x00001010,
 471         0x28350, 0xffffffff, 0x00000000,
 472         0xa008, 0xffffffff, 0x00010000,
 473         0x5cc, 0xffffffff, 0x00000001,
 474         0x9508, 0xffffffff, 0x00000002
 475 };
 476
 477 static const u32 cedar_mgcg_init[] =
 478 {
 479         0x802c, 0xffffffff, 0xc0000000,
 480         0x5448, 0xffffffff, 0x00000100,
 481         0x55e4, 0xffffffff, 0x00000100,
 482         0x160c, 0xffffffff, 0x00000100,
 483         0x5644, 0xffffffff, 0x00000100,
 484         0xc164, 0xffffffff, 0x00000100,
 485         0x8a18, 0xffffffff, 0x00000100,
 486         0x897c, 0xffffffff, 0x06000100,
 487         0x8b28, 0xffffffff, 0x00000100,
 488         0x9144, 0xffffffff, 0x00000100,
 489         0x9a60, 0xffffffff, 0x00000100,
 490         0x9868, 0xffffffff, 0x00000100,
 491         0x8d58, 0xffffffff, 0x00000100,
 492         0x9510, 0xffffffff, 0x00000100,
 493         0x949c, 0xffffffff, 0x00000100,
 494         0x9654, 0xffffffff, 0x00000100,
 495         0x9030, 0xffffffff, 0x00000100,
 496         0x9034, 0xffffffff, 0x00000100,
 497         0x9038, 0xffffffff, 0x00000100,
 498         0x903c, 0xffffffff, 0x00000100,
 499         0x9040, 0xffffffff, 0x00000100,
 500         0xa200, 0xffffffff, 0x00000100,
 501         0xa204, 0xffffffff, 0x00000100,
 502         0xa208, 0xffffffff, 0x00000100,
 503         0xa20c, 0xffffffff, 0x00000100,
 504         0x971c, 0xffffffff, 0x00000100,
 505         0x977c, 0xffffffff, 0x00000100,
 506         0x3f80, 0xffffffff, 0x00000100,
 507         0xa210, 0xffffffff, 0x00000100,
 508         0xa214, 0xffffffff, 0x00000100,
 509         0x4d8, 0xffffffff, 0x00000100,
 510         0x9784, 0xffffffff, 0x00000100,
 511         0x9698, 0xffffffff, 0x00000100,
 512         0x4d4, 0xffffffff, 0x00000200,
 513         0x30cc, 0xffffffff, 0x00000100,
 514         0xd0c0, 0xffffffff, 0xff000100,
 515         0x802c, 0xffffffff, 0x40000000,
 516         0x915c, 0xffffffff, 0x00010000,
 517         0x9178, 0xffffffff, 0x00050000,
 518         0x917c, 0xffffffff, 0x00030002,
 519         0x918c, 0xffffffff, 0x00010004,
 520         0x9190, 0xffffffff, 0x00070006,
 521         0x9194, 0xffffffff, 0x00050000,
 522         0x9198, 0xffffffff, 0x00030002,
 523         0x91a8, 0xffffffff, 0x00010004,
 524         0x91ac, 0xffffffff, 0x00070006,
 525         0x91e8, 0xffffffff, 0x00000000,
 526         0x9294, 0xffffffff, 0x00000000,
 527         0x929c, 0xffffffff, 0x00000001,
 528         0x802c, 0xffffffff, 0xc0000000
 529 };
 530
 531 static const u32 juniper_mgcg_init[] =
 532 {
 533         0x802c, 0xffffffff, 0xc0000000,
 534         0x5448, 0xffffffff, 0x00000100,
 535         0x55e4, 0xffffffff, 0x00000100,
 536         0x160c, 0xffffffff, 0x00000100,
 537         0x5644, 0xffffffff, 0x00000100,
 538         0xc164, 0xffffffff, 0x00000100,
 539         0x8a18, 0xffffffff, 0x00000100,
 540         0x897c, 0xffffffff, 0x06000100,
 541         0x8b28, 0xffffffff, 0x00000100,
 542         0x9144, 0xffffffff, 0x00000100,
 543         0x9a60, 0xffffffff, 0x00000100,
 544         0x9868, 0xffffffff, 0x00000100,
 545         0x8d58, 0xffffffff, 0x00000100,
 546         0x9510, 0xffffffff, 0x00000100,
 547         0x949c, 0xffffffff, 0x00000100,
 548         0x9654, 0xffffffff, 0x00000100,
 549         0x9030, 0xffffffff, 0x00000100,
 550         0x9034, 0xffffffff, 0x00000100,
 551         0x9038, 0xffffffff, 0x00000100,
 552         0x903c, 0xffffffff, 0x00000100,
 553         0x9040, 0xffffffff, 0x00000100,
 554         0xa200, 0xffffffff, 0x00000100,
 555         0xa204, 0xffffffff, 0x00000100,
 556         0xa208, 0xffffffff, 0x00000100,
 557         0xa20c, 0xffffffff, 0x00000100,
 558         0x971c, 0xffffffff, 0x00000100,
 559         0xd0c0, 0xffffffff, 0xff000100,
 560         0x802c, 0xffffffff, 0x40000000,
 561         0x915c, 0xffffffff, 0x00010000,
 562         0x9160, 0xffffffff, 0x00030002,
 563         0x9178, 0xffffffff, 0x00070000,
 564         0x917c, 0xffffffff, 0x00030002,
 565         0x9180, 0xffffffff, 0x00050004,
 566         0x918c, 0xffffffff, 0x00010006,
 567         0x9190, 0xffffffff, 0x00090008,
 568         0x9194, 0xffffffff, 0x00070000,
 569         0x9198, 0xffffffff, 0x00030002,
 570         0x919c, 0xffffffff, 0x00050004,
 571         0x91a8, 0xffffffff, 0x00010006,
 572         0x91ac, 0xffffffff, 0x00090008,
 573         0x91b0, 0xffffffff, 0x00070000,
 574         0x91b4, 0xffffffff, 0x00030002,
 575         0x91b8, 0xffffffff, 0x00050004,
 576         0x91c4, 0xffffffff, 0x00010006,
 577         0x91c8, 0xffffffff, 0x00090008,
 578         0x91cc, 0xffffffff, 0x00070000,
 579         0x91d0, 0xffffffff, 0x00030002,
 580         0x91d4, 0xffffffff, 0x00050004,
 581         0x91e0, 0xffffffff, 0x00010006,
 582         0x91e4, 0xffffffff, 0x00090008,
 583         0x91e8, 0xffffffff, 0x00000000,
 584         0x91ec, 0xffffffff, 0x00070000,
 585         0x91f0, 0xffffffff, 0x00030002,
 586         0x91f4, 0xffffffff, 0x00050004,
 587         0x9200, 0xffffffff, 0x00010006,
 588         0x9204, 0xffffffff, 0x00090008,
 589         0x9208, 0xffffffff, 0x00070000,
 590         0x920c, 0xffffffff, 0x00030002,
 591         0x9210, 0xffffffff, 0x00050004,
 592         0x921c, 0xffffffff, 0x00010006,
 593         0x9220, 0xffffffff, 0x00090008,
 594         0x9224, 0xffffffff, 0x00070000,
 595         0x9228, 0xffffffff, 0x00030002,
 596         0x922c, 0xffffffff, 0x00050004,
 597         0x9238, 0xffffffff, 0x00010006,
 598         0x923c, 0xffffffff, 0x00090008,
 599         0x9240, 0xffffffff, 0x00070000,
 600         0x9244, 0xffffffff, 0x00030002,
 601         0x9248, 0xffffffff, 0x00050004,
 602         0x9254, 0xffffffff, 0x00010006,
 603         0x9258, 0xffffffff, 0x00090008,
 604         0x925c, 0xffffffff, 0x00070000,
 605         0x9260, 0xffffffff, 0x00030002,
 606         0x9264, 0xffffffff, 0x00050004,
 607         0x9270, 0xffffffff, 0x00010006,
 608         0x9274, 0xffffffff, 0x00090008,
 609         0x9278, 0xffffffff, 0x00070000,
 610         0x927c, 0xffffffff, 0x00030002,
 611         0x9280, 0xffffffff, 0x00050004,
 612         0x928c, 0xffffffff, 0x00010006,
 613         0x9290, 0xffffffff, 0x00090008,
 614         0x9294, 0xffffffff, 0x00000000,
 615         0x929c, 0xffffffff, 0x00000001,
 616         0x802c, 0xffffffff, 0xc0000000,
 617         0x977c, 0xffffffff, 0x00000100,
 618         0x3f80, 0xffffffff, 0x00000100,
 619         0xa210, 0xffffffff, 0x00000100,
 620         0xa214, 0xffffffff, 0x00000100,
 621         0x4d8, 0xffffffff, 0x00000100,
 622         0x9784, 0xffffffff, 0x00000100,
 623         0x9698, 0xffffffff, 0x00000100,
 624         0x4d4, 0xffffffff, 0x00000200,
 625         0x30cc, 0xffffffff, 0x00000100,
 626         0x802c, 0xffffffff, 0xc0000000
 627 };
 628
 629 static const u32 supersumo_golden_registers[] =
 630 {
 631         0x5eb4, 0xffffffff, 0x00000002,
 632         0x5cc, 0xffffffff, 0x00000001,
 633         0x7030, 0xffffffff, 0x00000011,
 634         0x7c30, 0xffffffff, 0x00000011,
 635         0x6104, 0x01000300, 0x00000000,
 636         0x5bc0, 0x00300000, 0x00000000,
 637         0x8c04, 0xffffffff, 0x40600060,
 638         0x8c08, 0xffffffff, 0x001c001c,
 639         0x8c20, 0xffffffff, 0x00800080,
 640         0x8c24, 0xffffffff, 0x00800080,
 641         0x8c18, 0xffffffff, 0x20202078,
 642         0x8c1c, 0xffffffff, 0x00001010,
 643         0x918c, 0xffffffff, 0x00010006,
 644         0x91a8, 0xffffffff, 0x00010006,
 645         0x91c4, 0xffffffff, 0x00010006,
 646         0x91e0, 0xffffffff, 0x00010006,
 647         0x9200, 0xffffffff, 0x00010006,
 648         0x9150, 0xffffffff, 0x6e944040,
 649         0x917c, 0xffffffff, 0x00030002,
 650         0x9180, 0xffffffff, 0x00050004,
 651         0x9198, 0xffffffff, 0x00030002,
 652         0x919c, 0xffffffff, 0x00050004,
 653         0x91b4, 0xffffffff, 0x00030002,
 654         0x91b8, 0xffffffff, 0x00050004,
 655         0x91d0, 0xffffffff, 0x00030002,
 656         0x91d4, 0xffffffff, 0x00050004,
 657         0x91f0, 0xffffffff, 0x00030002,
 658         0x91f4, 0xffffffff, 0x00050004,
 659         0x915c, 0xffffffff, 0x00010000,
 660         0x9160, 0xffffffff, 0x00030002,
 661         0x3f90, 0xffff0000, 0xff000000,
 662         0x9178, 0xffffffff, 0x00070000,
 663         0x9194, 0xffffffff, 0x00070000,
 664         0x91b0, 0xffffffff, 0x00070000,
 665         0x91cc, 0xffffffff, 0x00070000,
 666         0x91ec, 0xffffffff, 0x00070000,
 667         0x9148, 0xffff0000, 0xff000000,
 668         0x9190, 0xffffffff, 0x00090008,
 669         0x91ac, 0xffffffff, 0x00090008,
 670         0x91c8, 0xffffffff, 0x00090008,
 671         0x91e4, 0xffffffff, 0x00090008,
 672         0x9204, 0xffffffff, 0x00090008,
 673         0x3f94, 0xffff0000, 0xff000000,
 674         0x914c, 0xffff0000, 0xff000000,
 675         0x929c, 0xffffffff, 0x00000001,
 676         0x8a18, 0xffffffff, 0x00000100,
 677         0x8b28, 0xffffffff, 0x00000100,
 678         0x9144, 0xffffffff, 0x00000100,
 679         0x5644, 0xffffffff, 0x00000100,
 680         0x9b7c, 0xffffffff, 0x00000000,
 681         0x8030, 0xffffffff, 0x0000100a,
 682         0x8a14, 0xffffffff, 0x00000007,
 683         0x8b24, 0xffffffff, 0x00ff0fff,
 684         0x8b10, 0xffffffff, 0x00000000,
 685         0x28a4c, 0x06000000, 0x06000000,
 686         0x4d8, 0xffffffff, 0x00000100,
 687         0x913c, 0xffff000f, 0x0100000a,
 688         0x960c, 0xffffffff, 0x54763210,
 689         0x88c4, 0xffffffff, 0x000000c2,
 690         0x88d4, 0xffffffff, 0x00000010,
 691         0x8974, 0xffffffff, 0x00000000,
 692         0xc78, 0x00000080, 0x00000080,
 693         0x5e78, 0xffffffff, 0x001000f0,
 694         0xd02c, 0xffffffff, 0x08421000,
 695         0xa008, 0xffffffff, 0x00010000,
 696         0x8d00, 0xffffffff, 0x100e4848,
 697         0x8d04, 0xffffffff, 0x00164745,
 698         0x8c00, 0xffffffff, 0xe4000003,
 699         0x8cf0, 0x1fffffff, 0x08e00620,
 700         0x28350, 0xffffffff, 0x00000000,
 701         0x9508, 0xffffffff, 0x00000002
 702 };
 703
 704 static const u32 sumo_golden_registers[] =
 705 {
 706         0x900c, 0x00ffffff, 0x0017071f,
 707         0x8c18, 0xffffffff, 0x10101060,
 708         0x8c1c, 0xffffffff, 0x00001010,
 709         0x8c30, 0x0000000f, 0x00000005,
 710         0x9688, 0x0000000f, 0x00000007
 711 };
 712
 713 static const u32 wrestler_golden_registers[] =
 714 {
 715         0x5eb4, 0xffffffff, 0x00000002,
 716         0x5cc, 0xffffffff, 0x00000001,
 717         0x7030, 0xffffffff, 0x00000011,
 718         0x7c30, 0xffffffff, 0x00000011,
 719         0x6104, 0x01000300, 0x00000000,
 720         0x5bc0, 0x00300000, 0x00000000,
 721         0x918c, 0xffffffff, 0x00010006,
 722         0x91a8, 0xffffffff, 0x00010006,
 723         0x9150, 0xffffffff, 0x6e944040,
 724         0x917c, 0xffffffff, 0x00030002,
 725         0x9198, 0xffffffff, 0x00030002,
 726         0x915c, 0xffffffff, 0x00010000,
 727         0x3f90, 0xffff0000, 0xff000000,
 728         0x9178, 0xffffffff, 0x00070000,
 729         0x9194, 0xffffffff, 0x00070000,
 730         0x9148, 0xffff0000, 0xff000000,
 731         0x9190, 0xffffffff, 0x00090008,
 732         0x91ac, 0xffffffff, 0x00090008,
 733         0x3f94, 0xffff0000, 0xff000000,
 734         0x914c, 0xffff0000, 0xff000000,
 735         0x929c, 0xffffffff, 0x00000001,
 736         0x8a18, 0xffffffff, 0x00000100,
 737         0x8b28, 0xffffffff, 0x00000100,
 738         0x9144, 0xffffffff, 0x00000100,
 739         0x9b7c, 0xffffffff, 0x00000000,
 740         0x8030, 0xffffffff, 0x0000100a,
 741         0x8a14, 0xffffffff, 0x00000001,
 742         0x8b24, 0xffffffff, 0x00ff0fff,
 743         0x8b10, 0xffffffff, 0x00000000,
 744         0x28a4c, 0x06000000, 0x06000000,
 745         0x4d8, 0xffffffff, 0x00000100,
 746         0x913c, 0xffff000f, 0x0100000a,
 747         0x960c, 0xffffffff, 0x54763210,
 748         0x88c4, 0xffffffff, 0x000000c2,
 749         0x88d4, 0xffffffff, 0x00000010,
 750         0x8974, 0xffffffff, 0x00000000,
 751         0xc78, 0x00000080, 0x00000080,
 752         0x5e78, 0xffffffff, 0x001000f0,
 753         0xd02c, 0xffffffff, 0x08421000,
 754         0xa008, 0xffffffff, 0x00010000,
 755         0x8d00, 0xffffffff, 0x100e4848,
 756         0x8d04, 0xffffffff, 0x00164745,
 757         0x8c00, 0xffffffff, 0xe4000003,
 758         0x8cf0, 0x1fffffff, 0x08e00410,
 759         0x28350, 0xffffffff, 0x00000000,
 760         0x9508, 0xffffffff, 0x00000002,
 761         0x900c, 0xffffffff, 0x0017071f,
 762         0x8c18, 0xffffffff, 0x10101060,
 763         0x8c1c, 0xffffffff, 0x00001010
 764 };
 765
 766 static const u32 barts_golden_registers[] =
 767 {
 768         0x5eb4, 0xffffffff, 0x00000002,
 769         0x5e78, 0x8f311ff1, 0x001000f0,
 770         0x3f90, 0xffff0000, 0xff000000,
 771         0x9148, 0xffff0000, 0xff000000,
 772         0x3f94, 0xffff0000, 0xff000000,
 773         0x914c, 0xffff0000, 0xff000000,
 774         0xc78, 0x00000080, 0x00000080,
 775         0xbd4, 0x70073777, 0x00010001,
 776         0xd02c, 0xbfffff1f, 0x08421000,
 777         0xd0b8, 0x03773777, 0x02011003,
 778         0x5bc0, 0x00200000, 0x50100000,
 779         0x98f8, 0x33773777, 0x02011003,
 780         0x98fc, 0xffffffff, 0x76543210,
 781         0x7030, 0x31000311, 0x00000011,
 782         0x2f48, 0x00000007, 0x02011003,
 783         0x6b28, 0x00000010, 0x00000012,
 784         0x7728, 0x00000010, 0x00000012,
 785         0x10328, 0x00000010, 0x00000012,
 786         0x10f28, 0x00000010, 0x00000012,
 787         0x11b28, 0x00000010, 0x00000012,
 788         0x12728, 0x00000010, 0x00000012,
 789         0x240c, 0x000007ff, 0x00000380,
 790         0x8a14, 0xf000001f, 0x00000007,
 791         0x8b24, 0x3fff3fff, 0x00ff0fff,
 792         0x8b10, 0x0000ff0f, 0x00000000,
 793         0x28a4c, 0x07ffffff, 0x06000000,
 794         0x10c, 0x00000001, 0x00010003,
 795         0xa02c, 0xffffffff, 0x0000009b,
 796         0x913c, 0x0000000f, 0x0100000a,
 797         0x8d00, 0xffff7f7f, 0x100e4848,
 798         0x8d04, 0x00ffffff, 0x00164745,
 799         0x8c00, 0xfffc0003, 0xe4000003,
 800         0x8c04, 0xf8ff00ff, 0x40600060,
 801         0x8c08, 0x00ff00ff, 0x001c001c,
 802         0x8cf0, 0x1fff1fff, 0x08e00620,
 803         0x8c20, 0x0fff0fff, 0x00800080,
 804         0x8c24, 0x0fff0fff, 0x00800080,
 805         0x8c18, 0xffffffff, 0x20202078,
 806         0x8c1c, 0x0000ffff, 0x00001010,
 807         0x28350, 0x00000f01, 0x00000000,
 808         0x9508, 0x3700001f, 0x00000002,
 809         0x960c, 0xffffffff, 0x54763210,
 810         0x88c4, 0x001f3ae3, 0x000000c2,
 811         0x88d4, 0x0000001f, 0x00000010,
 812         0x8974, 0xffffffff, 0x00000000
 813 };
 814
 815 static const u32 turks_golden_registers[] =
 816 {
 817         0x5eb4, 0xffffffff, 0x00000002,
 818         0x5e78, 0x8f311ff1, 0x001000f0,
 819         0x8c8, 0x00003000, 0x00001070,
 820         0x8cc, 0x000fffff, 0x00040035,
 821         0x3f90, 0xffff0000, 0xfff00000,
 822         0x9148, 0xffff0000, 0xfff00000,
 823         0x3f94, 0xffff0000, 0xfff00000,
 824         0x914c, 0xffff0000, 0xfff00000,
 825         0xc78, 0x00000080, 0x00000080,
 826         0xbd4, 0x00073007, 0x00010002,
 827         0xd02c, 0xbfffff1f, 0x08421000,
 828         0xd0b8, 0x03773777, 0x02010002,
 829         0x5bc0, 0x00200000, 0x50100000,
 830         0x98f8, 0x33773777, 0x00010002,
 831         0x98fc, 0xffffffff, 0x33221100,
 832         0x7030, 0x31000311, 0x00000011,
 833         0x2f48, 0x33773777, 0x00010002,
 834         0x6b28, 0x00000010, 0x00000012,
 835         0x7728, 0x00000010, 0x00000012,
 836         0x10328, 0x00000010, 0x00000012,
 837         0x10f28, 0x00000010, 0x00000012,
 838         0x11b28, 0x00000010, 0x00000012,
 839         0x12728, 0x00000010, 0x00000012,
 840         0x240c, 0x000007ff, 0x00000380,
 841         0x8a14, 0xf000001f, 0x00000007,
 842         0x8b24, 0x3fff3fff, 0x00ff0fff,
 843         0x8b10, 0x0000ff0f, 0x00000000,
 844         0x28a4c, 0x07ffffff, 0x06000000,
 845         0x10c, 0x00000001, 0x00010003,
 846         0xa02c, 0xffffffff, 0x0000009b,
 847         0x913c, 0x0000000f, 0x0100000a,
 848         0x8d00, 0xffff7f7f, 0x100e4848,
 849         0x8d04, 0x00ffffff, 0x00164745,
 850         0x8c00, 0xfffc0003, 0xe4000003,
 851         0x8c04, 0xf8ff00ff, 0x40600060,
 852         0x8c08, 0x00ff00ff, 0x001c001c,
 853         0x8cf0, 0x1fff1fff, 0x08e00410,
 854         0x8c20, 0x0fff0fff, 0x00800080,
 855         0x8c24, 0x0fff0fff, 0x00800080,
 856         0x8c18, 0xffffffff, 0x20202078,
 857         0x8c1c, 0x0000ffff, 0x00001010,
 858         0x28350, 0x00000f01, 0x00000000,
 859         0x9508, 0x3700001f, 0x00000002,
 860         0x960c, 0xffffffff, 0x54763210,
 861         0x88c4, 0x001f3ae3, 0x000000c2,
 862         0x88d4, 0x0000001f, 0x00000010,
 863         0x8974, 0xffffffff, 0x00000000
 864 };
 865
 866 static const u32 caicos_golden_registers[] =
 867 {
 868         0x5eb4, 0xffffffff, 0x00000002,
 869         0x5e78, 0x8f311ff1, 0x001000f0,
 870         0x8c8, 0x00003420, 0x00001450,
 871         0x8cc, 0x000fffff, 0x00040035,
 872         0x3f90, 0xffff0000, 0xfffc0000,
 873         0x9148, 0xffff0000, 0xfffc0000,
 874         0x3f94, 0xffff0000, 0xfffc0000,
 875         0x914c, 0xffff0000, 0xfffc0000,
 876         0xc78, 0x00000080, 0x00000080,
 877         0xbd4, 0x00073007, 0x00010001,
 878         0xd02c, 0xbfffff1f, 0x08421000,
 879         0xd0b8, 0x03773777, 0x02010001,
 880         0x5bc0, 0x00200000, 0x50100000,
 881         0x98f8, 0x33773777, 0x02010001,
 882         0x98fc, 0xffffffff, 0x33221100,
 883         0x7030, 0x31000311, 0x00000011,
 884         0x2f48, 0x33773777, 0x02010001,
 885         0x6b28, 0x00000010, 0x00000012,
 886         0x7728, 0x00000010, 0x00000012,
 887         0x10328, 0x00000010, 0x00000012,
 888         0x10f28, 0x00000010, 0x00000012,
 889         0x11b28, 0x00000010, 0x00000012,
 890         0x12728, 0x00000010, 0x00000012,
 891         0x240c, 0x000007ff, 0x00000380,
 892         0x8a14, 0xf000001f, 0x00000001,
 893         0x8b24, 0x3fff3fff, 0x00ff0fff,
 894         0x8b10, 0x0000ff0f, 0x00000000,
 895         0x28a4c, 0x07ffffff, 0x06000000,
 896         0x10c, 0x00000001, 0x00010003,
 897         0xa02c, 0xffffffff, 0x0000009b,
 898         0x913c, 0x0000000f, 0x0100000a,
 899         0x8d00, 0xffff7f7f, 0x100e4848,
 900         0x8d04, 0x00ffffff, 0x00164745,
 901         0x8c00, 0xfffc0003, 0xe4000003,
 902         0x8c04, 0xf8ff00ff, 0x40600060,
 903         0x8c08, 0x00ff00ff, 0x001c001c,
 904         0x8cf0, 0x1fff1fff, 0x08e00410,
 905         0x8c20, 0x0fff0fff, 0x00800080,
 906         0x8c24, 0x0fff0fff, 0x00800080,
 907         0x8c18, 0xffffffff, 0x20202078,
 908         0x8c1c, 0x0000ffff, 0x00001010,
 909         0x28350, 0x00000f01, 0x00000000,
 910         0x9508, 0x3700001f, 0x00000002,
 911         0x960c, 0xffffffff, 0x54763210,
 912         0x88c4, 0x001f3ae3, 0x000000c2,
 913         0x88d4, 0x0000001f, 0x00000010,
 914         0x8974, 0xffffffff, 0x00000000
 915 };
 916
 917 static void evergreen_init_golden_registers(struct radeon_device *rdev)
 918 {
 919         switch (rdev->family) {
 920         case CHIP_CYPRESS:
 921         case CHIP_HEMLOCK:
 922                 radeon_program_register_sequence(rdev,
 923                                                  evergreen_golden_registers,
 924                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers));
 925                 radeon_program_register_sequence(rdev,
 926                                                  evergreen_golden_registers2,
 927                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
 928                 radeon_program_register_sequence(rdev,
 929                                                  cypress_mgcg_init,
 930                                                  (const u32)ARRAY_SIZE(cypress_mgcg_init));
 931                 break;
 932         case CHIP_JUNIPER:
 933                 radeon_program_register_sequence(rdev,
 934                                                  evergreen_golden_registers,
 935                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers));
 936                 radeon_program_register_sequence(rdev,
 937                                                  evergreen_golden_registers2,
 938                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
 939                 radeon_program_register_sequence(rdev,
 940                                                  juniper_mgcg_init,
 941                                                  (const u32)ARRAY_SIZE(juniper_mgcg_init));
 942                 break;
 943         case CHIP_REDWOOD:
 944                 radeon_program_register_sequence(rdev,
 945                                                  evergreen_golden_registers,
 946                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers));
 947                 radeon_program_register_sequence(rdev,
 948                                                  evergreen_golden_registers2,
 949                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
 950                 radeon_program_register_sequence(rdev,
 951                                                  redwood_mgcg_init,
 952                                                  (const u32)ARRAY_SIZE(redwood_mgcg_init));
 953                 break;
 954         case CHIP_CEDAR:
 955                 radeon_program_register_sequence(rdev,
 956                                                  cedar_golden_registers,
 957                                                  (const u32)ARRAY_SIZE(cedar_golden_registers));
 958                 radeon_program_register_sequence(rdev,
 959                                                  evergreen_golden_registers2,
 960                                                  (const u32)ARRAY_SIZE(evergreen_golden_registers2));
 961                 radeon_program_register_sequence(rdev,
 962                                                  cedar_mgcg_init,
 963                                                  (const u32)ARRAY_SIZE(cedar_mgcg_init));
 964                 break;
 965         case CHIP_PALM:
 966                 radeon_program_register_sequence(rdev,
 967                                                  wrestler_golden_registers,
 968                                                  (const u32)ARRAY_SIZE(wrestler_golden_registers));
 969                 break;
 970         case CHIP_SUMO:
 971                 radeon_program_register_sequence(rdev,
 972                                                  supersumo_golden_registers,
 973                                                  (const u32)ARRAY_SIZE(supersumo_golden_registers));
 974                 break;
 975         case CHIP_SUMO2:
 976                 radeon_program_register_sequence(rdev,
 977                                                  supersumo_golden_registers,
 978                                                  (const u32)ARRAY_SIZE(supersumo_golden_registers));
 979                 radeon_program_register_sequence(rdev,
 980                                                  sumo_golden_registers,
 981                                                  (const u32)ARRAY_SIZE(sumo_golden_registers));
 982                 break;
 983         case CHIP_BARTS:
 984                 radeon_program_register_sequence(rdev,
 985                                                  barts_golden_registers,
 986                                                  (const u32)ARRAY_SIZE(barts_golden_registers));
 987                 break;
 988         case CHIP_TURKS:
 989                 radeon_program_register_sequence(rdev,
 990                                                  turks_golden_registers,
 991                                                  (const u32)ARRAY_SIZE(turks_golden_registers));
 992                 break;
 993         case CHIP_CAICOS:
 994                 radeon_program_register_sequence(rdev,
 995                                                  caicos_golden_registers,
 996                                                  (const u32)ARRAY_SIZE(caicos_golden_registers));
 997                 break;
 998         default:
 999                 break;
1000         }
1001 }
1002
1003 void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1004                              unsigned *bankh, unsigned *mtaspect,
1005                              unsigned *tile_split)
1006 {
1007         *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1008         *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1009         *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1010         *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1011         switch (*bankw) {
1012         default:
1013         case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1014         case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1015         case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1016         case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1017         }
1018         switch (*bankh) {
1019         default:
1020         case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1021         case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1022         case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1023         case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1024         }
1025         switch (*mtaspect) {
1026         default:
1027         case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1028         case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1029         case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1030         case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1031         }
1032 }
1033
1034 static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1035                               u32 cntl_reg, u32 status_reg)
1036 {
1037         int r, i;
1038         struct atom_clock_dividers dividers;
1039
1040         r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1041                                            clock, false, &dividers);
1042         if (r)
1043                 return r;
1044
1045         WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1046
1047         for (i = 0; i < 100; i++) {
1048                 if (RREG32(status_reg) & DCLK_STATUS)
1049                         break;
1050                 mdelay(10);
1051         }
1052         if (i == 100)
1053                 return -ETIMEDOUT;
1054
1055         return 0;
1056 }
1057
1058 int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1059 {
1060         int r = 0;
1061         u32 cg_scratch = RREG32(CG_SCRATCH1);
1062
1063         r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1064         if (r)
1065                 goto done;
1066         cg_scratch &= 0xffff0000;
1067         cg_scratch |= vclk / 100; /* Mhz */
1068
1069         r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1070         if (r)
1071                 goto done;
1072         cg_scratch &= 0x0000ffff;
1073         cg_scratch |= (dclk / 100) << 16; /* Mhz */
1074
1075 done:
1076         WREG32(CG_SCRATCH1, cg_scratch);
1077
1078         return r;
1079 }
1080
1081 int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1082 {
1083         /* start off with something large */
1084         unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
1085         int r;
1086
1087         /* bypass vclk and dclk with bclk */
1088         WREG32_P(CG_UPLL_FUNC_CNTL_2,
1089                 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1090                 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1091
1092         /* put PLL in bypass mode */
1093         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1094
1095         if (!vclk || !dclk) {
1096                 /* keep the Bypass mode, put PLL to sleep */
1097                 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1098                 return 0;
1099         }
1100
1101         r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1102                                           16384, 0x03FFFFFF, 0, 128, 5,
1103                                           &fb_div, &vclk_div, &dclk_div);
1104         if (r)
1105                 return r;
1106
1107         /* set VCO_MODE to 1 */
1108         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1109
1110         /* toggle UPLL_SLEEP to 1 then back to 0 */
1111         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1112         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1113
1114         /* deassert UPLL_RESET */
1115         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1116
1117         mdelay(1);
1118
1119         r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1120         if (r)
1121                 return r;
1122
1123         /* assert UPLL_RESET again */
1124         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1125
1126         /* disable spread spectrum. */
1127         WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1128
1129         /* set feedback divider */
1130         WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
1131
1132         /* set ref divider to 0 */
1133         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1134
1135         if (fb_div < 307200)
1136                 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1137         else
1138                 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1139
1140         /* set PDIV_A and PDIV_B */
1141         WREG32_P(CG_UPLL_FUNC_CNTL_2,
1142                 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
1143                 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1144
1145         /* give the PLL some time to settle */
1146         mdelay(15);
1147
1148         /* deassert PLL_RESET */
1149         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1150
1151         mdelay(15);
1152
1153         /* switch from bypass mode to normal mode */
1154         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1155
1156         r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1157         if (r)
1158                 return r;
1159
1160         /* switch VCLK and DCLK selection */
1161         WREG32_P(CG_UPLL_FUNC_CNTL_2,
1162                 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1163                 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1164
1165         mdelay(100);
1166
1167         return 0;
1168 }
1169
1170 void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1171 {
1172         u16 ctl, v;
1173         int err;
1174
1175         err = pcie_capability_read_word(rdev->pdev, PCI_EXP_DEVCTL, &ctl);
1176         if (err)
1177                 return;
1178
1179         v = (ctl & PCI_EXP_DEVCTL_READRQ) >> 12;
1180
1181         /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1182          * to avoid hangs or perfomance issues
1183          */
1184         if ((v == 0) || (v == 6) || (v == 7)) {
1185                 ctl &= ~PCI_EXP_DEVCTL_READRQ;
1186                 ctl |= (2 << 12);
1187                 pcie_capability_write_word(rdev->pdev, PCI_EXP_DEVCTL, ctl);
1188         }
1189 }
1190
1191 static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1192 {
1193         if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1194                 return true;
1195         else
1196                 return false;
1197 }
1198
1199 static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1200 {
1201         u32 pos1, pos2;
1202
1203         pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1204         pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1205
1206         if (pos1 != pos2)
1207                 return true;
1208         else
1209                 return false;
1210 }
1211
1212 /**
1213  * dce4_wait_for_vblank - vblank wait asic callback.
1214  *
1215  * @rdev: radeon_device pointer
1216  * @crtc: crtc to wait for vblank on
1217  *
1218  * Wait for vblank on the requested crtc (evergreen+).
1219  */
1220 void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1221 {
1222         unsigned i = 0;
1223
1224         if (crtc >= rdev->num_crtc)
1225                 return;
1226
1227         if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1228                 return;
1229
1230         /* depending on when we hit vblank, we may be close to active; if so,
1231          * wait for another frame.
1232          */
1233         while (dce4_is_in_vblank(rdev, crtc)) {
1234                 if (i++ % 100 == 0) {
1235                         if (!dce4_is_counter_moving(rdev, crtc))
1236                                 break;
1237                 }
1238         }
1239
1240         while (!dce4_is_in_vblank(rdev, crtc)) {
1241                 if (i++ % 100 == 0) {
1242                         if (!dce4_is_counter_moving(rdev, crtc))
1243                                 break;
1244                 }
1245         }
1246 }
1247
1248 /**
1249  * radeon_irq_kms_pflip_irq_get - pre-pageflip callback.
1250  *
1251  * @rdev: radeon_device pointer
1252  * @crtc: crtc to prepare for pageflip on
1253  *
1254  * Pre-pageflip callback (evergreen+).
1255  * Enables the pageflip irq (vblank irq).
1256  */
1257 void evergreen_pre_page_flip(struct radeon_device *rdev, int crtc)
1258 {
1259         /* enable the pflip int */
1260         radeon_irq_kms_pflip_irq_get(rdev, crtc);
1261 }
1262
1263 /**
1264  * evergreen_post_page_flip - pos-pageflip callback.
1265  *
1266  * @rdev: radeon_device pointer
1267  * @crtc: crtc to cleanup pageflip on
1268  *
1269  * Post-pageflip callback (evergreen+).
1270  * Disables the pageflip irq (vblank irq).
1271  */
1272 void evergreen_post_page_flip(struct radeon_device *rdev, int crtc)
1273 {
1274         /* disable the pflip int */
1275         radeon_irq_kms_pflip_irq_put(rdev, crtc);
1276 }
1277
1278 /**
1279  * evergreen_page_flip - pageflip callback.
1280  *
1281  * @rdev: radeon_device pointer
1282  * @crtc_id: crtc to cleanup pageflip on
1283  * @crtc_base: new address of the crtc (GPU MC address)
1284  *
1285  * Does the actual pageflip (evergreen+).
1286  * During vblank we take the crtc lock and wait for the update_pending
1287  * bit to go high, when it does, we release the lock, and allow the
1288  * double buffered update to take place.
1289  * Returns the current update pending status.
1290  */
1291 u32 evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
1292 {
1293         struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1294         u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
1295         int i;
1296
1297         /* Lock the graphics update lock */
1298         tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
1299         WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1300
1301         /* update the scanout addresses */
1302         WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1303                upper_32_bits(crtc_base));
1304         WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1305                (u32)crtc_base);
1306
1307         WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1308                upper_32_bits(crtc_base));
1309         WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1310                (u32)crtc_base);
1311
1312         /* Wait for update_pending to go high. */
1313         for (i = 0; i < rdev->usec_timeout; i++) {
1314                 if (RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING)
1315                         break;
1316                 udelay(1);
1317         }
1318         DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
1319
1320         /* Unlock the lock, so double-buffering can take place inside vblank */
1321         tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
1322         WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1323
1324         /* Return current update_pending status: */
1325         return RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING;
1326 }
1327
1328 /* get temperature in millidegrees */
1329 int evergreen_get_temp(struct radeon_device *rdev)
1330 {
1331         u32 temp, toffset;
1332         int actual_temp = 0;
1333
1334         if (rdev->family == CHIP_JUNIPER) {
1335                 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1336                         TOFFSET_SHIFT;
1337                 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1338                         TS0_ADC_DOUT_SHIFT;
1339
1340                 if (toffset & 0x100)
1341                         actual_temp = temp / 2 - (0x200 - toffset);
1342                 else
1343                         actual_temp = temp / 2 + toffset;
1344
1345                 actual_temp = actual_temp * 1000;
1346
1347         } else {
1348                 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1349                         ASIC_T_SHIFT;
1350
1351                 if (temp & 0x400)
1352                         actual_temp = -256;
1353                 else if (temp & 0x200)
1354                         actual_temp = 255;
1355                 else if (temp & 0x100) {
1356                         actual_temp = temp & 0x1ff;
1357                         actual_temp |= ~0x1ff;
1358                 } else
1359                         actual_temp = temp & 0xff;
1360
1361                 actual_temp = (actual_temp * 1000) / 2;
1362         }
1363
1364         return actual_temp;
1365 }
1366
1367 int sumo_get_temp(struct radeon_device *rdev)
1368 {
1369         u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
1370         int actual_temp = temp - 49;
1371
1372         return actual_temp * 1000;
1373 }
1374
1375 /**
1376  * sumo_pm_init_profile - Initialize power profiles callback.
1377  *
1378  * @rdev: radeon_device pointer
1379  *
1380  * Initialize the power states used in profile mode
1381  * (sumo, trinity, SI).
1382  * Used for profile mode only.
1383  */
1384 void sumo_pm_init_profile(struct radeon_device *rdev)
1385 {
1386         int idx;
1387
1388         /* default */
1389         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1390         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1391         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1392         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1393
1394         /* low,mid sh/mh */
1395         if (rdev->flags & RADEON_IS_MOBILITY)
1396                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1397         else
1398                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1399
1400         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1401         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1402         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1403         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1404
1405         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1406         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1407         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1408         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1409
1410         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1411         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1412         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1413         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1414
1415         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1416         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1417         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1418         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1419
1420         /* high sh/mh */
1421         idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1422         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1423         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1424         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1425         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1426                 rdev->pm.power_state[idx].num_clock_modes - 1;
1427
1428         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1429         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1430         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1431         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1432                 rdev->pm.power_state[idx].num_clock_modes - 1;
1433 }
1434
1435 /**
1436  * btc_pm_init_profile - Initialize power profiles callback.
1437  *
1438  * @rdev: radeon_device pointer
1439  *
1440  * Initialize the power states used in profile mode
1441  * (BTC, cayman).
1442  * Used for profile mode only.
1443  */
1444 void btc_pm_init_profile(struct radeon_device *rdev)
1445 {
1446         int idx;
1447
1448         /* default */
1449         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1450         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1451         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1452         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1453         /* starting with BTC, there is one state that is used for both
1454          * MH and SH.  Difference is that we always use the high clock index for
1455          * mclk.
1456          */
1457         if (rdev->flags & RADEON_IS_MOBILITY)
1458                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1459         else
1460                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1461         /* low sh */
1462         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1463         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1464         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1465         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1466         /* mid sh */
1467         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1468         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1469         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1470         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1471         /* high sh */
1472         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1473         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1474         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1475         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1476         /* low mh */
1477         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1478         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1479         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1480         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1481         /* mid mh */
1482         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1483         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1484         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1485         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1486         /* high mh */
1487         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1488         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1489         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1490         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1491 }
1492
1493 /**
1494  * evergreen_pm_misc - set additional pm hw parameters callback.
1495  *
1496  * @rdev: radeon_device pointer
1497  *
1498  * Set non-clock parameters associated with a power state
1499  * (voltage, etc.) (evergreen+).
1500  */
1501 void evergreen_pm_misc(struct radeon_device *rdev)
1502 {
1503         int req_ps_idx = rdev->pm.requested_power_state_index;
1504         int req_cm_idx = rdev->pm.requested_clock_mode_index;
1505         struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1506         struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
1507
1508         if (voltage->type == VOLTAGE_SW) {
1509                 /* 0xff0x are flags rather then an actual voltage */
1510                 if ((voltage->voltage & 0xff00) == 0xff00)
1511                         return;
1512                 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
1513                         radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
1514                         rdev->pm.current_vddc = voltage->voltage;
1515                         DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1516                 }
1517
1518                 /* starting with BTC, there is one state that is used for both
1519                  * MH and SH.  Difference is that we always use the high clock index for
1520                  * mclk and vddci.
1521                  */
1522                 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1523                     (rdev->family >= CHIP_BARTS) &&
1524                     rdev->pm.active_crtc_count &&
1525                     ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1526                      (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1527                         voltage = &rdev->pm.power_state[req_ps_idx].
1528                                 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1529
1530                 /* 0xff0x are flags rather then an actual voltage */
1531                 if ((voltage->vddci & 0xff00) == 0xff00)
1532                         return;
1533                 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1534                         radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1535                         rdev->pm.current_vddci = voltage->vddci;
1536                         DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
1537                 }
1538         }
1539 }
1540
1541 /**
1542  * evergreen_pm_prepare - pre-power state change callback.
1543  *
1544  * @rdev: radeon_device pointer
1545  *
1546  * Prepare for a power state change (evergreen+).
1547  */
1548 void evergreen_pm_prepare(struct radeon_device *rdev)
1549 {
1550         struct drm_device *ddev = rdev->ddev;
1551         struct drm_crtc *crtc;
1552         struct radeon_crtc *radeon_crtc;
1553         u32 tmp;
1554
1555         /* disable any active CRTCs */
1556         list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1557                 radeon_crtc = to_radeon_crtc(crtc);
1558                 if (radeon_crtc->enabled) {
1559                         tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1560                         tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1561                         WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1562                 }
1563         }
1564 }
1565
1566 /**
1567  * evergreen_pm_finish - post-power state change callback.
1568  *
1569  * @rdev: radeon_device pointer
1570  *
1571  * Clean up after a power state change (evergreen+).
1572  */
1573 void evergreen_pm_finish(struct radeon_device *rdev)
1574 {
1575         struct drm_device *ddev = rdev->ddev;
1576         struct drm_crtc *crtc;
1577         struct radeon_crtc *radeon_crtc;
1578         u32 tmp;
1579
1580         /* enable any active CRTCs */
1581         list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1582                 radeon_crtc = to_radeon_crtc(crtc);
1583                 if (radeon_crtc->enabled) {
1584                         tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1585                         tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1586                         WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1587                 }
1588         }
1589 }
1590
1591 /**
1592  * evergreen_hpd_sense - hpd sense callback.
1593  *
1594  * @rdev: radeon_device pointer
1595  * @hpd: hpd (hotplug detect) pin
1596  *
1597  * Checks if a digital monitor is connected (evergreen+).
1598  * Returns true if connected, false if not connected.
1599  */
1600 bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1601 {
1602         bool connected = false;
1603
1604         switch (hpd) {
1605         case RADEON_HPD_1:
1606                 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
1607                         connected = true;
1608                 break;
1609         case RADEON_HPD_2:
1610                 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
1611                         connected = true;
1612                 break;
1613         case RADEON_HPD_3:
1614                 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
1615                         connected = true;
1616                 break;
1617         case RADEON_HPD_4:
1618                 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
1619                         connected = true;
1620                 break;
1621         case RADEON_HPD_5:
1622                 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
1623                         connected = true;
1624                 break;
1625         case RADEON_HPD_6:
1626                 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
1627                         connected = true;
1628                         break;
1629         default:
1630                 break;
1631         }
1632
1633         return connected;
1634 }
1635
1636 /**
1637  * evergreen_hpd_set_polarity - hpd set polarity callback.
1638  *
1639  * @rdev: radeon_device pointer
1640  * @hpd: hpd (hotplug detect) pin
1641  *
1642  * Set the polarity of the hpd pin (evergreen+).
1643  */
1644 void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1645                                 enum radeon_hpd_id hpd)
1646 {
1647         u32 tmp;
1648         bool connected = evergreen_hpd_sense(rdev, hpd);
1649
1650         switch (hpd) {
1651         case RADEON_HPD_1:
1652                 tmp = RREG32(DC_HPD1_INT_CONTROL);
1653                 if (connected)
1654                         tmp &= ~DC_HPDx_INT_POLARITY;
1655                 else
1656                         tmp |= DC_HPDx_INT_POLARITY;
1657                 WREG32(DC_HPD1_INT_CONTROL, tmp);
1658                 break;
1659         case RADEON_HPD_2:
1660                 tmp = RREG32(DC_HPD2_INT_CONTROL);
1661                 if (connected)
1662                         tmp &= ~DC_HPDx_INT_POLARITY;
1663                 else
1664                         tmp |= DC_HPDx_INT_POLARITY;
1665                 WREG32(DC_HPD2_INT_CONTROL, tmp);
1666                 break;
1667         case RADEON_HPD_3:
1668                 tmp = RREG32(DC_HPD3_INT_CONTROL);
1669                 if (connected)
1670                         tmp &= ~DC_HPDx_INT_POLARITY;
1671                 else
1672                         tmp |= DC_HPDx_INT_POLARITY;
1673                 WREG32(DC_HPD3_INT_CONTROL, tmp);
1674                 break;
1675         case RADEON_HPD_4:
1676                 tmp = RREG32(DC_HPD4_INT_CONTROL);
1677                 if (connected)
1678                         tmp &= ~DC_HPDx_INT_POLARITY;
1679                 else
1680                         tmp |= DC_HPDx_INT_POLARITY;
1681                 WREG32(DC_HPD4_INT_CONTROL, tmp);
1682                 break;
1683         case RADEON_HPD_5:
1684                 tmp = RREG32(DC_HPD5_INT_CONTROL);
1685                 if (connected)
1686                         tmp &= ~DC_HPDx_INT_POLARITY;
1687                 else
1688                         tmp |= DC_HPDx_INT_POLARITY;
1689                 WREG32(DC_HPD5_INT_CONTROL, tmp);
1690                         break;
1691         case RADEON_HPD_6:
1692                 tmp = RREG32(DC_HPD6_INT_CONTROL);
1693                 if (connected)
1694                         tmp &= ~DC_HPDx_INT_POLARITY;
1695                 else
1696                         tmp |= DC_HPDx_INT_POLARITY;
1697                 WREG32(DC_HPD6_INT_CONTROL, tmp);
1698                 break;
1699         default:
1700                 break;
1701         }
1702 }
1703
1704 /**
1705  * evergreen_hpd_init - hpd setup callback.
1706  *
1707  * @rdev: radeon_device pointer
1708  *
1709  * Setup the hpd pins used by the card (evergreen+).
1710  * Enable the pin, set the polarity, and enable the hpd interrupts.
1711  */
1712 void evergreen_hpd_init(struct radeon_device *rdev)
1713 {
1714         struct drm_device *dev = rdev->ddev;
1715         struct drm_connector *connector;
1716         unsigned enabled = 0;
1717         u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1718                 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
1719
1720         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1721                 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1722
1723                 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1724                     connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1725                         /* don't try to enable hpd on eDP or LVDS avoid breaking the
1726                          * aux dp channel on imac and help (but not completely fix)
1727                          * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1728                          * also avoid interrupt storms during dpms.
1729                          */
1730                         continue;
1731                 }
1732                 switch (radeon_connector->hpd.hpd) {
1733                 case RADEON_HPD_1:
1734                         WREG32(DC_HPD1_CONTROL, tmp);
1735                         break;
1736                 case RADEON_HPD_2:
1737                         WREG32(DC_HPD2_CONTROL, tmp);
1738                         break;
1739                 case RADEON_HPD_3:
1740                         WREG32(DC_HPD3_CONTROL, tmp);
1741                         break;
1742                 case RADEON_HPD_4:
1743                         WREG32(DC_HPD4_CONTROL, tmp);
1744                         break;
1745                 case RADEON_HPD_5:
1746                         WREG32(DC_HPD5_CONTROL, tmp);
1747                         break;
1748                 case RADEON_HPD_6:
1749                         WREG32(DC_HPD6_CONTROL, tmp);
1750                         break;
1751                 default:
1752                         break;
1753                 }
1754                 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
1755                 enabled |= 1 << radeon_connector->hpd.hpd;
1756         }
1757         radeon_irq_kms_enable_hpd(rdev, enabled);
1758 }
1759
1760 /**
1761  * evergreen_hpd_fini - hpd tear down callback.
1762  *
1763  * @rdev: radeon_device pointer
1764  *
1765  * Tear down the hpd pins used by the card (evergreen+).
1766  * Disable the hpd interrupts.
1767  */
1768 void evergreen_hpd_fini(struct radeon_device *rdev)
1769 {
1770         struct drm_device *dev = rdev->ddev;
1771         struct drm_connector *connector;
1772         unsigned disabled = 0;
1773
1774         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1775                 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1776                 switch (radeon_connector->hpd.hpd) {
1777                 case RADEON_HPD_1:
1778                         WREG32(DC_HPD1_CONTROL, 0);
1779                         break;
1780                 case RADEON_HPD_2:
1781                         WREG32(DC_HPD2_CONTROL, 0);
1782                         break;
1783                 case RADEON_HPD_3:
1784                         WREG32(DC_HPD3_CONTROL, 0);
1785                         break;
1786                 case RADEON_HPD_4:
1787                         WREG32(DC_HPD4_CONTROL, 0);
1788                         break;
1789                 case RADEON_HPD_5:
1790                         WREG32(DC_HPD5_CONTROL, 0);
1791                         break;
1792                 case RADEON_HPD_6:
1793                         WREG32(DC_HPD6_CONTROL, 0);
1794                         break;
1795                 default:
1796                         break;
1797                 }
1798                 disabled |= 1 << radeon_connector->hpd.hpd;
1799         }
1800         radeon_irq_kms_disable_hpd(rdev, disabled);
1801 }
1802
1803 /* watermark setup */
1804
1805 static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1806                                         struct radeon_crtc *radeon_crtc,
1807                                         struct drm_display_mode *mode,
1808                                         struct drm_display_mode *other_mode)
1809 {
1810         u32 tmp;
1811         /*
1812          * Line Buffer Setup
1813          * There are 3 line buffers, each one shared by 2 display controllers.
1814          * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1815          * the display controllers.  The paritioning is done via one of four
1816          * preset allocations specified in bits 2:0:
1817          * first display controller
1818          *  0 - first half of lb (3840 * 2)
1819          *  1 - first 3/4 of lb (5760 * 2)
1820          *  2 - whole lb (7680 * 2), other crtc must be disabled
1821          *  3 - first 1/4 of lb (1920 * 2)
1822          * second display controller
1823          *  4 - second half of lb (3840 * 2)
1824          *  5 - second 3/4 of lb (5760 * 2)
1825          *  6 - whole lb (7680 * 2), other crtc must be disabled
1826          *  7 - last 1/4 of lb (1920 * 2)
1827          */
1828         /* this can get tricky if we have two large displays on a paired group
1829          * of crtcs.  Ideally for multiple large displays we'd assign them to
1830          * non-linked crtcs for maximum line buffer allocation.
1831          */
1832         if (radeon_crtc->base.enabled && mode) {
1833                 if (other_mode)
1834                         tmp = 0; /* 1/2 */
1835                 else
1836                         tmp = 2; /* whole */
1837         } else
1838                 tmp = 0;
1839
1840         /* second controller of the pair uses second half of the lb */
1841         if (radeon_crtc->crtc_id % 2)
1842                 tmp += 4;
1843         WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1844
1845         if (radeon_crtc->base.enabled && mode) {
1846                 switch (tmp) {
1847                 case 0:
1848                 case 4:
1849                 default:
1850                         if (ASIC_IS_DCE5(rdev))
1851                                 return 4096 * 2;
1852                         else
1853                                 return 3840 * 2;
1854                 case 1:
1855                 case 5:
1856                         if (ASIC_IS_DCE5(rdev))
1857                                 return 6144 * 2;
1858                         else
1859                                 return 5760 * 2;
1860                 case 2:
1861                 case 6:
1862                         if (ASIC_IS_DCE5(rdev))
1863                                 return 8192 * 2;
1864                         else
1865                                 return 7680 * 2;
1866                 case 3:
1867                 case 7:
1868                         if (ASIC_IS_DCE5(rdev))
1869                                 return 2048 * 2;
1870                         else
1871                                 return 1920 * 2;
1872                 }
1873         }
1874
1875         /* controller not enabled, so no lb used */
1876         return 0;
1877 }
1878
1879 u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
1880 {
1881         u32 tmp = RREG32(MC_SHARED_CHMAP);
1882
1883         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1884         case 0:
1885         default:
1886                 return 1;
1887         case 1:
1888                 return 2;
1889         case 2:
1890                 return 4;
1891         case 3:
1892                 return 8;
1893         }
1894 }
1895
1896 struct evergreen_wm_params {
1897         u32 dram_channels; /* number of dram channels */
1898         u32 yclk;          /* bandwidth per dram data pin in kHz */
1899         u32 sclk;          /* engine clock in kHz */
1900         u32 disp_clk;      /* display clock in kHz */
1901         u32 src_width;     /* viewport width */
1902         u32 active_time;   /* active display time in ns */
1903         u32 blank_time;    /* blank time in ns */
1904         bool interlaced;    /* mode is interlaced */
1905         fixed20_12 vsc;    /* vertical scale ratio */
1906         u32 num_heads;     /* number of active crtcs */
1907         u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1908         u32 lb_size;       /* line buffer allocated to pipe */
1909         u32 vtaps;         /* vertical scaler taps */
1910 };
1911
1912 static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1913 {
1914         /* Calculate DRAM Bandwidth and the part allocated to display. */
1915         fixed20_12 dram_efficiency; /* 0.7 */
1916         fixed20_12 yclk, dram_channels, bandwidth;
1917         fixed20_12 a;
1918
1919         a.full = dfixed_const(1000);
1920         yclk.full = dfixed_const(wm->yclk);
1921         yclk.full = dfixed_div(yclk, a);
1922         dram_channels.full = dfixed_const(wm->dram_channels * 4);
1923         a.full = dfixed_const(10);
1924         dram_efficiency.full = dfixed_const(7);
1925         dram_efficiency.full = dfixed_div(dram_efficiency, a);
1926         bandwidth.full = dfixed_mul(dram_channels, yclk);
1927         bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1928
1929         return dfixed_trunc(bandwidth);
1930 }
1931
1932 static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
1933 {
1934         /* Calculate DRAM Bandwidth and the part allocated to display. */
1935         fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1936         fixed20_12 yclk, dram_channels, bandwidth;
1937         fixed20_12 a;
1938
1939         a.full = dfixed_const(1000);
1940         yclk.full = dfixed_const(wm->yclk);
1941         yclk.full = dfixed_div(yclk, a);
1942         dram_channels.full = dfixed_const(wm->dram_channels * 4);
1943         a.full = dfixed_const(10);
1944         disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1945         disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1946         bandwidth.full = dfixed_mul(dram_channels, yclk);
1947         bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1948
1949         return dfixed_trunc(bandwidth);
1950 }
1951
1952 static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
1953 {
1954         /* Calculate the display Data return Bandwidth */
1955         fixed20_12 return_efficiency; /* 0.8 */
1956         fixed20_12 sclk, bandwidth;
1957         fixed20_12 a;
1958
1959         a.full = dfixed_const(1000);
1960         sclk.full = dfixed_const(wm->sclk);
1961         sclk.full = dfixed_div(sclk, a);
1962         a.full = dfixed_const(10);
1963         return_efficiency.full = dfixed_const(8);
1964         return_efficiency.full = dfixed_div(return_efficiency, a);
1965         a.full = dfixed_const(32);
1966         bandwidth.full = dfixed_mul(a, sclk);
1967         bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1968
1969         return dfixed_trunc(bandwidth);
1970 }
1971
1972 static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
1973 {
1974         /* Calculate the DMIF Request Bandwidth */
1975         fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1976         fixed20_12 disp_clk, bandwidth;
1977         fixed20_12 a;
1978
1979         a.full = dfixed_const(1000);
1980         disp_clk.full = dfixed_const(wm->disp_clk);
1981         disp_clk.full = dfixed_div(disp_clk, a);
1982         a.full = dfixed_const(10);
1983         disp_clk_request_efficiency.full = dfixed_const(8);
1984         disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
1985         a.full = dfixed_const(32);
1986         bandwidth.full = dfixed_mul(a, disp_clk);
1987         bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
1988
1989         return dfixed_trunc(bandwidth);
1990 }
1991
1992 static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
1993 {
1994         /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
1995         u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
1996         u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
1997         u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
1998
1999         return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2000 }
2001
2002 static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2003 {
2004         /* Calculate the display mode Average Bandwidth
2005          * DisplayMode should contain the source and destination dimensions,
2006          * timing, etc.
2007          */
2008         fixed20_12 bpp;
2009         fixed20_12 line_time;
2010         fixed20_12 src_width;
2011         fixed20_12 bandwidth;
2012         fixed20_12 a;
2013
2014         a.full = dfixed_const(1000);
2015         line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2016         line_time.full = dfixed_div(line_time, a);
2017         bpp.full = dfixed_const(wm->bytes_per_pixel);
2018         src_width.full = dfixed_const(wm->src_width);
2019         bandwidth.full = dfixed_mul(src_width, bpp);
2020         bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2021         bandwidth.full = dfixed_div(bandwidth, line_time);
2022
2023         return dfixed_trunc(bandwidth);
2024 }
2025
2026 static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2027 {
2028         /* First calcualte the latency in ns */
2029         u32 mc_latency = 2000; /* 2000 ns. */
2030         u32 available_bandwidth = evergreen_available_bandwidth(wm);
2031         u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2032         u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2033         u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2034         u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2035                 (wm->num_heads * cursor_line_pair_return_time);
2036         u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2037         u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2038         fixed20_12 a, b, c;
2039
2040         if (wm->num_heads == 0)
2041                 return 0;
2042
2043         a.full = dfixed_const(2);
2044         b.full = dfixed_const(1);
2045         if ((wm->vsc.full > a.full) ||
2046             ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2047             (wm->vtaps >= 5) ||
2048             ((wm->vsc.full >= a.full) && wm->interlaced))
2049                 max_src_lines_per_dst_line = 4;
2050         else
2051                 max_src_lines_per_dst_line = 2;
2052
2053         a.full = dfixed_const(available_bandwidth);
2054         b.full = dfixed_const(wm->num_heads);
2055         a.full = dfixed_div(a, b);
2056
2057         b.full = dfixed_const(1000);
2058         c.full = dfixed_const(wm->disp_clk);
2059         b.full = dfixed_div(c, b);
2060         c.full = dfixed_const(wm->bytes_per_pixel);
2061         b.full = dfixed_mul(b, c);
2062
2063         lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
2064
2065         a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2066         b.full = dfixed_const(1000);
2067         c.full = dfixed_const(lb_fill_bw);
2068         b.full = dfixed_div(c, b);
2069         a.full = dfixed_div(a, b);
2070         line_fill_time = dfixed_trunc(a);
2071
2072         if (line_fill_time < wm->active_time)
2073                 return latency;
2074         else
2075                 return latency + (line_fill_time - wm->active_time);
2076
2077 }
2078
2079 static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2080 {
2081         if (evergreen_average_bandwidth(wm) <=
2082             (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2083                 return true;
2084         else
2085                 return false;
2086 };
2087
2088 static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2089 {
2090         if (evergreen_average_bandwidth(wm) <=
2091             (evergreen_available_bandwidth(wm) / wm->num_heads))
2092                 return true;
2093         else
2094                 return false;
2095 };
2096
2097 static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2098 {
2099         u32 lb_partitions = wm->lb_size / wm->src_width;
2100         u32 line_time = wm->active_time + wm->blank_time;
2101         u32 latency_tolerant_lines;
2102         u32 latency_hiding;
2103         fixed20_12 a;
2104
2105         a.full = dfixed_const(1);
2106         if (wm->vsc.full > a.full)
2107                 latency_tolerant_lines = 1;
2108         else {
2109                 if (lb_partitions <= (wm->vtaps + 1))
2110                         latency_tolerant_lines = 1;
2111                 else
2112                         latency_tolerant_lines = 2;
2113         }
2114
2115         latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2116
2117         if (evergreen_latency_watermark(wm) <= latency_hiding)
2118                 return true;
2119         else
2120                 return false;
2121 }
2122
2123 static void evergreen_program_watermarks(struct radeon_device *rdev,
2124                                          struct radeon_crtc *radeon_crtc,
2125                                          u32 lb_size, u32 num_heads)
2126 {
2127         struct drm_display_mode *mode = &radeon_crtc->base.mode;
2128         struct evergreen_wm_params wm_low, wm_high;
2129         u32 dram_channels;
2130         u32 pixel_period;
2131         u32 line_time = 0;
2132         u32 latency_watermark_a = 0, latency_watermark_b = 0;
2133         u32 priority_a_mark = 0, priority_b_mark = 0;
2134         u32 priority_a_cnt = PRIORITY_OFF;
2135         u32 priority_b_cnt = PRIORITY_OFF;
2136         u32 pipe_offset = radeon_crtc->crtc_id * 16;
2137         u32 tmp, arb_control3;
2138         fixed20_12 a, b, c;
2139
2140         if (radeon_crtc->base.enabled && num_heads && mode) {
2141                 pixel_period = 1000000 / (u32)mode->clock;
2142                 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2143                 priority_a_cnt = 0;
2144                 priority_b_cnt = 0;
2145                 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2146
2147                 /* watermark for high clocks */
2148                 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2149                         wm_high.yclk =
2150                                 radeon_dpm_get_mclk(rdev, false) * 10;
2151                         wm_high.sclk =
2152                                 radeon_dpm_get_sclk(rdev, false) * 10;
2153                 } else {
2154                         wm_high.yclk = rdev->pm.current_mclk * 10;
2155                         wm_high.sclk = rdev->pm.current_sclk * 10;
2156                 }
2157
2158                 wm_high.disp_clk = mode->clock;
2159                 wm_high.src_width = mode->crtc_hdisplay;
2160                 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2161                 wm_high.blank_time = line_time - wm_high.active_time;
2162                 wm_high.interlaced = false;
2163                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2164                         wm_high.interlaced = true;
2165                 wm_high.vsc = radeon_crtc->vsc;
2166                 wm_high.vtaps = 1;
2167                 if (radeon_crtc->rmx_type != RMX_OFF)
2168                         wm_high.vtaps = 2;
2169                 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2170                 wm_high.lb_size = lb_size;
2171                 wm_high.dram_channels = dram_channels;
2172                 wm_high.num_heads = num_heads;
2173
2174                 /* watermark for low clocks */
2175                 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2176                         wm_low.yclk =
2177                                 radeon_dpm_get_mclk(rdev, true) * 10;
2178                         wm_low.sclk =
2179                                 radeon_dpm_get_sclk(rdev, true) * 10;
2180                 } else {
2181                         wm_low.yclk = rdev->pm.current_mclk * 10;
2182                         wm_low.sclk = rdev->pm.current_sclk * 10;
2183                 }
2184
2185                 wm_low.disp_clk = mode->clock;
2186                 wm_low.src_width = mode->crtc_hdisplay;
2187                 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2188                 wm_low.blank_time = line_time - wm_low.active_time;
2189                 wm_low.interlaced = false;
2190                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2191                         wm_low.interlaced = true;
2192                 wm_low.vsc = radeon_crtc->vsc;
2193                 wm_low.vtaps = 1;
2194                 if (radeon_crtc->rmx_type != RMX_OFF)
2195                         wm_low.vtaps = 2;
2196                 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2197                 wm_low.lb_size = lb_size;
2198                 wm_low.dram_channels = dram_channels;
2199                 wm_low.num_heads = num_heads;
2200
2201                 /* set for high clocks */
2202                 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
2203                 /* set for low clocks */
2204                 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
2205
2206                 /* possibly force display priority to high */
2207                 /* should really do this at mode validation time... */
2208                 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2209                     !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2210                     !evergreen_check_latency_hiding(&wm_high) ||
2211                     (rdev->disp_priority == 2)) {
2212                         DRM_DEBUG_KMS("force priority a to high\n");
2213                         priority_a_cnt |= PRIORITY_ALWAYS_ON;
2214                 }
2215                 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2216                     !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2217                     !evergreen_check_latency_hiding(&wm_low) ||
2218                     (rdev->disp_priority == 2)) {
2219                         DRM_DEBUG_KMS("force priority b to high\n");
2220                         priority_b_cnt |= PRIORITY_ALWAYS_ON;
2221                 }
2222
2223                 a.full = dfixed_const(1000);
2224                 b.full = dfixed_const(mode->clock);
2225                 b.full = dfixed_div(b, a);
2226                 c.full = dfixed_const(latency_watermark_a);
2227                 c.full = dfixed_mul(c, b);
2228                 c.full = dfixed_mul(c, radeon_crtc->hsc);
2229                 c.full = dfixed_div(c, a);
2230                 a.full = dfixed_const(16);
2231                 c.full = dfixed_div(c, a);
2232                 priority_a_mark = dfixed_trunc(c);
2233                 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2234
2235                 a.full = dfixed_const(1000);
2236                 b.full = dfixed_const(mode->clock);
2237                 b.full = dfixed_div(b, a);
2238                 c.full = dfixed_const(latency_watermark_b);
2239                 c.full = dfixed_mul(c, b);
2240                 c.full = dfixed_mul(c, radeon_crtc->hsc);
2241                 c.full = dfixed_div(c, a);
2242                 a.full = dfixed_const(16);
2243                 c.full = dfixed_div(c, a);
2244                 priority_b_mark = dfixed_trunc(c);
2245                 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2246         }
2247
2248         /* select wm A */
2249         arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2250         tmp = arb_control3;
2251         tmp &= ~LATENCY_WATERMARK_MASK(3);
2252         tmp |= LATENCY_WATERMARK_MASK(1);
2253         WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2254         WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2255                (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2256                 LATENCY_HIGH_WATERMARK(line_time)));
2257         /* select wm B */
2258         tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2259         tmp &= ~LATENCY_WATERMARK_MASK(3);
2260         tmp |= LATENCY_WATERMARK_MASK(2);
2261         WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2262         WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2263                (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2264                 LATENCY_HIGH_WATERMARK(line_time)));
2265         /* restore original selection */
2266         WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2267
2268         /* write the priority marks */
2269         WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2270         WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2271
2272         /* save values for DPM */
2273         radeon_crtc->line_time = line_time;
2274         radeon_crtc->wm_high = latency_watermark_a;
2275         radeon_crtc->wm_low = latency_watermark_b;
2276 }
2277
2278 /**
2279  * evergreen_bandwidth_update - update display watermarks callback.
2280  *
2281  * @rdev: radeon_device pointer
2282  *
2283  * Update the display watermarks based on the requested mode(s)
2284  * (evergreen+).
2285  */
2286 void evergreen_bandwidth_update(struct radeon_device *rdev)
2287 {
2288         struct drm_display_mode *mode0 = NULL;
2289         struct drm_display_mode *mode1 = NULL;
2290         u32 num_heads = 0, lb_size;
2291         int i;
2292
2293         radeon_update_display_priority(rdev);
2294
2295         for (i = 0; i < rdev->num_crtc; i++) {
2296                 if (rdev->mode_info.crtcs[i]->base.enabled)
2297                         num_heads++;
2298         }
2299         for (i = 0; i < rdev->num_crtc; i += 2) {
2300                 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2301                 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2302                 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2303                 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2304                 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2305                 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2306         }
2307 }
2308
2309 /**
2310  * evergreen_mc_wait_for_idle - wait for MC idle callback.
2311  *
2312  * @rdev: radeon_device pointer
2313  *
2314  * Wait for the MC (memory controller) to be idle.
2315  * (evergreen+).
2316  * Returns 0 if the MC is idle, -1 if not.
2317  */
2318 int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
2319 {
2320         unsigned i;
2321         u32 tmp;
2322
2323         for (i = 0; i < rdev->usec_timeout; i++) {
2324                 /* read MC_STATUS */
2325                 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2326                 if (!tmp)
2327                         return 0;
2328                 udelay(1);
2329         }
2330         return -1;
2331 }
2332
2333 /*
2334  * GART
2335  */
2336 void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2337 {
2338         unsigned i;
2339         u32 tmp;
2340
2341         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2342
2343         WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2344         for (i = 0; i < rdev->usec_timeout; i++) {
2345                 /* read MC_STATUS */
2346                 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2347                 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2348                 if (tmp == 2) {
2349                         printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
2350                         return;
2351                 }
2352                 if (tmp) {
2353                         return;
2354                 }
2355                 udelay(1);
2356         }
2357 }
2358
2359 static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
2360 {
2361         u32 tmp;
2362         int r;
2363
2364         if (rdev->gart.robj == NULL) {
2365                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2366                 return -EINVAL;
2367         }
2368         r = radeon_gart_table_vram_pin(rdev);
2369         if (r)
2370                 return r;
2371         radeon_gart_restore(rdev);
2372         /* Setup L2 cache */
2373         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2374                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2375                                 EFFECTIVE_L2_QUEUE_SIZE(7));
2376         WREG32(VM_L2_CNTL2, 0);
2377         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2378         /* Setup TLB control */
2379         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2380                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2381                 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2382                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2383         if (rdev->flags & RADEON_IS_IGP) {
2384                 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2385                 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2386                 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2387         } else {
2388                 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2389                 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2390                 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2391                 if ((rdev->family == CHIP_JUNIPER) ||
2392                     (rdev->family == CHIP_CYPRESS) ||
2393                     (rdev->family == CHIP_HEMLOCK) ||
2394                     (rdev->family == CHIP_BARTS))
2395                         WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
2396         }
2397         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2398         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2399         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2400         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2401         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2402         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2403         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2404         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2405                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2406         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2407                         (u32)(rdev->dummy_page.addr >> 12));
2408         WREG32(VM_CONTEXT1_CNTL, 0);
2409
2410         evergreen_pcie_gart_tlb_flush(rdev);
2411         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2412                  (unsigned)(rdev->mc.gtt_size >> 20),
2413                  (unsigned long long)rdev->gart.table_addr);
2414         rdev->gart.ready = true;
2415         return 0;
2416 }
2417
2418 static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
2419 {
2420         u32 tmp;
2421
2422         /* Disable all tables */
2423         WREG32(VM_CONTEXT0_CNTL, 0);
2424         WREG32(VM_CONTEXT1_CNTL, 0);
2425
2426         /* Setup L2 cache */
2427         WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2428                                 EFFECTIVE_L2_QUEUE_SIZE(7));
2429         WREG32(VM_L2_CNTL2, 0);
2430         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2431         /* Setup TLB control */
2432         tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2433         WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2434         WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2435         WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2436         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2437         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2438         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2439         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2440         radeon_gart_table_vram_unpin(rdev);
2441 }
2442
2443 static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
2444 {
2445         evergreen_pcie_gart_disable(rdev);
2446         radeon_gart_table_vram_free(rdev);
2447         radeon_gart_fini(rdev);
2448 }
2449
2450
2451 static void evergreen_agp_enable(struct radeon_device *rdev)
2452 {
2453         u32 tmp;
2454
2455         /* Setup L2 cache */
2456         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2457                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2458                                 EFFECTIVE_L2_QUEUE_SIZE(7));
2459         WREG32(VM_L2_CNTL2, 0);
2460         WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2461         /* Setup TLB control */
2462         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2463                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2464                 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2465                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2466         WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2467         WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2468         WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2469         WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2470         WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2471         WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2472         WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2473         WREG32(VM_CONTEXT0_CNTL, 0);
2474         WREG32(VM_CONTEXT1_CNTL, 0);
2475 }
2476
2477 void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
2478 {
2479         u32 crtc_enabled, tmp, frame_count, blackout;
2480         int i, j;
2481
2482         if (!ASIC_IS_NODCE(rdev)) {
2483                 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2484                 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
2485
2486                 /* disable VGA render */
2487                 WREG32(VGA_RENDER_CONTROL, 0);
2488         }
2489         /* blank the display controllers */
2490         for (i = 0; i < rdev->num_crtc; i++) {
2491                 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2492                 if (crtc_enabled) {
2493                         save->crtc_enabled[i] = true;
2494                         if (ASIC_IS_DCE6(rdev)) {
2495                                 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2496                                 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2497                                         radeon_wait_for_vblank(rdev, i);
2498                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2499                                         tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2500                                         WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2501                                 }
2502                         } else {
2503                                 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2504                                 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2505                                         radeon_wait_for_vblank(rdev, i);
2506                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2507                                         tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2508                                         WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2509                                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2510                                 }
2511                         }
2512                         /* wait for the next frame */
2513                         frame_count = radeon_get_vblank_counter(rdev, i);
2514                         for (j = 0; j < rdev->usec_timeout; j++) {
2515                                 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2516                                         break;
2517                                 udelay(1);
2518                         }
2519
2520                         /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2521                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2522                         tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2523                         tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2524                         WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2525                         WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2526                         save->crtc_enabled[i] = false;
2527                         /* ***** */
2528                 } else {
2529                         save->crtc_enabled[i] = false;
2530                 }
2531         }
2532
2533         radeon_mc_wait_for_idle(rdev);
2534
2535         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2536         if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2537                 /* Block CPU access */
2538                 WREG32(BIF_FB_EN, 0);
2539                 /* blackout the MC */
2540                 blackout &= ~BLACKOUT_MODE_MASK;
2541                 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
2542         }
2543         /* wait for the MC to settle */
2544         udelay(100);
2545
2546         /* lock double buffered regs */
2547         for (i = 0; i < rdev->num_crtc; i++) {
2548                 if (save->crtc_enabled[i]) {
2549                         tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2550                         if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2551                                 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2552                                 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2553                         }
2554                         tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2555                         if (!(tmp & 1)) {
2556                                 tmp |= 1;
2557                                 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2558                         }
2559                 }
2560         }
2561 }
2562
2563 void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
2564 {
2565         u32 tmp, frame_count;
2566         int i, j;
2567
2568         /* update crtc base addresses */
2569         for (i = 0; i < rdev->num_crtc; i++) {
2570                 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2571                        upper_32_bits(rdev->mc.vram_start));
2572                 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2573                        upper_32_bits(rdev->mc.vram_start));
2574                 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
2575                        (u32)rdev->mc.vram_start);
2576                 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
2577                        (u32)rdev->mc.vram_start);
2578         }
2579
2580         if (!ASIC_IS_NODCE(rdev)) {
2581                 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2582                 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2583         }
2584
2585         /* unlock regs and wait for update */
2586         for (i = 0; i < rdev->num_crtc; i++) {
2587                 if (save->crtc_enabled[i]) {
2588                         tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2589                         if ((tmp & 0x3) != 0) {
2590                                 tmp &= ~0x3;
2591                                 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2592                         }
2593                         tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2594                         if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2595                                 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2596                                 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2597                         }
2598                         tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2599                         if (tmp & 1) {
2600                                 tmp &= ~1;
2601                                 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2602                         }
2603                         for (j = 0; j < rdev->usec_timeout; j++) {
2604                                 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2605                                 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2606                                         break;
2607                                 udelay(1);
2608                         }
2609                 }
2610         }
2611
2612         /* unblackout the MC */
2613         tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2614         tmp &= ~BLACKOUT_MODE_MASK;
2615         WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2616         /* allow CPU access */
2617         WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2618
2619         for (i = 0; i < rdev->num_crtc; i++) {
2620                 if (save->crtc_enabled[i]) {
2621                         if (ASIC_IS_DCE6(rdev)) {
2622                                 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2623                                 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2624                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2625                                 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2626                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2627                         } else {
2628                                 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2629                                 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2630                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2631                                 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2632                                 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2633                         }
2634                         /* wait for the next frame */
2635                         frame_count = radeon_get_vblank_counter(rdev, i);
2636                         for (j = 0; j < rdev->usec_timeout; j++) {
2637                                 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2638                                         break;
2639                                 udelay(1);
2640                         }
2641                 }
2642         }
2643         if (!ASIC_IS_NODCE(rdev)) {
2644                 /* Unlock vga access */
2645                 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2646                 mdelay(1);
2647                 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2648         }
2649 }
2650
2651 void evergreen_mc_program(struct radeon_device *rdev)
2652 {
2653         struct evergreen_mc_save save;
2654         u32 tmp;
2655         int i, j;
2656
2657         /* Initialize HDP */
2658         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2659                 WREG32((0x2c14 + j), 0x00000000);
2660                 WREG32((0x2c18 + j), 0x00000000);
2661                 WREG32((0x2c1c + j), 0x00000000);
2662                 WREG32((0x2c20 + j), 0x00000000);
2663                 WREG32((0x2c24 + j), 0x00000000);
2664         }
2665         WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2666
2667         evergreen_mc_stop(rdev, &save);
2668         if (evergreen_mc_wait_for_idle(rdev)) {
2669                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2670         }
2671         /* Lockout access through VGA aperture*/
2672         WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2673         /* Update configuration */
2674         if (rdev->flags & RADEON_IS_AGP) {
2675                 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2676                         /* VRAM before AGP */
2677                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2678                                 rdev->mc.vram_start >> 12);
2679                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2680                                 rdev->mc.gtt_end >> 12);
2681                 } else {
2682                         /* VRAM after AGP */
2683                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2684                                 rdev->mc.gtt_start >> 12);
2685                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2686                                 rdev->mc.vram_end >> 12);
2687                 }
2688         } else {
2689                 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2690                         rdev->mc.vram_start >> 12);
2691                 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2692                         rdev->mc.vram_end >> 12);
2693         }
2694         WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
2695         /* llano/ontario only */
2696         if ((rdev->family == CHIP_PALM) ||
2697             (rdev->family == CHIP_SUMO) ||
2698             (rdev->family == CHIP_SUMO2)) {
2699                 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2700                 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2701                 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2702                 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2703         }
2704         tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2705         tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2706         WREG32(MC_VM_FB_LOCATION, tmp);
2707         WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
2708         WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
2709         WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
2710         if (rdev->flags & RADEON_IS_AGP) {
2711                 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2712                 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2713                 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2714         } else {
2715                 WREG32(MC_VM_AGP_BASE, 0);
2716                 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2717                 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2718         }
2719         if (evergreen_mc_wait_for_idle(rdev)) {
2720                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2721         }
2722         evergreen_mc_resume(rdev, &save);
2723         /* we need to own VRAM, so turn off the VGA renderer here
2724          * to stop it overwriting our objects */
2725         rv515_vga_render_disable(rdev);
2726 }
2727
2728 /*
2729  * CP.
2730  */
2731 void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2732 {
2733         struct radeon_ring *ring = &rdev->ring[ib->ring];
2734         u32 next_rptr;
2735
2736         /* set to DX10/11 mode */
2737         radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2738         radeon_ring_write(ring, 1);
2739
2740         if (ring->rptr_save_reg) {
2741                 next_rptr = ring->wptr + 3 + 4;
2742                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2743                 radeon_ring_write(ring, ((ring->rptr_save_reg -
2744                                           PACKET3_SET_CONFIG_REG_START) >> 2));
2745                 radeon_ring_write(ring, next_rptr);
2746         } else if (rdev->wb.enabled) {
2747                 next_rptr = ring->wptr + 5 + 4;
2748                 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2749                 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2750                 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2751                 radeon_ring_write(ring, next_rptr);
2752                 radeon_ring_write(ring, 0);
2753         }
2754
2755         radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2756         radeon_ring_write(ring,
2757 #ifdef __BIG_ENDIAN
2758                           (2 << 0) |
2759 #endif
2760                           (ib->gpu_addr & 0xFFFFFFFC));
2761         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2762         radeon_ring_write(ring, ib->length_dw);
2763 }
2764
2765
2766 static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2767 {
2768         const __be32 *fw_data;
2769         int i;
2770
2771         if (!rdev->me_fw || !rdev->pfp_fw)
2772                 return -EINVAL;
2773
2774         r700_cp_stop(rdev);
2775         WREG32(CP_RB_CNTL,
2776 #ifdef __BIG_ENDIAN
2777                BUF_SWAP_32BIT |
2778 #endif
2779                RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
2780
2781         fw_data = (const __be32 *)rdev->pfp_fw->data;
2782         WREG32(CP_PFP_UCODE_ADDR, 0);
2783         for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2784                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2785         WREG32(CP_PFP_UCODE_ADDR, 0);
2786
2787         fw_data = (const __be32 *)rdev->me_fw->data;
2788         WREG32(CP_ME_RAM_WADDR, 0);
2789         for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2790                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2791
2792         WREG32(CP_PFP_UCODE_ADDR, 0);
2793         WREG32(CP_ME_RAM_WADDR, 0);
2794         WREG32(CP_ME_RAM_RADDR, 0);
2795         return 0;
2796 }
2797
2798 static int evergreen_cp_start(struct radeon_device *rdev)
2799 {
2800         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2801         int r, i;
2802         uint32_t cp_me;
2803
2804         r = radeon_ring_lock(rdev, ring, 7);
2805         if (r) {
2806                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2807                 return r;
2808         }
2809         radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2810         radeon_ring_write(ring, 0x1);
2811         radeon_ring_write(ring, 0x0);
2812         radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
2813         radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2814         radeon_ring_write(ring, 0);
2815         radeon_ring_write(ring, 0);
2816         radeon_ring_unlock_commit(rdev, ring);
2817
2818         cp_me = 0xff;
2819         WREG32(CP_ME_CNTL, cp_me);
2820
2821         r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
2822         if (r) {
2823                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2824                 return r;
2825         }
2826
2827         /* setup clear context state */
2828         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2829         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2830
2831         for (i = 0; i < evergreen_default_size; i++)
2832                 radeon_ring_write(ring, evergreen_default_state[i]);
2833
2834         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2835         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2836
2837         /* set clear context state */
2838         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2839         radeon_ring_write(ring, 0);
2840
2841         /* SQ_VTX_BASE_VTX_LOC */
2842         radeon_ring_write(ring, 0xc0026f00);
2843         radeon_ring_write(ring, 0x00000000);
2844         radeon_ring_write(ring, 0x00000000);
2845         radeon_ring_write(ring, 0x00000000);
2846
2847         /* Clear consts */
2848         radeon_ring_write(ring, 0xc0036f00);
2849         radeon_ring_write(ring, 0x00000bc4);
2850         radeon_ring_write(ring, 0xffffffff);
2851         radeon_ring_write(ring, 0xffffffff);
2852         radeon_ring_write(ring, 0xffffffff);
2853
2854         radeon_ring_write(ring, 0xc0026900);
2855         radeon_ring_write(ring, 0x00000316);
2856         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
2857         radeon_ring_write(ring, 0x00000010); /*  */
2858
2859         radeon_ring_unlock_commit(rdev, ring);
2860
2861         return 0;
2862 }
2863
2864 static int evergreen_cp_resume(struct radeon_device *rdev)
2865 {
2866         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2867         u32 tmp;
2868         u32 rb_bufsz;
2869         int r;
2870
2871         /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
2872         WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
2873                                  SOFT_RESET_PA |
2874                                  SOFT_RESET_SH |
2875                                  SOFT_RESET_VGT |
2876                                  SOFT_RESET_SPI |
2877                                  SOFT_RESET_SX));
2878         RREG32(GRBM_SOFT_RESET);
2879         mdelay(15);
2880         WREG32(GRBM_SOFT_RESET, 0);
2881         RREG32(GRBM_SOFT_RESET);
2882
2883         /* Set ring buffer size */
2884         rb_bufsz = drm_order(ring->ring_size / 8);
2885         tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
2886 #ifdef __BIG_ENDIAN
2887         tmp |= BUF_SWAP_32BIT;
2888 #endif
2889         WREG32(CP_RB_CNTL, tmp);
2890         WREG32(CP_SEM_WAIT_TIMER, 0x0);
2891         WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
2892
2893         /* Set the write pointer delay */
2894         WREG32(CP_RB_WPTR_DELAY, 0);
2895
2896         /* Initialize the ring buffer's read and write pointers */
2897         WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
2898         WREG32(CP_RB_RPTR_WR, 0);
2899         ring->wptr = 0;
2900         WREG32(CP_RB_WPTR, ring->wptr);
2901
2902         /* set the wb address whether it's enabled or not */
2903         WREG32(CP_RB_RPTR_ADDR,
2904                ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
2905         WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2906         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2907
2908         if (rdev->wb.enabled)
2909                 WREG32(SCRATCH_UMSK, 0xff);
2910         else {
2911                 tmp |= RB_NO_UPDATE;
2912                 WREG32(SCRATCH_UMSK, 0);
2913         }
2914
2915         mdelay(1);
2916         WREG32(CP_RB_CNTL, tmp);
2917
2918         WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
2919         WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
2920
2921         ring->rptr = RREG32(CP_RB_RPTR);
2922
2923         evergreen_cp_start(rdev);
2924         ring->ready = true;
2925         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
2926         if (r) {
2927                 ring->ready = false;
2928                 return r;
2929         }
2930         return 0;
2931 }
2932
2933 /*
2934  * Core functions
2935  */
2936 static void evergreen_gpu_init(struct radeon_device *rdev)
2937 {
2938         u32 gb_addr_config;
2939         u32 mc_shared_chmap, mc_arb_ramcfg;
2940         u32 sx_debug_1;
2941         u32 smx_dc_ctl0;
2942         u32 sq_config;
2943         u32 sq_lds_resource_mgmt;
2944         u32 sq_gpr_resource_mgmt_1;
2945         u32 sq_gpr_resource_mgmt_2;
2946         u32 sq_gpr_resource_mgmt_3;
2947         u32 sq_thread_resource_mgmt;
2948         u32 sq_thread_resource_mgmt_2;
2949         u32 sq_stack_resource_mgmt_1;
2950         u32 sq_stack_resource_mgmt_2;
2951         u32 sq_stack_resource_mgmt_3;
2952         u32 vgt_cache_invalidation;
2953         u32 hdp_host_path_cntl, tmp;
2954         u32 disabled_rb_mask;
2955         int i, j, num_shader_engines, ps_thread_count;
2956
2957         switch (rdev->family) {
2958         case CHIP_CYPRESS:
2959         case CHIP_HEMLOCK:
2960                 rdev->config.evergreen.num_ses = 2;
2961                 rdev->config.evergreen.max_pipes = 4;
2962                 rdev->config.evergreen.max_tile_pipes = 8;
2963                 rdev->config.evergreen.max_simds = 10;
2964                 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
2965                 rdev->config.evergreen.max_gprs = 256;
2966                 rdev->config.evergreen.max_threads = 248;
2967                 rdev->config.evergreen.max_gs_threads = 32;
2968                 rdev->config.evergreen.max_stack_entries = 512;
2969                 rdev->config.evergreen.sx_num_of_sets = 4;
2970                 rdev->config.evergreen.sx_max_export_size = 256;
2971                 rdev->config.evergreen.sx_max_export_pos_size = 64;
2972                 rdev->config.evergreen.sx_max_export_smx_size = 192;
2973                 rdev->config.evergreen.max_hw_contexts = 8;
2974                 rdev->config.evergreen.sq_num_cf_insts = 2;
2975
2976                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
2977                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
2978                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
2979                 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
2980                 break;
2981         case CHIP_JUNIPER:
2982                 rdev->config.evergreen.num_ses = 1;
2983                 rdev->config.evergreen.max_pipes = 4;
2984                 rdev->config.evergreen.max_tile_pipes = 4;
2985                 rdev->config.evergreen.max_simds = 10;
2986                 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
2987                 rdev->config.evergreen.max_gprs = 256;
2988                 rdev->config.evergreen.max_threads = 248;
2989                 rdev->config.evergreen.max_gs_threads = 32;
2990                 rdev->config.evergreen.max_stack_entries = 512;
2991                 rdev->config.evergreen.sx_num_of_sets = 4;
2992                 rdev->config.evergreen.sx_max_export_size = 256;
2993                 rdev->config.evergreen.sx_max_export_pos_size = 64;
2994                 rdev->config.evergreen.sx_max_export_smx_size = 192;
2995                 rdev->config.evergreen.max_hw_contexts = 8;
2996                 rdev->config.evergreen.sq_num_cf_insts = 2;
2997
2998                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
2999                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3000                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3001                 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
3002                 break;
3003         case CHIP_REDWOOD:
3004                 rdev->config.evergreen.num_ses = 1;
3005                 rdev->config.evergreen.max_pipes = 4;
3006                 rdev->config.evergreen.max_tile_pipes = 4;
3007                 rdev->config.evergreen.max_simds = 5;
3008                 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3009                 rdev->config.evergreen.max_gprs = 256;
3010                 rdev->config.evergreen.max_threads = 248;
3011                 rdev->config.evergreen.max_gs_threads = 32;
3012                 rdev->config.evergreen.max_stack_entries = 256;
3013                 rdev->config.evergreen.sx_num_of_sets = 4;
3014                 rdev->config.evergreen.sx_max_export_size = 256;
3015                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3016                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3017                 rdev->config.evergreen.max_hw_contexts = 8;
3018                 rdev->config.evergreen.sq_num_cf_insts = 2;
3019
3020                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3021                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3022                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3023                 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
3024                 break;
3025         case CHIP_CEDAR:
3026         default:
3027                 rdev->config.evergreen.num_ses = 1;
3028                 rdev->config.evergreen.max_pipes = 2;
3029                 rdev->config.evergreen.max_tile_pipes = 2;
3030                 rdev->config.evergreen.max_simds = 2;
3031                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3032                 rdev->config.evergreen.max_gprs = 256;
3033                 rdev->config.evergreen.max_threads = 192;
3034                 rdev->config.evergreen.max_gs_threads = 16;
3035                 rdev->config.evergreen.max_stack_entries = 256;
3036                 rdev->config.evergreen.sx_num_of_sets = 4;
3037                 rdev->config.evergreen.sx_max_export_size = 128;
3038                 rdev->config.evergreen.sx_max_export_pos_size = 32;
3039                 rdev->config.evergreen.sx_max_export_smx_size = 96;
3040                 rdev->config.evergreen.max_hw_contexts = 4;
3041                 rdev->config.evergreen.sq_num_cf_insts = 1;
3042
3043                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3044                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3045                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3046                 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3047                 break;
3048         case CHIP_PALM:
3049                 rdev->config.evergreen.num_ses = 1;
3050                 rdev->config.evergreen.max_pipes = 2;
3051                 rdev->config.evergreen.max_tile_pipes = 2;
3052                 rdev->config.evergreen.max_simds = 2;
3053                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3054                 rdev->config.evergreen.max_gprs = 256;
3055                 rdev->config.evergreen.max_threads = 192;
3056                 rdev->config.evergreen.max_gs_threads = 16;
3057                 rdev->config.evergreen.max_stack_entries = 256;
3058                 rdev->config.evergreen.sx_num_of_sets = 4;
3059                 rdev->config.evergreen.sx_max_export_size = 128;
3060                 rdev->config.evergreen.sx_max_export_pos_size = 32;
3061                 rdev->config.evergreen.sx_max_export_smx_size = 96;
3062                 rdev->config.evergreen.max_hw_contexts = 4;
3063                 rdev->config.evergreen.sq_num_cf_insts = 1;
3064
3065                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3066                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3067                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3068                 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3069                 break;
3070         case CHIP_SUMO:
3071                 rdev->config.evergreen.num_ses = 1;
3072                 rdev->config.evergreen.max_pipes = 4;
3073                 rdev->config.evergreen.max_tile_pipes = 4;
3074                 if (rdev->pdev->device == 0x9648)
3075                         rdev->config.evergreen.max_simds = 3;
3076                 else if ((rdev->pdev->device == 0x9647) ||
3077                          (rdev->pdev->device == 0x964a))
3078                         rdev->config.evergreen.max_simds = 4;
3079                 else
3080                         rdev->config.evergreen.max_simds = 5;
3081                 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3082                 rdev->config.evergreen.max_gprs = 256;
3083                 rdev->config.evergreen.max_threads = 248;
3084                 rdev->config.evergreen.max_gs_threads = 32;
3085                 rdev->config.evergreen.max_stack_entries = 256;
3086                 rdev->config.evergreen.sx_num_of_sets = 4;
3087                 rdev->config.evergreen.sx_max_export_size = 256;
3088                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3089                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3090                 rdev->config.evergreen.max_hw_contexts = 8;
3091                 rdev->config.evergreen.sq_num_cf_insts = 2;
3092
3093                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3094                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3095                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3096                 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
3097                 break;
3098         case CHIP_SUMO2:
3099                 rdev->config.evergreen.num_ses = 1;
3100                 rdev->config.evergreen.max_pipes = 4;
3101                 rdev->config.evergreen.max_tile_pipes = 4;
3102                 rdev->config.evergreen.max_simds = 2;
3103                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3104                 rdev->config.evergreen.max_gprs = 256;
3105                 rdev->config.evergreen.max_threads = 248;
3106                 rdev->config.evergreen.max_gs_threads = 32;
3107                 rdev->config.evergreen.max_stack_entries = 512;
3108                 rdev->config.evergreen.sx_num_of_sets = 4;
3109                 rdev->config.evergreen.sx_max_export_size = 256;
3110                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3111                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3112                 rdev->config.evergreen.max_hw_contexts = 8;
3113                 rdev->config.evergreen.sq_num_cf_insts = 2;
3114
3115                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3116                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3117                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3118                 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
3119                 break;
3120         case CHIP_BARTS:
3121                 rdev->config.evergreen.num_ses = 2;
3122                 rdev->config.evergreen.max_pipes = 4;
3123                 rdev->config.evergreen.max_tile_pipes = 8;
3124                 rdev->config.evergreen.max_simds = 7;
3125                 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3126                 rdev->config.evergreen.max_gprs = 256;
3127                 rdev->config.evergreen.max_threads = 248;
3128                 rdev->config.evergreen.max_gs_threads = 32;
3129                 rdev->config.evergreen.max_stack_entries = 512;
3130                 rdev->config.evergreen.sx_num_of_sets = 4;
3131                 rdev->config.evergreen.sx_max_export_size = 256;
3132                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3133                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3134                 rdev->config.evergreen.max_hw_contexts = 8;
3135                 rdev->config.evergreen.sq_num_cf_insts = 2;
3136
3137                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3138                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3139                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3140                 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
3141                 break;
3142         case CHIP_TURKS:
3143                 rdev->config.evergreen.num_ses = 1;
3144                 rdev->config.evergreen.max_pipes = 4;
3145                 rdev->config.evergreen.max_tile_pipes = 4;
3146                 rdev->config.evergreen.max_simds = 6;
3147                 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3148                 rdev->config.evergreen.max_gprs = 256;
3149                 rdev->config.evergreen.max_threads = 248;
3150                 rdev->config.evergreen.max_gs_threads = 32;
3151                 rdev->config.evergreen.max_stack_entries = 256;
3152                 rdev->config.evergreen.sx_num_of_sets = 4;
3153                 rdev->config.evergreen.sx_max_export_size = 256;
3154                 rdev->config.evergreen.sx_max_export_pos_size = 64;
3155                 rdev->config.evergreen.sx_max_export_smx_size = 192;
3156                 rdev->config.evergreen.max_hw_contexts = 8;
3157                 rdev->config.evergreen.sq_num_cf_insts = 2;
3158
3159                 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3160                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3161                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3162                 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
3163                 break;
3164         case CHIP_CAICOS:
3165                 rdev->config.evergreen.num_ses = 1;
3166                 rdev->config.evergreen.max_pipes = 2;
3167                 rdev->config.evergreen.max_tile_pipes = 2;
3168                 rdev->config.evergreen.max_simds = 2;
3169                 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3170                 rdev->config.evergreen.max_gprs = 256;
3171                 rdev->config.evergreen.max_threads = 192;
3172                 rdev->config.evergreen.max_gs_threads = 16;
3173                 rdev->config.evergreen.max_stack_entries = 256;
3174                 rdev->config.evergreen.sx_num_of_sets = 4;
3175                 rdev->config.evergreen.sx_max_export_size = 128;
3176                 rdev->config.evergreen.sx_max_export_pos_size = 32;
3177                 rdev->config.evergreen.sx_max_export_smx_size = 96;
3178                 rdev->config.evergreen.max_hw_contexts = 4;
3179                 rdev->config.evergreen.sq_num_cf_insts = 1;
3180
3181                 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3182                 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3183                 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3184                 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
3185                 break;
3186         }
3187
3188         /* Initialize HDP */
3189         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3190                 WREG32((0x2c14 + j), 0x00000000);
3191                 WREG32((0x2c18 + j), 0x00000000);
3192                 WREG32((0x2c1c + j), 0x00000000);
3193                 WREG32((0x2c20 + j), 0x00000000);
3194                 WREG32((0x2c24 + j), 0x00000000);
3195         }
3196
3197         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3198
3199         evergreen_fix_pci_max_read_req_size(rdev);
3200
3201         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3202         if ((rdev->family == CHIP_PALM) ||
3203             (rdev->family == CHIP_SUMO) ||
3204             (rdev->family == CHIP_SUMO2))
3205                 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3206         else
3207                 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3208
3209         /* setup tiling info dword.  gb_addr_config is not adequate since it does
3210          * not have bank info, so create a custom tiling dword.
3211          * bits 3:0   num_pipes
3212          * bits 7:4   num_banks
3213          * bits 11:8  group_size
3214          * bits 15:12 row_size
3215          */
3216         rdev->config.evergreen.tile_config = 0;
3217         switch (rdev->config.evergreen.max_tile_pipes) {
3218         case 1:
3219         default:
3220                 rdev->config.evergreen.tile_config |= (0 << 0);
3221                 break;
3222         case 2:
3223                 rdev->config.evergreen.tile_config |= (1 << 0);
3224                 break;
3225         case 4:
3226                 rdev->config.evergreen.tile_config |= (2 << 0);
3227                 break;
3228         case 8:
3229                 rdev->config.evergreen.tile_config |= (3 << 0);
3230                 break;
3231         }
3232         /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
3233         if (rdev->flags & RADEON_IS_IGP)
3234                 rdev->config.evergreen.tile_config |= 1 << 4;
3235         else {
3236                 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3237                 case 0: /* four banks */
3238                         rdev->config.evergreen.tile_config |= 0 << 4;
3239                         break;
3240                 case 1: /* eight banks */
3241                         rdev->config.evergreen.tile_config |= 1 << 4;
3242                         break;
3243                 case 2: /* sixteen banks */
3244                 default:
3245                         rdev->config.evergreen.tile_config |= 2 << 4;
3246                         break;
3247                 }
3248         }
3249         rdev->config.evergreen.tile_config |= 0 << 8;
3250         rdev->config.evergreen.tile_config |=
3251                 ((gb_addr_config & 0x30000000) >> 28) << 12;
3252
3253         num_shader_engines = (gb_addr_config & NUM_SHADER_ENGINES(3) >> 12) + 1;
3254
3255         if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3256                 u32 efuse_straps_4;
3257                 u32 efuse_straps_3;
3258
3259                 efuse_straps_4 = RREG32_RCU(0x204);
3260                 efuse_straps_3 = RREG32_RCU(0x203);
3261                 tmp = (((efuse_straps_4 & 0xf) << 4) |
3262                       ((efuse_straps_3 & 0xf0000000) >> 28));
3263         } else {
3264                 tmp = 0;
3265                 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3266                         u32 rb_disable_bitmap;
3267
3268                         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3269                         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3270                         rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3271                         tmp <<= 4;
3272                         tmp |= rb_disable_bitmap;
3273                 }
3274         }
3275         /* enabled rb are just the one not disabled :) */
3276         disabled_rb_mask = tmp;
3277         tmp = 0;
3278         for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3279                 tmp |= (1 << i);
3280         /* if all the backends are disabled, fix it up here */
3281         if ((disabled_rb_mask & tmp) == tmp) {
3282                 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3283                         disabled_rb_mask &= ~(1 << i);
3284         }
3285
3286         WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3287         WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3288
3289         WREG32(GB_ADDR_CONFIG, gb_addr_config);
3290         WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3291         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3292         WREG32(DMA_TILING_CONFIG, gb_addr_config);
3293         WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3294         WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3295         WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3296
3297         if ((rdev->config.evergreen.max_backends == 1) &&
3298             (rdev->flags & RADEON_IS_IGP)) {
3299                 if ((disabled_rb_mask & 3) == 1) {
3300                         /* RB0 disabled, RB1 enabled */
3301                         tmp = 0x11111111;
3302                 } else {
3303                         /* RB1 disabled, RB0 enabled */
3304                         tmp = 0x00000000;
3305                 }
3306         } else {
3307                 tmp = gb_addr_config & NUM_PIPES_MASK;
3308                 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3309                                                 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3310         }
3311         WREG32(GB_BACKEND_MAP, tmp);
3312
3313         WREG32(CGTS_SYS_TCC_DISABLE, 0);
3314         WREG32(CGTS_TCC_DISABLE, 0);
3315         WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3316         WREG32(CGTS_USER_TCC_DISABLE, 0);
3317
3318         /* set HW defaults for 3D engine */
3319         WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3320                                      ROQ_IB2_START(0x2b)));
3321
3322         WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3323
3324         WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3325                              SYNC_GRADIENT |
3326                              SYNC_WALKER |
3327                              SYNC_ALIGNER));
3328
3329         sx_debug_1 = RREG32(SX_DEBUG_1);
3330         sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3331         WREG32(SX_DEBUG_1, sx_debug_1);
3332
3333
3334         smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3335         smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3336         smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3337         WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3338
3339         if (rdev->family <= CHIP_SUMO2)
3340                 WREG32(SMX_SAR_CTL0, 0x00010000);
3341
3342         WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3343                                         POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3344                                         SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3345
3346         WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3347                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3348                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3349
3350         WREG32(VGT_NUM_INSTANCES, 1);
3351         WREG32(SPI_CONFIG_CNTL, 0);
3352         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3353         WREG32(CP_PERFMON_CNTL, 0);
3354
3355         WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3356                                   FETCH_FIFO_HIWATER(0x4) |
3357                                   DONE_FIFO_HIWATER(0xe0) |
3358                                   ALU_UPDATE_FIFO_HIWATER(0x8)));
3359
3360         sq_config = RREG32(SQ_CONFIG);
3361         sq_config &= ~(PS_PRIO(3) |
3362                        VS_PRIO(3) |
3363                        GS_PRIO(3) |
3364                        ES_PRIO(3));
3365         sq_config |= (VC_ENABLE |
3366                       EXPORT_SRC_C |
3367                       PS_PRIO(0) |
3368                       VS_PRIO(1) |
3369                       GS_PRIO(2) |
3370                       ES_PRIO(3));
3371
3372         switch (rdev->family) {
3373         case CHIP_CEDAR:
3374         case CHIP_PALM:
3375         case CHIP_SUMO:
3376         case CHIP_SUMO2:
3377         case CHIP_CAICOS:
3378                 /* no vertex cache */
3379                 sq_config &= ~VC_ENABLE;
3380                 break;
3381         default:
3382                 break;
3383         }
3384
3385         sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3386
3387         sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3388         sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3389         sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3390         sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3391         sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3392         sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3393         sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3394
3395         switch (rdev->family) {
3396         case CHIP_CEDAR:
3397         case CHIP_PALM:
3398         case CHIP_SUMO:
3399         case CHIP_SUMO2:
3400                 ps_thread_count = 96;
3401                 break;
3402         default:
3403                 ps_thread_count = 128;
3404                 break;
3405         }
3406
3407         sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
3408         sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3409         sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3410         sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3411         sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3412         sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3413
3414         sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3415         sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3416         sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3417         sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3418         sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3419         sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3420
3421         WREG32(SQ_CONFIG, sq_config);
3422         WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3423         WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3424         WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3425         WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3426         WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3427         WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3428         WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3429         WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3430         WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3431         WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3432
3433         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3434                                           FORCE_EOV_MAX_REZ_CNT(255)));
3435
3436         switch (rdev->family) {
3437         case CHIP_CEDAR:
3438         case CHIP_PALM:
3439         case CHIP_SUMO:
3440         case CHIP_SUMO2:
3441         case CHIP_CAICOS:
3442                 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
3443                 break;
3444         default:
3445                 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
3446                 break;
3447         }
3448         vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3449         WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3450
3451         WREG32(VGT_GS_VERTEX_REUSE, 16);
3452         WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
3453         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3454
3455         WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3456         WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3457
3458         WREG32(CB_PERF_CTR0_SEL_0, 0);
3459         WREG32(CB_PERF_CTR0_SEL_1, 0);
3460         WREG32(CB_PERF_CTR1_SEL_0, 0);
3461         WREG32(CB_PERF_CTR1_SEL_1, 0);
3462         WREG32(CB_PERF_CTR2_SEL_0, 0);
3463         WREG32(CB_PERF_CTR2_SEL_1, 0);
3464         WREG32(CB_PERF_CTR3_SEL_0, 0);
3465         WREG32(CB_PERF_CTR3_SEL_1, 0);
3466
3467         /* clear render buffer base addresses */
3468         WREG32(CB_COLOR0_BASE, 0);
3469         WREG32(CB_COLOR1_BASE, 0);
3470         WREG32(CB_COLOR2_BASE, 0);
3471         WREG32(CB_COLOR3_BASE, 0);
3472         WREG32(CB_COLOR4_BASE, 0);
3473         WREG32(CB_COLOR5_BASE, 0);
3474         WREG32(CB_COLOR6_BASE, 0);
3475         WREG32(CB_COLOR7_BASE, 0);
3476         WREG32(CB_COLOR8_BASE, 0);
3477         WREG32(CB_COLOR9_BASE, 0);
3478         WREG32(CB_COLOR10_BASE, 0);
3479         WREG32(CB_COLOR11_BASE, 0);
3480
3481         /* set the shader const cache sizes to 0 */
3482         for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3483                 WREG32(i, 0);
3484         for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3485                 WREG32(i, 0);
3486
3487         tmp = RREG32(HDP_MISC_CNTL);
3488         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3489         WREG32(HDP_MISC_CNTL, tmp);
3490
3491         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3492         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3493
3494         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3495
3496         udelay(50);
3497
3498 }
3499
3500 int evergreen_mc_init(struct radeon_device *rdev)
3501 {
3502         u32 tmp;
3503         int chansize, numchan;
3504
3505         /* Get VRAM informations */
3506         rdev->mc.vram_is_ddr = true;
3507         if ((rdev->family == CHIP_PALM) ||
3508             (rdev->family == CHIP_SUMO) ||
3509             (rdev->family == CHIP_SUMO2))
3510                 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3511         else
3512                 tmp = RREG32(MC_ARB_RAMCFG);
3513         if (tmp & CHANSIZE_OVERRIDE) {
3514                 chansize = 16;
3515         } else if (tmp & CHANSIZE_MASK) {
3516                 chansize = 64;
3517         } else {
3518                 chansize = 32;
3519         }
3520         tmp = RREG32(MC_SHARED_CHMAP);
3521         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3522         case 0:
3523         default:
3524                 numchan = 1;
3525                 break;
3526         case 1:
3527                 numchan = 2;
3528                 break;
3529         case 2:
3530                 numchan = 4;
3531                 break;
3532         case 3:
3533                 numchan = 8;
3534                 break;
3535         }
3536         rdev->mc.vram_width = numchan * chansize;
3537         /* Could aper size report 0 ? */
3538         rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3539         rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3540         /* Setup GPU memory space */
3541         if ((rdev->family == CHIP_PALM) ||
3542             (rdev->family == CHIP_SUMO) ||
3543             (rdev->family == CHIP_SUMO2)) {
3544                 /* size in bytes on fusion */
3545                 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3546                 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3547         } else {
3548                 /* size in MB on evergreen/cayman/tn */
3549                 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3550                 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3551         }
3552         rdev->mc.visible_vram_size = rdev->mc.aper_size;
3553         r700_vram_gtt_location(rdev, &rdev->mc);
3554         radeon_update_bandwidth_info(rdev);
3555
3556         return 0;
3557 }
3558
3559 void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
3560 {
3561         dev_info(rdev->dev, "  GRBM_STATUS               = 0x%08X\n",
3562                 RREG32(GRBM_STATUS));
3563         dev_info(rdev->dev, "  GRBM_STATUS_SE0           = 0x%08X\n",
3564                 RREG32(GRBM_STATUS_SE0));
3565         dev_info(rdev->dev, "  GRBM_STATUS_SE1           = 0x%08X\n",
3566                 RREG32(GRBM_STATUS_SE1));
3567         dev_info(rdev->dev, "  SRBM_STATUS               = 0x%08X\n",
3568                 RREG32(SRBM_STATUS));
3569         dev_info(rdev->dev, "  SRBM_STATUS2              = 0x%08X\n",
3570                 RREG32(SRBM_STATUS2));
3571         dev_info(rdev->dev, "  R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3572                 RREG32(CP_STALLED_STAT1));
3573         dev_info(rdev->dev, "  R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3574                 RREG32(CP_STALLED_STAT2));
3575         dev_info(rdev->dev, "  R_00867C_CP_BUSY_STAT     = 0x%08X\n",
3576                 RREG32(CP_BUSY_STAT));
3577         dev_info(rdev->dev, "  R_008680_CP_STAT          = 0x%08X\n",
3578                 RREG32(CP_STAT));
3579         dev_info(rdev->dev, "  R_00D034_DMA_STATUS_REG   = 0x%08X\n",
3580                 RREG32(DMA_STATUS_REG));
3581         if (rdev->family >= CHIP_CAYMAN) {
3582                 dev_info(rdev->dev, "  R_00D834_DMA_STATUS_REG   = 0x%08X\n",
3583                          RREG32(DMA_STATUS_REG + 0x800));
3584         }
3585 }
3586
3587 bool evergreen_is_display_hung(struct radeon_device *rdev)
3588 {
3589         u32 crtc_hung = 0;
3590         u32 crtc_status[6];
3591         u32 i, j, tmp;
3592
3593         for (i = 0; i < rdev->num_crtc; i++) {
3594                 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3595                         crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3596                         crtc_hung |= (1 << i);
3597                 }
3598         }
3599
3600         for (j = 0; j < 10; j++) {
3601                 for (i = 0; i < rdev->num_crtc; i++) {
3602                         if (crtc_hung & (1 << i)) {
3603                                 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3604                                 if (tmp != crtc_status[i])
3605                                         crtc_hung &= ~(1 << i);
3606                         }
3607                 }
3608                 if (crtc_hung == 0)
3609                         return false;
3610                 udelay(100);
3611         }
3612
3613         return true;
3614 }
3615
3616 static u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3617 {
3618         u32 reset_mask = 0;
3619         u32 tmp;
3620
3621         /* GRBM_STATUS */
3622         tmp = RREG32(GRBM_STATUS);
3623         if (tmp & (PA_BUSY | SC_BUSY |
3624                    SH_BUSY | SX_BUSY |
3625                    TA_BUSY | VGT_BUSY |
3626                    DB_BUSY | CB_BUSY |
3627                    SPI_BUSY | VGT_BUSY_NO_DMA))
3628                 reset_mask |= RADEON_RESET_GFX;
3629
3630         if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3631                    CP_BUSY | CP_COHERENCY_BUSY))
3632                 reset_mask |= RADEON_RESET_CP;
3633
3634         if (tmp & GRBM_EE_BUSY)
3635                 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3636
3637         /* DMA_STATUS_REG */
3638         tmp = RREG32(DMA_STATUS_REG);
3639         if (!(tmp & DMA_IDLE))
3640                 reset_mask |= RADEON_RESET_DMA;
3641
3642         /* SRBM_STATUS2 */
3643         tmp = RREG32(SRBM_STATUS2);
3644         if (tmp & DMA_BUSY)
3645                 reset_mask |= RADEON_RESET_DMA;
3646
3647         /* SRBM_STATUS */
3648         tmp = RREG32(SRBM_STATUS);
3649         if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3650                 reset_mask |= RADEON_RESET_RLC;
3651
3652         if (tmp & IH_BUSY)
3653                 reset_mask |= RADEON_RESET_IH;
3654
3655         if (tmp & SEM_BUSY)
3656                 reset_mask |= RADEON_RESET_SEM;
3657
3658         if (tmp & GRBM_RQ_PENDING)
3659                 reset_mask |= RADEON_RESET_GRBM;
3660
3661         if (tmp & VMC_BUSY)
3662                 reset_mask |= RADEON_RESET_VMC;
3663
3664         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3665                    MCC_BUSY | MCD_BUSY))
3666                 reset_mask |= RADEON_RESET_MC;
3667
3668         if (evergreen_is_display_hung(rdev))
3669                 reset_mask |= RADEON_RESET_DISPLAY;
3670
3671         /* VM_L2_STATUS */
3672         tmp = RREG32(VM_L2_STATUS);
3673         if (tmp & L2_BUSY)
3674                 reset_mask |= RADEON_RESET_VMC;
3675
3676         /* Skip MC reset as it's mostly likely not hung, just busy */
3677         if (reset_mask & RADEON_RESET_MC) {
3678                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3679                 reset_mask &= ~RADEON_RESET_MC;
3680         }
3681
3682         return reset_mask;
3683 }
3684
3685 static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3686 {
3687         struct evergreen_mc_save save;
3688         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3689         u32 tmp;
3690
3691         if (reset_mask == 0)
3692                 return;
3693
3694         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3695
3696         evergreen_print_gpu_status_regs(rdev);
3697
3698         /* Disable CP parsing/prefetching */
3699         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3700
3701         if (reset_mask & RADEON_RESET_DMA) {
3702                 /* Disable DMA */
3703                 tmp = RREG32(DMA_RB_CNTL);
3704                 tmp &= ~DMA_RB_ENABLE;
3705                 WREG32(DMA_RB_CNTL, tmp);
3706         }
3707
3708         udelay(50);
3709
3710         evergreen_mc_stop(rdev, &save);
3711         if (evergreen_mc_wait_for_idle(rdev)) {
3712                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3713         }
3714
3715         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3716                 grbm_soft_reset |= SOFT_RESET_DB |
3717                         SOFT_RESET_CB |
3718                         SOFT_RESET_PA |
3719                         SOFT_RESET_SC |
3720                         SOFT_RESET_SPI |
3721                         SOFT_RESET_SX |
3722                         SOFT_RESET_SH |
3723                         SOFT_RESET_TC |
3724                         SOFT_RESET_TA |
3725                         SOFT_RESET_VC |
3726                         SOFT_RESET_VGT;
3727         }
3728
3729         if (reset_mask & RADEON_RESET_CP) {
3730                 grbm_soft_reset |= SOFT_RESET_CP |
3731                         SOFT_RESET_VGT;
3732
3733                 srbm_soft_reset |= SOFT_RESET_GRBM;
3734         }
3735
3736         if (reset_mask & RADEON_RESET_DMA)
3737                 srbm_soft_reset |= SOFT_RESET_DMA;
3738
3739         if (reset_mask & RADEON_RESET_DISPLAY)
3740                 srbm_soft_reset |= SOFT_RESET_DC;
3741
3742         if (reset_mask & RADEON_RESET_RLC)
3743                 srbm_soft_reset |= SOFT_RESET_RLC;
3744
3745         if (reset_mask & RADEON_RESET_SEM)
3746                 srbm_soft_reset |= SOFT_RESET_SEM;
3747
3748         if (reset_mask & RADEON_RESET_IH)
3749                 srbm_soft_reset |= SOFT_RESET_IH;
3750
3751         if (reset_mask & RADEON_RESET_GRBM)
3752                 srbm_soft_reset |= SOFT_RESET_GRBM;
3753
3754         if (reset_mask & RADEON_RESET_VMC)
3755                 srbm_soft_reset |= SOFT_RESET_VMC;
3756
3757         if (!(rdev->flags & RADEON_IS_IGP)) {
3758                 if (reset_mask & RADEON_RESET_MC)
3759                         srbm_soft_reset |= SOFT_RESET_MC;
3760         }
3761
3762         if (grbm_soft_reset) {
3763                 tmp = RREG32(GRBM_SOFT_RESET);
3764                 tmp |= grbm_soft_reset;
3765                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3766                 WREG32(GRBM_SOFT_RESET, tmp);
3767                 tmp = RREG32(GRBM_SOFT_RESET);
3768
3769                 udelay(50);
3770
3771                 tmp &= ~grbm_soft_reset;
3772                 WREG32(GRBM_SOFT_RESET, tmp);
3773                 tmp = RREG32(GRBM_SOFT_RESET);
3774         }
3775
3776         if (srbm_soft_reset) {
3777                 tmp = RREG32(SRBM_SOFT_RESET);
3778                 tmp |= srbm_soft_reset;
3779                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3780                 WREG32(SRBM_SOFT_RESET, tmp);
3781                 tmp = RREG32(SRBM_SOFT_RESET);
3782
3783                 udelay(50);
3784
3785                 tmp &= ~srbm_soft_reset;
3786                 WREG32(SRBM_SOFT_RESET, tmp);
3787                 tmp = RREG32(SRBM_SOFT_RESET);
3788         }
3789
3790         /* Wait a little for things to settle down */
3791         udelay(50);
3792
3793         evergreen_mc_resume(rdev, &save);
3794         udelay(50);
3795
3796         evergreen_print_gpu_status_regs(rdev);
3797 }
3798
3799 int evergreen_asic_reset(struct radeon_device *rdev)
3800 {
3801         u32 reset_mask;
3802
3803         reset_mask = evergreen_gpu_check_soft_reset(rdev);
3804
3805         if (reset_mask)
3806                 r600_set_bios_scratch_engine_hung(rdev, true);
3807
3808         evergreen_gpu_soft_reset(rdev, reset_mask);
3809
3810         reset_mask = evergreen_gpu_check_soft_reset(rdev);
3811
3812         if (!reset_mask)
3813                 r600_set_bios_scratch_engine_hung(rdev, false);
3814
3815         return 0;
3816 }
3817
3818 /**
3819  * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
3820  *
3821  * @rdev: radeon_device pointer
3822  * @ring: radeon_ring structure holding ring information
3823  *
3824  * Check if the GFX engine is locked up.
3825  * Returns true if the engine appears to be locked up, false if not.
3826  */
3827 bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3828 {
3829         u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3830
3831         if (!(reset_mask & (RADEON_RESET_GFX |
3832                             RADEON_RESET_COMPUTE |
3833                             RADEON_RESET_CP))) {
3834                 radeon_ring_lockup_update(ring);
3835                 return false;
3836         }
3837         /* force CP activities */
3838         radeon_ring_force_activity(rdev, ring);
3839         return radeon_ring_test_lockup(rdev, ring);
3840 }
3841
3842 /**
3843  * evergreen_dma_is_lockup - Check if the DMA engine is locked up
3844  *
3845  * @rdev: radeon_device pointer
3846  * @ring: radeon_ring structure holding ring information
3847  *
3848  * Check if the async DMA engine is locked up.
3849  * Returns true if the engine appears to be locked up, false if not.
3850  */
3851 bool evergreen_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3852 {
3853         u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3854
3855         if (!(reset_mask & RADEON_RESET_DMA)) {
3856                 radeon_ring_lockup_update(ring);
3857                 return false;
3858         }
3859         /* force ring activities */
3860         radeon_ring_force_activity(rdev, ring);
3861         return radeon_ring_test_lockup(rdev, ring);
3862 }
3863
3864 /*
3865  * RLC
3866  */
3867 #define RLC_SAVE_RESTORE_LIST_END_MARKER    0x00000000
3868 #define RLC_CLEAR_STATE_END_MARKER          0x00000001
3869
3870 void sumo_rlc_fini(struct radeon_device *rdev)
3871 {
3872         int r;
3873
3874         /* save restore block */
3875         if (rdev->rlc.save_restore_obj) {
3876                 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
3877                 if (unlikely(r != 0))
3878                         dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
3879                 radeon_bo_unpin(rdev->rlc.save_restore_obj);
3880                 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3881
3882                 radeon_bo_unref(&rdev->rlc.save_restore_obj);
3883                 rdev->rlc.save_restore_obj = NULL;
3884         }
3885
3886         /* clear state block */
3887         if (rdev->rlc.clear_state_obj) {
3888                 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
3889                 if (unlikely(r != 0))
3890                         dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
3891                 radeon_bo_unpin(rdev->rlc.clear_state_obj);
3892                 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
3893
3894                 radeon_bo_unref(&rdev->rlc.clear_state_obj);
3895                 rdev->rlc.clear_state_obj = NULL;
3896         }
3897 }
3898
3899 int sumo_rlc_init(struct radeon_device *rdev)
3900 {
3901         u32 *src_ptr;
3902         volatile u32 *dst_ptr;
3903         u32 dws, data, i, j, k, reg_num;
3904         u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index;
3905         u64 reg_list_mc_addr;
3906         struct cs_section_def *cs_data;
3907         int r;
3908
3909         src_ptr = rdev->rlc.reg_list;
3910         dws = rdev->rlc.reg_list_size;
3911         cs_data = rdev->rlc.cs_data;
3912
3913         /* save restore block */
3914         if (rdev->rlc.save_restore_obj == NULL) {
3915                 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
3916                                      RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.save_restore_obj);
3917                 if (r) {
3918                         dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
3919                         return r;
3920                 }
3921         }
3922
3923         r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
3924         if (unlikely(r != 0)) {
3925                 sumo_rlc_fini(rdev);
3926                 return r;
3927         }
3928         r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
3929                           &rdev->rlc.save_restore_gpu_addr);
3930         if (r) {
3931                 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3932                 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
3933                 sumo_rlc_fini(rdev);
3934                 return r;
3935         }
3936         r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
3937         if (r) {
3938                 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
3939                 sumo_rlc_fini(rdev);
3940                 return r;
3941         }
3942         /* write the sr buffer */
3943         dst_ptr = rdev->rlc.sr_ptr;
3944         /* format:
3945          * dw0: (reg2 << 16) | reg1
3946          * dw1: reg1 save space
3947          * dw2: reg2 save space
3948          */
3949         for (i = 0; i < dws; i++) {
3950                 data = src_ptr[i] >> 2;
3951                 i++;
3952                 if (i < dws)
3953                         data |= (src_ptr[i] >> 2) << 16;
3954                 j = (((i - 1) * 3) / 2);
3955                 dst_ptr[j] = data;
3956         }
3957         j = ((i * 3) / 2);
3958         dst_ptr[j] = RLC_SAVE_RESTORE_LIST_END_MARKER;
3959
3960         radeon_bo_kunmap(rdev->rlc.save_restore_obj);
3961         radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3962
3963         /* clear state block */
3964         reg_list_num = 0;
3965         dws = 0;
3966         for (i = 0; cs_data[i].section != NULL; i++) {
3967                 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
3968                         reg_list_num++;
3969                         dws += cs_data[i].section[j].reg_count;
3970                 }
3971         }
3972         reg_list_blk_index = (3 * reg_list_num + 2);
3973         dws += reg_list_blk_index;
3974
3975         if (rdev->rlc.clear_state_obj == NULL) {
3976                 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
3977                                      RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.clear_state_obj);
3978                 if (r) {
3979                         dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
3980                         sumo_rlc_fini(rdev);
3981                         return r;
3982                 }
3983         }
3984         r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
3985         if (unlikely(r != 0)) {
3986                 sumo_rlc_fini(rdev);
3987                 return r;
3988         }
3989         r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
3990                           &rdev->rlc.clear_state_gpu_addr);
3991         if (r) {
3992
3993                 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
3994                 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
3995                 sumo_rlc_fini(rdev);
3996                 return r;
3997         }
3998         r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
3999         if (r) {
4000                 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4001                 sumo_rlc_fini(rdev);
4002                 return r;
4003         }
4004         /* set up the cs buffer */
4005         dst_ptr = rdev->rlc.cs_ptr;
4006         reg_list_hdr_blk_index = 0;
4007         reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4008         data = upper_32_bits(reg_list_mc_addr);
4009         dst_ptr[reg_list_hdr_blk_index] = data;
4010         reg_list_hdr_blk_index++;
4011         for (i = 0; cs_data[i].section != NULL; i++) {
4012                 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4013                         reg_num = cs_data[i].section[j].reg_count;
4014                         data = reg_list_mc_addr & 0xffffffff;
4015                         dst_ptr[reg_list_hdr_blk_index] = data;
4016                         reg_list_hdr_blk_index++;
4017
4018                         data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4019                         dst_ptr[reg_list_hdr_blk_index] = data;
4020                         reg_list_hdr_blk_index++;
4021
4022                         data = 0x08000000 | (reg_num * 4);
4023                         dst_ptr[reg_list_hdr_blk_index] = data;
4024                         reg_list_hdr_blk_index++;
4025
4026                         for (k = 0; k < reg_num; k++) {
4027                                 data = cs_data[i].section[j].extent[k];
4028                                 dst_ptr[reg_list_blk_index + k] = data;
4029                         }
4030                         reg_list_mc_addr += reg_num * 4;
4031                         reg_list_blk_index += reg_num;
4032                 }
4033         }
4034         dst_ptr[reg_list_hdr_blk_index] = RLC_CLEAR_STATE_END_MARKER;
4035
4036         radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4037         radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4038
4039         return 0;
4040 }
4041
4042 static void evergreen_rlc_start(struct radeon_device *rdev)
4043 {
4044         u32 mask = RLC_ENABLE;
4045
4046         if (rdev->flags & RADEON_IS_IGP) {
4047                 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
4048         }
4049
4050         WREG32(RLC_CNTL, mask);
4051 }
4052
4053 int evergreen_rlc_resume(struct radeon_device *rdev)
4054 {
4055         u32 i;
4056         const __be32 *fw_data;
4057
4058         if (!rdev->rlc_fw)
4059                 return -EINVAL;
4060
4061         r600_rlc_stop(rdev);
4062
4063         WREG32(RLC_HB_CNTL, 0);
4064
4065         if (rdev->flags & RADEON_IS_IGP) {
4066                 if (rdev->family == CHIP_ARUBA) {
4067                         u32 always_on_bitmap =
4068                                 3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4069                         /* find out the number of active simds */
4070                         u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4071                         tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4072                         tmp = hweight32(~tmp);
4073                         if (tmp == rdev->config.cayman.max_simds_per_se) {
4074                                 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4075                                 WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4076                                 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4077                                 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4078                                 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4079                         }
4080                 } else {
4081                         WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4082                         WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4083                 }
4084                 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4085                 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4086         } else {
4087                 WREG32(RLC_HB_BASE, 0);
4088                 WREG32(RLC_HB_RPTR, 0);
4089                 WREG32(RLC_HB_WPTR, 0);
4090                 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4091                 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4092         }
4093         WREG32(RLC_MC_CNTL, 0);
4094         WREG32(RLC_UCODE_CNTL, 0);
4095
4096         fw_data = (const __be32 *)rdev->rlc_fw->data;
4097         if (rdev->family >= CHIP_ARUBA) {
4098                 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4099                         WREG32(RLC_UCODE_ADDR, i);
4100                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4101                 }
4102         } else if (rdev->family >= CHIP_CAYMAN) {
4103                 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4104                         WREG32(RLC_UCODE_ADDR, i);
4105                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4106                 }
4107         } else {
4108                 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4109                         WREG32(RLC_UCODE_ADDR, i);
4110                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4111                 }
4112         }
4113         WREG32(RLC_UCODE_ADDR, 0);
4114
4115         evergreen_rlc_start(rdev);
4116
4117         return 0;
4118 }
4119
4120 /* Interrupts */
4121
4122 u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4123 {
4124         if (crtc >= rdev->num_crtc)
4125                 return 0;
4126         else
4127                 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
4128 }
4129
4130 void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4131 {
4132         u32 tmp;
4133
4134         if (rdev->family >= CHIP_CAYMAN) {
4135                 cayman_cp_int_cntl_setup(rdev, 0,
4136                                          CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4137                 cayman_cp_int_cntl_setup(rdev, 1, 0);
4138                 cayman_cp_int_cntl_setup(rdev, 2, 0);
4139                 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4140                 WREG32(CAYMAN_DMA1_CNTL, tmp);
4141         } else
4142                 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4143         tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4144         WREG32(DMA_CNTL, tmp);
4145         WREG32(GRBM_INT_CNTL, 0);
4146         WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4147         WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4148         if (rdev->num_crtc >= 4) {
4149                 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4150                 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4151         }
4152         if (rdev->num_crtc >= 6) {
4153                 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4154                 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4155         }
4156
4157         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4158         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4159         if (rdev->num_crtc >= 4) {
4160                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4161                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4162         }
4163         if (rdev->num_crtc >= 6) {
4164                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4165                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4166         }
4167
4168         /* only one DAC on DCE6 */
4169         if (!ASIC_IS_DCE6(rdev))
4170                 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4171         WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4172
4173         tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4174         WREG32(DC_HPD1_INT_CONTROL, tmp);
4175         tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4176         WREG32(DC_HPD2_INT_CONTROL, tmp);
4177         tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4178         WREG32(DC_HPD3_INT_CONTROL, tmp);
4179         tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4180         WREG32(DC_HPD4_INT_CONTROL, tmp);
4181         tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4182         WREG32(DC_HPD5_INT_CONTROL, tmp);
4183         tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4184         WREG32(DC_HPD6_INT_CONTROL, tmp);
4185
4186 }
4187
4188 int evergreen_irq_set(struct radeon_device *rdev)
4189 {
4190         u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4191         u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
4192         u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
4193         u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
4194         u32 grbm_int_cntl = 0;
4195         u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
4196         u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0;
4197         u32 dma_cntl, dma_cntl1 = 0;
4198         u32 thermal_int = 0;
4199
4200         if (!rdev->irq.installed) {
4201                 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4202                 return -EINVAL;
4203         }
4204         /* don't enable anything if the ih is disabled */
4205         if (!rdev->ih.enabled) {
4206                 r600_disable_interrupts(rdev);
4207                 /* force the active interrupt state to all disabled */
4208                 evergreen_disable_interrupt_state(rdev);
4209                 return 0;
4210         }
4211
4212         hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
4213         hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
4214         hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
4215         hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
4216         hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
4217         hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
4218         if (rdev->family == CHIP_ARUBA)
4219                 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4220                         ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4221         else
4222                 thermal_int = RREG32(CG_THERMAL_INT) &
4223                         ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4224
4225         afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4226         afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4227         afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4228         afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4229         afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4230         afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4231
4232         dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4233
4234         if (rdev->family >= CHIP_CAYMAN) {
4235                 /* enable CP interrupts on all rings */
4236                 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4237                         DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4238                         cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4239                 }
4240                 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
4241                         DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4242                         cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4243                 }
4244                 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
4245                         DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4246                         cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4247                 }
4248         } else {
4249                 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4250                         DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4251                         cp_int_cntl |= RB_INT_ENABLE;
4252                         cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4253                 }
4254         }
4255
4256         if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4257                 DRM_DEBUG("r600_irq_set: sw int dma\n");
4258                 dma_cntl |= TRAP_ENABLE;
4259         }
4260
4261         if (rdev->family >= CHIP_CAYMAN) {
4262                 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4263                 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4264                         DRM_DEBUG("r600_irq_set: sw int dma1\n");
4265                         dma_cntl1 |= TRAP_ENABLE;
4266                 }
4267         }
4268
4269         if (rdev->irq.dpm_thermal) {
4270                 DRM_DEBUG("dpm thermal\n");
4271                 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4272         }
4273
4274         if (rdev->irq.crtc_vblank_int[0] ||
4275             atomic_read(&rdev->irq.pflip[0])) {
4276                 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
4277                 crtc1 |= VBLANK_INT_MASK;
4278         }
4279         if (rdev->irq.crtc_vblank_int[1] ||
4280             atomic_read(&rdev->irq.pflip[1])) {
4281                 DRM_DEBUG("evergreen_irq_set: vblank 1\n");
4282                 crtc2 |= VBLANK_INT_MASK;
4283         }
4284         if (rdev->irq.crtc_vblank_int[2] ||
4285             atomic_read(&rdev->irq.pflip[2])) {
4286                 DRM_DEBUG("evergreen_irq_set: vblank 2\n");
4287                 crtc3 |= VBLANK_INT_MASK;
4288         }
4289         if (rdev->irq.crtc_vblank_int[3] ||
4290             atomic_read(&rdev->irq.pflip[3])) {
4291                 DRM_DEBUG("evergreen_irq_set: vblank 3\n");
4292                 crtc4 |= VBLANK_INT_MASK;
4293         }
4294         if (rdev->irq.crtc_vblank_int[4] ||
4295             atomic_read(&rdev->irq.pflip[4])) {
4296                 DRM_DEBUG("evergreen_irq_set: vblank 4\n");
4297                 crtc5 |= VBLANK_INT_MASK;
4298         }
4299         if (rdev->irq.crtc_vblank_int[5] ||
4300             atomic_read(&rdev->irq.pflip[5])) {
4301                 DRM_DEBUG("evergreen_irq_set: vblank 5\n");
4302                 crtc6 |= VBLANK_INT_MASK;
4303         }
4304         if (rdev->irq.hpd[0]) {
4305                 DRM_DEBUG("evergreen_irq_set: hpd 1\n");
4306                 hpd1 |= DC_HPDx_INT_EN;
4307         }
4308         if (rdev->irq.hpd[1]) {
4309                 DRM_DEBUG("evergreen_irq_set: hpd 2\n");
4310                 hpd2 |= DC_HPDx_INT_EN;
4311         }
4312         if (rdev->irq.hpd[2]) {
4313                 DRM_DEBUG("evergreen_irq_set: hpd 3\n");
4314                 hpd3 |= DC_HPDx_INT_EN;
4315         }
4316         if (rdev->irq.hpd[3]) {
4317                 DRM_DEBUG("evergreen_irq_set: hpd 4\n");
4318                 hpd4 |= DC_HPDx_INT_EN;
4319         }
4320         if (rdev->irq.hpd[4]) {
4321                 DRM_DEBUG("evergreen_irq_set: hpd 5\n");
4322                 hpd5 |= DC_HPDx_INT_EN;
4323         }
4324         if (rdev->irq.hpd[5]) {
4325                 DRM_DEBUG("evergreen_irq_set: hpd 6\n");
4326                 hpd6 |= DC_HPDx_INT_EN;
4327         }
4328         if (rdev->irq.afmt[0]) {
4329                 DRM_DEBUG("evergreen_irq_set: hdmi 0\n");
4330                 afmt1 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4331         }
4332         if (rdev->irq.afmt[1]) {
4333                 DRM_DEBUG("evergreen_irq_set: hdmi 1\n");
4334                 afmt2 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4335         }
4336         if (rdev->irq.afmt[2]) {
4337                 DRM_DEBUG("evergreen_irq_set: hdmi 2\n");
4338                 afmt3 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4339         }
4340         if (rdev->irq.afmt[3]) {
4341                 DRM_DEBUG("evergreen_irq_set: hdmi 3\n");
4342                 afmt4 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4343         }
4344         if (rdev->irq.afmt[4]) {
4345                 DRM_DEBUG("evergreen_irq_set: hdmi 4\n");
4346                 afmt5 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4347         }
4348         if (rdev->irq.afmt[5]) {
4349                 DRM_DEBUG("evergreen_irq_set: hdmi 5\n");
4350                 afmt6 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4351         }
4352
4353         if (rdev->family >= CHIP_CAYMAN) {
4354                 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4355                 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4356                 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4357         } else
4358                 WREG32(CP_INT_CNTL, cp_int_cntl);
4359
4360         WREG32(DMA_CNTL, dma_cntl);
4361
4362         if (rdev->family >= CHIP_CAYMAN)
4363                 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4364
4365         WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4366
4367         WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
4368         WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
4369         if (rdev->num_crtc >= 4) {
4370                 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
4371                 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
4372         }
4373         if (rdev->num_crtc >= 6) {
4374                 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
4375                 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
4376         }
4377
4378         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
4379         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
4380         if (rdev->num_crtc >= 4) {
4381                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
4382                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
4383         }
4384         if (rdev->num_crtc >= 6) {
4385                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
4386                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
4387         }
4388
4389         WREG32(DC_HPD1_INT_CONTROL, hpd1);
4390         WREG32(DC_HPD2_INT_CONTROL, hpd2);
4391         WREG32(DC_HPD3_INT_CONTROL, hpd3);
4392         WREG32(DC_HPD4_INT_CONTROL, hpd4);
4393         WREG32(DC_HPD5_INT_CONTROL, hpd5);
4394         WREG32(DC_HPD6_INT_CONTROL, hpd6);
4395         if (rdev->family == CHIP_ARUBA)
4396                 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4397         else
4398                 WREG32(CG_THERMAL_INT, thermal_int);
4399
4400         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1);
4401         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2);
4402         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3);
4403         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4);
4404         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5);
4405         WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6);
4406
4407         return 0;
4408 }
4409
4410 static void evergreen_irq_ack(struct radeon_device *rdev)
4411 {
4412         u32 tmp;
4413
4414         rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4415         rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4416         rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
4417         rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
4418         rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
4419         rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
4420         rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4421         rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4422         if (rdev->num_crtc >= 4) {
4423                 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4424                 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4425         }
4426         if (rdev->num_crtc >= 6) {
4427                 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4428                 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4429         }
4430
4431         rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4432         rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4433         rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4434         rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4435         rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4436         rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4437
4438         if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
4439                 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4440         if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
4441                 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4442         if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
4443                 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
4444         if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
4445                 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
4446         if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
4447                 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
4448         if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
4449                 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
4450
4451         if (rdev->num_crtc >= 4) {
4452                 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
4453                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4454                 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
4455                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4456                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
4457                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
4458                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
4459                         WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
4460                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
4461                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
4462                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
4463                         WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
4464         }
4465
4466         if (rdev->num_crtc >= 6) {
4467                 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
4468                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4469                 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
4470                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4471                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
4472                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
4473                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
4474                         WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
4475                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
4476                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
4477                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
4478                         WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
4479         }
4480
4481         if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4482                 tmp = RREG32(DC_HPD1_INT_CONTROL);
4483                 tmp |= DC_HPDx_INT_ACK;
4484                 WREG32(DC_HPD1_INT_CONTROL, tmp);
4485         }
4486         if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4487                 tmp = RREG32(DC_HPD2_INT_CONTROL);
4488                 tmp |= DC_HPDx_INT_ACK;
4489                 WREG32(DC_HPD2_INT_CONTROL, tmp);
4490         }
4491         if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4492                 tmp = RREG32(DC_HPD3_INT_CONTROL);
4493                 tmp |= DC_HPDx_INT_ACK;
4494                 WREG32(DC_HPD3_INT_CONTROL, tmp);
4495         }
4496         if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4497                 tmp = RREG32(DC_HPD4_INT_CONTROL);
4498                 tmp |= DC_HPDx_INT_ACK;
4499                 WREG32(DC_HPD4_INT_CONTROL, tmp);
4500         }
4501         if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4502                 tmp = RREG32(DC_HPD5_INT_CONTROL);
4503                 tmp |= DC_HPDx_INT_ACK;
4504                 WREG32(DC_HPD5_INT_CONTROL, tmp);
4505         }
4506         if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4507                 tmp = RREG32(DC_HPD5_INT_CONTROL);
4508                 tmp |= DC_HPDx_INT_ACK;
4509                 WREG32(DC_HPD6_INT_CONTROL, tmp);
4510         }
4511         if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4512                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
4513                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4514                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp);
4515         }
4516         if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4517                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
4518                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4519                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp);
4520         }
4521         if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4522                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
4523                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4524                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp);
4525         }
4526         if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4527                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
4528                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4529                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp);
4530         }
4531         if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4532                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
4533                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4534                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp);
4535         }
4536         if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4537                 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
4538                 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4539                 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp);
4540         }
4541 }
4542
4543 static void evergreen_irq_disable(struct radeon_device *rdev)
4544 {
4545         r600_disable_interrupts(rdev);
4546         /* Wait and acknowledge irq */
4547         mdelay(1);
4548         evergreen_irq_ack(rdev);
4549         evergreen_disable_interrupt_state(rdev);
4550 }
4551
4552 void evergreen_irq_suspend(struct radeon_device *rdev)
4553 {
4554         evergreen_irq_disable(rdev);
4555         r600_rlc_stop(rdev);
4556 }
4557
4558 static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
4559 {
4560         u32 wptr, tmp;
4561
4562         if (rdev->wb.enabled)
4563                 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4564         else
4565                 wptr = RREG32(IH_RB_WPTR);
4566
4567         if (wptr & RB_OVERFLOW) {
4568                 /* When a ring buffer overflow happen start parsing interrupt
4569                  * from the last not overwritten vector (wptr + 16). Hopefully
4570                  * this should allow us to catchup.
4571                  */
4572                 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
4573                         wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
4574                 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4575                 tmp = RREG32(IH_RB_CNTL);
4576                 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4577                 WREG32(IH_RB_CNTL, tmp);
4578         }
4579         return (wptr & rdev->ih.ptr_mask);
4580 }
4581
4582 int evergreen_irq_process(struct radeon_device *rdev)
4583 {
4584         u32 wptr;
4585         u32 rptr;
4586         u32 src_id, src_data;
4587         u32 ring_index;
4588         bool queue_hotplug = false;
4589         bool queue_hdmi = false;
4590         bool queue_thermal = false;
4591         u32 status, addr;
4592
4593         if (!rdev->ih.enabled || rdev->shutdown)
4594                 return IRQ_NONE;
4595
4596         wptr = evergreen_get_ih_wptr(rdev);
4597
4598 restart_ih:
4599         /* is somebody else already processing irqs? */
4600         if (atomic_xchg(&rdev->ih.lock, 1))
4601                 return IRQ_NONE;
4602
4603         rptr = rdev->ih.rptr;
4604         DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4605
4606         /* Order reading of wptr vs. reading of IH ring data */
4607         rmb();
4608
4609         /* display interrupts */
4610         evergreen_irq_ack(rdev);
4611
4612         while (rptr != wptr) {
4613                 /* wptr/rptr are in bytes! */
4614                 ring_index = rptr / 4;
4615                 src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4616                 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4617
4618                 switch (src_id) {
4619                 case 1: /* D1 vblank/vline */
4620                         switch (src_data) {
4621                         case 0: /* D1 vblank */
4622                                 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
4623                                         if (rdev->irq.crtc_vblank_int[0]) {
4624                                                 drm_handle_vblank(rdev->ddev, 0);
4625                                                 rdev->pm.vblank_sync = true;
4626                                                 wake_up(&rdev->irq.vblank_queue);
4627                                         }
4628                                         if (atomic_read(&rdev->irq.pflip[0]))
4629                                                 radeon_crtc_handle_flip(rdev, 0);
4630                                         rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
4631                                         DRM_DEBUG("IH: D1 vblank\n");
4632                                 }
4633                                 break;
4634                         case 1: /* D1 vline */
4635                                 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
4636                                         rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
4637                                         DRM_DEBUG("IH: D1 vline\n");
4638                                 }
4639                                 break;
4640                         default:
4641                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4642                                 break;
4643                         }
4644                         break;
4645                 case 2: /* D2 vblank/vline */
4646                         switch (src_data) {
4647                         case 0: /* D2 vblank */
4648                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
4649                                         if (rdev->irq.crtc_vblank_int[1]) {
4650                                                 drm_handle_vblank(rdev->ddev, 1);
4651                                                 rdev->pm.vblank_sync = true;
4652                                                 wake_up(&rdev->irq.vblank_queue);
4653                                         }
4654                                         if (atomic_read(&rdev->irq.pflip[1]))
4655                                                 radeon_crtc_handle_flip(rdev, 1);
4656                                         rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
4657                                         DRM_DEBUG("IH: D2 vblank\n");
4658                                 }
4659                                 break;
4660                         case 1: /* D2 vline */
4661                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
4662                                         rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
4663                                         DRM_DEBUG("IH: D2 vline\n");
4664                                 }
4665                                 break;
4666                         default:
4667                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4668                                 break;
4669                         }
4670                         break;
4671                 case 3: /* D3 vblank/vline */
4672                         switch (src_data) {
4673                         case 0: /* D3 vblank */
4674                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
4675                                         if (rdev->irq.crtc_vblank_int[2]) {
4676                                                 drm_handle_vblank(rdev->ddev, 2);
4677                                                 rdev->pm.vblank_sync = true;
4678                                                 wake_up(&rdev->irq.vblank_queue);
4679                                         }
4680                                         if (atomic_read(&rdev->irq.pflip[2]))
4681                                                 radeon_crtc_handle_flip(rdev, 2);
4682                                         rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
4683                                         DRM_DEBUG("IH: D3 vblank\n");
4684                                 }
4685                                 break;
4686                         case 1: /* D3 vline */
4687                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
4688                                         rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
4689                                         DRM_DEBUG("IH: D3 vline\n");
4690                                 }
4691                                 break;
4692                         default:
4693                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4694                                 break;
4695                         }
4696                         break;
4697                 case 4: /* D4 vblank/vline */
4698                         switch (src_data) {
4699                         case 0: /* D4 vblank */
4700                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
4701                                         if (rdev->irq.crtc_vblank_int[3]) {
4702                                                 drm_handle_vblank(rdev->ddev, 3);
4703                                                 rdev->pm.vblank_sync = true;
4704                                                 wake_up(&rdev->irq.vblank_queue);
4705                                         }
4706                                         if (atomic_read(&rdev->irq.pflip[3]))
4707                                                 radeon_crtc_handle_flip(rdev, 3);
4708                                         rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
4709                                         DRM_DEBUG("IH: D4 vblank\n");
4710                                 }
4711                                 break;
4712                         case 1: /* D4 vline */
4713                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
4714                                         rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
4715                                         DRM_DEBUG("IH: D4 vline\n");
4716                                 }
4717                                 break;
4718                         default:
4719                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4720                                 break;
4721                         }
4722                         break;
4723                 case 5: /* D5 vblank/vline */
4724                         switch (src_data) {
4725                         case 0: /* D5 vblank */
4726                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
4727                                         if (rdev->irq.crtc_vblank_int[4]) {
4728                                                 drm_handle_vblank(rdev->ddev, 4);
4729                                                 rdev->pm.vblank_sync = true;
4730                                                 wake_up(&rdev->irq.vblank_queue);
4731                                         }
4732                                         if (atomic_read(&rdev->irq.pflip[4]))
4733                                                 radeon_crtc_handle_flip(rdev, 4);
4734                                         rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
4735                                         DRM_DEBUG("IH: D5 vblank\n");
4736                                 }
4737                                 break;
4738                         case 1: /* D5 vline */
4739                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
4740                                         rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
4741                                         DRM_DEBUG("IH: D5 vline\n");
4742                                 }
4743                                 break;
4744                         default:
4745                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4746                                 break;
4747                         }
4748                         break;
4749                 case 6: /* D6 vblank/vline */
4750                         switch (src_data) {
4751                         case 0: /* D6 vblank */
4752                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
4753                                         if (rdev->irq.crtc_vblank_int[5]) {
4754                                                 drm_handle_vblank(rdev->ddev, 5);
4755                                                 rdev->pm.vblank_sync = true;
4756                                                 wake_up(&rdev->irq.vblank_queue);
4757                                         }
4758                                         if (atomic_read(&rdev->irq.pflip[5]))
4759                                                 radeon_crtc_handle_flip(rdev, 5);
4760                                         rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
4761                                         DRM_DEBUG("IH: D6 vblank\n");
4762                                 }
4763                                 break;
4764                         case 1: /* D6 vline */
4765                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
4766                                         rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
4767                                         DRM_DEBUG("IH: D6 vline\n");
4768                                 }
4769                                 break;
4770                         default:
4771                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4772                                 break;
4773                         }
4774                         break;
4775                 case 42: /* HPD hotplug */
4776                         switch (src_data) {
4777                         case 0:
4778                                 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4779                                         rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
4780                                         queue_hotplug = true;
4781                                         DRM_DEBUG("IH: HPD1\n");
4782                                 }
4783                                 break;
4784                         case 1:
4785                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4786                                         rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
4787                                         queue_hotplug = true;
4788                                         DRM_DEBUG("IH: HPD2\n");
4789                                 }
4790                                 break;
4791                         case 2:
4792                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4793                                         rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
4794                                         queue_hotplug = true;
4795                                         DRM_DEBUG("IH: HPD3\n");
4796                                 }
4797                                 break;
4798                         case 3:
4799                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4800                                         rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
4801                                         queue_hotplug = true;
4802                                         DRM_DEBUG("IH: HPD4\n");
4803                                 }
4804                                 break;
4805                         case 4:
4806                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4807                                         rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
4808                                         queue_hotplug = true;
4809                                         DRM_DEBUG("IH: HPD5\n");
4810                                 }
4811                                 break;
4812                         case 5:
4813                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4814                                         rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
4815                                         queue_hotplug = true;
4816                                         DRM_DEBUG("IH: HPD6\n");
4817                                 }
4818                                 break;
4819                         default:
4820                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4821                                 break;
4822                         }
4823                         break;
4824                 case 44: /* hdmi */
4825                         switch (src_data) {
4826                         case 0:
4827                                 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4828                                         rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG;
4829                                         queue_hdmi = true;
4830                                         DRM_DEBUG("IH: HDMI0\n");
4831                                 }
4832                                 break;
4833                         case 1:
4834                                 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4835                                         rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG;
4836                                         queue_hdmi = true;
4837                                         DRM_DEBUG("IH: HDMI1\n");
4838                                 }
4839                                 break;
4840                         case 2:
4841                                 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4842                                         rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG;
4843                                         queue_hdmi = true;
4844                                         DRM_DEBUG("IH: HDMI2\n");
4845                                 }
4846                                 break;
4847                         case 3:
4848                                 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4849                                         rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG;
4850                                         queue_hdmi = true;
4851                                         DRM_DEBUG("IH: HDMI3\n");
4852                                 }
4853                                 break;
4854                         case 4:
4855                                 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4856                                         rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG;
4857                                         queue_hdmi = true;
4858                                         DRM_DEBUG("IH: HDMI4\n");
4859                                 }
4860                                 break;
4861                         case 5:
4862                                 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4863                                         rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG;
4864                                         queue_hdmi = true;
4865                                         DRM_DEBUG("IH: HDMI5\n");
4866                                 }
4867                                 break;
4868                         default:
4869                                 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
4870                                 break;
4871                         }
4872                 case 124: /* UVD */
4873                         DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
4874                         radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
4875                         break;
4876                 case 146:
4877                 case 147:
4878                         addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
4879                         status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
4880                         dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
4881                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
4882                                 addr);
4883                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4884                                 status);
4885                         cayman_vm_decode_fault(rdev, status, addr);
4886                         /* reset addr and status */
4887                         WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
4888                         break;
4889                 case 176: /* CP_INT in ring buffer */
4890                 case 177: /* CP_INT in IB1 */
4891                 case 178: /* CP_INT in IB2 */
4892                         DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
4893                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4894                         break;
4895                 case 181: /* CP EOP event */
4896                         DRM_DEBUG("IH: CP EOP\n");
4897                         if (rdev->family >= CHIP_CAYMAN) {
4898                                 switch (src_data) {
4899                                 case 0:
4900                                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4901                                         break;
4902                                 case 1:
4903                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
4904                                         break;
4905                                 case 2:
4906                                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
4907                                         break;
4908                                 }
4909                         } else
4910                                 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4911                         break;
4912                 case 224: /* DMA trap event */
4913                         DRM_DEBUG("IH: DMA trap\n");
4914                         radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
4915                         break;
4916                 case 230: /* thermal low to high */
4917                         DRM_DEBUG("IH: thermal low to high\n");
4918                         rdev->pm.dpm.thermal.high_to_low = false;
4919                         queue_thermal = true;
4920                         break;
4921                 case 231: /* thermal high to low */
4922                         DRM_DEBUG("IH: thermal high to low\n");
4923                         rdev->pm.dpm.thermal.high_to_low = true;
4924                         queue_thermal = true;
4925                         break;
4926                 case 233: /* GUI IDLE */
4927                         DRM_DEBUG("IH: GUI idle\n");
4928                         break;
4929                 case 244: /* DMA trap event */
4930                         if (rdev->family >= CHIP_CAYMAN) {
4931                                 DRM_DEBUG("IH: DMA1 trap\n");
4932                                 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
4933                         }
4934                         break;
4935                 default:
4936                         DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4937                         break;
4938                 }
4939
4940                 /* wptr/rptr are in bytes! */
4941                 rptr += 16;
4942                 rptr &= rdev->ih.ptr_mask;
4943         }
4944         if (queue_hotplug)
4945                 schedule_work(&rdev->hotplug_work);
4946         if (queue_hdmi)
4947                 schedule_work(&rdev->audio_work);
4948         if (queue_thermal && rdev->pm.dpm_enabled)
4949                 schedule_work(&rdev->pm.dpm.thermal.work);
4950         rdev->ih.rptr = rptr;
4951         WREG32(IH_RB_RPTR, rdev->ih.rptr);
4952         atomic_set(&rdev->ih.lock, 0);
4953
4954         /* make sure wptr hasn't changed while processing */
4955         wptr = evergreen_get_ih_wptr(rdev);
4956         if (wptr != rptr)
4957                 goto restart_ih;
4958
4959         return IRQ_HANDLED;
4960 }
4961
4962 /**
4963  * evergreen_dma_fence_ring_emit - emit a fence on the DMA ring
4964  *
4965  * @rdev: radeon_device pointer
4966  * @fence: radeon fence object
4967  *
4968  * Add a DMA fence packet to the ring to write
4969  * the fence seq number and DMA trap packet to generate
4970  * an interrupt if needed (evergreen-SI).
4971  */
4972 void evergreen_dma_fence_ring_emit(struct radeon_device *rdev,
4973                                    struct radeon_fence *fence)
4974 {
4975         struct radeon_ring *ring = &rdev->ring[fence->ring];
4976         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
4977         /* write the fence */
4978         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_FENCE, 0, 0));
4979         radeon_ring_write(ring, addr & 0xfffffffc);
4980         radeon_ring_write(ring, (upper_32_bits(addr) & 0xff));
4981         radeon_ring_write(ring, fence->seq);
4982         /* generate an interrupt */
4983         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_TRAP, 0, 0));
4984         /* flush HDP */
4985         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0));
4986         radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2));
4987         radeon_ring_write(ring, 1);
4988 }
4989
4990 /**
4991  * evergreen_dma_ring_ib_execute - schedule an IB on the DMA engine
4992  *
4993  * @rdev: radeon_device pointer
4994  * @ib: IB object to schedule
4995  *
4996  * Schedule an IB in the DMA ring (evergreen).
4997  */
4998 void evergreen_dma_ring_ib_execute(struct radeon_device *rdev,
4999                                    struct radeon_ib *ib)
5000 {
5001         struct radeon_ring *ring = &rdev->ring[ib->ring];
5002
5003         if (rdev->wb.enabled) {
5004                 u32 next_rptr = ring->wptr + 4;
5005                 while ((next_rptr & 7) != 5)
5006                         next_rptr++;
5007                 next_rptr += 3;
5008                 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 1));
5009                 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
5010                 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xff);
5011                 radeon_ring_write(ring, next_rptr);
5012         }
5013
5014         /* The indirect buffer packet must end on an 8 DW boundary in the DMA ring.
5015          * Pad as necessary with NOPs.
5016          */
5017         while ((ring->wptr & 7) != 5)
5018                 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0));
5019         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_INDIRECT_BUFFER, 0, 0));
5020         radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0));
5021         radeon_ring_write(ring, (ib->length_dw << 12) | (upper_32_bits(ib->gpu_addr) & 0xFF));
5022
5023 }
5024
5025 /**
5026  * evergreen_copy_dma - copy pages using the DMA engine
5027  *
5028  * @rdev: radeon_device pointer
5029  * @src_offset: src GPU address
5030  * @dst_offset: dst GPU address
5031  * @num_gpu_pages: number of GPU pages to xfer
5032  * @fence: radeon fence object
5033  *
5034  * Copy GPU paging using the DMA engine (evergreen-cayman).
5035  * Used by the radeon ttm implementation to move pages if
5036  * registered as the asic copy callback.
5037  */
5038 int evergreen_copy_dma(struct radeon_device *rdev,
5039                        uint64_t src_offset, uint64_t dst_offset,
5040                        unsigned num_gpu_pages,
5041                        struct radeon_fence **fence)
5042 {
5043         struct radeon_semaphore *sem = NULL;
5044         int ring_index = rdev->asic->copy.dma_ring_index;
5045         struct radeon_ring *ring = &rdev->ring[ring_index];
5046         u32 size_in_dw, cur_size_in_dw;
5047         int i, num_loops;
5048         int r = 0;
5049
5050         r = radeon_semaphore_create(rdev, &sem);
5051         if (r) {
5052                 DRM_ERROR("radeon: moving bo (%d).\n", r);
5053                 return r;
5054         }
5055
5056         size_in_dw = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT) / 4;
5057         num_loops = DIV_ROUND_UP(size_in_dw, 0xfffff);
5058         r = radeon_ring_lock(rdev, ring, num_loops * 5 + 11);
5059         if (r) {
5060                 DRM_ERROR("radeon: moving bo (%d).\n", r);
5061                 radeon_semaphore_free(rdev, &sem, NULL);
5062                 return r;
5063         }
5064
5065         if (radeon_fence_need_sync(*fence, ring->idx)) {
5066                 radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
5067                                             ring->idx);
5068                 radeon_fence_note_sync(*fence, ring->idx);
5069         } else {
5070                 radeon_semaphore_free(rdev, &sem, NULL);
5071         }
5072
5073         for (i = 0; i < num_loops; i++) {
5074                 cur_size_in_dw = size_in_dw;
5075                 if (cur_size_in_dw > 0xFFFFF)
5076                         cur_size_in_dw = 0xFFFFF;
5077                 size_in_dw -= cur_size_in_dw;
5078                 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 0, cur_size_in_dw));
5079                 radeon_ring_write(ring, dst_offset & 0xfffffffc);
5080                 radeon_ring_write(ring, src_offset & 0xfffffffc);
5081                 radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff);
5082                 radeon_ring_write(ring, upper_32_bits(src_offset) & 0xff);
5083                 src_offset += cur_size_in_dw * 4;
5084                 dst_offset += cur_size_in_dw * 4;
5085         }
5086
5087         r = radeon_fence_emit(rdev, fence, ring->idx);
5088         if (r) {
5089                 radeon_ring_unlock_undo(rdev, ring);
5090                 return r;
5091         }
5092
5093         radeon_ring_unlock_commit(rdev, ring);
5094         radeon_semaphore_free(rdev, &sem, *fence);
5095
5096         return r;
5097 }
5098
5099 static int evergreen_startup(struct radeon_device *rdev)
5100 {
5101         struct radeon_ring *ring;
5102         int r;
5103
5104         /* enable pcie gen2 link */
5105         evergreen_pcie_gen2_enable(rdev);
5106         /* enable aspm */
5107         evergreen_program_aspm(rdev);
5108
5109         if (ASIC_IS_DCE5(rdev)) {
5110                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5111                         r = ni_init_microcode(rdev);
5112                         if (r) {
5113                                 DRM_ERROR("Failed to load firmware!\n");
5114                                 return r;
5115                         }
5116                 }
5117                 r = ni_mc_load_microcode(rdev);
5118                 if (r) {
5119                         DRM_ERROR("Failed to load MC firmware!\n");
5120                         return r;
5121                 }
5122         } else {
5123                 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5124                         r = r600_init_microcode(rdev);
5125                         if (r) {
5126                                 DRM_ERROR("Failed to load firmware!\n");
5127                                 return r;
5128                         }
5129                 }
5130         }
5131
5132         r = r600_vram_scratch_init(rdev);
5133         if (r)
5134                 return r;
5135
5136         evergreen_mc_program(rdev);
5137         if (rdev->flags & RADEON_IS_AGP) {
5138                 evergreen_agp_enable(rdev);
5139         } else {
5140                 r = evergreen_pcie_gart_enable(rdev);
5141                 if (r)
5142                         return r;
5143         }
5144         evergreen_gpu_init(rdev);
5145
5146         r = evergreen_blit_init(rdev);
5147         if (r) {
5148                 r600_blit_fini(rdev);
5149                 rdev->asic->copy.copy = NULL;
5150                 dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
5151         }
5152
5153         /* allocate rlc buffers */
5154         if (rdev->flags & RADEON_IS_IGP) {
5155                 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
5156                 rdev->rlc.reg_list_size = sumo_rlc_save_restore_register_list_size;
5157                 rdev->rlc.cs_data = evergreen_cs_data;
5158                 r = sumo_rlc_init(rdev);
5159                 if (r) {
5160                         DRM_ERROR("Failed to init rlc BOs!\n");
5161                         return r;
5162                 }
5163         }
5164
5165         /* allocate wb buffer */
5166         r = radeon_wb_init(rdev);
5167         if (r)
5168                 return r;
5169
5170         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5171         if (r) {
5172                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5173                 return r;
5174         }
5175
5176         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5177         if (r) {
5178                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5179                 return r;
5180         }
5181
5182         r = rv770_uvd_resume(rdev);
5183         if (!r) {
5184                 r = radeon_fence_driver_start_ring(rdev,
5185                                                    R600_RING_TYPE_UVD_INDEX);
5186                 if (r)
5187                         dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
5188         }
5189
5190         if (r)
5191                 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
5192
5193         /* Enable IRQ */
5194         if (!rdev->irq.installed) {
5195                 r = radeon_irq_kms_init(rdev);
5196                 if (r)
5197                         return r;
5198         }
5199
5200         r = r600_irq_init(rdev);
5201         if (r) {
5202                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5203                 radeon_irq_kms_fini(rdev);
5204                 return r;
5205         }
5206         evergreen_irq_set(rdev);
5207
5208         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5209         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5210                              R600_CP_RB_RPTR, R600_CP_RB_WPTR,
5211                              0, 0xfffff, RADEON_CP_PACKET2);
5212         if (r)
5213                 return r;
5214
5215         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5216         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5217                              DMA_RB_RPTR, DMA_RB_WPTR,
5218                              2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0));
5219         if (r)
5220                 return r;
5221
5222         r = evergreen_cp_load_microcode(rdev);
5223         if (r)
5224                 return r;
5225         r = evergreen_cp_resume(rdev);
5226         if (r)
5227                 return r;
5228         r = r600_dma_resume(rdev);
5229         if (r)
5230                 return r;
5231
5232         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5233         if (ring->ring_size) {
5234                 r = radeon_ring_init(rdev, ring, ring->ring_size,
5235                                      R600_WB_UVD_RPTR_OFFSET,
5236                                      UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
5237                                      0, 0xfffff, RADEON_CP_PACKET2);
5238                 if (!r)
5239                         r = r600_uvd_init(rdev);
5240
5241                 if (r)
5242                         DRM_ERROR("radeon: error initializing UVD (%d).\n", r);
5243         }
5244
5245         r = radeon_ib_pool_init(rdev);
5246         if (r) {
5247                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
5248                 return r;
5249         }
5250
5251         r = r600_audio_init(rdev);
5252         if (r) {
5253                 DRM_ERROR("radeon: audio init failed\n");
5254                 return r;
5255         }
5256
5257         return 0;
5258 }
5259
5260 int evergreen_resume(struct radeon_device *rdev)
5261 {
5262         int r;
5263
5264         /* reset the asic, the gfx blocks are often in a bad state
5265          * after the driver is unloaded or after a resume
5266          */
5267         if (radeon_asic_reset(rdev))
5268                 dev_warn(rdev->dev, "GPU reset failed !\n");
5269         /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5270          * posting will perform necessary task to bring back GPU into good
5271          * shape.
5272          */
5273         /* post card */
5274         atom_asic_init(rdev->mode_info.atom_context);
5275
5276         /* init golden registers */
5277         evergreen_init_golden_registers(rdev);
5278
5279         rdev->accel_working = true;
5280         r = evergreen_startup(rdev);
5281         if (r) {
5282                 DRM_ERROR("evergreen startup failed on resume\n");
5283                 rdev->accel_working = false;
5284                 return r;
5285         }
5286
5287         return r;
5288
5289 }
5290
5291 int evergreen_suspend(struct radeon_device *rdev)
5292 {
5293         r600_audio_fini(rdev);
5294         r600_uvd_stop(rdev);
5295         radeon_uvd_suspend(rdev);
5296         r700_cp_stop(rdev);
5297         r600_dma_stop(rdev);
5298         evergreen_irq_suspend(rdev);
5299         radeon_wb_disable(rdev);
5300         evergreen_pcie_gart_disable(rdev);
5301
5302         return 0;
5303 }
5304
5305 /* Plan is to move initialization in that function and use
5306  * helper function so that radeon_device_init pretty much
5307  * do nothing more than calling asic specific function. This
5308  * should also allow to remove a bunch of callback function
5309  * like vram_info.
5310  */
5311 int evergreen_init(struct radeon_device *rdev)
5312 {
5313         int r;
5314
5315         /* Read BIOS */
5316         if (!radeon_get_bios(rdev)) {
5317                 if (ASIC_IS_AVIVO(rdev))
5318                         return -EINVAL;
5319         }
5320         /* Must be an ATOMBIOS */
5321         if (!rdev->is_atom_bios) {
5322                 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
5323                 return -EINVAL;
5324         }
5325         r = radeon_atombios_init(rdev);
5326         if (r)
5327                 return r;
5328         /* reset the asic, the gfx blocks are often in a bad state
5329          * after the driver is unloaded or after a resume
5330          */
5331         if (radeon_asic_reset(rdev))
5332                 dev_warn(rdev->dev, "GPU reset failed !\n");
5333         /* Post card if necessary */
5334         if (!radeon_card_posted(rdev)) {
5335                 if (!rdev->bios) {
5336                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5337                         return -EINVAL;
5338                 }
5339                 DRM_INFO("GPU not posted. posting now...\n");
5340                 atom_asic_init(rdev->mode_info.atom_context);
5341         }
5342         /* init golden registers */
5343         evergreen_init_golden_registers(rdev);
5344         /* Initialize scratch registers */
5345         r600_scratch_init(rdev);
5346         /* Initialize surface registers */
5347         radeon_surface_init(rdev);
5348         /* Initialize clocks */
5349         radeon_get_clock_info(rdev->ddev);
5350         /* Fence driver */
5351         r = radeon_fence_driver_init(rdev);
5352         if (r)
5353                 return r;
5354         /* initialize AGP */
5355         if (rdev->flags & RADEON_IS_AGP) {
5356                 r = radeon_agp_init(rdev);
5357                 if (r)
5358                         radeon_agp_disable(rdev);
5359         }
5360         /* initialize memory controller */
5361         r = evergreen_mc_init(rdev);
5362         if (r)
5363                 return r;
5364         /* Memory manager */
5365         r = radeon_bo_init(rdev);
5366         if (r)
5367                 return r;
5368
5369         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5370         r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
5371
5372         rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5373         r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5374
5375         r = radeon_uvd_init(rdev);
5376         if (!r) {
5377                 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
5378                 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX],
5379                                4096);
5380         }
5381
5382         rdev->ih.ring_obj = NULL;
5383         r600_ih_ring_init(rdev, 64 * 1024);
5384
5385         r = r600_pcie_gart_init(rdev);
5386         if (r)
5387                 return r;
5388
5389         rdev->accel_working = true;
5390         r = evergreen_startup(rdev);
5391         if (r) {
5392                 dev_err(rdev->dev, "disabling GPU acceleration\n");
5393                 r700_cp_fini(rdev);
5394                 r600_dma_fini(rdev);
5395                 r600_irq_fini(rdev);
5396                 if (rdev->flags & RADEON_IS_IGP)
5397                         sumo_rlc_fini(rdev);
5398                 radeon_wb_fini(rdev);
5399                 radeon_ib_pool_fini(rdev);
5400                 radeon_irq_kms_fini(rdev);
5401                 evergreen_pcie_gart_fini(rdev);
5402                 rdev->accel_working = false;
5403         }
5404
5405         /* Don't start up if the MC ucode is missing on BTC parts.
5406          * The default clocks and voltages before the MC ucode
5407          * is loaded are not suffient for advanced operations.
5408          */
5409         if (ASIC_IS_DCE5(rdev)) {
5410                 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5411                         DRM_ERROR("radeon: MC ucode required for NI+.\n");
5412                         return -EINVAL;
5413                 }
5414         }
5415
5416         return 0;
5417 }
5418
5419 void evergreen_fini(struct radeon_device *rdev)
5420 {
5421         r600_audio_fini(rdev);
5422         r600_blit_fini(rdev);
5423         r700_cp_fini(rdev);
5424         r600_dma_fini(rdev);
5425         r600_irq_fini(rdev);
5426         if (rdev->flags & RADEON_IS_IGP)
5427                 sumo_rlc_fini(rdev);
5428         radeon_wb_fini(rdev);
5429         radeon_ib_pool_fini(rdev);
5430         radeon_irq_kms_fini(rdev);
5431         evergreen_pcie_gart_fini(rdev);
5432         r600_uvd_stop(rdev);
5433         radeon_uvd_fini(rdev);
5434         r600_vram_scratch_fini(rdev);
5435         radeon_gem_fini(rdev);
5436         radeon_fence_driver_fini(rdev);
5437         radeon_agp_fini(rdev);
5438         radeon_bo_fini(rdev);
5439         radeon_atombios_fini(rdev);
5440         kfree(rdev->bios);
5441         rdev->bios = NULL;
5442 }
5443
5444 void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
5445 {
5446         u32 link_width_cntl, speed_cntl;
5447
5448         if (radeon_pcie_gen2 == 0)
5449                 return;
5450
5451         if (rdev->flags & RADEON_IS_IGP)
5452                 return;
5453
5454         if (!(rdev->flags & RADEON_IS_PCIE))
5455                 return;
5456
5457         /* x2 cards have a special sequence */
5458         if (ASIC_IS_X2(rdev))
5459                 return;
5460
5461         if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5462                 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
5463                 return;
5464
5465         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5466         if (speed_cntl & LC_CURRENT_DATA_RATE) {
5467                 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5468                 return;
5469         }
5470
5471         DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5472
5473         if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5474             (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5475
5476                 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5477                 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5478                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5479
5480                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5481                 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
5482                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5483
5484                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5485                 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
5486                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5487
5488                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5489                 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
5490                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5491
5492                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5493                 speed_cntl |= LC_GEN2_EN_STRAP;
5494                 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5495
5496         } else {
5497                 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5498                 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5499                 if (1)
5500                         link_width_cntl |= LC_UPCONFIGURE_DIS;
5501                 else
5502                         link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5503                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5504         }
5505 }
5506
5507 void evergreen_program_aspm(struct radeon_device *rdev)
5508 {
5509         u32 data, orig;
5510         u32 pcie_lc_cntl, pcie_lc_cntl_old;
5511         bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5512         /* fusion_platform = true
5513          * if the system is a fusion system
5514          * (APU or DGPU in a fusion system).
5515          * todo: check if the system is a fusion platform.
5516          */
5517         bool fusion_platform = false;
5518
5519         if (radeon_aspm == 0)
5520                 return;
5521
5522         if (!(rdev->flags & RADEON_IS_PCIE))
5523                 return;
5524
5525         switch (rdev->family) {
5526         case CHIP_CYPRESS:
5527         case CHIP_HEMLOCK:
5528         case CHIP_JUNIPER:
5529         case CHIP_REDWOOD:
5530         case CHIP_CEDAR:
5531         case CHIP_SUMO:
5532         case CHIP_SUMO2:
5533         case CHIP_PALM:
5534         case CHIP_ARUBA:
5535                 disable_l0s = true;
5536                 break;
5537         default:
5538                 disable_l0s = false;
5539                 break;
5540         }
5541
5542         if (rdev->flags & RADEON_IS_IGP)
5543                 fusion_platform = true; /* XXX also dGPUs in a fusion system */
5544
5545         data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5546         if (fusion_platform)
5547                 data &= ~MULTI_PIF;
5548         else
5549                 data |= MULTI_PIF;
5550         if (data != orig)
5551                 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5552
5553         data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5554         if (fusion_platform)
5555                 data &= ~MULTI_PIF;
5556         else
5557                 data |= MULTI_PIF;
5558         if (data != orig)
5559                 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5560
5561         pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5562         pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5563         if (!disable_l0s) {
5564                 if (rdev->family >= CHIP_BARTS)
5565                         pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5566                 else
5567                         pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5568         }
5569
5570         if (!disable_l1) {
5571                 if (rdev->family >= CHIP_BARTS)
5572                         pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5573                 else
5574                         pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5575
5576                 if (!disable_plloff_in_l1) {
5577                         data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5578                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5579                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5580                         if (data != orig)
5581                                 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5582
5583                         data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5584                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5585                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5586                         if (data != orig)
5587                                 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5588
5589                         data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5590                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5591                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5592                         if (data != orig)
5593                                 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5594
5595                         data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5596                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5597                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5598                         if (data != orig)
5599                                 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5600
5601                         if (rdev->family >= CHIP_BARTS) {
5602                                 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5603                                 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5604                                 data |= PLL_RAMP_UP_TIME_0(4);
5605                                 if (data != orig)
5606                                         WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5607
5608                                 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5609                                 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5610                                 data |= PLL_RAMP_UP_TIME_1(4);
5611                                 if (data != orig)
5612                                         WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5613
5614                                 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5615                                 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5616                                 data |= PLL_RAMP_UP_TIME_0(4);
5617                                 if (data != orig)
5618                                         WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5619
5620                                 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5621                                 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5622                                 data |= PLL_RAMP_UP_TIME_1(4);
5623                                 if (data != orig)
5624                                         WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5625                         }
5626
5627                         data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5628                         data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5629                         data |= LC_DYN_LANES_PWR_STATE(3);
5630                         if (data != orig)
5631                                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5632
5633                         if (rdev->family >= CHIP_BARTS) {
5634                                 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5635                                 data &= ~LS2_EXIT_TIME_MASK;
5636                                 data |= LS2_EXIT_TIME(1);
5637                                 if (data != orig)
5638                                         WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5639
5640                                 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5641                                 data &= ~LS2_EXIT_TIME_MASK;
5642                                 data |= LS2_EXIT_TIME(1);
5643                                 if (data != orig)
5644                                         WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5645                         }
5646                 }
5647         }
5648
5649         /* evergreen parts only */
5650         if (rdev->family < CHIP_BARTS)
5651                 pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5652
5653         if (pcie_lc_cntl != pcie_lc_cntl_old)
5654                 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);
5655 }