drivers/gpu/drm/radeon/si.c

   1 /*
   2  * Copyright 2011 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/slab.h>
  26 #include <linux/module.h>
  27 #include <drm/drmP.h>
  28 #include "radeon.h"
  29 #include "radeon_asic.h"
  30 #include <drm/radeon_drm.h>
  31 #include "sid.h"
  32 #include "atom.h"
  33 #include "si_blit_shaders.h"
  34 #include "clearstate_si.h"
  35 #include "radeon_ucode.h"
  36
  37
  38 MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
  39 MODULE_FIRMWARE("radeon/TAHITI_me.bin");
  40 MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
  41 MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
  42 MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
  43 MODULE_FIRMWARE("radeon/TAHITI_smc.bin");
  44 MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
  45 MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
  46 MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
  47 MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
  48 MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
  49 MODULE_FIRMWARE("radeon/PITCAIRN_smc.bin");
  50 MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
  51 MODULE_FIRMWARE("radeon/VERDE_me.bin");
  52 MODULE_FIRMWARE("radeon/VERDE_ce.bin");
  53 MODULE_FIRMWARE("radeon/VERDE_mc.bin");
  54 MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
  55 MODULE_FIRMWARE("radeon/VERDE_smc.bin");
  56 MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
  57 MODULE_FIRMWARE("radeon/OLAND_me.bin");
  58 MODULE_FIRMWARE("radeon/OLAND_ce.bin");
  59 MODULE_FIRMWARE("radeon/OLAND_mc.bin");
  60 MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
  61 MODULE_FIRMWARE("radeon/OLAND_smc.bin");
  62 MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
  63 MODULE_FIRMWARE("radeon/HAINAN_me.bin");
  64 MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
  65 MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
  66 MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
  67 MODULE_FIRMWARE("radeon/HAINAN_smc.bin");
  68
  69 static void si_pcie_gen3_enable(struct radeon_device *rdev);
  70 static void si_program_aspm(struct radeon_device *rdev);
  71 extern void sumo_rlc_fini(struct radeon_device *rdev);
  72 extern int sumo_rlc_init(struct radeon_device *rdev);
  73 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
  74 extern void r600_ih_ring_fini(struct radeon_device *rdev);
  75 extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
  76 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
  77 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
  78 extern u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev);
  79 extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
  80 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
  81 static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
  82                                          bool enable);
  83 static void si_fini_pg(struct radeon_device *rdev);
  84 static void si_fini_cg(struct radeon_device *rdev);
  85 static void si_rlc_stop(struct radeon_device *rdev);
  86
  87 static const u32 verde_rlc_save_restore_register_list[] =
  88 {
  89         (0x8000 << 16) | (0x98f4 >> 2),
  90         0x00000000,
  91         (0x8040 << 16) | (0x98f4 >> 2),
  92         0x00000000,
  93         (0x8000 << 16) | (0xe80 >> 2),
  94         0x00000000,
  95         (0x8040 << 16) | (0xe80 >> 2),
  96         0x00000000,
  97         (0x8000 << 16) | (0x89bc >> 2),
  98         0x00000000,
  99         (0x8040 << 16) | (0x89bc >> 2),
 100         0x00000000,
 101         (0x8000 << 16) | (0x8c1c >> 2),
 102         0x00000000,
 103         (0x8040 << 16) | (0x8c1c >> 2),
 104         0x00000000,
 105         (0x9c00 << 16) | (0x98f0 >> 2),
 106         0x00000000,
 107         (0x9c00 << 16) | (0xe7c >> 2),
 108         0x00000000,
 109         (0x8000 << 16) | (0x9148 >> 2),
 110         0x00000000,
 111         (0x8040 << 16) | (0x9148 >> 2),
 112         0x00000000,
 113         (0x9c00 << 16) | (0x9150 >> 2),
 114         0x00000000,
 115         (0x9c00 << 16) | (0x897c >> 2),
 116         0x00000000,
 117         (0x9c00 << 16) | (0x8d8c >> 2),
 118         0x00000000,
 119         (0x9c00 << 16) | (0xac54 >> 2),
 120         0X00000000,
 121         0x3,
 122         (0x9c00 << 16) | (0x98f8 >> 2),
 123         0x00000000,
 124         (0x9c00 << 16) | (0x9910 >> 2),
 125         0x00000000,
 126         (0x9c00 << 16) | (0x9914 >> 2),
 127         0x00000000,
 128         (0x9c00 << 16) | (0x9918 >> 2),
 129         0x00000000,
 130         (0x9c00 << 16) | (0x991c >> 2),
 131         0x00000000,
 132         (0x9c00 << 16) | (0x9920 >> 2),
 133         0x00000000,
 134         (0x9c00 << 16) | (0x9924 >> 2),
 135         0x00000000,
 136         (0x9c00 << 16) | (0x9928 >> 2),
 137         0x00000000,
 138         (0x9c00 << 16) | (0x992c >> 2),
 139         0x00000000,
 140         (0x9c00 << 16) | (0x9930 >> 2),
 141         0x00000000,
 142         (0x9c00 << 16) | (0x9934 >> 2),
 143         0x00000000,
 144         (0x9c00 << 16) | (0x9938 >> 2),
 145         0x00000000,
 146         (0x9c00 << 16) | (0x993c >> 2),
 147         0x00000000,
 148         (0x9c00 << 16) | (0x9940 >> 2),
 149         0x00000000,
 150         (0x9c00 << 16) | (0x9944 >> 2),
 151         0x00000000,
 152         (0x9c00 << 16) | (0x9948 >> 2),
 153         0x00000000,
 154         (0x9c00 << 16) | (0x994c >> 2),
 155         0x00000000,
 156         (0x9c00 << 16) | (0x9950 >> 2),
 157         0x00000000,
 158         (0x9c00 << 16) | (0x9954 >> 2),
 159         0x00000000,
 160         (0x9c00 << 16) | (0x9958 >> 2),
 161         0x00000000,
 162         (0x9c00 << 16) | (0x995c >> 2),
 163         0x00000000,
 164         (0x9c00 << 16) | (0x9960 >> 2),
 165         0x00000000,
 166         (0x9c00 << 16) | (0x9964 >> 2),
 167         0x00000000,
 168         (0x9c00 << 16) | (0x9968 >> 2),
 169         0x00000000,
 170         (0x9c00 << 16) | (0x996c >> 2),
 171         0x00000000,
 172         (0x9c00 << 16) | (0x9970 >> 2),
 173         0x00000000,
 174         (0x9c00 << 16) | (0x9974 >> 2),
 175         0x00000000,
 176         (0x9c00 << 16) | (0x9978 >> 2),
 177         0x00000000,
 178         (0x9c00 << 16) | (0x997c >> 2),
 179         0x00000000,
 180         (0x9c00 << 16) | (0x9980 >> 2),
 181         0x00000000,
 182         (0x9c00 << 16) | (0x9984 >> 2),
 183         0x00000000,
 184         (0x9c00 << 16) | (0x9988 >> 2),
 185         0x00000000,
 186         (0x9c00 << 16) | (0x998c >> 2),
 187         0x00000000,
 188         (0x9c00 << 16) | (0x8c00 >> 2),
 189         0x00000000,
 190         (0x9c00 << 16) | (0x8c14 >> 2),
 191         0x00000000,
 192         (0x9c00 << 16) | (0x8c04 >> 2),
 193         0x00000000,
 194         (0x9c00 << 16) | (0x8c08 >> 2),
 195         0x00000000,
 196         (0x8000 << 16) | (0x9b7c >> 2),
 197         0x00000000,
 198         (0x8040 << 16) | (0x9b7c >> 2),
 199         0x00000000,
 200         (0x8000 << 16) | (0xe84 >> 2),
 201         0x00000000,
 202         (0x8040 << 16) | (0xe84 >> 2),
 203         0x00000000,
 204         (0x8000 << 16) | (0x89c0 >> 2),
 205         0x00000000,
 206         (0x8040 << 16) | (0x89c0 >> 2),
 207         0x00000000,
 208         (0x8000 << 16) | (0x914c >> 2),
 209         0x00000000,
 210         (0x8040 << 16) | (0x914c >> 2),
 211         0x00000000,
 212         (0x8000 << 16) | (0x8c20 >> 2),
 213         0x00000000,
 214         (0x8040 << 16) | (0x8c20 >> 2),
 215         0x00000000,
 216         (0x8000 << 16) | (0x9354 >> 2),
 217         0x00000000,
 218         (0x8040 << 16) | (0x9354 >> 2),
 219         0x00000000,
 220         (0x9c00 << 16) | (0x9060 >> 2),
 221         0x00000000,
 222         (0x9c00 << 16) | (0x9364 >> 2),
 223         0x00000000,
 224         (0x9c00 << 16) | (0x9100 >> 2),
 225         0x00000000,
 226         (0x9c00 << 16) | (0x913c >> 2),
 227         0x00000000,
 228         (0x8000 << 16) | (0x90e0 >> 2),
 229         0x00000000,
 230         (0x8000 << 16) | (0x90e4 >> 2),
 231         0x00000000,
 232         (0x8000 << 16) | (0x90e8 >> 2),
 233         0x00000000,
 234         (0x8040 << 16) | (0x90e0 >> 2),
 235         0x00000000,
 236         (0x8040 << 16) | (0x90e4 >> 2),
 237         0x00000000,
 238         (0x8040 << 16) | (0x90e8 >> 2),
 239         0x00000000,
 240         (0x9c00 << 16) | (0x8bcc >> 2),
 241         0x00000000,
 242         (0x9c00 << 16) | (0x8b24 >> 2),
 243         0x00000000,
 244         (0x9c00 << 16) | (0x88c4 >> 2),
 245         0x00000000,
 246         (0x9c00 << 16) | (0x8e50 >> 2),
 247         0x00000000,
 248         (0x9c00 << 16) | (0x8c0c >> 2),
 249         0x00000000,
 250         (0x9c00 << 16) | (0x8e58 >> 2),
 251         0x00000000,
 252         (0x9c00 << 16) | (0x8e5c >> 2),
 253         0x00000000,
 254         (0x9c00 << 16) | (0x9508 >> 2),
 255         0x00000000,
 256         (0x9c00 << 16) | (0x950c >> 2),
 257         0x00000000,
 258         (0x9c00 << 16) | (0x9494 >> 2),
 259         0x00000000,
 260         (0x9c00 << 16) | (0xac0c >> 2),
 261         0x00000000,
 262         (0x9c00 << 16) | (0xac10 >> 2),
 263         0x00000000,
 264         (0x9c00 << 16) | (0xac14 >> 2),
 265         0x00000000,
 266         (0x9c00 << 16) | (0xae00 >> 2),
 267         0x00000000,
 268         (0x9c00 << 16) | (0xac08 >> 2),
 269         0x00000000,
 270         (0x9c00 << 16) | (0x88d4 >> 2),
 271         0x00000000,
 272         (0x9c00 << 16) | (0x88c8 >> 2),
 273         0x00000000,
 274         (0x9c00 << 16) | (0x88cc >> 2),
 275         0x00000000,
 276         (0x9c00 << 16) | (0x89b0 >> 2),
 277         0x00000000,
 278         (0x9c00 << 16) | (0x8b10 >> 2),
 279         0x00000000,
 280         (0x9c00 << 16) | (0x8a14 >> 2),
 281         0x00000000,
 282         (0x9c00 << 16) | (0x9830 >> 2),
 283         0x00000000,
 284         (0x9c00 << 16) | (0x9834 >> 2),
 285         0x00000000,
 286         (0x9c00 << 16) | (0x9838 >> 2),
 287         0x00000000,
 288         (0x9c00 << 16) | (0x9a10 >> 2),
 289         0x00000000,
 290         (0x8000 << 16) | (0x9870 >> 2),
 291         0x00000000,
 292         (0x8000 << 16) | (0x9874 >> 2),
 293         0x00000000,
 294         (0x8001 << 16) | (0x9870 >> 2),
 295         0x00000000,
 296         (0x8001 << 16) | (0x9874 >> 2),
 297         0x00000000,
 298         (0x8040 << 16) | (0x9870 >> 2),
 299         0x00000000,
 300         (0x8040 << 16) | (0x9874 >> 2),
 301         0x00000000,
 302         (0x8041 << 16) | (0x9870 >> 2),
 303         0x00000000,
 304         (0x8041 << 16) | (0x9874 >> 2),
 305         0x00000000,
 306         0x00000000
 307 };
 308
 309 static const u32 tahiti_golden_rlc_registers[] =
 310 {
 311         0xc424, 0xffffffff, 0x00601005,
 312         0xc47c, 0xffffffff, 0x10104040,
 313         0xc488, 0xffffffff, 0x0100000a,
 314         0xc314, 0xffffffff, 0x00000800,
 315         0xc30c, 0xffffffff, 0x800000f4,
 316         0xf4a8, 0xffffffff, 0x00000000
 317 };
 318
 319 static const u32 tahiti_golden_registers[] =
 320 {
 321         0x9a10, 0x00010000, 0x00018208,
 322         0x9830, 0xffffffff, 0x00000000,
 323         0x9834, 0xf00fffff, 0x00000400,
 324         0x9838, 0x0002021c, 0x00020200,
 325         0xc78, 0x00000080, 0x00000000,
 326         0xd030, 0x000300c0, 0x00800040,
 327         0xd830, 0x000300c0, 0x00800040,
 328         0x5bb0, 0x000000f0, 0x00000070,
 329         0x5bc0, 0x00200000, 0x50100000,
 330         0x7030, 0x31000311, 0x00000011,
 331         0x277c, 0x00000003, 0x000007ff,
 332         0x240c, 0x000007ff, 0x00000000,
 333         0x8a14, 0xf000001f, 0x00000007,
 334         0x8b24, 0xffffffff, 0x00ffffff,
 335         0x8b10, 0x0000ff0f, 0x00000000,
 336         0x28a4c, 0x07ffffff, 0x4e000000,
 337         0x28350, 0x3f3f3fff, 0x2a00126a,
 338         0x30, 0x000000ff, 0x0040,
 339         0x34, 0x00000040, 0x00004040,
 340         0x9100, 0x07ffffff, 0x03000000,
 341         0x8e88, 0x01ff1f3f, 0x00000000,
 342         0x8e84, 0x01ff1f3f, 0x00000000,
 343         0x9060, 0x0000007f, 0x00000020,
 344         0x9508, 0x00010000, 0x00010000,
 345         0xac14, 0x00000200, 0x000002fb,
 346         0xac10, 0xffffffff, 0x0000543b,
 347         0xac0c, 0xffffffff, 0xa9210876,
 348         0x88d0, 0xffffffff, 0x000fff40,
 349         0x88d4, 0x0000001f, 0x00000010,
 350         0x1410, 0x20000000, 0x20fffed8,
 351         0x15c0, 0x000c0fc0, 0x000c0400
 352 };
 353
 354 static const u32 tahiti_golden_registers2[] =
 355 {
 356         0xc64, 0x00000001, 0x00000001
 357 };
 358
 359 static const u32 pitcairn_golden_rlc_registers[] =
 360 {
 361         0xc424, 0xffffffff, 0x00601004,
 362         0xc47c, 0xffffffff, 0x10102020,
 363         0xc488, 0xffffffff, 0x01000020,
 364         0xc314, 0xffffffff, 0x00000800,
 365         0xc30c, 0xffffffff, 0x800000a4
 366 };
 367
 368 static const u32 pitcairn_golden_registers[] =
 369 {
 370         0x9a10, 0x00010000, 0x00018208,
 371         0x9830, 0xffffffff, 0x00000000,
 372         0x9834, 0xf00fffff, 0x00000400,
 373         0x9838, 0x0002021c, 0x00020200,
 374         0xc78, 0x00000080, 0x00000000,
 375         0xd030, 0x000300c0, 0x00800040,
 376         0xd830, 0x000300c0, 0x00800040,
 377         0x5bb0, 0x000000f0, 0x00000070,
 378         0x5bc0, 0x00200000, 0x50100000,
 379         0x7030, 0x31000311, 0x00000011,
 380         0x2ae4, 0x00073ffe, 0x000022a2,
 381         0x240c, 0x000007ff, 0x00000000,
 382         0x8a14, 0xf000001f, 0x00000007,
 383         0x8b24, 0xffffffff, 0x00ffffff,
 384         0x8b10, 0x0000ff0f, 0x00000000,
 385         0x28a4c, 0x07ffffff, 0x4e000000,
 386         0x28350, 0x3f3f3fff, 0x2a00126a,
 387         0x30, 0x000000ff, 0x0040,
 388         0x34, 0x00000040, 0x00004040,
 389         0x9100, 0x07ffffff, 0x03000000,
 390         0x9060, 0x0000007f, 0x00000020,
 391         0x9508, 0x00010000, 0x00010000,
 392         0xac14, 0x000003ff, 0x000000f7,
 393         0xac10, 0xffffffff, 0x00000000,
 394         0xac0c, 0xffffffff, 0x32761054,
 395         0x88d4, 0x0000001f, 0x00000010,
 396         0x15c0, 0x000c0fc0, 0x000c0400
 397 };
 398
 399 static const u32 verde_golden_rlc_registers[] =
 400 {
 401         0xc424, 0xffffffff, 0x033f1005,
 402         0xc47c, 0xffffffff, 0x10808020,
 403         0xc488, 0xffffffff, 0x00800008,
 404         0xc314, 0xffffffff, 0x00001000,
 405         0xc30c, 0xffffffff, 0x80010014
 406 };
 407
 408 static const u32 verde_golden_registers[] =
 409 {
 410         0x9a10, 0x00010000, 0x00018208,
 411         0x9830, 0xffffffff, 0x00000000,
 412         0x9834, 0xf00fffff, 0x00000400,
 413         0x9838, 0x0002021c, 0x00020200,
 414         0xc78, 0x00000080, 0x00000000,
 415         0xd030, 0x000300c0, 0x00800040,
 416         0xd030, 0x000300c0, 0x00800040,
 417         0xd830, 0x000300c0, 0x00800040,
 418         0xd830, 0x000300c0, 0x00800040,
 419         0x5bb0, 0x000000f0, 0x00000070,
 420         0x5bc0, 0x00200000, 0x50100000,
 421         0x7030, 0x31000311, 0x00000011,
 422         0x2ae4, 0x00073ffe, 0x000022a2,
 423         0x2ae4, 0x00073ffe, 0x000022a2,
 424         0x2ae4, 0x00073ffe, 0x000022a2,
 425         0x240c, 0x000007ff, 0x00000000,
 426         0x240c, 0x000007ff, 0x00000000,
 427         0x240c, 0x000007ff, 0x00000000,
 428         0x8a14, 0xf000001f, 0x00000007,
 429         0x8a14, 0xf000001f, 0x00000007,
 430         0x8a14, 0xf000001f, 0x00000007,
 431         0x8b24, 0xffffffff, 0x00ffffff,
 432         0x8b10, 0x0000ff0f, 0x00000000,
 433         0x28a4c, 0x07ffffff, 0x4e000000,
 434         0x28350, 0x3f3f3fff, 0x0000124a,
 435         0x28350, 0x3f3f3fff, 0x0000124a,
 436         0x28350, 0x3f3f3fff, 0x0000124a,
 437         0x30, 0x000000ff, 0x0040,
 438         0x34, 0x00000040, 0x00004040,
 439         0x9100, 0x07ffffff, 0x03000000,
 440         0x9100, 0x07ffffff, 0x03000000,
 441         0x8e88, 0x01ff1f3f, 0x00000000,
 442         0x8e88, 0x01ff1f3f, 0x00000000,
 443         0x8e88, 0x01ff1f3f, 0x00000000,
 444         0x8e84, 0x01ff1f3f, 0x00000000,
 445         0x8e84, 0x01ff1f3f, 0x00000000,
 446         0x8e84, 0x01ff1f3f, 0x00000000,
 447         0x9060, 0x0000007f, 0x00000020,
 448         0x9508, 0x00010000, 0x00010000,
 449         0xac14, 0x000003ff, 0x00000003,
 450         0xac14, 0x000003ff, 0x00000003,
 451         0xac14, 0x000003ff, 0x00000003,
 452         0xac10, 0xffffffff, 0x00000000,
 453         0xac10, 0xffffffff, 0x00000000,
 454         0xac10, 0xffffffff, 0x00000000,
 455         0xac0c, 0xffffffff, 0x00001032,
 456         0xac0c, 0xffffffff, 0x00001032,
 457         0xac0c, 0xffffffff, 0x00001032,
 458         0x88d4, 0x0000001f, 0x00000010,
 459         0x88d4, 0x0000001f, 0x00000010,
 460         0x88d4, 0x0000001f, 0x00000010,
 461         0x15c0, 0x000c0fc0, 0x000c0400
 462 };
 463
 464 static const u32 oland_golden_rlc_registers[] =
 465 {
 466         0xc424, 0xffffffff, 0x00601005,
 467         0xc47c, 0xffffffff, 0x10104040,
 468         0xc488, 0xffffffff, 0x0100000a,
 469         0xc314, 0xffffffff, 0x00000800,
 470         0xc30c, 0xffffffff, 0x800000f4
 471 };
 472
 473 static const u32 oland_golden_registers[] =
 474 {
 475         0x9a10, 0x00010000, 0x00018208,
 476         0x9830, 0xffffffff, 0x00000000,
 477         0x9834, 0xf00fffff, 0x00000400,
 478         0x9838, 0x0002021c, 0x00020200,
 479         0xc78, 0x00000080, 0x00000000,
 480         0xd030, 0x000300c0, 0x00800040,
 481         0xd830, 0x000300c0, 0x00800040,
 482         0x5bb0, 0x000000f0, 0x00000070,
 483         0x5bc0, 0x00200000, 0x50100000,
 484         0x7030, 0x31000311, 0x00000011,
 485         0x2ae4, 0x00073ffe, 0x000022a2,
 486         0x240c, 0x000007ff, 0x00000000,
 487         0x8a14, 0xf000001f, 0x00000007,
 488         0x8b24, 0xffffffff, 0x00ffffff,
 489         0x8b10, 0x0000ff0f, 0x00000000,
 490         0x28a4c, 0x07ffffff, 0x4e000000,
 491         0x28350, 0x3f3f3fff, 0x00000082,
 492         0x30, 0x000000ff, 0x0040,
 493         0x34, 0x00000040, 0x00004040,
 494         0x9100, 0x07ffffff, 0x03000000,
 495         0x9060, 0x0000007f, 0x00000020,
 496         0x9508, 0x00010000, 0x00010000,
 497         0xac14, 0x000003ff, 0x000000f3,
 498         0xac10, 0xffffffff, 0x00000000,
 499         0xac0c, 0xffffffff, 0x00003210,
 500         0x88d4, 0x0000001f, 0x00000010,
 501         0x15c0, 0x000c0fc0, 0x000c0400
 502 };
 503
 504 static const u32 hainan_golden_registers[] =
 505 {
 506         0x9a10, 0x00010000, 0x00018208,
 507         0x9830, 0xffffffff, 0x00000000,
 508         0x9834, 0xf00fffff, 0x00000400,
 509         0x9838, 0x0002021c, 0x00020200,
 510         0xd0c0, 0xff000fff, 0x00000100,
 511         0xd030, 0x000300c0, 0x00800040,
 512         0xd8c0, 0xff000fff, 0x00000100,
 513         0xd830, 0x000300c0, 0x00800040,
 514         0x2ae4, 0x00073ffe, 0x000022a2,
 515         0x240c, 0x000007ff, 0x00000000,
 516         0x8a14, 0xf000001f, 0x00000007,
 517         0x8b24, 0xffffffff, 0x00ffffff,
 518         0x8b10, 0x0000ff0f, 0x00000000,
 519         0x28a4c, 0x07ffffff, 0x4e000000,
 520         0x28350, 0x3f3f3fff, 0x00000000,
 521         0x30, 0x000000ff, 0x0040,
 522         0x34, 0x00000040, 0x00004040,
 523         0x9100, 0x03e00000, 0x03600000,
 524         0x9060, 0x0000007f, 0x00000020,
 525         0x9508, 0x00010000, 0x00010000,
 526         0xac14, 0x000003ff, 0x000000f1,
 527         0xac10, 0xffffffff, 0x00000000,
 528         0xac0c, 0xffffffff, 0x00003210,
 529         0x88d4, 0x0000001f, 0x00000010,
 530         0x15c0, 0x000c0fc0, 0x000c0400
 531 };
 532
 533 static const u32 hainan_golden_registers2[] =
 534 {
 535         0x98f8, 0xffffffff, 0x02010001
 536 };
 537
 538 static const u32 tahiti_mgcg_cgcg_init[] =
 539 {
 540         0xc400, 0xffffffff, 0xfffffffc,
 541         0x802c, 0xffffffff, 0xe0000000,
 542         0x9a60, 0xffffffff, 0x00000100,
 543         0x92a4, 0xffffffff, 0x00000100,
 544         0xc164, 0xffffffff, 0x00000100,
 545         0x9774, 0xffffffff, 0x00000100,
 546         0x8984, 0xffffffff, 0x06000100,
 547         0x8a18, 0xffffffff, 0x00000100,
 548         0x92a0, 0xffffffff, 0x00000100,
 549         0xc380, 0xffffffff, 0x00000100,
 550         0x8b28, 0xffffffff, 0x00000100,
 551         0x9144, 0xffffffff, 0x00000100,
 552         0x8d88, 0xffffffff, 0x00000100,
 553         0x8d8c, 0xffffffff, 0x00000100,
 554         0x9030, 0xffffffff, 0x00000100,
 555         0x9034, 0xffffffff, 0x00000100,
 556         0x9038, 0xffffffff, 0x00000100,
 557         0x903c, 0xffffffff, 0x00000100,
 558         0xad80, 0xffffffff, 0x00000100,
 559         0xac54, 0xffffffff, 0x00000100,
 560         0x897c, 0xffffffff, 0x06000100,
 561         0x9868, 0xffffffff, 0x00000100,
 562         0x9510, 0xffffffff, 0x00000100,
 563         0xaf04, 0xffffffff, 0x00000100,
 564         0xae04, 0xffffffff, 0x00000100,
 565         0x949c, 0xffffffff, 0x00000100,
 566         0x802c, 0xffffffff, 0xe0000000,
 567         0x9160, 0xffffffff, 0x00010000,
 568         0x9164, 0xffffffff, 0x00030002,
 569         0x9168, 0xffffffff, 0x00040007,
 570         0x916c, 0xffffffff, 0x00060005,
 571         0x9170, 0xffffffff, 0x00090008,
 572         0x9174, 0xffffffff, 0x00020001,
 573         0x9178, 0xffffffff, 0x00040003,
 574         0x917c, 0xffffffff, 0x00000007,
 575         0x9180, 0xffffffff, 0x00060005,
 576         0x9184, 0xffffffff, 0x00090008,
 577         0x9188, 0xffffffff, 0x00030002,
 578         0x918c, 0xffffffff, 0x00050004,
 579         0x9190, 0xffffffff, 0x00000008,
 580         0x9194, 0xffffffff, 0x00070006,
 581         0x9198, 0xffffffff, 0x000a0009,
 582         0x919c, 0xffffffff, 0x00040003,
 583         0x91a0, 0xffffffff, 0x00060005,
 584         0x91a4, 0xffffffff, 0x00000009,
 585         0x91a8, 0xffffffff, 0x00080007,
 586         0x91ac, 0xffffffff, 0x000b000a,
 587         0x91b0, 0xffffffff, 0x00050004,
 588         0x91b4, 0xffffffff, 0x00070006,
 589         0x91b8, 0xffffffff, 0x0008000b,
 590         0x91bc, 0xffffffff, 0x000a0009,
 591         0x91c0, 0xffffffff, 0x000d000c,
 592         0x91c4, 0xffffffff, 0x00060005,
 593         0x91c8, 0xffffffff, 0x00080007,
 594         0x91cc, 0xffffffff, 0x0000000b,
 595         0x91d0, 0xffffffff, 0x000a0009,
 596         0x91d4, 0xffffffff, 0x000d000c,
 597         0x91d8, 0xffffffff, 0x00070006,
 598         0x91dc, 0xffffffff, 0x00090008,
 599         0x91e0, 0xffffffff, 0x0000000c,
 600         0x91e4, 0xffffffff, 0x000b000a,
 601         0x91e8, 0xffffffff, 0x000e000d,
 602         0x91ec, 0xffffffff, 0x00080007,
 603         0x91f0, 0xffffffff, 0x000a0009,
 604         0x91f4, 0xffffffff, 0x0000000d,
 605         0x91f8, 0xffffffff, 0x000c000b,
 606         0x91fc, 0xffffffff, 0x000f000e,
 607         0x9200, 0xffffffff, 0x00090008,
 608         0x9204, 0xffffffff, 0x000b000a,
 609         0x9208, 0xffffffff, 0x000c000f,
 610         0x920c, 0xffffffff, 0x000e000d,
 611         0x9210, 0xffffffff, 0x00110010,
 612         0x9214, 0xffffffff, 0x000a0009,
 613         0x9218, 0xffffffff, 0x000c000b,
 614         0x921c, 0xffffffff, 0x0000000f,
 615         0x9220, 0xffffffff, 0x000e000d,
 616         0x9224, 0xffffffff, 0x00110010,
 617         0x9228, 0xffffffff, 0x000b000a,
 618         0x922c, 0xffffffff, 0x000d000c,
 619         0x9230, 0xffffffff, 0x00000010,
 620         0x9234, 0xffffffff, 0x000f000e,
 621         0x9238, 0xffffffff, 0x00120011,
 622         0x923c, 0xffffffff, 0x000c000b,
 623         0x9240, 0xffffffff, 0x000e000d,
 624         0x9244, 0xffffffff, 0x00000011,
 625         0x9248, 0xffffffff, 0x0010000f,
 626         0x924c, 0xffffffff, 0x00130012,
 627         0x9250, 0xffffffff, 0x000d000c,
 628         0x9254, 0xffffffff, 0x000f000e,
 629         0x9258, 0xffffffff, 0x00100013,
 630         0x925c, 0xffffffff, 0x00120011,
 631         0x9260, 0xffffffff, 0x00150014,
 632         0x9264, 0xffffffff, 0x000e000d,
 633         0x9268, 0xffffffff, 0x0010000f,
 634         0x926c, 0xffffffff, 0x00000013,
 635         0x9270, 0xffffffff, 0x00120011,
 636         0x9274, 0xffffffff, 0x00150014,
 637         0x9278, 0xffffffff, 0x000f000e,
 638         0x927c, 0xffffffff, 0x00110010,
 639         0x9280, 0xffffffff, 0x00000014,
 640         0x9284, 0xffffffff, 0x00130012,
 641         0x9288, 0xffffffff, 0x00160015,
 642         0x928c, 0xffffffff, 0x0010000f,
 643         0x9290, 0xffffffff, 0x00120011,
 644         0x9294, 0xffffffff, 0x00000015,
 645         0x9298, 0xffffffff, 0x00140013,
 646         0x929c, 0xffffffff, 0x00170016,
 647         0x9150, 0xffffffff, 0x96940200,
 648         0x8708, 0xffffffff, 0x00900100,
 649         0xc478, 0xffffffff, 0x00000080,
 650         0xc404, 0xffffffff, 0x0020003f,
 651         0x30, 0xffffffff, 0x0000001c,
 652         0x34, 0x000f0000, 0x000f0000,
 653         0x160c, 0xffffffff, 0x00000100,
 654         0x1024, 0xffffffff, 0x00000100,
 655         0x102c, 0x00000101, 0x00000000,
 656         0x20a8, 0xffffffff, 0x00000104,
 657         0x264c, 0x000c0000, 0x000c0000,
 658         0x2648, 0x000c0000, 0x000c0000,
 659         0x55e4, 0xff000fff, 0x00000100,
 660         0x55e8, 0x00000001, 0x00000001,
 661         0x2f50, 0x00000001, 0x00000001,
 662         0x30cc, 0xc0000fff, 0x00000104,
 663         0xc1e4, 0x00000001, 0x00000001,
 664         0xd0c0, 0xfffffff0, 0x00000100,
 665         0xd8c0, 0xfffffff0, 0x00000100
 666 };
 667
 668 static const u32 pitcairn_mgcg_cgcg_init[] =
 669 {
 670         0xc400, 0xffffffff, 0xfffffffc,
 671         0x802c, 0xffffffff, 0xe0000000,
 672         0x9a60, 0xffffffff, 0x00000100,
 673         0x92a4, 0xffffffff, 0x00000100,
 674         0xc164, 0xffffffff, 0x00000100,
 675         0x9774, 0xffffffff, 0x00000100,
 676         0x8984, 0xffffffff, 0x06000100,
 677         0x8a18, 0xffffffff, 0x00000100,
 678         0x92a0, 0xffffffff, 0x00000100,
 679         0xc380, 0xffffffff, 0x00000100,
 680         0x8b28, 0xffffffff, 0x00000100,
 681         0x9144, 0xffffffff, 0x00000100,
 682         0x8d88, 0xffffffff, 0x00000100,
 683         0x8d8c, 0xffffffff, 0x00000100,
 684         0x9030, 0xffffffff, 0x00000100,
 685         0x9034, 0xffffffff, 0x00000100,
 686         0x9038, 0xffffffff, 0x00000100,
 687         0x903c, 0xffffffff, 0x00000100,
 688         0xad80, 0xffffffff, 0x00000100,
 689         0xac54, 0xffffffff, 0x00000100,
 690         0x897c, 0xffffffff, 0x06000100,
 691         0x9868, 0xffffffff, 0x00000100,
 692         0x9510, 0xffffffff, 0x00000100,
 693         0xaf04, 0xffffffff, 0x00000100,
 694         0xae04, 0xffffffff, 0x00000100,
 695         0x949c, 0xffffffff, 0x00000100,
 696         0x802c, 0xffffffff, 0xe0000000,
 697         0x9160, 0xffffffff, 0x00010000,
 698         0x9164, 0xffffffff, 0x00030002,
 699         0x9168, 0xffffffff, 0x00040007,
 700         0x916c, 0xffffffff, 0x00060005,
 701         0x9170, 0xffffffff, 0x00090008,
 702         0x9174, 0xffffffff, 0x00020001,
 703         0x9178, 0xffffffff, 0x00040003,
 704         0x917c, 0xffffffff, 0x00000007,
 705         0x9180, 0xffffffff, 0x00060005,
 706         0x9184, 0xffffffff, 0x00090008,
 707         0x9188, 0xffffffff, 0x00030002,
 708         0x918c, 0xffffffff, 0x00050004,
 709         0x9190, 0xffffffff, 0x00000008,
 710         0x9194, 0xffffffff, 0x00070006,
 711         0x9198, 0xffffffff, 0x000a0009,
 712         0x919c, 0xffffffff, 0x00040003,
 713         0x91a0, 0xffffffff, 0x00060005,
 714         0x91a4, 0xffffffff, 0x00000009,
 715         0x91a8, 0xffffffff, 0x00080007,
 716         0x91ac, 0xffffffff, 0x000b000a,
 717         0x91b0, 0xffffffff, 0x00050004,
 718         0x91b4, 0xffffffff, 0x00070006,
 719         0x91b8, 0xffffffff, 0x0008000b,
 720         0x91bc, 0xffffffff, 0x000a0009,
 721         0x91c0, 0xffffffff, 0x000d000c,
 722         0x9200, 0xffffffff, 0x00090008,
 723         0x9204, 0xffffffff, 0x000b000a,
 724         0x9208, 0xffffffff, 0x000c000f,
 725         0x920c, 0xffffffff, 0x000e000d,
 726         0x9210, 0xffffffff, 0x00110010,
 727         0x9214, 0xffffffff, 0x000a0009,
 728         0x9218, 0xffffffff, 0x000c000b,
 729         0x921c, 0xffffffff, 0x0000000f,
 730         0x9220, 0xffffffff, 0x000e000d,
 731         0x9224, 0xffffffff, 0x00110010,
 732         0x9228, 0xffffffff, 0x000b000a,
 733         0x922c, 0xffffffff, 0x000d000c,
 734         0x9230, 0xffffffff, 0x00000010,
 735         0x9234, 0xffffffff, 0x000f000e,
 736         0x9238, 0xffffffff, 0x00120011,
 737         0x923c, 0xffffffff, 0x000c000b,
 738         0x9240, 0xffffffff, 0x000e000d,
 739         0x9244, 0xffffffff, 0x00000011,
 740         0x9248, 0xffffffff, 0x0010000f,
 741         0x924c, 0xffffffff, 0x00130012,
 742         0x9250, 0xffffffff, 0x000d000c,
 743         0x9254, 0xffffffff, 0x000f000e,
 744         0x9258, 0xffffffff, 0x00100013,
 745         0x925c, 0xffffffff, 0x00120011,
 746         0x9260, 0xffffffff, 0x00150014,
 747         0x9150, 0xffffffff, 0x96940200,
 748         0x8708, 0xffffffff, 0x00900100,
 749         0xc478, 0xffffffff, 0x00000080,
 750         0xc404, 0xffffffff, 0x0020003f,
 751         0x30, 0xffffffff, 0x0000001c,
 752         0x34, 0x000f0000, 0x000f0000,
 753         0x160c, 0xffffffff, 0x00000100,
 754         0x1024, 0xffffffff, 0x00000100,
 755         0x102c, 0x00000101, 0x00000000,
 756         0x20a8, 0xffffffff, 0x00000104,
 757         0x55e4, 0xff000fff, 0x00000100,
 758         0x55e8, 0x00000001, 0x00000001,
 759         0x2f50, 0x00000001, 0x00000001,
 760         0x30cc, 0xc0000fff, 0x00000104,
 761         0xc1e4, 0x00000001, 0x00000001,
 762         0xd0c0, 0xfffffff0, 0x00000100,
 763         0xd8c0, 0xfffffff0, 0x00000100
 764 };
 765
 766 static const u32 verde_mgcg_cgcg_init[] =
 767 {
 768         0xc400, 0xffffffff, 0xfffffffc,
 769         0x802c, 0xffffffff, 0xe0000000,
 770         0x9a60, 0xffffffff, 0x00000100,
 771         0x92a4, 0xffffffff, 0x00000100,
 772         0xc164, 0xffffffff, 0x00000100,
 773         0x9774, 0xffffffff, 0x00000100,
 774         0x8984, 0xffffffff, 0x06000100,
 775         0x8a18, 0xffffffff, 0x00000100,
 776         0x92a0, 0xffffffff, 0x00000100,
 777         0xc380, 0xffffffff, 0x00000100,
 778         0x8b28, 0xffffffff, 0x00000100,
 779         0x9144, 0xffffffff, 0x00000100,
 780         0x8d88, 0xffffffff, 0x00000100,
 781         0x8d8c, 0xffffffff, 0x00000100,
 782         0x9030, 0xffffffff, 0x00000100,
 783         0x9034, 0xffffffff, 0x00000100,
 784         0x9038, 0xffffffff, 0x00000100,
 785         0x903c, 0xffffffff, 0x00000100,
 786         0xad80, 0xffffffff, 0x00000100,
 787         0xac54, 0xffffffff, 0x00000100,
 788         0x897c, 0xffffffff, 0x06000100,
 789         0x9868, 0xffffffff, 0x00000100,
 790         0x9510, 0xffffffff, 0x00000100,
 791         0xaf04, 0xffffffff, 0x00000100,
 792         0xae04, 0xffffffff, 0x00000100,
 793         0x949c, 0xffffffff, 0x00000100,
 794         0x802c, 0xffffffff, 0xe0000000,
 795         0x9160, 0xffffffff, 0x00010000,
 796         0x9164, 0xffffffff, 0x00030002,
 797         0x9168, 0xffffffff, 0x00040007,
 798         0x916c, 0xffffffff, 0x00060005,
 799         0x9170, 0xffffffff, 0x00090008,
 800         0x9174, 0xffffffff, 0x00020001,
 801         0x9178, 0xffffffff, 0x00040003,
 802         0x917c, 0xffffffff, 0x00000007,
 803         0x9180, 0xffffffff, 0x00060005,
 804         0x9184, 0xffffffff, 0x00090008,
 805         0x9188, 0xffffffff, 0x00030002,
 806         0x918c, 0xffffffff, 0x00050004,
 807         0x9190, 0xffffffff, 0x00000008,
 808         0x9194, 0xffffffff, 0x00070006,
 809         0x9198, 0xffffffff, 0x000a0009,
 810         0x919c, 0xffffffff, 0x00040003,
 811         0x91a0, 0xffffffff, 0x00060005,
 812         0x91a4, 0xffffffff, 0x00000009,
 813         0x91a8, 0xffffffff, 0x00080007,
 814         0x91ac, 0xffffffff, 0x000b000a,
 815         0x91b0, 0xffffffff, 0x00050004,
 816         0x91b4, 0xffffffff, 0x00070006,
 817         0x91b8, 0xffffffff, 0x0008000b,
 818         0x91bc, 0xffffffff, 0x000a0009,
 819         0x91c0, 0xffffffff, 0x000d000c,
 820         0x9200, 0xffffffff, 0x00090008,
 821         0x9204, 0xffffffff, 0x000b000a,
 822         0x9208, 0xffffffff, 0x000c000f,
 823         0x920c, 0xffffffff, 0x000e000d,
 824         0x9210, 0xffffffff, 0x00110010,
 825         0x9214, 0xffffffff, 0x000a0009,
 826         0x9218, 0xffffffff, 0x000c000b,
 827         0x921c, 0xffffffff, 0x0000000f,
 828         0x9220, 0xffffffff, 0x000e000d,
 829         0x9224, 0xffffffff, 0x00110010,
 830         0x9228, 0xffffffff, 0x000b000a,
 831         0x922c, 0xffffffff, 0x000d000c,
 832         0x9230, 0xffffffff, 0x00000010,
 833         0x9234, 0xffffffff, 0x000f000e,
 834         0x9238, 0xffffffff, 0x00120011,
 835         0x923c, 0xffffffff, 0x000c000b,
 836         0x9240, 0xffffffff, 0x000e000d,
 837         0x9244, 0xffffffff, 0x00000011,
 838         0x9248, 0xffffffff, 0x0010000f,
 839         0x924c, 0xffffffff, 0x00130012,
 840         0x9250, 0xffffffff, 0x000d000c,
 841         0x9254, 0xffffffff, 0x000f000e,
 842         0x9258, 0xffffffff, 0x00100013,
 843         0x925c, 0xffffffff, 0x00120011,
 844         0x9260, 0xffffffff, 0x00150014,
 845         0x9150, 0xffffffff, 0x96940200,
 846         0x8708, 0xffffffff, 0x00900100,
 847         0xc478, 0xffffffff, 0x00000080,
 848         0xc404, 0xffffffff, 0x0020003f,
 849         0x30, 0xffffffff, 0x0000001c,
 850         0x34, 0x000f0000, 0x000f0000,
 851         0x160c, 0xffffffff, 0x00000100,
 852         0x1024, 0xffffffff, 0x00000100,
 853         0x102c, 0x00000101, 0x00000000,
 854         0x20a8, 0xffffffff, 0x00000104,
 855         0x264c, 0x000c0000, 0x000c0000,
 856         0x2648, 0x000c0000, 0x000c0000,
 857         0x55e4, 0xff000fff, 0x00000100,
 858         0x55e8, 0x00000001, 0x00000001,
 859         0x2f50, 0x00000001, 0x00000001,
 860         0x30cc, 0xc0000fff, 0x00000104,
 861         0xc1e4, 0x00000001, 0x00000001,
 862         0xd0c0, 0xfffffff0, 0x00000100,
 863         0xd8c0, 0xfffffff0, 0x00000100
 864 };
 865
 866 static const u32 oland_mgcg_cgcg_init[] =
 867 {
 868         0xc400, 0xffffffff, 0xfffffffc,
 869         0x802c, 0xffffffff, 0xe0000000,
 870         0x9a60, 0xffffffff, 0x00000100,
 871         0x92a4, 0xffffffff, 0x00000100,
 872         0xc164, 0xffffffff, 0x00000100,
 873         0x9774, 0xffffffff, 0x00000100,
 874         0x8984, 0xffffffff, 0x06000100,
 875         0x8a18, 0xffffffff, 0x00000100,
 876         0x92a0, 0xffffffff, 0x00000100,
 877         0xc380, 0xffffffff, 0x00000100,
 878         0x8b28, 0xffffffff, 0x00000100,
 879         0x9144, 0xffffffff, 0x00000100,
 880         0x8d88, 0xffffffff, 0x00000100,
 881         0x8d8c, 0xffffffff, 0x00000100,
 882         0x9030, 0xffffffff, 0x00000100,
 883         0x9034, 0xffffffff, 0x00000100,
 884         0x9038, 0xffffffff, 0x00000100,
 885         0x903c, 0xffffffff, 0x00000100,
 886         0xad80, 0xffffffff, 0x00000100,
 887         0xac54, 0xffffffff, 0x00000100,
 888         0x897c, 0xffffffff, 0x06000100,
 889         0x9868, 0xffffffff, 0x00000100,
 890         0x9510, 0xffffffff, 0x00000100,
 891         0xaf04, 0xffffffff, 0x00000100,
 892         0xae04, 0xffffffff, 0x00000100,
 893         0x949c, 0xffffffff, 0x00000100,
 894         0x802c, 0xffffffff, 0xe0000000,
 895         0x9160, 0xffffffff, 0x00010000,
 896         0x9164, 0xffffffff, 0x00030002,
 897         0x9168, 0xffffffff, 0x00040007,
 898         0x916c, 0xffffffff, 0x00060005,
 899         0x9170, 0xffffffff, 0x00090008,
 900         0x9174, 0xffffffff, 0x00020001,
 901         0x9178, 0xffffffff, 0x00040003,
 902         0x917c, 0xffffffff, 0x00000007,
 903         0x9180, 0xffffffff, 0x00060005,
 904         0x9184, 0xffffffff, 0x00090008,
 905         0x9188, 0xffffffff, 0x00030002,
 906         0x918c, 0xffffffff, 0x00050004,
 907         0x9190, 0xffffffff, 0x00000008,
 908         0x9194, 0xffffffff, 0x00070006,
 909         0x9198, 0xffffffff, 0x000a0009,
 910         0x919c, 0xffffffff, 0x00040003,
 911         0x91a0, 0xffffffff, 0x00060005,
 912         0x91a4, 0xffffffff, 0x00000009,
 913         0x91a8, 0xffffffff, 0x00080007,
 914         0x91ac, 0xffffffff, 0x000b000a,
 915         0x91b0, 0xffffffff, 0x00050004,
 916         0x91b4, 0xffffffff, 0x00070006,
 917         0x91b8, 0xffffffff, 0x0008000b,
 918         0x91bc, 0xffffffff, 0x000a0009,
 919         0x91c0, 0xffffffff, 0x000d000c,
 920         0x91c4, 0xffffffff, 0x00060005,
 921         0x91c8, 0xffffffff, 0x00080007,
 922         0x91cc, 0xffffffff, 0x0000000b,
 923         0x91d0, 0xffffffff, 0x000a0009,
 924         0x91d4, 0xffffffff, 0x000d000c,
 925         0x9150, 0xffffffff, 0x96940200,
 926         0x8708, 0xffffffff, 0x00900100,
 927         0xc478, 0xffffffff, 0x00000080,
 928         0xc404, 0xffffffff, 0x0020003f,
 929         0x30, 0xffffffff, 0x0000001c,
 930         0x34, 0x000f0000, 0x000f0000,
 931         0x160c, 0xffffffff, 0x00000100,
 932         0x1024, 0xffffffff, 0x00000100,
 933         0x102c, 0x00000101, 0x00000000,
 934         0x20a8, 0xffffffff, 0x00000104,
 935         0x264c, 0x000c0000, 0x000c0000,
 936         0x2648, 0x000c0000, 0x000c0000,
 937         0x55e4, 0xff000fff, 0x00000100,
 938         0x55e8, 0x00000001, 0x00000001,
 939         0x2f50, 0x00000001, 0x00000001,
 940         0x30cc, 0xc0000fff, 0x00000104,
 941         0xc1e4, 0x00000001, 0x00000001,
 942         0xd0c0, 0xfffffff0, 0x00000100,
 943         0xd8c0, 0xfffffff0, 0x00000100
 944 };
 945
 946 static const u32 hainan_mgcg_cgcg_init[] =
 947 {
 948         0xc400, 0xffffffff, 0xfffffffc,
 949         0x802c, 0xffffffff, 0xe0000000,
 950         0x9a60, 0xffffffff, 0x00000100,
 951         0x92a4, 0xffffffff, 0x00000100,
 952         0xc164, 0xffffffff, 0x00000100,
 953         0x9774, 0xffffffff, 0x00000100,
 954         0x8984, 0xffffffff, 0x06000100,
 955         0x8a18, 0xffffffff, 0x00000100,
 956         0x92a0, 0xffffffff, 0x00000100,
 957         0xc380, 0xffffffff, 0x00000100,
 958         0x8b28, 0xffffffff, 0x00000100,
 959         0x9144, 0xffffffff, 0x00000100,
 960         0x8d88, 0xffffffff, 0x00000100,
 961         0x8d8c, 0xffffffff, 0x00000100,
 962         0x9030, 0xffffffff, 0x00000100,
 963         0x9034, 0xffffffff, 0x00000100,
 964         0x9038, 0xffffffff, 0x00000100,
 965         0x903c, 0xffffffff, 0x00000100,
 966         0xad80, 0xffffffff, 0x00000100,
 967         0xac54, 0xffffffff, 0x00000100,
 968         0x897c, 0xffffffff, 0x06000100,
 969         0x9868, 0xffffffff, 0x00000100,
 970         0x9510, 0xffffffff, 0x00000100,
 971         0xaf04, 0xffffffff, 0x00000100,
 972         0xae04, 0xffffffff, 0x00000100,
 973         0x949c, 0xffffffff, 0x00000100,
 974         0x802c, 0xffffffff, 0xe0000000,
 975         0x9160, 0xffffffff, 0x00010000,
 976         0x9164, 0xffffffff, 0x00030002,
 977         0x9168, 0xffffffff, 0x00040007,
 978         0x916c, 0xffffffff, 0x00060005,
 979         0x9170, 0xffffffff, 0x00090008,
 980         0x9174, 0xffffffff, 0x00020001,
 981         0x9178, 0xffffffff, 0x00040003,
 982         0x917c, 0xffffffff, 0x00000007,
 983         0x9180, 0xffffffff, 0x00060005,
 984         0x9184, 0xffffffff, 0x00090008,
 985         0x9188, 0xffffffff, 0x00030002,
 986         0x918c, 0xffffffff, 0x00050004,
 987         0x9190, 0xffffffff, 0x00000008,
 988         0x9194, 0xffffffff, 0x00070006,
 989         0x9198, 0xffffffff, 0x000a0009,
 990         0x919c, 0xffffffff, 0x00040003,
 991         0x91a0, 0xffffffff, 0x00060005,
 992         0x91a4, 0xffffffff, 0x00000009,
 993         0x91a8, 0xffffffff, 0x00080007,
 994         0x91ac, 0xffffffff, 0x000b000a,
 995         0x91b0, 0xffffffff, 0x00050004,
 996         0x91b4, 0xffffffff, 0x00070006,
 997         0x91b8, 0xffffffff, 0x0008000b,
 998         0x91bc, 0xffffffff, 0x000a0009,
 999         0x91c0, 0xffffffff, 0x000d000c,
1000         0x91c4, 0xffffffff, 0x00060005,
1001         0x91c8, 0xffffffff, 0x00080007,
1002         0x91cc, 0xffffffff, 0x0000000b,
1003         0x91d0, 0xffffffff, 0x000a0009,
1004         0x91d4, 0xffffffff, 0x000d000c,
1005         0x9150, 0xffffffff, 0x96940200,
1006         0x8708, 0xffffffff, 0x00900100,
1007         0xc478, 0xffffffff, 0x00000080,
1008         0xc404, 0xffffffff, 0x0020003f,
1009         0x30, 0xffffffff, 0x0000001c,
1010         0x34, 0x000f0000, 0x000f0000,
1011         0x160c, 0xffffffff, 0x00000100,
1012         0x1024, 0xffffffff, 0x00000100,
1013         0x20a8, 0xffffffff, 0x00000104,
1014         0x264c, 0x000c0000, 0x000c0000,
1015         0x2648, 0x000c0000, 0x000c0000,
1016         0x2f50, 0x00000001, 0x00000001,
1017         0x30cc, 0xc0000fff, 0x00000104,
1018         0xc1e4, 0x00000001, 0x00000001,
1019         0xd0c0, 0xfffffff0, 0x00000100,
1020         0xd8c0, 0xfffffff0, 0x00000100
1021 };
1022
1023 static u32 verde_pg_init[] =
1024 {
1025         0x353c, 0xffffffff, 0x40000,
1026         0x3538, 0xffffffff, 0x200010ff,
1027         0x353c, 0xffffffff, 0x0,
1028         0x353c, 0xffffffff, 0x0,
1029         0x353c, 0xffffffff, 0x0,
1030         0x353c, 0xffffffff, 0x0,
1031         0x353c, 0xffffffff, 0x0,
1032         0x353c, 0xffffffff, 0x7007,
1033         0x3538, 0xffffffff, 0x300010ff,
1034         0x353c, 0xffffffff, 0x0,
1035         0x353c, 0xffffffff, 0x0,
1036         0x353c, 0xffffffff, 0x0,
1037         0x353c, 0xffffffff, 0x0,
1038         0x353c, 0xffffffff, 0x0,
1039         0x353c, 0xffffffff, 0x400000,
1040         0x3538, 0xffffffff, 0x100010ff,
1041         0x353c, 0xffffffff, 0x0,
1042         0x353c, 0xffffffff, 0x0,
1043         0x353c, 0xffffffff, 0x0,
1044         0x353c, 0xffffffff, 0x0,
1045         0x353c, 0xffffffff, 0x0,
1046         0x353c, 0xffffffff, 0x120200,
1047         0x3538, 0xffffffff, 0x500010ff,
1048         0x353c, 0xffffffff, 0x0,
1049         0x353c, 0xffffffff, 0x0,
1050         0x353c, 0xffffffff, 0x0,
1051         0x353c, 0xffffffff, 0x0,
1052         0x353c, 0xffffffff, 0x0,
1053         0x353c, 0xffffffff, 0x1e1e16,
1054         0x3538, 0xffffffff, 0x600010ff,
1055         0x353c, 0xffffffff, 0x0,
1056         0x353c, 0xffffffff, 0x0,
1057         0x353c, 0xffffffff, 0x0,
1058         0x353c, 0xffffffff, 0x0,
1059         0x353c, 0xffffffff, 0x0,
1060         0x353c, 0xffffffff, 0x171f1e,
1061         0x3538, 0xffffffff, 0x700010ff,
1062         0x353c, 0xffffffff, 0x0,
1063         0x353c, 0xffffffff, 0x0,
1064         0x353c, 0xffffffff, 0x0,
1065         0x353c, 0xffffffff, 0x0,
1066         0x353c, 0xffffffff, 0x0,
1067         0x353c, 0xffffffff, 0x0,
1068         0x3538, 0xffffffff, 0x9ff,
1069         0x3500, 0xffffffff, 0x0,
1070         0x3504, 0xffffffff, 0x10000800,
1071         0x3504, 0xffffffff, 0xf,
1072         0x3504, 0xffffffff, 0xf,
1073         0x3500, 0xffffffff, 0x4,
1074         0x3504, 0xffffffff, 0x1000051e,
1075         0x3504, 0xffffffff, 0xffff,
1076         0x3504, 0xffffffff, 0xffff,
1077         0x3500, 0xffffffff, 0x8,
1078         0x3504, 0xffffffff, 0x80500,
1079         0x3500, 0xffffffff, 0x12,
1080         0x3504, 0xffffffff, 0x9050c,
1081         0x3500, 0xffffffff, 0x1d,
1082         0x3504, 0xffffffff, 0xb052c,
1083         0x3500, 0xffffffff, 0x2a,
1084         0x3504, 0xffffffff, 0x1053e,
1085         0x3500, 0xffffffff, 0x2d,
1086         0x3504, 0xffffffff, 0x10546,
1087         0x3500, 0xffffffff, 0x30,
1088         0x3504, 0xffffffff, 0xa054e,
1089         0x3500, 0xffffffff, 0x3c,
1090         0x3504, 0xffffffff, 0x1055f,
1091         0x3500, 0xffffffff, 0x3f,
1092         0x3504, 0xffffffff, 0x10567,
1093         0x3500, 0xffffffff, 0x42,
1094         0x3504, 0xffffffff, 0x1056f,
1095         0x3500, 0xffffffff, 0x45,
1096         0x3504, 0xffffffff, 0x10572,
1097         0x3500, 0xffffffff, 0x48,
1098         0x3504, 0xffffffff, 0x20575,
1099         0x3500, 0xffffffff, 0x4c,
1100         0x3504, 0xffffffff, 0x190801,
1101         0x3500, 0xffffffff, 0x67,
1102         0x3504, 0xffffffff, 0x1082a,
1103         0x3500, 0xffffffff, 0x6a,
1104         0x3504, 0xffffffff, 0x1b082d,
1105         0x3500, 0xffffffff, 0x87,
1106         0x3504, 0xffffffff, 0x310851,
1107         0x3500, 0xffffffff, 0xba,
1108         0x3504, 0xffffffff, 0x891,
1109         0x3500, 0xffffffff, 0xbc,
1110         0x3504, 0xffffffff, 0x893,
1111         0x3500, 0xffffffff, 0xbe,
1112         0x3504, 0xffffffff, 0x20895,
1113         0x3500, 0xffffffff, 0xc2,
1114         0x3504, 0xffffffff, 0x20899,
1115         0x3500, 0xffffffff, 0xc6,
1116         0x3504, 0xffffffff, 0x2089d,
1117         0x3500, 0xffffffff, 0xca,
1118         0x3504, 0xffffffff, 0x8a1,
1119         0x3500, 0xffffffff, 0xcc,
1120         0x3504, 0xffffffff, 0x8a3,
1121         0x3500, 0xffffffff, 0xce,
1122         0x3504, 0xffffffff, 0x308a5,
1123         0x3500, 0xffffffff, 0xd3,
1124         0x3504, 0xffffffff, 0x6d08cd,
1125         0x3500, 0xffffffff, 0x142,
1126         0x3504, 0xffffffff, 0x2000095a,
1127         0x3504, 0xffffffff, 0x1,
1128         0x3500, 0xffffffff, 0x144,
1129         0x3504, 0xffffffff, 0x301f095b,
1130         0x3500, 0xffffffff, 0x165,
1131         0x3504, 0xffffffff, 0xc094d,
1132         0x3500, 0xffffffff, 0x173,
1133         0x3504, 0xffffffff, 0xf096d,
1134         0x3500, 0xffffffff, 0x184,
1135         0x3504, 0xffffffff, 0x15097f,
1136         0x3500, 0xffffffff, 0x19b,
1137         0x3504, 0xffffffff, 0xc0998,
1138         0x3500, 0xffffffff, 0x1a9,
1139         0x3504, 0xffffffff, 0x409a7,
1140         0x3500, 0xffffffff, 0x1af,
1141         0x3504, 0xffffffff, 0xcdc,
1142         0x3500, 0xffffffff, 0x1b1,
1143         0x3504, 0xffffffff, 0x800,
1144         0x3508, 0xffffffff, 0x6c9b2000,
1145         0x3510, 0xfc00, 0x2000,
1146         0x3544, 0xffffffff, 0xfc0,
1147         0x28d4, 0x00000100, 0x100
1148 };
1149
1150 static void si_init_golden_registers(struct radeon_device *rdev)
1151 {
1152         switch (rdev->family) {
1153         case CHIP_TAHITI:
1154                 radeon_program_register_sequence(rdev,
1155                                                  tahiti_golden_registers,
1156                                                  (const u32)ARRAY_SIZE(tahiti_golden_registers));
1157                 radeon_program_register_sequence(rdev,
1158                                                  tahiti_golden_rlc_registers,
1159                                                  (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
1160                 radeon_program_register_sequence(rdev,
1161                                                  tahiti_mgcg_cgcg_init,
1162                                                  (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
1163                 radeon_program_register_sequence(rdev,
1164                                                  tahiti_golden_registers2,
1165                                                  (const u32)ARRAY_SIZE(tahiti_golden_registers2));
1166                 break;
1167         case CHIP_PITCAIRN:
1168                 radeon_program_register_sequence(rdev,
1169                                                  pitcairn_golden_registers,
1170                                                  (const u32)ARRAY_SIZE(pitcairn_golden_registers));
1171                 radeon_program_register_sequence(rdev,
1172                                                  pitcairn_golden_rlc_registers,
1173                                                  (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
1174                 radeon_program_register_sequence(rdev,
1175                                                  pitcairn_mgcg_cgcg_init,
1176                                                  (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
1177                 break;
1178         case CHIP_VERDE:
1179                 radeon_program_register_sequence(rdev,
1180                                                  verde_golden_registers,
1181                                                  (const u32)ARRAY_SIZE(verde_golden_registers));
1182                 radeon_program_register_sequence(rdev,
1183                                                  verde_golden_rlc_registers,
1184                                                  (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
1185                 radeon_program_register_sequence(rdev,
1186                                                  verde_mgcg_cgcg_init,
1187                                                  (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
1188                 radeon_program_register_sequence(rdev,
1189                                                  verde_pg_init,
1190                                                  (const u32)ARRAY_SIZE(verde_pg_init));
1191                 break;
1192         case CHIP_OLAND:
1193                 radeon_program_register_sequence(rdev,
1194                                                  oland_golden_registers,
1195                                                  (const u32)ARRAY_SIZE(oland_golden_registers));
1196                 radeon_program_register_sequence(rdev,
1197                                                  oland_golden_rlc_registers,
1198                                                  (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
1199                 radeon_program_register_sequence(rdev,
1200                                                  oland_mgcg_cgcg_init,
1201                                                  (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
1202                 break;
1203         case CHIP_HAINAN:
1204                 radeon_program_register_sequence(rdev,
1205                                                  hainan_golden_registers,
1206                                                  (const u32)ARRAY_SIZE(hainan_golden_registers));
1207                 radeon_program_register_sequence(rdev,
1208                                                  hainan_golden_registers2,
1209                                                  (const u32)ARRAY_SIZE(hainan_golden_registers2));
1210                 radeon_program_register_sequence(rdev,
1211                                                  hainan_mgcg_cgcg_init,
1212                                                  (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
1213                 break;
1214         default:
1215                 break;
1216         }
1217 }
1218
1219 #define PCIE_BUS_CLK                10000
1220 #define TCLK                        (PCIE_BUS_CLK / 10)
1221
1222 /**
1223  * si_get_xclk - get the xclk
1224  *
1225  * @rdev: radeon_device pointer
1226  *
1227  * Returns the reference clock used by the gfx engine
1228  * (SI).
1229  */
1230 u32 si_get_xclk(struct radeon_device *rdev)
1231 {
1232         u32 reference_clock = rdev->clock.spll.reference_freq;
1233         u32 tmp;
1234
1235         tmp = RREG32(CG_CLKPIN_CNTL_2);
1236         if (tmp & MUX_TCLK_TO_XCLK)
1237                 return TCLK;
1238
1239         tmp = RREG32(CG_CLKPIN_CNTL);
1240         if (tmp & XTALIN_DIVIDE)
1241                 return reference_clock / 4;
1242
1243         return reference_clock;
1244 }
1245
1246 /* get temperature in millidegrees */
1247 int si_get_temp(struct radeon_device *rdev)
1248 {
1249         u32 temp;
1250         int actual_temp = 0;
1251
1252         temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
1253                 CTF_TEMP_SHIFT;
1254
1255         if (temp & 0x200)
1256                 actual_temp = 255;
1257         else
1258                 actual_temp = temp & 0x1ff;
1259
1260         actual_temp = (actual_temp * 1000);
1261
1262         return actual_temp;
1263 }
1264
1265 #define TAHITI_IO_MC_REGS_SIZE 36
1266
1267 static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1268         {0x0000006f, 0x03044000},
1269         {0x00000070, 0x0480c018},
1270         {0x00000071, 0x00000040},
1271         {0x00000072, 0x01000000},
1272         {0x00000074, 0x000000ff},
1273         {0x00000075, 0x00143400},
1274         {0x00000076, 0x08ec0800},
1275         {0x00000077, 0x040000cc},
1276         {0x00000079, 0x00000000},
1277         {0x0000007a, 0x21000409},
1278         {0x0000007c, 0x00000000},
1279         {0x0000007d, 0xe8000000},
1280         {0x0000007e, 0x044408a8},
1281         {0x0000007f, 0x00000003},
1282         {0x00000080, 0x00000000},
1283         {0x00000081, 0x01000000},
1284         {0x00000082, 0x02000000},
1285         {0x00000083, 0x00000000},
1286         {0x00000084, 0xe3f3e4f4},
1287         {0x00000085, 0x00052024},
1288         {0x00000087, 0x00000000},
1289         {0x00000088, 0x66036603},
1290         {0x00000089, 0x01000000},
1291         {0x0000008b, 0x1c0a0000},
1292         {0x0000008c, 0xff010000},
1293         {0x0000008e, 0xffffefff},
1294         {0x0000008f, 0xfff3efff},
1295         {0x00000090, 0xfff3efbf},
1296         {0x00000094, 0x00101101},
1297         {0x00000095, 0x00000fff},
1298         {0x00000096, 0x00116fff},
1299         {0x00000097, 0x60010000},
1300         {0x00000098, 0x10010000},
1301         {0x00000099, 0x00006000},
1302         {0x0000009a, 0x00001000},
1303         {0x0000009f, 0x00a77400}
1304 };
1305
1306 static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1307         {0x0000006f, 0x03044000},
1308         {0x00000070, 0x0480c018},
1309         {0x00000071, 0x00000040},
1310         {0x00000072, 0x01000000},
1311         {0x00000074, 0x000000ff},
1312         {0x00000075, 0x00143400},
1313         {0x00000076, 0x08ec0800},
1314         {0x00000077, 0x040000cc},
1315         {0x00000079, 0x00000000},
1316         {0x0000007a, 0x21000409},
1317         {0x0000007c, 0x00000000},
1318         {0x0000007d, 0xe8000000},
1319         {0x0000007e, 0x044408a8},
1320         {0x0000007f, 0x00000003},
1321         {0x00000080, 0x00000000},
1322         {0x00000081, 0x01000000},
1323         {0x00000082, 0x02000000},
1324         {0x00000083, 0x00000000},
1325         {0x00000084, 0xe3f3e4f4},
1326         {0x00000085, 0x00052024},
1327         {0x00000087, 0x00000000},
1328         {0x00000088, 0x66036603},
1329         {0x00000089, 0x01000000},
1330         {0x0000008b, 0x1c0a0000},
1331         {0x0000008c, 0xff010000},
1332         {0x0000008e, 0xffffefff},
1333         {0x0000008f, 0xfff3efff},
1334         {0x00000090, 0xfff3efbf},
1335         {0x00000094, 0x00101101},
1336         {0x00000095, 0x00000fff},
1337         {0x00000096, 0x00116fff},
1338         {0x00000097, 0x60010000},
1339         {0x00000098, 0x10010000},
1340         {0x00000099, 0x00006000},
1341         {0x0000009a, 0x00001000},
1342         {0x0000009f, 0x00a47400}
1343 };
1344
1345 static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1346         {0x0000006f, 0x03044000},
1347         {0x00000070, 0x0480c018},
1348         {0x00000071, 0x00000040},
1349         {0x00000072, 0x01000000},
1350         {0x00000074, 0x000000ff},
1351         {0x00000075, 0x00143400},
1352         {0x00000076, 0x08ec0800},
1353         {0x00000077, 0x040000cc},
1354         {0x00000079, 0x00000000},
1355         {0x0000007a, 0x21000409},
1356         {0x0000007c, 0x00000000},
1357         {0x0000007d, 0xe8000000},
1358         {0x0000007e, 0x044408a8},
1359         {0x0000007f, 0x00000003},
1360         {0x00000080, 0x00000000},
1361         {0x00000081, 0x01000000},
1362         {0x00000082, 0x02000000},
1363         {0x00000083, 0x00000000},
1364         {0x00000084, 0xe3f3e4f4},
1365         {0x00000085, 0x00052024},
1366         {0x00000087, 0x00000000},
1367         {0x00000088, 0x66036603},
1368         {0x00000089, 0x01000000},
1369         {0x0000008b, 0x1c0a0000},
1370         {0x0000008c, 0xff010000},
1371         {0x0000008e, 0xffffefff},
1372         {0x0000008f, 0xfff3efff},
1373         {0x00000090, 0xfff3efbf},
1374         {0x00000094, 0x00101101},
1375         {0x00000095, 0x00000fff},
1376         {0x00000096, 0x00116fff},
1377         {0x00000097, 0x60010000},
1378         {0x00000098, 0x10010000},
1379         {0x00000099, 0x00006000},
1380         {0x0000009a, 0x00001000},
1381         {0x0000009f, 0x00a37400}
1382 };
1383
1384 static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1385         {0x0000006f, 0x03044000},
1386         {0x00000070, 0x0480c018},
1387         {0x00000071, 0x00000040},
1388         {0x00000072, 0x01000000},
1389         {0x00000074, 0x000000ff},
1390         {0x00000075, 0x00143400},
1391         {0x00000076, 0x08ec0800},
1392         {0x00000077, 0x040000cc},
1393         {0x00000079, 0x00000000},
1394         {0x0000007a, 0x21000409},
1395         {0x0000007c, 0x00000000},
1396         {0x0000007d, 0xe8000000},
1397         {0x0000007e, 0x044408a8},
1398         {0x0000007f, 0x00000003},
1399         {0x00000080, 0x00000000},
1400         {0x00000081, 0x01000000},
1401         {0x00000082, 0x02000000},
1402         {0x00000083, 0x00000000},
1403         {0x00000084, 0xe3f3e4f4},
1404         {0x00000085, 0x00052024},
1405         {0x00000087, 0x00000000},
1406         {0x00000088, 0x66036603},
1407         {0x00000089, 0x01000000},
1408         {0x0000008b, 0x1c0a0000},
1409         {0x0000008c, 0xff010000},
1410         {0x0000008e, 0xffffefff},
1411         {0x0000008f, 0xfff3efff},
1412         {0x00000090, 0xfff3efbf},
1413         {0x00000094, 0x00101101},
1414         {0x00000095, 0x00000fff},
1415         {0x00000096, 0x00116fff},
1416         {0x00000097, 0x60010000},
1417         {0x00000098, 0x10010000},
1418         {0x00000099, 0x00006000},
1419         {0x0000009a, 0x00001000},
1420         {0x0000009f, 0x00a17730}
1421 };
1422
1423 static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1424         {0x0000006f, 0x03044000},
1425         {0x00000070, 0x0480c018},
1426         {0x00000071, 0x00000040},
1427         {0x00000072, 0x01000000},
1428         {0x00000074, 0x000000ff},
1429         {0x00000075, 0x00143400},
1430         {0x00000076, 0x08ec0800},
1431         {0x00000077, 0x040000cc},
1432         {0x00000079, 0x00000000},
1433         {0x0000007a, 0x21000409},
1434         {0x0000007c, 0x00000000},
1435         {0x0000007d, 0xe8000000},
1436         {0x0000007e, 0x044408a8},
1437         {0x0000007f, 0x00000003},
1438         {0x00000080, 0x00000000},
1439         {0x00000081, 0x01000000},
1440         {0x00000082, 0x02000000},
1441         {0x00000083, 0x00000000},
1442         {0x00000084, 0xe3f3e4f4},
1443         {0x00000085, 0x00052024},
1444         {0x00000087, 0x00000000},
1445         {0x00000088, 0x66036603},
1446         {0x00000089, 0x01000000},
1447         {0x0000008b, 0x1c0a0000},
1448         {0x0000008c, 0xff010000},
1449         {0x0000008e, 0xffffefff},
1450         {0x0000008f, 0xfff3efff},
1451         {0x00000090, 0xfff3efbf},
1452         {0x00000094, 0x00101101},
1453         {0x00000095, 0x00000fff},
1454         {0x00000096, 0x00116fff},
1455         {0x00000097, 0x60010000},
1456         {0x00000098, 0x10010000},
1457         {0x00000099, 0x00006000},
1458         {0x0000009a, 0x00001000},
1459         {0x0000009f, 0x00a07730}
1460 };
1461
1462 /* ucode loading */
1463 static int si_mc_load_microcode(struct radeon_device *rdev)
1464 {
1465         const __be32 *fw_data;
1466         u32 running, blackout = 0;
1467         u32 *io_mc_regs;
1468         int i, ucode_size, regs_size;
1469
1470         if (!rdev->mc_fw)
1471                 return -EINVAL;
1472
1473         switch (rdev->family) {
1474         case CHIP_TAHITI:
1475                 io_mc_regs = (u32 *)&tahiti_io_mc_regs;
1476                 ucode_size = SI_MC_UCODE_SIZE;
1477                 regs_size = TAHITI_IO_MC_REGS_SIZE;
1478                 break;
1479         case CHIP_PITCAIRN:
1480                 io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
1481                 ucode_size = SI_MC_UCODE_SIZE;
1482                 regs_size = TAHITI_IO_MC_REGS_SIZE;
1483                 break;
1484         case CHIP_VERDE:
1485         default:
1486                 io_mc_regs = (u32 *)&verde_io_mc_regs;
1487                 ucode_size = SI_MC_UCODE_SIZE;
1488                 regs_size = TAHITI_IO_MC_REGS_SIZE;
1489                 break;
1490         case CHIP_OLAND:
1491                 io_mc_regs = (u32 *)&oland_io_mc_regs;
1492                 ucode_size = OLAND_MC_UCODE_SIZE;
1493                 regs_size = TAHITI_IO_MC_REGS_SIZE;
1494                 break;
1495         case CHIP_HAINAN:
1496                 io_mc_regs = (u32 *)&hainan_io_mc_regs;
1497                 ucode_size = OLAND_MC_UCODE_SIZE;
1498                 regs_size = TAHITI_IO_MC_REGS_SIZE;
1499                 break;
1500         }
1501
1502         running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1503
1504         if (running == 0) {
1505                 if (running) {
1506                         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1507                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1508                 }
1509
1510                 /* reset the engine and set to writable */
1511                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1512                 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1513
1514                 /* load mc io regs */
1515                 for (i = 0; i < regs_size; i++) {
1516                         WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1517                         WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1518                 }
1519                 /* load the MC ucode */
1520                 fw_data = (const __be32 *)rdev->mc_fw->data;
1521                 for (i = 0; i < ucode_size; i++)
1522                         WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1523
1524                 /* put the engine back into the active state */
1525                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1526                 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1527                 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1528
1529                 /* wait for training to complete */
1530                 for (i = 0; i < rdev->usec_timeout; i++) {
1531                         if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1532                                 break;
1533                         udelay(1);
1534                 }
1535                 for (i = 0; i < rdev->usec_timeout; i++) {
1536                         if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1537                                 break;
1538                         udelay(1);
1539                 }
1540
1541                 if (running)
1542                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1543         }
1544
1545         return 0;
1546 }
1547
1548 static int si_init_microcode(struct radeon_device *rdev)
1549 {
1550         const char *chip_name;
1551         const char *rlc_chip_name;
1552         size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
1553         size_t smc_req_size;
1554         char fw_name[30];
1555         int err;
1556
1557         DRM_DEBUG("\n");
1558
1559         switch (rdev->family) {
1560         case CHIP_TAHITI:
1561                 chip_name = "TAHITI";
1562                 rlc_chip_name = "TAHITI";
1563                 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1564                 me_req_size = SI_PM4_UCODE_SIZE * 4;
1565                 ce_req_size = SI_CE_UCODE_SIZE * 4;
1566                 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1567                 mc_req_size = SI_MC_UCODE_SIZE * 4;
1568                 smc_req_size = ALIGN(TAHITI_SMC_UCODE_SIZE, 4);
1569                 break;
1570         case CHIP_PITCAIRN:
1571                 chip_name = "PITCAIRN";
1572                 rlc_chip_name = "PITCAIRN";
1573                 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1574                 me_req_size = SI_PM4_UCODE_SIZE * 4;
1575                 ce_req_size = SI_CE_UCODE_SIZE * 4;
1576                 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1577                 mc_req_size = SI_MC_UCODE_SIZE * 4;
1578                 smc_req_size = ALIGN(PITCAIRN_SMC_UCODE_SIZE, 4);
1579                 break;
1580         case CHIP_VERDE:
1581                 chip_name = "VERDE";
1582                 rlc_chip_name = "VERDE";
1583                 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1584                 me_req_size = SI_PM4_UCODE_SIZE * 4;
1585                 ce_req_size = SI_CE_UCODE_SIZE * 4;
1586                 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1587                 mc_req_size = SI_MC_UCODE_SIZE * 4;
1588                 smc_req_size = ALIGN(VERDE_SMC_UCODE_SIZE, 4);
1589                 break;
1590         case CHIP_OLAND:
1591                 chip_name = "OLAND";
1592                 rlc_chip_name = "OLAND";
1593                 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1594                 me_req_size = SI_PM4_UCODE_SIZE * 4;
1595                 ce_req_size = SI_CE_UCODE_SIZE * 4;
1596                 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1597                 mc_req_size = OLAND_MC_UCODE_SIZE * 4;
1598                 smc_req_size = ALIGN(OLAND_SMC_UCODE_SIZE, 4);
1599                 break;
1600         case CHIP_HAINAN:
1601                 chip_name = "HAINAN";
1602                 rlc_chip_name = "HAINAN";
1603                 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1604                 me_req_size = SI_PM4_UCODE_SIZE * 4;
1605                 ce_req_size = SI_CE_UCODE_SIZE * 4;
1606                 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1607                 mc_req_size = OLAND_MC_UCODE_SIZE * 4;
1608                 smc_req_size = ALIGN(HAINAN_SMC_UCODE_SIZE, 4);
1609                 break;
1610         default: BUG();
1611         }
1612
1613         DRM_INFO("Loading %s Microcode\n", chip_name);
1614
1615         snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1616         err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1617         if (err)
1618                 goto out;
1619         if (rdev->pfp_fw->size != pfp_req_size) {
1620                 printk(KERN_ERR
1621                        "si_cp: Bogus length %zu in firmware \"%s\"\n",
1622                        rdev->pfp_fw->size, fw_name);
1623                 err = -EINVAL;
1624                 goto out;
1625         }
1626
1627         snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1628         err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1629         if (err)
1630                 goto out;
1631         if (rdev->me_fw->size != me_req_size) {
1632                 printk(KERN_ERR
1633                        "si_cp: Bogus length %zu in firmware \"%s\"\n",
1634                        rdev->me_fw->size, fw_name);
1635                 err = -EINVAL;
1636         }
1637
1638         snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1639         err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1640         if (err)
1641                 goto out;
1642         if (rdev->ce_fw->size != ce_req_size) {
1643                 printk(KERN_ERR
1644                        "si_cp: Bogus length %zu in firmware \"%s\"\n",
1645                        rdev->ce_fw->size, fw_name);
1646                 err = -EINVAL;
1647         }
1648
1649         snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
1650         err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1651         if (err)
1652                 goto out;
1653         if (rdev->rlc_fw->size != rlc_req_size) {
1654                 printk(KERN_ERR
1655                        "si_rlc: Bogus length %zu in firmware \"%s\"\n",
1656                        rdev->rlc_fw->size, fw_name);
1657                 err = -EINVAL;
1658         }
1659
1660         snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1661         err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1662         if (err)
1663                 goto out;
1664         if (rdev->mc_fw->size != mc_req_size) {
1665                 printk(KERN_ERR
1666                        "si_mc: Bogus length %zu in firmware \"%s\"\n",
1667                        rdev->mc_fw->size, fw_name);
1668                 err = -EINVAL;
1669         }
1670
1671         snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1672         err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1673         if (err) {
1674                 printk(KERN_ERR
1675                        "smc: error loading firmware \"%s\"\n",
1676                        fw_name);
1677                 release_firmware(rdev->smc_fw);
1678                 rdev->smc_fw = NULL;
1679                 err = 0;
1680         } else if (rdev->smc_fw->size != smc_req_size) {
1681                 printk(KERN_ERR
1682                        "si_smc: Bogus length %zu in firmware \"%s\"\n",
1683                        rdev->smc_fw->size, fw_name);
1684                 err = -EINVAL;
1685         }
1686
1687 out:
1688         if (err) {
1689                 if (err != -EINVAL)
1690                         printk(KERN_ERR
1691                                "si_cp: Failed to load firmware \"%s\"\n",
1692                                fw_name);
1693                 release_firmware(rdev->pfp_fw);
1694                 rdev->pfp_fw = NULL;
1695                 release_firmware(rdev->me_fw);
1696                 rdev->me_fw = NULL;
1697                 release_firmware(rdev->ce_fw);
1698                 rdev->ce_fw = NULL;
1699                 release_firmware(rdev->rlc_fw);
1700                 rdev->rlc_fw = NULL;
1701                 release_firmware(rdev->mc_fw);
1702                 rdev->mc_fw = NULL;
1703                 release_firmware(rdev->smc_fw);
1704                 rdev->smc_fw = NULL;
1705         }
1706         return err;
1707 }
1708
1709 /* watermark setup */
1710 static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
1711                                    struct radeon_crtc *radeon_crtc,
1712                                    struct drm_display_mode *mode,
1713                                    struct drm_display_mode *other_mode)
1714 {
1715         u32 tmp, buffer_alloc, i;
1716         u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1717         /*
1718          * Line Buffer Setup
1719          * There are 3 line buffers, each one shared by 2 display controllers.
1720          * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1721          * the display controllers.  The paritioning is done via one of four
1722          * preset allocations specified in bits 21:20:
1723          *  0 - half lb
1724          *  2 - whole lb, other crtc must be disabled
1725          */
1726         /* this can get tricky if we have two large displays on a paired group
1727          * of crtcs.  Ideally for multiple large displays we'd assign them to
1728          * non-linked crtcs for maximum line buffer allocation.
1729          */
1730         if (radeon_crtc->base.enabled && mode) {
1731                 if (other_mode) {
1732                         tmp = 0; /* 1/2 */
1733                         buffer_alloc = 1;
1734                 } else {
1735                         tmp = 2; /* whole */
1736                         buffer_alloc = 2;
1737                 }
1738         } else {
1739                 tmp = 0;
1740                 buffer_alloc = 0;
1741         }
1742
1743         WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
1744                DC_LB_MEMORY_CONFIG(tmp));
1745
1746         WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1747                DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1748         for (i = 0; i < rdev->usec_timeout; i++) {
1749                 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1750                     DMIF_BUFFERS_ALLOCATED_COMPLETED)
1751                         break;
1752                 udelay(1);
1753         }
1754
1755         if (radeon_crtc->base.enabled && mode) {
1756                 switch (tmp) {
1757                 case 0:
1758                 default:
1759                         return 4096 * 2;
1760                 case 2:
1761                         return 8192 * 2;
1762                 }
1763         }
1764
1765         /* controller not enabled, so no lb used */
1766         return 0;
1767 }
1768
1769 static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
1770 {
1771         u32 tmp = RREG32(MC_SHARED_CHMAP);
1772
1773         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1774         case 0:
1775         default:
1776                 return 1;
1777         case 1:
1778                 return 2;
1779         case 2:
1780                 return 4;
1781         case 3:
1782                 return 8;
1783         case 4:
1784                 return 3;
1785         case 5:
1786                 return 6;
1787         case 6:
1788                 return 10;
1789         case 7:
1790                 return 12;
1791         case 8:
1792                 return 16;
1793         }
1794 }
1795
1796 struct dce6_wm_params {
1797         u32 dram_channels; /* number of dram channels */
1798         u32 yclk;          /* bandwidth per dram data pin in kHz */
1799         u32 sclk;          /* engine clock in kHz */
1800         u32 disp_clk;      /* display clock in kHz */
1801         u32 src_width;     /* viewport width */
1802         u32 active_time;   /* active display time in ns */
1803         u32 blank_time;    /* blank time in ns */
1804         bool interlaced;    /* mode is interlaced */
1805         fixed20_12 vsc;    /* vertical scale ratio */
1806         u32 num_heads;     /* number of active crtcs */
1807         u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1808         u32 lb_size;       /* line buffer allocated to pipe */
1809         u32 vtaps;         /* vertical scaler taps */
1810 };
1811
1812 static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
1813 {
1814         /* Calculate raw DRAM Bandwidth */
1815         fixed20_12 dram_efficiency; /* 0.7 */
1816         fixed20_12 yclk, dram_channels, bandwidth;
1817         fixed20_12 a;
1818
1819         a.full = dfixed_const(1000);
1820         yclk.full = dfixed_const(wm->yclk);
1821         yclk.full = dfixed_div(yclk, a);
1822         dram_channels.full = dfixed_const(wm->dram_channels * 4);
1823         a.full = dfixed_const(10);
1824         dram_efficiency.full = dfixed_const(7);
1825         dram_efficiency.full = dfixed_div(dram_efficiency, a);
1826         bandwidth.full = dfixed_mul(dram_channels, yclk);
1827         bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1828
1829         return dfixed_trunc(bandwidth);
1830 }
1831
1832 static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
1833 {
1834         /* Calculate DRAM Bandwidth and the part allocated to display. */
1835         fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1836         fixed20_12 yclk, dram_channels, bandwidth;
1837         fixed20_12 a;
1838
1839         a.full = dfixed_const(1000);
1840         yclk.full = dfixed_const(wm->yclk);
1841         yclk.full = dfixed_div(yclk, a);
1842         dram_channels.full = dfixed_const(wm->dram_channels * 4);
1843         a.full = dfixed_const(10);
1844         disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1845         disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1846         bandwidth.full = dfixed_mul(dram_channels, yclk);
1847         bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1848
1849         return dfixed_trunc(bandwidth);
1850 }
1851
1852 static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
1853 {
1854         /* Calculate the display Data return Bandwidth */
1855         fixed20_12 return_efficiency; /* 0.8 */
1856         fixed20_12 sclk, bandwidth;
1857         fixed20_12 a;
1858
1859         a.full = dfixed_const(1000);
1860         sclk.full = dfixed_const(wm->sclk);
1861         sclk.full = dfixed_div(sclk, a);
1862         a.full = dfixed_const(10);
1863         return_efficiency.full = dfixed_const(8);
1864         return_efficiency.full = dfixed_div(return_efficiency, a);
1865         a.full = dfixed_const(32);
1866         bandwidth.full = dfixed_mul(a, sclk);
1867         bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1868
1869         return dfixed_trunc(bandwidth);
1870 }
1871
1872 static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
1873 {
1874         return 32;
1875 }
1876
1877 static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
1878 {
1879         /* Calculate the DMIF Request Bandwidth */
1880         fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1881         fixed20_12 disp_clk, sclk, bandwidth;
1882         fixed20_12 a, b1, b2;
1883         u32 min_bandwidth;
1884
1885         a.full = dfixed_const(1000);
1886         disp_clk.full = dfixed_const(wm->disp_clk);
1887         disp_clk.full = dfixed_div(disp_clk, a);
1888         a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
1889         b1.full = dfixed_mul(a, disp_clk);
1890
1891         a.full = dfixed_const(1000);
1892         sclk.full = dfixed_const(wm->sclk);
1893         sclk.full = dfixed_div(sclk, a);
1894         a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
1895         b2.full = dfixed_mul(a, sclk);
1896
1897         a.full = dfixed_const(10);
1898         disp_clk_request_efficiency.full = dfixed_const(8);
1899         disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
1900
1901         min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
1902
1903         a.full = dfixed_const(min_bandwidth);
1904         bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
1905
1906         return dfixed_trunc(bandwidth);
1907 }
1908
1909 static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
1910 {
1911         /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
1912         u32 dram_bandwidth = dce6_dram_bandwidth(wm);
1913         u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
1914         u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
1915
1916         return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
1917 }
1918
1919 static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
1920 {
1921         /* Calculate the display mode Average Bandwidth
1922          * DisplayMode should contain the source and destination dimensions,
1923          * timing, etc.
1924          */
1925         fixed20_12 bpp;
1926         fixed20_12 line_time;
1927         fixed20_12 src_width;
1928         fixed20_12 bandwidth;
1929         fixed20_12 a;
1930
1931         a.full = dfixed_const(1000);
1932         line_time.full = dfixed_const(wm->active_time + wm->blank_time);
1933         line_time.full = dfixed_div(line_time, a);
1934         bpp.full = dfixed_const(wm->bytes_per_pixel);
1935         src_width.full = dfixed_const(wm->src_width);
1936         bandwidth.full = dfixed_mul(src_width, bpp);
1937         bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
1938         bandwidth.full = dfixed_div(bandwidth, line_time);
1939
1940         return dfixed_trunc(bandwidth);
1941 }
1942
1943 static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
1944 {
1945         /* First calcualte the latency in ns */
1946         u32 mc_latency = 2000; /* 2000 ns. */
1947         u32 available_bandwidth = dce6_available_bandwidth(wm);
1948         u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
1949         u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
1950         u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
1951         u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
1952                 (wm->num_heads * cursor_line_pair_return_time);
1953         u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
1954         u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
1955         u32 tmp, dmif_size = 12288;
1956         fixed20_12 a, b, c;
1957
1958         if (wm->num_heads == 0)
1959                 return 0;
1960
1961         a.full = dfixed_const(2);
1962         b.full = dfixed_const(1);
1963         if ((wm->vsc.full > a.full) ||
1964             ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
1965             (wm->vtaps >= 5) ||
1966             ((wm->vsc.full >= a.full) && wm->interlaced))
1967                 max_src_lines_per_dst_line = 4;
1968         else
1969                 max_src_lines_per_dst_line = 2;
1970
1971         a.full = dfixed_const(available_bandwidth);
1972         b.full = dfixed_const(wm->num_heads);
1973         a.full = dfixed_div(a, b);
1974
1975         b.full = dfixed_const(mc_latency + 512);
1976         c.full = dfixed_const(wm->disp_clk);
1977         b.full = dfixed_div(b, c);
1978
1979         c.full = dfixed_const(dmif_size);
1980         b.full = dfixed_div(c, b);
1981
1982         tmp = min(dfixed_trunc(a), dfixed_trunc(b));
1983
1984         b.full = dfixed_const(1000);
1985         c.full = dfixed_const(wm->disp_clk);
1986         b.full = dfixed_div(c, b);
1987         c.full = dfixed_const(wm->bytes_per_pixel);
1988         b.full = dfixed_mul(b, c);
1989
1990         lb_fill_bw = min(tmp, dfixed_trunc(b));
1991
1992         a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
1993         b.full = dfixed_const(1000);
1994         c.full = dfixed_const(lb_fill_bw);
1995         b.full = dfixed_div(c, b);
1996         a.full = dfixed_div(a, b);
1997         line_fill_time = dfixed_trunc(a);
1998
1999         if (line_fill_time < wm->active_time)
2000                 return latency;
2001         else
2002                 return latency + (line_fill_time - wm->active_time);
2003
2004 }
2005
2006 static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
2007 {
2008         if (dce6_average_bandwidth(wm) <=
2009             (dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
2010                 return true;
2011         else
2012                 return false;
2013 };
2014
2015 static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
2016 {
2017         if (dce6_average_bandwidth(wm) <=
2018             (dce6_available_bandwidth(wm) / wm->num_heads))
2019                 return true;
2020         else
2021                 return false;
2022 };
2023
2024 static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
2025 {
2026         u32 lb_partitions = wm->lb_size / wm->src_width;
2027         u32 line_time = wm->active_time + wm->blank_time;
2028         u32 latency_tolerant_lines;
2029         u32 latency_hiding;
2030         fixed20_12 a;
2031
2032         a.full = dfixed_const(1);
2033         if (wm->vsc.full > a.full)
2034                 latency_tolerant_lines = 1;
2035         else {
2036                 if (lb_partitions <= (wm->vtaps + 1))
2037                         latency_tolerant_lines = 1;
2038                 else
2039                         latency_tolerant_lines = 2;
2040         }
2041
2042         latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2043
2044         if (dce6_latency_watermark(wm) <= latency_hiding)
2045                 return true;
2046         else
2047                 return false;
2048 }
2049
2050 static void dce6_program_watermarks(struct radeon_device *rdev,
2051                                          struct radeon_crtc *radeon_crtc,
2052                                          u32 lb_size, u32 num_heads)
2053 {
2054         struct drm_display_mode *mode = &radeon_crtc->base.mode;
2055         struct dce6_wm_params wm_low, wm_high;
2056         u32 dram_channels;
2057         u32 pixel_period;
2058         u32 line_time = 0;
2059         u32 latency_watermark_a = 0, latency_watermark_b = 0;
2060         u32 priority_a_mark = 0, priority_b_mark = 0;
2061         u32 priority_a_cnt = PRIORITY_OFF;
2062         u32 priority_b_cnt = PRIORITY_OFF;
2063         u32 tmp, arb_control3;
2064         fixed20_12 a, b, c;
2065
2066         if (radeon_crtc->base.enabled && num_heads && mode) {
2067                 pixel_period = 1000000 / (u32)mode->clock;
2068                 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2069                 priority_a_cnt = 0;
2070                 priority_b_cnt = 0;
2071
2072                 if (rdev->family == CHIP_ARUBA)
2073                         dram_channels = evergreen_get_number_of_dram_channels(rdev);
2074                 else
2075                         dram_channels = si_get_number_of_dram_channels(rdev);
2076
2077                 /* watermark for high clocks */
2078                 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2079                         wm_high.yclk =
2080                                 radeon_dpm_get_mclk(rdev, false) * 10;
2081                         wm_high.sclk =
2082                                 radeon_dpm_get_sclk(rdev, false) * 10;
2083                 } else {
2084                         wm_high.yclk = rdev->pm.current_mclk * 10;
2085                         wm_high.sclk = rdev->pm.current_sclk * 10;
2086                 }
2087
2088                 wm_high.disp_clk = mode->clock;
2089                 wm_high.src_width = mode->crtc_hdisplay;
2090                 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2091                 wm_high.blank_time = line_time - wm_high.active_time;
2092                 wm_high.interlaced = false;
2093                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2094                         wm_high.interlaced = true;
2095                 wm_high.vsc = radeon_crtc->vsc;
2096                 wm_high.vtaps = 1;
2097                 if (radeon_crtc->rmx_type != RMX_OFF)
2098                         wm_high.vtaps = 2;
2099                 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2100                 wm_high.lb_size = lb_size;
2101                 wm_high.dram_channels = dram_channels;
2102                 wm_high.num_heads = num_heads;
2103
2104                 /* watermark for low clocks */
2105                 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2106                         wm_low.yclk =
2107                                 radeon_dpm_get_mclk(rdev, true) * 10;
2108                         wm_low.sclk =
2109                                 radeon_dpm_get_sclk(rdev, true) * 10;
2110                 } else {
2111                         wm_low.yclk = rdev->pm.current_mclk * 10;
2112                         wm_low.sclk = rdev->pm.current_sclk * 10;
2113                 }
2114
2115                 wm_low.disp_clk = mode->clock;
2116                 wm_low.src_width = mode->crtc_hdisplay;
2117                 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2118                 wm_low.blank_time = line_time - wm_low.active_time;
2119                 wm_low.interlaced = false;
2120                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2121                         wm_low.interlaced = true;
2122                 wm_low.vsc = radeon_crtc->vsc;
2123                 wm_low.vtaps = 1;
2124                 if (radeon_crtc->rmx_type != RMX_OFF)
2125                         wm_low.vtaps = 2;
2126                 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2127                 wm_low.lb_size = lb_size;
2128                 wm_low.dram_channels = dram_channels;
2129                 wm_low.num_heads = num_heads;
2130
2131                 /* set for high clocks */
2132                 latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535);
2133                 /* set for low clocks */
2134                 latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535);
2135
2136                 /* possibly force display priority to high */
2137                 /* should really do this at mode validation time... */
2138                 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2139                     !dce6_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2140                     !dce6_check_latency_hiding(&wm_high) ||
2141                     (rdev->disp_priority == 2)) {
2142                         DRM_DEBUG_KMS("force priority to high\n");
2143                         priority_a_cnt |= PRIORITY_ALWAYS_ON;
2144                         priority_b_cnt |= PRIORITY_ALWAYS_ON;
2145                 }
2146                 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2147                     !dce6_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2148                     !dce6_check_latency_hiding(&wm_low) ||
2149                     (rdev->disp_priority == 2)) {
2150                         DRM_DEBUG_KMS("force priority to high\n");
2151                         priority_a_cnt |= PRIORITY_ALWAYS_ON;
2152                         priority_b_cnt |= PRIORITY_ALWAYS_ON;
2153                 }
2154
2155                 a.full = dfixed_const(1000);
2156                 b.full = dfixed_const(mode->clock);
2157                 b.full = dfixed_div(b, a);
2158                 c.full = dfixed_const(latency_watermark_a);
2159                 c.full = dfixed_mul(c, b);
2160                 c.full = dfixed_mul(c, radeon_crtc->hsc);
2161                 c.full = dfixed_div(c, a);
2162                 a.full = dfixed_const(16);
2163                 c.full = dfixed_div(c, a);
2164                 priority_a_mark = dfixed_trunc(c);
2165                 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2166
2167                 a.full = dfixed_const(1000);
2168                 b.full = dfixed_const(mode->clock);
2169                 b.full = dfixed_div(b, a);
2170                 c.full = dfixed_const(latency_watermark_b);
2171                 c.full = dfixed_mul(c, b);
2172                 c.full = dfixed_mul(c, radeon_crtc->hsc);
2173                 c.full = dfixed_div(c, a);
2174                 a.full = dfixed_const(16);
2175                 c.full = dfixed_div(c, a);
2176                 priority_b_mark = dfixed_trunc(c);
2177                 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2178         }
2179
2180         /* select wm A */
2181         arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2182         tmp = arb_control3;
2183         tmp &= ~LATENCY_WATERMARK_MASK(3);
2184         tmp |= LATENCY_WATERMARK_MASK(1);
2185         WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2186         WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2187                (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2188                 LATENCY_HIGH_WATERMARK(line_time)));
2189         /* select wm B */
2190         tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2191         tmp &= ~LATENCY_WATERMARK_MASK(3);
2192         tmp |= LATENCY_WATERMARK_MASK(2);
2193         WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2194         WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2195                (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2196                 LATENCY_HIGH_WATERMARK(line_time)));
2197         /* restore original selection */
2198         WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
2199
2200         /* write the priority marks */
2201         WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2202         WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2203
2204         /* save values for DPM */
2205         radeon_crtc->line_time = line_time;
2206         radeon_crtc->wm_high = latency_watermark_a;
2207         radeon_crtc->wm_low = latency_watermark_b;
2208 }
2209
2210 void dce6_bandwidth_update(struct radeon_device *rdev)
2211 {
2212         struct drm_display_mode *mode0 = NULL;
2213         struct drm_display_mode *mode1 = NULL;
2214         u32 num_heads = 0, lb_size;
2215         int i;
2216
2217         radeon_update_display_priority(rdev);
2218
2219         for (i = 0; i < rdev->num_crtc; i++) {
2220                 if (rdev->mode_info.crtcs[i]->base.enabled)
2221                         num_heads++;
2222         }
2223         for (i = 0; i < rdev->num_crtc; i += 2) {
2224                 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2225                 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2226                 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2227                 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2228                 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2229                 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2230         }
2231 }
2232
2233 /*
2234  * Core functions
2235  */
2236 static void si_tiling_mode_table_init(struct radeon_device *rdev)
2237 {
2238         const u32 num_tile_mode_states = 32;
2239         u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2240
2241         switch (rdev->config.si.mem_row_size_in_kb) {
2242         case 1:
2243                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2244                 break;
2245         case 2:
2246         default:
2247                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2248                 break;
2249         case 4:
2250                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2251                 break;
2252         }
2253
2254         if ((rdev->family == CHIP_TAHITI) ||
2255             (rdev->family == CHIP_PITCAIRN)) {
2256                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2257                         switch (reg_offset) {
2258                         case 0:  /* non-AA compressed depth or any compressed stencil */
2259                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2260                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2261                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2262                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2263                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2264                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2265                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2266                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2267                                 break;
2268                         case 1:  /* 2xAA/4xAA compressed depth only */
2269                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2270                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2271                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2272                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2273                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2274                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2275                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2276                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2277                                 break;
2278                         case 2:  /* 8xAA compressed depth only */
2279                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2280                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2281                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2282                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2283                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2284                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2285                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2286                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2287                                 break;
2288                         case 3:  /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2289                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2290                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2291                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2292                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2293                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2294                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2295                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2296                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2297                                 break;
2298                         case 4:  /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2299                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2300                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2301                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2302                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2303                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2304                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2305                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2306                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2307                                 break;
2308                         case 5:  /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2309                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2310                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2311                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2312                                                  TILE_SPLIT(split_equal_to_row_size) |
2313                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2314                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2315                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2316                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2317                                 break;
2318                         case 6:  /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2319                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2320                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2321                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2322                                                  TILE_SPLIT(split_equal_to_row_size) |
2323                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2324                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2325                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2326                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2327                                 break;
2328                         case 7:  /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2329                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2330                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2331                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2332                                                  TILE_SPLIT(split_equal_to_row_size) |
2333                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2334                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2335                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2336                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2337                                 break;
2338                         case 8:  /* 1D and 1D Array Surfaces */
2339                                 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2340                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2341                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2342                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2343                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2344                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2345                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2346                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2347                                 break;
2348                         case 9:  /* Displayable maps. */
2349                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2350                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2351                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2352                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2353                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2354                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2355                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2356                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2357                                 break;
2358                         case 10:  /* Display 8bpp. */
2359                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2360                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2361                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2362                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2363                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2364                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2365                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2366                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2367                                 break;
2368                         case 11:  /* Display 16bpp. */
2369                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2370                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2371                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2372                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2373                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2374                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2375                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2376                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2377                                 break;
2378                         case 12:  /* Display 32bpp. */
2379                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2380                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2381                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2382                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2383                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2384                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2385                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2386                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2387                                 break;
2388                         case 13:  /* Thin. */
2389                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2390                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2391                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2392                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2393                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2394                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2395                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2396                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2397                                 break;
2398                         case 14:  /* Thin 8 bpp. */
2399                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2400                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2401                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2402                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2403                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2404                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2405                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2406                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2407                                 break;
2408                         case 15:  /* Thin 16 bpp. */
2409                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2410                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2411                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2412                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2413                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2414                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2415                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2416                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2417                                 break;
2418                         case 16:  /* Thin 32 bpp. */
2419                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2420                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2421                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2422                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2423                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2424                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2425                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2426                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2427                                 break;
2428                         case 17:  /* Thin 64 bpp. */
2429                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2430                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2431                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2432                                                  TILE_SPLIT(split_equal_to_row_size) |
2433                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2434                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2435                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2436                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2437                                 break;
2438                         case 21:  /* 8 bpp PRT. */
2439                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2440                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2441                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2442                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2443                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2444                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2445                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2446                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2447                                 break;
2448                         case 22:  /* 16 bpp PRT */
2449                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2450                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2451                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2452                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2453                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2454                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2455                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2456                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2457                                 break;
2458                         case 23:  /* 32 bpp PRT */
2459                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2460                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2461                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2462                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2463                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2464                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2465                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2466                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2467                                 break;
2468                         case 24:  /* 64 bpp PRT */
2469                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2470                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2471                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2472                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2473                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2474                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2475                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2476                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2477                                 break;
2478                         case 25:  /* 128 bpp PRT */
2479                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2480                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2481                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2482                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2483                                                  NUM_BANKS(ADDR_SURF_8_BANK) |
2484                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2485                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2486                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2487                                 break;
2488                         default:
2489                                 gb_tile_moden = 0;
2490                                 break;
2491                         }
2492                         rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
2493                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2494                 }
2495         } else if ((rdev->family == CHIP_VERDE) ||
2496                    (rdev->family == CHIP_OLAND) ||
2497                    (rdev->family == CHIP_HAINAN)) {
2498                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2499                         switch (reg_offset) {
2500                         case 0:  /* non-AA compressed depth or any compressed stencil */
2501                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2502                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2503                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2504                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2505                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2506                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2507                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2508                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2509                                 break;
2510                         case 1:  /* 2xAA/4xAA compressed depth only */
2511                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2512                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2513                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2514                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2515                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2516                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2517                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2518                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2519                                 break;
2520                         case 2:  /* 8xAA compressed depth only */
2521                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2522                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2523                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2524                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2525                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2526                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2527                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2528                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2529                                 break;
2530                         case 3:  /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2531                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2532                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2533                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2534                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2535                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2536                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2537                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2538                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2539                                 break;
2540                         case 4:  /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2541                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2542                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2543                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2544                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2545                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2546                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2547                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2548                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2549                                 break;
2550                         case 5:  /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2551                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2552                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2553                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2554                                                  TILE_SPLIT(split_equal_to_row_size) |
2555                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2556                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2557                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2558                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2559                                 break;
2560                         case 6:  /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2561                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2562                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2563                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2564                                                  TILE_SPLIT(split_equal_to_row_size) |
2565                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2566                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2567                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2568                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2569                                 break;
2570                         case 7:  /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2571                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2572                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2573                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2574                                                  TILE_SPLIT(split_equal_to_row_size) |
2575                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2576                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2577                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2578                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2579                                 break;
2580                         case 8:  /* 1D and 1D Array Surfaces */
2581                                 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2582                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2583                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2584                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2585                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2586                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2587                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2588                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2589                                 break;
2590                         case 9:  /* Displayable maps. */
2591                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2592                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2593                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2594                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2595                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2596                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2597                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2598                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2599                                 break;
2600                         case 10:  /* Display 8bpp. */
2601                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2602                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2603                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2604                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2605                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2606                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2607                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2608                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2609                                 break;
2610                         case 11:  /* Display 16bpp. */
2611                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2612                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2613                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2614                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2615                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2616                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2617                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2618                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2619                                 break;
2620                         case 12:  /* Display 32bpp. */
2621                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2622                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2623                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2624                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2625                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2626                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2627                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2628                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2629                                 break;
2630                         case 13:  /* Thin. */
2631                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2632                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2633                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2634                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2635                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2636                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2637                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2638                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2639                                 break;
2640                         case 14:  /* Thin 8 bpp. */
2641                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2642                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2643                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2644                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2645                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2646                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2647                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2648                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2649                                 break;
2650                         case 15:  /* Thin 16 bpp. */
2651                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2652                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2653                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2654                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2655                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2656                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2657                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2658                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2659                                 break;
2660                         case 16:  /* Thin 32 bpp. */
2661                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2662                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2663                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2664                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2665                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2666                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2667                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2668                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2669                                 break;
2670                         case 17:  /* Thin 64 bpp. */
2671                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2672                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2673                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2674                                                  TILE_SPLIT(split_equal_to_row_size) |
2675                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2676                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2677                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2678                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2679                                 break;
2680                         case 21:  /* 8 bpp PRT. */
2681                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2682                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2683                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2684                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2685                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2686                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2687                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2688                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2689                                 break;
2690                         case 22:  /* 16 bpp PRT */
2691                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2692                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2693                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2694                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2695                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2696                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2697                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2698                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2699                                 break;
2700                         case 23:  /* 32 bpp PRT */
2701                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2702                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2703                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2704                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2705                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2706                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2707                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2708                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2709                                 break;
2710                         case 24:  /* 64 bpp PRT */
2711                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2712                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2713                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2714                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2715                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2716                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2717                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2718                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2719                                 break;
2720                         case 25:  /* 128 bpp PRT */
2721                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2722                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2723                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2724                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2725                                                  NUM_BANKS(ADDR_SURF_8_BANK) |
2726                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2727                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2728                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2729                                 break;
2730                         default:
2731                                 gb_tile_moden = 0;
2732                                 break;
2733                         }
2734                         rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
2735                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2736                 }
2737         } else
2738                 DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
2739 }
2740
2741 static void si_select_se_sh(struct radeon_device *rdev,
2742                             u32 se_num, u32 sh_num)
2743 {
2744         u32 data = INSTANCE_BROADCAST_WRITES;
2745
2746         if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
2747                 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
2748         else if (se_num == 0xffffffff)
2749                 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2750         else if (sh_num == 0xffffffff)
2751                 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2752         else
2753                 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2754         WREG32(GRBM_GFX_INDEX, data);
2755 }
2756
2757 static u32 si_create_bitmask(u32 bit_width)
2758 {
2759         u32 i, mask = 0;
2760
2761         for (i = 0; i < bit_width; i++) {
2762                 mask <<= 1;
2763                 mask |= 1;
2764         }
2765         return mask;
2766 }
2767
2768 static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
2769 {
2770         u32 data, mask;
2771
2772         data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
2773         if (data & 1)
2774                 data &= INACTIVE_CUS_MASK;
2775         else
2776                 data = 0;
2777         data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
2778
2779         data >>= INACTIVE_CUS_SHIFT;
2780
2781         mask = si_create_bitmask(cu_per_sh);
2782
2783         return ~data & mask;
2784 }
2785
2786 static void si_setup_spi(struct radeon_device *rdev,
2787                          u32 se_num, u32 sh_per_se,
2788                          u32 cu_per_sh)
2789 {
2790         int i, j, k;
2791         u32 data, mask, active_cu;
2792
2793         for (i = 0; i < se_num; i++) {
2794                 for (j = 0; j < sh_per_se; j++) {
2795                         si_select_se_sh(rdev, i, j);
2796                         data = RREG32(SPI_STATIC_THREAD_MGMT_3);
2797                         active_cu = si_get_cu_enabled(rdev, cu_per_sh);
2798
2799                         mask = 1;
2800                         for (k = 0; k < 16; k++) {
2801                                 mask <<= k;
2802                                 if (active_cu & mask) {
2803                                         data &= ~mask;
2804                                         WREG32(SPI_STATIC_THREAD_MGMT_3, data);
2805                                         break;
2806                                 }
2807                         }
2808                 }
2809         }
2810         si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2811 }
2812
2813 static u32 si_get_rb_disabled(struct radeon_device *rdev,
2814                               u32 max_rb_num, u32 se_num,
2815                               u32 sh_per_se)
2816 {
2817         u32 data, mask;
2818
2819         data = RREG32(CC_RB_BACKEND_DISABLE);
2820         if (data & 1)
2821                 data &= BACKEND_DISABLE_MASK;
2822         else
2823                 data = 0;
2824         data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
2825
2826         data >>= BACKEND_DISABLE_SHIFT;
2827
2828         mask = si_create_bitmask(max_rb_num / se_num / sh_per_se);
2829
2830         return data & mask;
2831 }
2832
2833 static void si_setup_rb(struct radeon_device *rdev,
2834                         u32 se_num, u32 sh_per_se,
2835                         u32 max_rb_num)
2836 {
2837         int i, j;
2838         u32 data, mask;
2839         u32 disabled_rbs = 0;
2840         u32 enabled_rbs = 0;
2841
2842         for (i = 0; i < se_num; i++) {
2843                 for (j = 0; j < sh_per_se; j++) {
2844                         si_select_se_sh(rdev, i, j);
2845                         data = si_get_rb_disabled(rdev, max_rb_num, se_num, sh_per_se);
2846                         disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
2847                 }
2848         }
2849         si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2850
2851         mask = 1;
2852         for (i = 0; i < max_rb_num; i++) {
2853                 if (!(disabled_rbs & mask))
2854                         enabled_rbs |= mask;
2855                 mask <<= 1;
2856         }
2857
2858         for (i = 0; i < se_num; i++) {
2859                 si_select_se_sh(rdev, i, 0xffffffff);
2860                 data = 0;
2861                 for (j = 0; j < sh_per_se; j++) {
2862                         switch (enabled_rbs & 3) {
2863                         case 1:
2864                                 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
2865                                 break;
2866                         case 2:
2867                                 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
2868                                 break;
2869                         case 3:
2870                         default:
2871                                 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
2872                                 break;
2873                         }
2874                         enabled_rbs >>= 2;
2875                 }
2876                 WREG32(PA_SC_RASTER_CONFIG, data);
2877         }
2878         si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2879 }
2880
2881 static void si_gpu_init(struct radeon_device *rdev)
2882 {
2883         u32 gb_addr_config = 0;
2884         u32 mc_shared_chmap, mc_arb_ramcfg;
2885         u32 sx_debug_1;
2886         u32 hdp_host_path_cntl;
2887         u32 tmp;
2888         int i, j;
2889
2890         switch (rdev->family) {
2891         case CHIP_TAHITI:
2892                 rdev->config.si.max_shader_engines = 2;
2893                 rdev->config.si.max_tile_pipes = 12;
2894                 rdev->config.si.max_cu_per_sh = 8;
2895                 rdev->config.si.max_sh_per_se = 2;
2896                 rdev->config.si.max_backends_per_se = 4;
2897                 rdev->config.si.max_texture_channel_caches = 12;
2898                 rdev->config.si.max_gprs = 256;
2899                 rdev->config.si.max_gs_threads = 32;
2900                 rdev->config.si.max_hw_contexts = 8;
2901
2902                 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2903                 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2904                 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2905                 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2906                 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
2907                 break;
2908         case CHIP_PITCAIRN:
2909                 rdev->config.si.max_shader_engines = 2;
2910                 rdev->config.si.max_tile_pipes = 8;
2911                 rdev->config.si.max_cu_per_sh = 5;
2912                 rdev->config.si.max_sh_per_se = 2;
2913                 rdev->config.si.max_backends_per_se = 4;
2914                 rdev->config.si.max_texture_channel_caches = 8;
2915                 rdev->config.si.max_gprs = 256;
2916                 rdev->config.si.max_gs_threads = 32;
2917                 rdev->config.si.max_hw_contexts = 8;
2918
2919                 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2920                 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2921                 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2922                 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2923                 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
2924                 break;
2925         case CHIP_VERDE:
2926         default:
2927                 rdev->config.si.max_shader_engines = 1;
2928                 rdev->config.si.max_tile_pipes = 4;
2929                 rdev->config.si.max_cu_per_sh = 5;
2930                 rdev->config.si.max_sh_per_se = 2;
2931                 rdev->config.si.max_backends_per_se = 4;
2932                 rdev->config.si.max_texture_channel_caches = 4;
2933                 rdev->config.si.max_gprs = 256;
2934                 rdev->config.si.max_gs_threads = 32;
2935                 rdev->config.si.max_hw_contexts = 8;
2936
2937                 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2938                 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2939                 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2940                 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2941                 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
2942                 break;
2943         case CHIP_OLAND:
2944                 rdev->config.si.max_shader_engines = 1;
2945                 rdev->config.si.max_tile_pipes = 4;
2946                 rdev->config.si.max_cu_per_sh = 6;
2947                 rdev->config.si.max_sh_per_se = 1;
2948                 rdev->config.si.max_backends_per_se = 2;
2949                 rdev->config.si.max_texture_channel_caches = 4;
2950                 rdev->config.si.max_gprs = 256;
2951                 rdev->config.si.max_gs_threads = 16;
2952                 rdev->config.si.max_hw_contexts = 8;
2953
2954                 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2955                 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2956                 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2957                 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2958                 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
2959                 break;
2960         case CHIP_HAINAN:
2961                 rdev->config.si.max_shader_engines = 1;
2962                 rdev->config.si.max_tile_pipes = 4;
2963                 rdev->config.si.max_cu_per_sh = 5;
2964                 rdev->config.si.max_sh_per_se = 1;
2965                 rdev->config.si.max_backends_per_se = 1;
2966                 rdev->config.si.max_texture_channel_caches = 2;
2967                 rdev->config.si.max_gprs = 256;
2968                 rdev->config.si.max_gs_threads = 16;
2969                 rdev->config.si.max_hw_contexts = 8;
2970
2971                 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2972                 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2973                 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2974                 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2975                 gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
2976                 break;
2977         }
2978
2979         /* Initialize HDP */
2980         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2981                 WREG32((0x2c14 + j), 0x00000000);
2982                 WREG32((0x2c18 + j), 0x00000000);
2983                 WREG32((0x2c1c + j), 0x00000000);
2984                 WREG32((0x2c20 + j), 0x00000000);
2985                 WREG32((0x2c24 + j), 0x00000000);
2986         }
2987
2988         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
2989
2990         evergreen_fix_pci_max_read_req_size(rdev);
2991
2992         WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2993
2994         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
2995         mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
2996
2997         rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
2998         rdev->config.si.mem_max_burst_length_bytes = 256;
2999         tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3000         rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3001         if (rdev->config.si.mem_row_size_in_kb > 4)
3002                 rdev->config.si.mem_row_size_in_kb = 4;
3003         /* XXX use MC settings? */
3004         rdev->config.si.shader_engine_tile_size = 32;
3005         rdev->config.si.num_gpus = 1;
3006         rdev->config.si.multi_gpu_tile_size = 64;
3007
3008         /* fix up row size */
3009         gb_addr_config &= ~ROW_SIZE_MASK;
3010         switch (rdev->config.si.mem_row_size_in_kb) {
3011         case 1:
3012         default:
3013                 gb_addr_config |= ROW_SIZE(0);
3014                 break;
3015         case 2:
3016                 gb_addr_config |= ROW_SIZE(1);
3017                 break;
3018         case 4:
3019                 gb_addr_config |= ROW_SIZE(2);
3020                 break;
3021         }
3022
3023         /* setup tiling info dword.  gb_addr_config is not adequate since it does
3024          * not have bank info, so create a custom tiling dword.
3025          * bits 3:0   num_pipes
3026          * bits 7:4   num_banks
3027          * bits 11:8  group_size
3028          * bits 15:12 row_size
3029          */
3030         rdev->config.si.tile_config = 0;
3031         switch (rdev->config.si.num_tile_pipes) {
3032         case 1:
3033                 rdev->config.si.tile_config |= (0 << 0);
3034                 break;
3035         case 2:
3036                 rdev->config.si.tile_config |= (1 << 0);
3037                 break;
3038         case 4:
3039                 rdev->config.si.tile_config |= (2 << 0);
3040                 break;
3041         case 8:
3042         default:
3043                 /* XXX what about 12? */
3044                 rdev->config.si.tile_config |= (3 << 0);
3045                 break;
3046         }
3047         switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3048         case 0: /* four banks */
3049                 rdev->config.si.tile_config |= 0 << 4;
3050                 break;
3051         case 1: /* eight banks */
3052                 rdev->config.si.tile_config |= 1 << 4;
3053                 break;
3054         case 2: /* sixteen banks */
3055         default:
3056                 rdev->config.si.tile_config |= 2 << 4;
3057                 break;
3058         }
3059         rdev->config.si.tile_config |=
3060                 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3061         rdev->config.si.tile_config |=
3062                 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3063
3064         WREG32(GB_ADDR_CONFIG, gb_addr_config);
3065         WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3066         WREG32(DMIF_ADDR_CALC, gb_addr_config);
3067         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3068         WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
3069         WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
3070         if (rdev->has_uvd) {
3071                 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3072                 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3073                 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3074         }
3075
3076         si_tiling_mode_table_init(rdev);
3077
3078         si_setup_rb(rdev, rdev->config.si.max_shader_engines,
3079                     rdev->config.si.max_sh_per_se,
3080                     rdev->config.si.max_backends_per_se);
3081
3082         si_setup_spi(rdev, rdev->config.si.max_shader_engines,
3083                      rdev->config.si.max_sh_per_se,
3084                      rdev->config.si.max_cu_per_sh);
3085
3086
3087         /* set HW defaults for 3D engine */
3088         WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3089                                      ROQ_IB2_START(0x2b)));
3090         WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3091
3092         sx_debug_1 = RREG32(SX_DEBUG_1);
3093         WREG32(SX_DEBUG_1, sx_debug_1);
3094
3095         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3096
3097         WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
3098                                  SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
3099                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
3100                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
3101
3102         WREG32(VGT_NUM_INSTANCES, 1);
3103
3104         WREG32(CP_PERFMON_CNTL, 0);
3105
3106         WREG32(SQ_CONFIG, 0);
3107
3108         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3109                                           FORCE_EOV_MAX_REZ_CNT(255)));
3110
3111         WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3112                AUTO_INVLD_EN(ES_AND_GS_AUTO));
3113
3114         WREG32(VGT_GS_VERTEX_REUSE, 16);
3115         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3116
3117         WREG32(CB_PERFCOUNTER0_SELECT0, 0);
3118         WREG32(CB_PERFCOUNTER0_SELECT1, 0);
3119         WREG32(CB_PERFCOUNTER1_SELECT0, 0);
3120         WREG32(CB_PERFCOUNTER1_SELECT1, 0);
3121         WREG32(CB_PERFCOUNTER2_SELECT0, 0);
3122         WREG32(CB_PERFCOUNTER2_SELECT1, 0);
3123         WREG32(CB_PERFCOUNTER3_SELECT0, 0);
3124         WREG32(CB_PERFCOUNTER3_SELECT1, 0);
3125
3126         tmp = RREG32(HDP_MISC_CNTL);
3127         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3128         WREG32(HDP_MISC_CNTL, tmp);
3129
3130         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3131         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3132
3133         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3134
3135         udelay(50);
3136 }
3137
3138 /*
3139  * GPU scratch registers helpers function.
3140  */
3141 static void si_scratch_init(struct radeon_device *rdev)
3142 {
3143         int i;
3144
3145         rdev->scratch.num_reg = 7;
3146         rdev->scratch.reg_base = SCRATCH_REG0;
3147         for (i = 0; i < rdev->scratch.num_reg; i++) {
3148                 rdev->scratch.free[i] = true;
3149                 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3150         }
3151 }
3152
3153 void si_fence_ring_emit(struct radeon_device *rdev,
3154                         struct radeon_fence *fence)
3155 {
3156         struct radeon_ring *ring = &rdev->ring[fence->ring];
3157         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3158
3159         /* flush read cache over gart */
3160         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3161         radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3162         radeon_ring_write(ring, 0);
3163         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3164         radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3165                           PACKET3_TC_ACTION_ENA |
3166                           PACKET3_SH_KCACHE_ACTION_ENA |
3167                           PACKET3_SH_ICACHE_ACTION_ENA);
3168         radeon_ring_write(ring, 0xFFFFFFFF);
3169         radeon_ring_write(ring, 0);
3170         radeon_ring_write(ring, 10); /* poll interval */
3171         /* EVENT_WRITE_EOP - flush caches, send int */
3172         radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3173         radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
3174         radeon_ring_write(ring, addr & 0xffffffff);
3175         radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
3176         radeon_ring_write(ring, fence->seq);
3177         radeon_ring_write(ring, 0);
3178 }
3179
3180 /*
3181  * IB stuff
3182  */
3183 void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3184 {
3185         struct radeon_ring *ring = &rdev->ring[ib->ring];
3186         u32 header;
3187
3188         if (ib->is_const_ib) {
3189                 /* set switch buffer packet before const IB */
3190                 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3191                 radeon_ring_write(ring, 0);
3192
3193                 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3194         } else {
3195                 u32 next_rptr;
3196                 if (ring->rptr_save_reg) {
3197                         next_rptr = ring->wptr + 3 + 4 + 8;
3198                         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3199                         radeon_ring_write(ring, ((ring->rptr_save_reg -
3200                                                   PACKET3_SET_CONFIG_REG_START) >> 2));
3201                         radeon_ring_write(ring, next_rptr);
3202                 } else if (rdev->wb.enabled) {
3203                         next_rptr = ring->wptr + 5 + 4 + 8;
3204                         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3205                         radeon_ring_write(ring, (1 << 8));
3206                         radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3207                         radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
3208                         radeon_ring_write(ring, next_rptr);
3209                 }
3210
3211                 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3212         }
3213
3214         radeon_ring_write(ring, header);
3215         radeon_ring_write(ring,
3216 #ifdef __BIG_ENDIAN
3217                           (2 << 0) |
3218 #endif
3219                           (ib->gpu_addr & 0xFFFFFFFC));
3220         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3221         radeon_ring_write(ring, ib->length_dw |
3222                           (ib->vm ? (ib->vm->id << 24) : 0));
3223
3224         if (!ib->is_const_ib) {
3225                 /* flush read cache over gart for this vmid */
3226                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3227                 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3228                 radeon_ring_write(ring, ib->vm ? ib->vm->id : 0);
3229                 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3230                 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3231                                   PACKET3_TC_ACTION_ENA |
3232                                   PACKET3_SH_KCACHE_ACTION_ENA |
3233                                   PACKET3_SH_ICACHE_ACTION_ENA);
3234                 radeon_ring_write(ring, 0xFFFFFFFF);
3235                 radeon_ring_write(ring, 0);
3236                 radeon_ring_write(ring, 10); /* poll interval */
3237         }
3238 }
3239
3240 /*
3241  * CP.
3242  */
3243 static void si_cp_enable(struct radeon_device *rdev, bool enable)
3244 {
3245         if (enable)
3246                 WREG32(CP_ME_CNTL, 0);
3247         else {
3248                 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3249                 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3250                 WREG32(SCRATCH_UMSK, 0);
3251                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3252                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3253                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3254         }
3255         udelay(50);
3256 }
3257
3258 static int si_cp_load_microcode(struct radeon_device *rdev)
3259 {
3260         const __be32 *fw_data;
3261         int i;
3262
3263         if (!rdev->me_fw || !rdev->pfp_fw)
3264                 return -EINVAL;
3265
3266         si_cp_enable(rdev, false);
3267
3268         /* PFP */
3269         fw_data = (const __be32 *)rdev->pfp_fw->data;
3270         WREG32(CP_PFP_UCODE_ADDR, 0);
3271         for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
3272                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3273         WREG32(CP_PFP_UCODE_ADDR, 0);
3274
3275         /* CE */
3276         fw_data = (const __be32 *)rdev->ce_fw->data;
3277         WREG32(CP_CE_UCODE_ADDR, 0);
3278         for (i = 0; i < SI_CE_UCODE_SIZE; i++)
3279                 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3280         WREG32(CP_CE_UCODE_ADDR, 0);
3281
3282         /* ME */
3283         fw_data = (const __be32 *)rdev->me_fw->data;
3284         WREG32(CP_ME_RAM_WADDR, 0);
3285         for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
3286                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3287         WREG32(CP_ME_RAM_WADDR, 0);
3288
3289         WREG32(CP_PFP_UCODE_ADDR, 0);
3290         WREG32(CP_CE_UCODE_ADDR, 0);
3291         WREG32(CP_ME_RAM_WADDR, 0);
3292         WREG32(CP_ME_RAM_RADDR, 0);
3293         return 0;
3294 }
3295
3296 static int si_cp_start(struct radeon_device *rdev)
3297 {
3298         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3299         int r, i;
3300
3301         r = radeon_ring_lock(rdev, ring, 7 + 4);
3302         if (r) {
3303                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3304                 return r;
3305         }
3306         /* init the CP */
3307         radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3308         radeon_ring_write(ring, 0x1);
3309         radeon_ring_write(ring, 0x0);
3310         radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
3311         radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3312         radeon_ring_write(ring, 0);
3313         radeon_ring_write(ring, 0);
3314
3315         /* init the CE partitions */
3316         radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3317         radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3318         radeon_ring_write(ring, 0xc000);
3319         radeon_ring_write(ring, 0xe000);
3320         radeon_ring_unlock_commit(rdev, ring);
3321
3322         si_cp_enable(rdev, true);
3323
3324         r = radeon_ring_lock(rdev, ring, si_default_size + 10);
3325         if (r) {
3326                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3327                 return r;
3328         }
3329
3330         /* setup clear context state */
3331         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3332         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3333
3334         for (i = 0; i < si_default_size; i++)
3335                 radeon_ring_write(ring, si_default_state[i]);
3336
3337         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3338         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3339
3340         /* set clear context state */
3341         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3342         radeon_ring_write(ring, 0);
3343
3344         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3345         radeon_ring_write(ring, 0x00000316);
3346         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3347         radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3348
3349         radeon_ring_unlock_commit(rdev, ring);
3350
3351         for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
3352                 ring = &rdev->ring[i];
3353                 r = radeon_ring_lock(rdev, ring, 2);
3354
3355                 /* clear the compute context state */
3356                 radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
3357                 radeon_ring_write(ring, 0);
3358
3359                 radeon_ring_unlock_commit(rdev, ring);
3360         }
3361
3362         return 0;
3363 }
3364
3365 static void si_cp_fini(struct radeon_device *rdev)
3366 {
3367         struct radeon_ring *ring;
3368         si_cp_enable(rdev, false);
3369
3370         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3371         radeon_ring_fini(rdev, ring);
3372         radeon_scratch_free(rdev, ring->rptr_save_reg);
3373
3374         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3375         radeon_ring_fini(rdev, ring);
3376         radeon_scratch_free(rdev, ring->rptr_save_reg);
3377
3378         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3379         radeon_ring_fini(rdev, ring);
3380         radeon_scratch_free(rdev, ring->rptr_save_reg);
3381 }
3382
3383 static int si_cp_resume(struct radeon_device *rdev)
3384 {
3385         struct radeon_ring *ring;
3386         u32 tmp;
3387         u32 rb_bufsz;
3388         int r;
3389
3390         si_enable_gui_idle_interrupt(rdev, false);
3391
3392         WREG32(CP_SEM_WAIT_TIMER, 0x0);
3393         WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3394
3395         /* Set the write pointer delay */
3396         WREG32(CP_RB_WPTR_DELAY, 0);
3397
3398         WREG32(CP_DEBUG, 0);
3399         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3400
3401         /* ring 0 - compute and gfx */
3402         /* Set ring buffer size */
3403         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3404         rb_bufsz = order_base_2(ring->ring_size / 8);
3405         tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3406 #ifdef __BIG_ENDIAN
3407         tmp |= BUF_SWAP_32BIT;
3408 #endif
3409         WREG32(CP_RB0_CNTL, tmp);
3410
3411         /* Initialize the ring buffer's read and write pointers */
3412         WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3413         ring->wptr = 0;
3414         WREG32(CP_RB0_WPTR, ring->wptr);
3415
3416         /* set the wb address whether it's enabled or not */
3417         WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3418         WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3419
3420         if (rdev->wb.enabled)
3421                 WREG32(SCRATCH_UMSK, 0xff);
3422         else {
3423                 tmp |= RB_NO_UPDATE;
3424                 WREG32(SCRATCH_UMSK, 0);
3425         }
3426
3427         mdelay(1);
3428         WREG32(CP_RB0_CNTL, tmp);
3429
3430         WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
3431
3432         ring->rptr = RREG32(CP_RB0_RPTR);
3433
3434         /* ring1  - compute only */
3435         /* Set ring buffer size */
3436         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3437         rb_bufsz = order_base_2(ring->ring_size / 8);
3438         tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3439 #ifdef __BIG_ENDIAN
3440         tmp |= BUF_SWAP_32BIT;
3441 #endif
3442         WREG32(CP_RB1_CNTL, tmp);
3443
3444         /* Initialize the ring buffer's read and write pointers */
3445         WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
3446         ring->wptr = 0;
3447         WREG32(CP_RB1_WPTR, ring->wptr);
3448
3449         /* set the wb address whether it's enabled or not */
3450         WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
3451         WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
3452
3453         mdelay(1);
3454         WREG32(CP_RB1_CNTL, tmp);
3455
3456         WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
3457
3458         ring->rptr = RREG32(CP_RB1_RPTR);
3459
3460         /* ring2 - compute only */
3461         /* Set ring buffer size */
3462         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3463         rb_bufsz = order_base_2(ring->ring_size / 8);
3464         tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3465 #ifdef __BIG_ENDIAN
3466         tmp |= BUF_SWAP_32BIT;
3467 #endif
3468         WREG32(CP_RB2_CNTL, tmp);
3469
3470         /* Initialize the ring buffer's read and write pointers */
3471         WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
3472         ring->wptr = 0;
3473         WREG32(CP_RB2_WPTR, ring->wptr);
3474
3475         /* set the wb address whether it's enabled or not */
3476         WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
3477         WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
3478
3479         mdelay(1);
3480         WREG32(CP_RB2_CNTL, tmp);
3481
3482         WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
3483
3484         ring->rptr = RREG32(CP_RB2_RPTR);
3485
3486         /* start the rings */
3487         si_cp_start(rdev);
3488         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3489         rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
3490         rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
3491         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3492         if (r) {
3493                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3494                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3495                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3496                 return r;
3497         }
3498         r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
3499         if (r) {
3500                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3501         }
3502         r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
3503         if (r) {
3504                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3505         }
3506
3507         si_enable_gui_idle_interrupt(rdev, true);
3508
3509         return 0;
3510 }
3511
3512 u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
3513 {
3514         u32 reset_mask = 0;
3515         u32 tmp;
3516
3517         /* GRBM_STATUS */
3518         tmp = RREG32(GRBM_STATUS);
3519         if (tmp & (PA_BUSY | SC_BUSY |
3520                    BCI_BUSY | SX_BUSY |
3521                    TA_BUSY | VGT_BUSY |
3522                    DB_BUSY | CB_BUSY |
3523                    GDS_BUSY | SPI_BUSY |
3524                    IA_BUSY | IA_BUSY_NO_DMA))
3525                 reset_mask |= RADEON_RESET_GFX;
3526
3527         if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3528                    CP_BUSY | CP_COHERENCY_BUSY))
3529                 reset_mask |= RADEON_RESET_CP;
3530
3531         if (tmp & GRBM_EE_BUSY)
3532                 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3533
3534         /* GRBM_STATUS2 */
3535         tmp = RREG32(GRBM_STATUS2);
3536         if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3537                 reset_mask |= RADEON_RESET_RLC;
3538
3539         /* DMA_STATUS_REG 0 */
3540         tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
3541         if (!(tmp & DMA_IDLE))
3542                 reset_mask |= RADEON_RESET_DMA;
3543
3544         /* DMA_STATUS_REG 1 */
3545         tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
3546         if (!(tmp & DMA_IDLE))
3547                 reset_mask |= RADEON_RESET_DMA1;
3548
3549         /* SRBM_STATUS2 */
3550         tmp = RREG32(SRBM_STATUS2);
3551         if (tmp & DMA_BUSY)
3552                 reset_mask |= RADEON_RESET_DMA;
3553
3554         if (tmp & DMA1_BUSY)
3555                 reset_mask |= RADEON_RESET_DMA1;
3556
3557         /* SRBM_STATUS */
3558         tmp = RREG32(SRBM_STATUS);
3559
3560         if (tmp & IH_BUSY)
3561                 reset_mask |= RADEON_RESET_IH;
3562
3563         if (tmp & SEM_BUSY)
3564                 reset_mask |= RADEON_RESET_SEM;
3565
3566         if (tmp & GRBM_RQ_PENDING)
3567                 reset_mask |= RADEON_RESET_GRBM;
3568
3569         if (tmp & VMC_BUSY)
3570                 reset_mask |= RADEON_RESET_VMC;
3571
3572         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3573                    MCC_BUSY | MCD_BUSY))
3574                 reset_mask |= RADEON_RESET_MC;
3575
3576         if (evergreen_is_display_hung(rdev))
3577                 reset_mask |= RADEON_RESET_DISPLAY;
3578
3579         /* VM_L2_STATUS */
3580         tmp = RREG32(VM_L2_STATUS);
3581         if (tmp & L2_BUSY)
3582                 reset_mask |= RADEON_RESET_VMC;
3583
3584         /* Skip MC reset as it's mostly likely not hung, just busy */
3585         if (reset_mask & RADEON_RESET_MC) {
3586                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3587                 reset_mask &= ~RADEON_RESET_MC;
3588         }
3589
3590         return reset_mask;
3591 }
3592
3593 static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3594 {
3595         struct evergreen_mc_save save;
3596         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3597         u32 tmp;
3598
3599         if (reset_mask == 0)
3600                 return;
3601
3602         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3603
3604         evergreen_print_gpu_status_regs(rdev);
3605         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
3606                  RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3607         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3608                  RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3609
3610         /* disable PG/CG */
3611         si_fini_pg(rdev);
3612         si_fini_cg(rdev);
3613
3614         /* stop the rlc */
3615         si_rlc_stop(rdev);
3616
3617         /* Disable CP parsing/prefetching */
3618         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3619
3620         if (reset_mask & RADEON_RESET_DMA) {
3621                 /* dma0 */
3622                 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3623                 tmp &= ~DMA_RB_ENABLE;
3624                 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
3625         }
3626         if (reset_mask & RADEON_RESET_DMA1) {
3627                 /* dma1 */
3628                 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3629                 tmp &= ~DMA_RB_ENABLE;
3630                 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3631         }
3632
3633         udelay(50);
3634
3635         evergreen_mc_stop(rdev, &save);
3636         if (evergreen_mc_wait_for_idle(rdev)) {
3637                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3638         }
3639
3640         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) {
3641                 grbm_soft_reset = SOFT_RESET_CB |
3642                         SOFT_RESET_DB |
3643                         SOFT_RESET_GDS |
3644                         SOFT_RESET_PA |
3645                         SOFT_RESET_SC |
3646                         SOFT_RESET_BCI |
3647                         SOFT_RESET_SPI |
3648                         SOFT_RESET_SX |
3649                         SOFT_RESET_TC |
3650                         SOFT_RESET_TA |
3651                         SOFT_RESET_VGT |
3652                         SOFT_RESET_IA;
3653         }
3654
3655         if (reset_mask & RADEON_RESET_CP) {
3656                 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
3657
3658                 srbm_soft_reset |= SOFT_RESET_GRBM;
3659         }
3660
3661         if (reset_mask & RADEON_RESET_DMA)
3662                 srbm_soft_reset |= SOFT_RESET_DMA;
3663
3664         if (reset_mask & RADEON_RESET_DMA1)
3665                 srbm_soft_reset |= SOFT_RESET_DMA1;
3666
3667         if (reset_mask & RADEON_RESET_DISPLAY)
3668                 srbm_soft_reset |= SOFT_RESET_DC;
3669
3670         if (reset_mask & RADEON_RESET_RLC)
3671                 grbm_soft_reset |= SOFT_RESET_RLC;
3672
3673         if (reset_mask & RADEON_RESET_SEM)
3674                 srbm_soft_reset |= SOFT_RESET_SEM;
3675
3676         if (reset_mask & RADEON_RESET_IH)
3677                 srbm_soft_reset |= SOFT_RESET_IH;
3678
3679         if (reset_mask & RADEON_RESET_GRBM)
3680                 srbm_soft_reset |= SOFT_RESET_GRBM;
3681
3682         if (reset_mask & RADEON_RESET_VMC)
3683                 srbm_soft_reset |= SOFT_RESET_VMC;
3684
3685         if (reset_mask & RADEON_RESET_MC)
3686                 srbm_soft_reset |= SOFT_RESET_MC;
3687
3688         if (grbm_soft_reset) {
3689                 tmp = RREG32(GRBM_SOFT_RESET);
3690                 tmp |= grbm_soft_reset;
3691                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3692                 WREG32(GRBM_SOFT_RESET, tmp);
3693                 tmp = RREG32(GRBM_SOFT_RESET);
3694
3695                 udelay(50);
3696
3697                 tmp &= ~grbm_soft_reset;
3698                 WREG32(GRBM_SOFT_RESET, tmp);
3699                 tmp = RREG32(GRBM_SOFT_RESET);
3700         }
3701
3702         if (srbm_soft_reset) {
3703                 tmp = RREG32(SRBM_SOFT_RESET);
3704                 tmp |= srbm_soft_reset;
3705                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3706                 WREG32(SRBM_SOFT_RESET, tmp);
3707                 tmp = RREG32(SRBM_SOFT_RESET);
3708
3709                 udelay(50);
3710
3711                 tmp &= ~srbm_soft_reset;
3712                 WREG32(SRBM_SOFT_RESET, tmp);
3713                 tmp = RREG32(SRBM_SOFT_RESET);
3714         }
3715
3716         /* Wait a little for things to settle down */
3717         udelay(50);
3718
3719         evergreen_mc_resume(rdev, &save);
3720         udelay(50);
3721
3722         evergreen_print_gpu_status_regs(rdev);
3723 }
3724
3725 int si_asic_reset(struct radeon_device *rdev)
3726 {
3727         u32 reset_mask;
3728
3729         reset_mask = si_gpu_check_soft_reset(rdev);
3730
3731         if (reset_mask)
3732                 r600_set_bios_scratch_engine_hung(rdev, true);
3733
3734         si_gpu_soft_reset(rdev, reset_mask);
3735
3736         reset_mask = si_gpu_check_soft_reset(rdev);
3737
3738         if (!reset_mask)
3739                 r600_set_bios_scratch_engine_hung(rdev, false);
3740
3741         return 0;
3742 }
3743
3744 /**
3745  * si_gfx_is_lockup - Check if the GFX engine is locked up
3746  *
3747  * @rdev: radeon_device pointer
3748  * @ring: radeon_ring structure holding ring information
3749  *
3750  * Check if the GFX engine is locked up.
3751  * Returns true if the engine appears to be locked up, false if not.
3752  */
3753 bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3754 {
3755         u32 reset_mask = si_gpu_check_soft_reset(rdev);
3756
3757         if (!(reset_mask & (RADEON_RESET_GFX |
3758                             RADEON_RESET_COMPUTE |
3759                             RADEON_RESET_CP))) {
3760                 radeon_ring_lockup_update(ring);
3761                 return false;
3762         }
3763         /* force CP activities */
3764         radeon_ring_force_activity(rdev, ring);
3765         return radeon_ring_test_lockup(rdev, ring);
3766 }
3767
3768 /* MC */
3769 static void si_mc_program(struct radeon_device *rdev)
3770 {
3771         struct evergreen_mc_save save;
3772         u32 tmp;
3773         int i, j;
3774
3775         /* Initialize HDP */
3776         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3777                 WREG32((0x2c14 + j), 0x00000000);
3778                 WREG32((0x2c18 + j), 0x00000000);
3779                 WREG32((0x2c1c + j), 0x00000000);
3780                 WREG32((0x2c20 + j), 0x00000000);
3781                 WREG32((0x2c24 + j), 0x00000000);
3782         }
3783         WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
3784
3785         evergreen_mc_stop(rdev, &save);
3786         if (radeon_mc_wait_for_idle(rdev)) {
3787                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3788         }
3789         if (!ASIC_IS_NODCE(rdev))
3790                 /* Lockout access through VGA aperture*/
3791                 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
3792         /* Update configuration */
3793         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
3794                rdev->mc.vram_start >> 12);
3795         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
3796                rdev->mc.vram_end >> 12);
3797         WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
3798                rdev->vram_scratch.gpu_addr >> 12);
3799         tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
3800         tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
3801         WREG32(MC_VM_FB_LOCATION, tmp);
3802         /* XXX double check these! */
3803         WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
3804         WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
3805         WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
3806         WREG32(MC_VM_AGP_BASE, 0);
3807         WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
3808         WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
3809         if (radeon_mc_wait_for_idle(rdev)) {
3810                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3811         }
3812         evergreen_mc_resume(rdev, &save);
3813         if (!ASIC_IS_NODCE(rdev)) {
3814                 /* we need to own VRAM, so turn off the VGA renderer here
3815                  * to stop it overwriting our objects */
3816                 rv515_vga_render_disable(rdev);
3817         }
3818 }
3819
3820 void si_vram_gtt_location(struct radeon_device *rdev,
3821                           struct radeon_mc *mc)
3822 {
3823         if (mc->mc_vram_size > 0xFFC0000000ULL) {
3824                 /* leave room for at least 1024M GTT */
3825                 dev_warn(rdev->dev, "limiting VRAM\n");
3826                 mc->real_vram_size = 0xFFC0000000ULL;
3827                 mc->mc_vram_size = 0xFFC0000000ULL;
3828         }
3829         radeon_vram_location(rdev, &rdev->mc, 0);
3830         rdev->mc.gtt_base_align = 0;
3831         radeon_gtt_location(rdev, mc);
3832 }
3833
3834 static int si_mc_init(struct radeon_device *rdev)
3835 {
3836         u32 tmp;
3837         int chansize, numchan;
3838
3839         /* Get VRAM informations */
3840         rdev->mc.vram_is_ddr = true;
3841         tmp = RREG32(MC_ARB_RAMCFG);
3842         if (tmp & CHANSIZE_OVERRIDE) {
3843                 chansize = 16;
3844         } else if (tmp & CHANSIZE_MASK) {
3845                 chansize = 64;
3846         } else {
3847                 chansize = 32;
3848         }
3849         tmp = RREG32(MC_SHARED_CHMAP);
3850         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3851         case 0:
3852         default:
3853                 numchan = 1;
3854                 break;
3855         case 1:
3856                 numchan = 2;
3857                 break;
3858         case 2:
3859                 numchan = 4;
3860                 break;
3861         case 3:
3862                 numchan = 8;
3863                 break;
3864         case 4:
3865                 numchan = 3;
3866                 break;
3867         case 5:
3868                 numchan = 6;
3869                 break;
3870         case 6:
3871                 numchan = 10;
3872                 break;
3873         case 7:
3874                 numchan = 12;
3875                 break;
3876         case 8:
3877                 numchan = 16;
3878                 break;
3879         }
3880         rdev->mc.vram_width = numchan * chansize;
3881         /* Could aper size report 0 ? */
3882         rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3883         rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3884         /* size in MB on si */
3885         rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3886         rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3887         rdev->mc.visible_vram_size = rdev->mc.aper_size;
3888         si_vram_gtt_location(rdev, &rdev->mc);
3889         radeon_update_bandwidth_info(rdev);
3890
3891         return 0;
3892 }
3893
3894 /*
3895  * GART
3896  */
3897 void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
3898 {
3899         /* flush hdp cache */
3900         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
3901
3902         /* bits 0-15 are the VM contexts0-15 */
3903         WREG32(VM_INVALIDATE_REQUEST, 1);
3904 }
3905
3906 static int si_pcie_gart_enable(struct radeon_device *rdev)
3907 {
3908         int r, i;
3909
3910         if (rdev->gart.robj == NULL) {
3911                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
3912                 return -EINVAL;
3913         }
3914         r = radeon_gart_table_vram_pin(rdev);
3915         if (r)
3916                 return r;
3917         radeon_gart_restore(rdev);
3918         /* Setup TLB control */
3919         WREG32(MC_VM_MX_L1_TLB_CNTL,
3920                (0xA << 7) |
3921                ENABLE_L1_TLB |
3922                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
3923                ENABLE_ADVANCED_DRIVER_MODEL |
3924                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
3925         /* Setup L2 cache */
3926         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
3927                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
3928                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
3929                EFFECTIVE_L2_QUEUE_SIZE(7) |
3930                CONTEXT1_IDENTITY_ACCESS_MODE(1));
3931         WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
3932         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
3933                L2_CACHE_BIGK_FRAGMENT_SIZE(0));
3934         /* setup context0 */
3935         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
3936         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
3937         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
3938         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
3939                         (u32)(rdev->dummy_page.addr >> 12));
3940         WREG32(VM_CONTEXT0_CNTL2, 0);
3941         WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
3942                                   RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
3943
3944         WREG32(0x15D4, 0);
3945         WREG32(0x15D8, 0);
3946         WREG32(0x15DC, 0);
3947
3948         /* empty context1-15 */
3949         /* set vm size, must be a multiple of 4 */
3950         WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
3951         WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
3952         /* Assign the pt base to something valid for now; the pts used for
3953          * the VMs are determined by the application and setup and assigned
3954          * on the fly in the vm part of radeon_gart.c
3955          */
3956         for (i = 1; i < 16; i++) {
3957                 if (i < 8)
3958                         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
3959                                rdev->gart.table_addr >> 12);
3960                 else
3961                         WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
3962                                rdev->gart.table_addr >> 12);
3963         }
3964
3965         /* enable context1-15 */
3966         WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
3967                (u32)(rdev->dummy_page.addr >> 12));
3968         WREG32(VM_CONTEXT1_CNTL2, 4);
3969         WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
3970                                 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3971                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
3972                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3973                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
3974                                 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
3975                                 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
3976                                 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
3977                                 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
3978                                 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
3979                                 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
3980                                 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3981                                 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
3982
3983         si_pcie_gart_tlb_flush(rdev);
3984         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
3985                  (unsigned)(rdev->mc.gtt_size >> 20),
3986                  (unsigned long long)rdev->gart.table_addr);
3987         rdev->gart.ready = true;
3988         return 0;
3989 }
3990
3991 static void si_pcie_gart_disable(struct radeon_device *rdev)
3992 {
3993         /* Disable all tables */
3994         WREG32(VM_CONTEXT0_CNTL, 0);
3995         WREG32(VM_CONTEXT1_CNTL, 0);
3996         /* Setup TLB control */
3997         WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
3998                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
3999         /* Setup L2 cache */
4000         WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4001                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4002                EFFECTIVE_L2_QUEUE_SIZE(7) |
4003                CONTEXT1_IDENTITY_ACCESS_MODE(1));
4004         WREG32(VM_L2_CNTL2, 0);
4005         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4006                L2_CACHE_BIGK_FRAGMENT_SIZE(0));
4007         radeon_gart_table_vram_unpin(rdev);
4008 }
4009
4010 static void si_pcie_gart_fini(struct radeon_device *rdev)
4011 {
4012         si_pcie_gart_disable(rdev);
4013         radeon_gart_table_vram_free(rdev);
4014         radeon_gart_fini(rdev);
4015 }
4016
4017 /* vm parser */
4018 static bool si_vm_reg_valid(u32 reg)
4019 {
4020         /* context regs are fine */
4021         if (reg >= 0x28000)
4022                 return true;
4023
4024         /* check config regs */
4025         switch (reg) {
4026         case GRBM_GFX_INDEX:
4027         case CP_STRMOUT_CNTL:
4028         case VGT_VTX_VECT_EJECT_REG:
4029         case VGT_CACHE_INVALIDATION:
4030         case VGT_ESGS_RING_SIZE:
4031         case VGT_GSVS_RING_SIZE:
4032         case VGT_GS_VERTEX_REUSE:
4033         case VGT_PRIMITIVE_TYPE:
4034         case VGT_INDEX_TYPE:
4035         case VGT_NUM_INDICES:
4036         case VGT_NUM_INSTANCES:
4037         case VGT_TF_RING_SIZE:
4038         case VGT_HS_OFFCHIP_PARAM:
4039         case VGT_TF_MEMORY_BASE:
4040         case PA_CL_ENHANCE:
4041         case PA_SU_LINE_STIPPLE_VALUE:
4042         case PA_SC_LINE_STIPPLE_STATE:
4043         case PA_SC_ENHANCE:
4044         case SQC_CACHES:
4045         case SPI_STATIC_THREAD_MGMT_1:
4046         case SPI_STATIC_THREAD_MGMT_2:
4047         case SPI_STATIC_THREAD_MGMT_3:
4048         case SPI_PS_MAX_WAVE_ID:
4049         case SPI_CONFIG_CNTL:
4050         case SPI_CONFIG_CNTL_1:
4051         case TA_CNTL_AUX:
4052                 return true;
4053         default:
4054                 DRM_ERROR("Invalid register 0x%x in CS\n", reg);
4055                 return false;
4056         }
4057 }
4058
4059 static int si_vm_packet3_ce_check(struct radeon_device *rdev,
4060                                   u32 *ib, struct radeon_cs_packet *pkt)
4061 {
4062         switch (pkt->opcode) {
4063         case PACKET3_NOP:
4064         case PACKET3_SET_BASE:
4065         case PACKET3_SET_CE_DE_COUNTERS:
4066         case PACKET3_LOAD_CONST_RAM:
4067         case PACKET3_WRITE_CONST_RAM:
4068         case PACKET3_WRITE_CONST_RAM_OFFSET:
4069         case PACKET3_DUMP_CONST_RAM:
4070         case PACKET3_INCREMENT_CE_COUNTER:
4071         case PACKET3_WAIT_ON_DE_COUNTER:
4072         case PACKET3_CE_WRITE:
4073                 break;
4074         default:
4075                 DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
4076                 return -EINVAL;
4077         }
4078         return 0;
4079 }
4080
4081 static int si_vm_packet3_cp_dma_check(u32 *ib, u32 idx)
4082 {
4083         u32 start_reg, reg, i;
4084         u32 command = ib[idx + 4];
4085         u32 info = ib[idx + 1];
4086         u32 idx_value = ib[idx];
4087         if (command & PACKET3_CP_DMA_CMD_SAS) {
4088                 /* src address space is register */
4089                 if (((info & 0x60000000) >> 29) == 0) {
4090                         start_reg = idx_value << 2;
4091                         if (command & PACKET3_CP_DMA_CMD_SAIC) {
4092                                 reg = start_reg;
4093                                 if (!si_vm_reg_valid(reg)) {
4094                                         DRM_ERROR("CP DMA Bad SRC register\n");
4095                                         return -EINVAL;
4096                                 }
4097                         } else {
4098                                 for (i = 0; i < (command & 0x1fffff); i++) {
4099                                         reg = start_reg + (4 * i);
4100                                         if (!si_vm_reg_valid(reg)) {
4101                                                 DRM_ERROR("CP DMA Bad SRC register\n");
4102                                                 return -EINVAL;
4103                                         }
4104                                 }
4105                         }
4106                 }
4107         }
4108         if (command & PACKET3_CP_DMA_CMD_DAS) {
4109                 /* dst address space is register */
4110                 if (((info & 0x00300000) >> 20) == 0) {
4111                         start_reg = ib[idx + 2];
4112                         if (command & PACKET3_CP_DMA_CMD_DAIC) {
4113                                 reg = start_reg;
4114                                 if (!si_vm_reg_valid(reg)) {
4115                                         DRM_ERROR("CP DMA Bad DST register\n");
4116                                         return -EINVAL;
4117                                 }
4118                         } else {
4119                                 for (i = 0; i < (command & 0x1fffff); i++) {
4120                                         reg = start_reg + (4 * i);
4121                                 if (!si_vm_reg_valid(reg)) {
4122                                                 DRM_ERROR("CP DMA Bad DST register\n");
4123                                                 return -EINVAL;
4124                                         }
4125                                 }
4126                         }
4127                 }
4128         }
4129         return 0;
4130 }
4131
4132 static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
4133                                    u32 *ib, struct radeon_cs_packet *pkt)
4134 {
4135         int r;
4136         u32 idx = pkt->idx + 1;
4137         u32 idx_value = ib[idx];
4138         u32 start_reg, end_reg, reg, i;
4139
4140         switch (pkt->opcode) {
4141         case PACKET3_NOP:
4142         case PACKET3_SET_BASE:
4143         case PACKET3_CLEAR_STATE:
4144         case PACKET3_INDEX_BUFFER_SIZE:
4145         case PACKET3_DISPATCH_DIRECT:
4146         case PACKET3_DISPATCH_INDIRECT:
4147         case PACKET3_ALLOC_GDS:
4148         case PACKET3_WRITE_GDS_RAM:
4149         case PACKET3_ATOMIC_GDS:
4150         case PACKET3_ATOMIC:
4151         case PACKET3_OCCLUSION_QUERY:
4152         case PACKET3_SET_PREDICATION:
4153         case PACKET3_COND_EXEC:
4154         case PACKET3_PRED_EXEC:
4155         case PACKET3_DRAW_INDIRECT:
4156         case PACKET3_DRAW_INDEX_INDIRECT:
4157         case PACKET3_INDEX_BASE:
4158         case PACKET3_DRAW_INDEX_2:
4159         case PACKET3_CONTEXT_CONTROL:
4160         case PACKET3_INDEX_TYPE:
4161         case PACKET3_DRAW_INDIRECT_MULTI:
4162         case PACKET3_DRAW_INDEX_AUTO:
4163         case PACKET3_DRAW_INDEX_IMMD:
4164         case PACKET3_NUM_INSTANCES:
4165         case PACKET3_DRAW_INDEX_MULTI_AUTO:
4166         case PACKET3_STRMOUT_BUFFER_UPDATE:
4167         case PACKET3_DRAW_INDEX_OFFSET_2:
4168         case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
4169         case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
4170         case PACKET3_MPEG_INDEX:
4171         case PACKET3_WAIT_REG_MEM:
4172         case PACKET3_MEM_WRITE:
4173         case PACKET3_PFP_SYNC_ME:
4174         case PACKET3_SURFACE_SYNC:
4175         case PACKET3_EVENT_WRITE:
4176         case PACKET3_EVENT_WRITE_EOP:
4177         case PACKET3_EVENT_WRITE_EOS:
4178         case PACKET3_SET_CONTEXT_REG:
4179         case PACKET3_SET_CONTEXT_REG_INDIRECT:
4180         case PACKET3_SET_SH_REG:
4181         case PACKET3_SET_SH_REG_OFFSET:
4182         case PACKET3_INCREMENT_DE_COUNTER:
4183         case PACKET3_WAIT_ON_CE_COUNTER:
4184         case PACKET3_WAIT_ON_AVAIL_BUFFER:
4185         case PACKET3_ME_WRITE:
4186                 break;
4187         case PACKET3_COPY_DATA:
4188                 if ((idx_value & 0xf00) == 0) {
4189                         reg = ib[idx + 3] * 4;
4190                         if (!si_vm_reg_valid(reg))
4191                                 return -EINVAL;
4192                 }
4193                 break;
4194         case PACKET3_WRITE_DATA:
4195                 if ((idx_value & 0xf00) == 0) {
4196                         start_reg = ib[idx + 1] * 4;
4197                         if (idx_value & 0x10000) {
4198                                 if (!si_vm_reg_valid(start_reg))
4199                                         return -EINVAL;
4200                         } else {
4201                                 for (i = 0; i < (pkt->count - 2); i++) {
4202                                         reg = start_reg + (4 * i);
4203                                         if (!si_vm_reg_valid(reg))
4204                                                 return -EINVAL;
4205                                 }
4206                         }
4207                 }
4208                 break;
4209         case PACKET3_COND_WRITE:
4210                 if (idx_value & 0x100) {
4211                         reg = ib[idx + 5] * 4;
4212                         if (!si_vm_reg_valid(reg))
4213                                 return -EINVAL;
4214                 }
4215                 break;
4216         case PACKET3_COPY_DW:
4217                 if (idx_value & 0x2) {
4218                         reg = ib[idx + 3] * 4;
4219                         if (!si_vm_reg_valid(reg))
4220                                 return -EINVAL;
4221                 }
4222                 break;
4223         case PACKET3_SET_CONFIG_REG:
4224                 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
4225                 end_reg = 4 * pkt->count + start_reg - 4;
4226                 if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
4227                     (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
4228                     (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
4229                         DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
4230                         return -EINVAL;
4231                 }
4232                 for (i = 0; i < pkt->count; i++) {
4233                         reg = start_reg + (4 * i);
4234                         if (!si_vm_reg_valid(reg))
4235                                 return -EINVAL;
4236                 }
4237                 break;
4238         case PACKET3_CP_DMA:
4239                 r = si_vm_packet3_cp_dma_check(ib, idx);
4240                 if (r)
4241                         return r;
4242                 break;
4243         default:
4244                 DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
4245                 return -EINVAL;
4246         }
4247         return 0;
4248 }
4249
4250 static int si_vm_packet3_compute_check(struct radeon_device *rdev,
4251                                        u32 *ib, struct radeon_cs_packet *pkt)
4252 {
4253         int r;
4254         u32 idx = pkt->idx + 1;
4255         u32 idx_value = ib[idx];
4256         u32 start_reg, reg, i;
4257
4258         switch (pkt->opcode) {
4259         case PACKET3_NOP:
4260         case PACKET3_SET_BASE:
4261         case PACKET3_CLEAR_STATE:
4262         case PACKET3_DISPATCH_DIRECT:
4263         case PACKET3_DISPATCH_INDIRECT:
4264         case PACKET3_ALLOC_GDS:
4265         case PACKET3_WRITE_GDS_RAM:
4266         case PACKET3_ATOMIC_GDS:
4267         case PACKET3_ATOMIC:
4268         case PACKET3_OCCLUSION_QUERY:
4269         case PACKET3_SET_PREDICATION:
4270         case PACKET3_COND_EXEC:
4271         case PACKET3_PRED_EXEC:
4272         case PACKET3_CONTEXT_CONTROL:
4273         case PACKET3_STRMOUT_BUFFER_UPDATE:
4274         case PACKET3_WAIT_REG_MEM:
4275         case PACKET3_MEM_WRITE:
4276         case PACKET3_PFP_SYNC_ME:
4277         case PACKET3_SURFACE_SYNC:
4278         case PACKET3_EVENT_WRITE:
4279         case PACKET3_EVENT_WRITE_EOP:
4280         case PACKET3_EVENT_WRITE_EOS:
4281         case PACKET3_SET_CONTEXT_REG:
4282         case PACKET3_SET_CONTEXT_REG_INDIRECT:
4283         case PACKET3_SET_SH_REG:
4284         case PACKET3_SET_SH_REG_OFFSET:
4285         case PACKET3_INCREMENT_DE_COUNTER:
4286         case PACKET3_WAIT_ON_CE_COUNTER:
4287         case PACKET3_WAIT_ON_AVAIL_BUFFER:
4288         case PACKET3_ME_WRITE:
4289                 break;
4290         case PACKET3_COPY_DATA:
4291                 if ((idx_value & 0xf00) == 0) {
4292                         reg = ib[idx + 3] * 4;
4293                         if (!si_vm_reg_valid(reg))
4294                                 return -EINVAL;
4295                 }
4296                 break;
4297         case PACKET3_WRITE_DATA:
4298                 if ((idx_value & 0xf00) == 0) {
4299                         start_reg = ib[idx + 1] * 4;
4300                         if (idx_value & 0x10000) {
4301                                 if (!si_vm_reg_valid(start_reg))
4302                                         return -EINVAL;
4303                         } else {
4304                                 for (i = 0; i < (pkt->count - 2); i++) {
4305                                         reg = start_reg + (4 * i);
4306                                         if (!si_vm_reg_valid(reg))
4307                                                 return -EINVAL;
4308                                 }
4309                         }
4310                 }
4311                 break;
4312         case PACKET3_COND_WRITE:
4313                 if (idx_value & 0x100) {
4314                         reg = ib[idx + 5] * 4;
4315                         if (!si_vm_reg_valid(reg))
4316                                 return -EINVAL;
4317                 }
4318                 break;
4319         case PACKET3_COPY_DW:
4320                 if (idx_value & 0x2) {
4321                         reg = ib[idx + 3] * 4;
4322                         if (!si_vm_reg_valid(reg))
4323                                 return -EINVAL;
4324                 }
4325                 break;
4326         case PACKET3_CP_DMA:
4327                 r = si_vm_packet3_cp_dma_check(ib, idx);
4328                 if (r)
4329                         return r;
4330                 break;
4331         default:
4332                 DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
4333                 return -EINVAL;
4334         }
4335         return 0;
4336 }
4337
4338 int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4339 {
4340         int ret = 0;
4341         u32 idx = 0;
4342         struct radeon_cs_packet pkt;
4343
4344         do {
4345                 pkt.idx = idx;
4346                 pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
4347                 pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
4348                 pkt.one_reg_wr = 0;
4349                 switch (pkt.type) {
4350                 case RADEON_PACKET_TYPE0:
4351                         dev_err(rdev->dev, "Packet0 not allowed!\n");
4352                         ret = -EINVAL;
4353                         break;
4354                 case RADEON_PACKET_TYPE2:
4355                         idx += 1;
4356                         break;
4357                 case RADEON_PACKET_TYPE3:
4358                         pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
4359                         if (ib->is_const_ib)
4360                                 ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
4361                         else {
4362                                 switch (ib->ring) {
4363                                 case RADEON_RING_TYPE_GFX_INDEX:
4364                                         ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
4365                                         break;
4366                                 case CAYMAN_RING_TYPE_CP1_INDEX:
4367                                 case CAYMAN_RING_TYPE_CP2_INDEX:
4368                                         ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
4369                                         break;
4370                                 default:
4371                                         dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
4372                                         ret = -EINVAL;
4373                                         break;
4374                                 }
4375                         }
4376                         idx += pkt.count + 2;
4377                         break;
4378                 default:
4379                         dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
4380                         ret = -EINVAL;
4381                         break;
4382                 }
4383                 if (ret)
4384                         break;
4385         } while (idx < ib->length_dw);
4386
4387         return ret;
4388 }
4389
4390 /*
4391  * vm
4392  */
4393 int si_vm_init(struct radeon_device *rdev)
4394 {
4395         /* number of VMs */
4396         rdev->vm_manager.nvm = 16;
4397         /* base offset of vram pages */
4398         rdev->vm_manager.vram_base_offset = 0;
4399
4400         return 0;
4401 }
4402
4403 void si_vm_fini(struct radeon_device *rdev)
4404 {
4405 }
4406
4407 /**
4408  * si_vm_decode_fault - print human readable fault info
4409  *
4410  * @rdev: radeon_device pointer
4411  * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
4412  * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
4413  *
4414  * Print human readable fault information (SI).
4415  */
4416 static void si_vm_decode_fault(struct radeon_device *rdev,
4417                                u32 status, u32 addr)
4418 {
4419         u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
4420         u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
4421         u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
4422         char *block;
4423
4424         if (rdev->family == CHIP_TAHITI) {
4425                 switch (mc_id) {
4426                 case 160:
4427                 case 144:
4428                 case 96:
4429                 case 80:
4430                 case 224:
4431                 case 208:
4432                 case 32:
4433                 case 16:
4434                         block = "CB";
4435                         break;
4436                 case 161:
4437                 case 145:
4438                 case 97:
4439                 case 81:
4440                 case 225:
4441                 case 209:
4442                 case 33:
4443                 case 17:
4444                         block = "CB_FMASK";
4445                         break;
4446                 case 162:
4447                 case 146:
4448                 case 98:
4449                 case 82:
4450                 case 226:
4451                 case 210:
4452                 case 34:
4453                 case 18:
4454                         block = "CB_CMASK";
4455                         break;
4456                 case 163:
4457                 case 147:
4458                 case 99:
4459                 case 83:
4460                 case 227:
4461                 case 211:
4462                 case 35:
4463                 case 19:
4464                         block = "CB_IMMED";
4465                         break;
4466                 case 164:
4467                 case 148:
4468                 case 100:
4469                 case 84:
4470                 case 228:
4471                 case 212:
4472                 case 36:
4473                 case 20:
4474                         block = "DB";
4475                         break;
4476                 case 165:
4477                 case 149:
4478                 case 101:
4479                 case 85:
4480                 case 229:
4481                 case 213:
4482                 case 37:
4483                 case 21:
4484                         block = "DB_HTILE";
4485                         break;
4486                 case 167:
4487                 case 151:
4488                 case 103:
4489                 case 87:
4490                 case 231:
4491                 case 215:
4492                 case 39:
4493                 case 23:
4494                         block = "DB_STEN";
4495                         break;
4496                 case 72:
4497                 case 68:
4498                 case 64:
4499                 case 8:
4500                 case 4:
4501                 case 0:
4502                 case 136:
4503                 case 132:
4504                 case 128:
4505                 case 200:
4506                 case 196:
4507                 case 192:
4508                         block = "TC";
4509                         break;
4510                 case 112:
4511                 case 48:
4512                         block = "CP";
4513                         break;
4514                 case 49:
4515                 case 177:
4516                 case 50:
4517                 case 178:
4518                         block = "SH";
4519                         break;
4520                 case 53:
4521                 case 190:
4522                         block = "VGT";
4523                         break;
4524                 case 117:
4525                         block = "IH";
4526                         break;
4527                 case 51:
4528                 case 115:
4529                         block = "RLC";
4530                         break;
4531                 case 119:
4532                 case 183:
4533                         block = "DMA0";
4534                         break;
4535                 case 61:
4536                         block = "DMA1";
4537                         break;
4538                 case 248:
4539                 case 120:
4540                         block = "HDP";
4541                         break;
4542                 default:
4543                         block = "unknown";
4544                         break;
4545                 }
4546         } else {
4547                 switch (mc_id) {
4548                 case 32:
4549                 case 16:
4550                 case 96:
4551                 case 80:
4552                 case 160:
4553                 case 144:
4554                 case 224:
4555                 case 208:
4556                         block = "CB";
4557                         break;
4558                 case 33:
4559                 case 17:
4560                 case 97:
4561                 case 81:
4562                 case 161:
4563                 case 145:
4564                 case 225:
4565                 case 209:
4566                         block = "CB_FMASK";
4567                         break;
4568                 case 34:
4569                 case 18:
4570                 case 98:
4571                 case 82:
4572                 case 162:
4573                 case 146:
4574                 case 226:
4575                 case 210:
4576                         block = "CB_CMASK";
4577                         break;
4578                 case 35:
4579                 case 19:
4580                 case 99:
4581                 case 83:
4582                 case 163:
4583                 case 147:
4584                 case 227:
4585                 case 211:
4586                         block = "CB_IMMED";
4587                         break;
4588                 case 36:
4589                 case 20:
4590                 case 100:
4591                 case 84:
4592                 case 164:
4593                 case 148:
4594                 case 228:
4595                 case 212:
4596                         block = "DB";
4597                         break;
4598                 case 37:
4599                 case 21:
4600                 case 101:
4601                 case 85:
4602                 case 165:
4603                 case 149:
4604                 case 229:
4605                 case 213:
4606                         block = "DB_HTILE";
4607                         break;
4608                 case 39:
4609                 case 23:
4610                 case 103:
4611                 case 87:
4612                 case 167:
4613                 case 151:
4614                 case 231:
4615                 case 215:
4616                         block = "DB_STEN";
4617                         break;
4618                 case 72:
4619                 case 68:
4620                 case 8:
4621                 case 4:
4622                 case 136:
4623                 case 132:
4624                 case 200:
4625                 case 196:
4626                         block = "TC";
4627                         break;
4628                 case 112:
4629                 case 48:
4630                         block = "CP";
4631                         break;
4632                 case 49:
4633                 case 177:
4634                 case 50:
4635                 case 178:
4636                         block = "SH";
4637                         break;
4638                 case 53:
4639                         block = "VGT";
4640                         break;
4641                 case 117:
4642                         block = "IH";
4643                         break;
4644                 case 51:
4645                 case 115:
4646                         block = "RLC";
4647                         break;
4648                 case 119:
4649                 case 183:
4650                         block = "DMA0";
4651                         break;
4652                 case 61:
4653                         block = "DMA1";
4654                         break;
4655                 case 248:
4656                 case 120:
4657                         block = "HDP";
4658                         break;
4659                 default:
4660                         block = "unknown";
4661                         break;
4662                 }
4663         }
4664
4665         printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
4666                protections, vmid, addr,
4667                (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
4668                block, mc_id);
4669 }
4670
4671 void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
4672 {
4673         struct radeon_ring *ring = &rdev->ring[ridx];
4674
4675         if (vm == NULL)
4676                 return;
4677
4678         /* write new base address */
4679         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4680         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4681                                  WRITE_DATA_DST_SEL(0)));
4682
4683         if (vm->id < 8) {
4684                 radeon_ring_write(ring,
4685                                   (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
4686         } else {
4687                 radeon_ring_write(ring,
4688                                   (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
4689         }
4690         radeon_ring_write(ring, 0);
4691         radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
4692
4693         /* flush hdp cache */
4694         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4695         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4696                                  WRITE_DATA_DST_SEL(0)));
4697         radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
4698         radeon_ring_write(ring, 0);
4699         radeon_ring_write(ring, 0x1);
4700
4701         /* bits 0-15 are the VM contexts0-15 */
4702         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4703         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4704                                  WRITE_DATA_DST_SEL(0)));
4705         radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
4706         radeon_ring_write(ring, 0);
4707         radeon_ring_write(ring, 1 << vm->id);
4708
4709         /* sync PFP to ME, otherwise we might get invalid PFP reads */
4710         radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
4711         radeon_ring_write(ring, 0x0);
4712 }
4713
4714 /*
4715  *  Power and clock gating
4716  */
4717 static void si_wait_for_rlc_serdes(struct radeon_device *rdev)
4718 {
4719         int i;
4720
4721         for (i = 0; i < rdev->usec_timeout; i++) {
4722                 if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0)
4723                         break;
4724                 udelay(1);
4725         }
4726
4727         for (i = 0; i < rdev->usec_timeout; i++) {
4728                 if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0)
4729                         break;
4730                 udelay(1);
4731         }
4732 }
4733
4734 static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
4735                                          bool enable)
4736 {
4737         u32 tmp = RREG32(CP_INT_CNTL_RING0);
4738         u32 mask;
4739         int i;
4740
4741         if (enable)
4742                 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4743         else
4744                 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4745         WREG32(CP_INT_CNTL_RING0, tmp);
4746
4747         if (!enable) {
4748                 /* read a gfx register */
4749                 tmp = RREG32(DB_DEPTH_INFO);
4750
4751                 mask = RLC_BUSY_STATUS | GFX_POWER_STATUS | GFX_CLOCK_STATUS | GFX_LS_STATUS;
4752                 for (i = 0; i < rdev->usec_timeout; i++) {
4753                         if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS))
4754                                 break;
4755                         udelay(1);
4756                 }
4757         }
4758 }
4759
4760 static void si_set_uvd_dcm(struct radeon_device *rdev,
4761                            bool sw_mode)
4762 {
4763         u32 tmp, tmp2;
4764
4765         tmp = RREG32(UVD_CGC_CTRL);
4766         tmp &= ~(CLK_OD_MASK | CG_DT_MASK);
4767         tmp |= DCM | CG_DT(1) | CLK_OD(4);
4768
4769         if (sw_mode) {
4770                 tmp &= ~0x7ffff800;
4771                 tmp2 = DYN_OR_EN | DYN_RR_EN | G_DIV_ID(7);
4772         } else {
4773                 tmp |= 0x7ffff800;
4774                 tmp2 = 0;
4775         }
4776
4777         WREG32(UVD_CGC_CTRL, tmp);
4778         WREG32_UVD_CTX(UVD_CGC_CTRL2, tmp2);
4779 }
4780
4781 void si_init_uvd_internal_cg(struct radeon_device *rdev)
4782 {
4783         bool hw_mode = true;
4784
4785         if (hw_mode) {
4786                 si_set_uvd_dcm(rdev, false);
4787         } else {
4788                 u32 tmp = RREG32(UVD_CGC_CTRL);
4789                 tmp &= ~DCM;
4790                 WREG32(UVD_CGC_CTRL, tmp);
4791         }
4792 }
4793
4794 static u32 si_halt_rlc(struct radeon_device *rdev)
4795 {
4796         u32 data, orig;
4797
4798         orig = data = RREG32(RLC_CNTL);
4799
4800         if (data & RLC_ENABLE) {
4801                 data &= ~RLC_ENABLE;
4802                 WREG32(RLC_CNTL, data);
4803
4804                 si_wait_for_rlc_serdes(rdev);
4805         }
4806
4807         return orig;
4808 }
4809
4810 static void si_update_rlc(struct radeon_device *rdev, u32 rlc)
4811 {
4812         u32 tmp;
4813
4814         tmp = RREG32(RLC_CNTL);
4815         if (tmp != rlc)
4816                 WREG32(RLC_CNTL, rlc);
4817 }
4818
4819 static void si_enable_dma_pg(struct radeon_device *rdev, bool enable)
4820 {
4821         u32 data, orig;
4822
4823         orig = data = RREG32(DMA_PG);
4824         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA))
4825                 data |= PG_CNTL_ENABLE;
4826         else
4827                 data &= ~PG_CNTL_ENABLE;
4828         if (orig != data)
4829                 WREG32(DMA_PG, data);
4830 }
4831
4832 static void si_init_dma_pg(struct radeon_device *rdev)
4833 {
4834         u32 tmp;
4835
4836         WREG32(DMA_PGFSM_WRITE,  0x00002000);
4837         WREG32(DMA_PGFSM_CONFIG, 0x100010ff);
4838
4839         for (tmp = 0; tmp < 5; tmp++)
4840                 WREG32(DMA_PGFSM_WRITE, 0);
4841 }
4842
4843 static void si_enable_gfx_cgpg(struct radeon_device *rdev,
4844                                bool enable)
4845 {
4846         u32 tmp;
4847
4848         if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
4849                 tmp = RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10);
4850                 WREG32(RLC_TTOP_D, tmp);
4851
4852                 tmp = RREG32(RLC_PG_CNTL);
4853                 tmp |= GFX_PG_ENABLE;
4854                 WREG32(RLC_PG_CNTL, tmp);
4855
4856                 tmp = RREG32(RLC_AUTO_PG_CTRL);
4857                 tmp |= AUTO_PG_EN;
4858                 WREG32(RLC_AUTO_PG_CTRL, tmp);
4859         } else {
4860                 tmp = RREG32(RLC_AUTO_PG_CTRL);
4861                 tmp &= ~AUTO_PG_EN;
4862                 WREG32(RLC_AUTO_PG_CTRL, tmp);
4863
4864                 tmp = RREG32(DB_RENDER_CONTROL);
4865         }
4866 }
4867
4868 static void si_init_gfx_cgpg(struct radeon_device *rdev)
4869 {
4870         u32 tmp;
4871
4872         WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4873
4874         tmp = RREG32(RLC_PG_CNTL);
4875         tmp |= GFX_PG_SRC;
4876         WREG32(RLC_PG_CNTL, tmp);
4877
4878         WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4879
4880         tmp = RREG32(RLC_AUTO_PG_CTRL);
4881
4882         tmp &= ~GRBM_REG_SGIT_MASK;
4883         tmp |= GRBM_REG_SGIT(0x700);
4884         tmp &= ~PG_AFTER_GRBM_REG_ST_MASK;
4885         WREG32(RLC_AUTO_PG_CTRL, tmp);
4886 }
4887
4888 static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
4889 {
4890         u32 mask = 0, tmp, tmp1;
4891         int i;
4892
4893         si_select_se_sh(rdev, se, sh);
4894         tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
4895         tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
4896         si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
4897
4898         tmp &= 0xffff0000;
4899
4900         tmp |= tmp1;
4901         tmp >>= 16;
4902
4903         for (i = 0; i < rdev->config.si.max_cu_per_sh; i ++) {
4904                 mask <<= 1;
4905                 mask |= 1;
4906         }
4907
4908         return (~tmp) & mask;
4909 }
4910
4911 static void si_init_ao_cu_mask(struct radeon_device *rdev)
4912 {
4913         u32 i, j, k, active_cu_number = 0;
4914         u32 mask, counter, cu_bitmap;
4915         u32 tmp = 0;
4916
4917         for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
4918                 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
4919                         mask = 1;
4920                         cu_bitmap = 0;
4921                         counter  = 0;
4922                         for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) {
4923                                 if (si_get_cu_active_bitmap(rdev, i, j) & mask) {
4924                                         if (counter < 2)
4925                                                 cu_bitmap |= mask;
4926                                         counter++;
4927                                 }
4928                                 mask <<= 1;
4929                         }
4930
4931                         active_cu_number += counter;
4932                         tmp |= (cu_bitmap << (i * 16 + j * 8));
4933                 }
4934         }
4935
4936         WREG32(RLC_PG_AO_CU_MASK, tmp);
4937
4938         tmp = RREG32(RLC_MAX_PG_CU);
4939         tmp &= ~MAX_PU_CU_MASK;
4940         tmp |= MAX_PU_CU(active_cu_number);
4941         WREG32(RLC_MAX_PG_CU, tmp);
4942 }
4943
4944 static void si_enable_cgcg(struct radeon_device *rdev,
4945                            bool enable)
4946 {
4947         u32 data, orig, tmp;
4948
4949         orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
4950
4951         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
4952                 si_enable_gui_idle_interrupt(rdev, true);
4953
4954                 WREG32(RLC_GCPM_GENERAL_3, 0x00000080);
4955
4956                 tmp = si_halt_rlc(rdev);
4957
4958                 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
4959                 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
4960                 WREG32(RLC_SERDES_WR_CTRL, 0x00b000ff);
4961
4962                 si_wait_for_rlc_serdes(rdev);
4963
4964                 si_update_rlc(rdev, tmp);
4965
4966                 WREG32(RLC_SERDES_WR_CTRL, 0x007000ff);
4967
4968                 data |= CGCG_EN | CGLS_EN;
4969         } else {
4970                 si_enable_gui_idle_interrupt(rdev, false);
4971
4972                 RREG32(CB_CGTT_SCLK_CTRL);
4973                 RREG32(CB_CGTT_SCLK_CTRL);
4974                 RREG32(CB_CGTT_SCLK_CTRL);
4975                 RREG32(CB_CGTT_SCLK_CTRL);
4976
4977                 data &= ~(CGCG_EN | CGLS_EN);
4978         }
4979
4980         if (orig != data)
4981                 WREG32(RLC_CGCG_CGLS_CTRL, data);
4982 }
4983
4984 static void si_enable_mgcg(struct radeon_device *rdev,
4985                            bool enable)
4986 {
4987         u32 data, orig, tmp = 0;
4988
4989         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
4990                 orig = data = RREG32(CGTS_SM_CTRL_REG);
4991                 data = 0x96940200;
4992                 if (orig != data)
4993                         WREG32(CGTS_SM_CTRL_REG, data);
4994
4995                 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
4996                         orig = data = RREG32(CP_MEM_SLP_CNTL);
4997                         data |= CP_MEM_LS_EN;
4998                         if (orig != data)
4999                                 WREG32(CP_MEM_SLP_CNTL, data);
5000                 }
5001
5002                 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5003                 data &= 0xffffffc0;
5004                 if (orig != data)
5005                         WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5006
5007                 tmp = si_halt_rlc(rdev);
5008
5009                 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5010                 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5011                 WREG32(RLC_SERDES_WR_CTRL, 0x00d000ff);
5012
5013                 si_update_rlc(rdev, tmp);
5014         } else {
5015                 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5016                 data |= 0x00000003;
5017                 if (orig != data)
5018                         WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5019
5020                 data = RREG32(CP_MEM_SLP_CNTL);
5021                 if (data & CP_MEM_LS_EN) {
5022                         data &= ~CP_MEM_LS_EN;
5023                         WREG32(CP_MEM_SLP_CNTL, data);
5024                 }
5025                 orig = data = RREG32(CGTS_SM_CTRL_REG);
5026                 data |= LS_OVERRIDE | OVERRIDE;
5027                 if (orig != data)
5028                         WREG32(CGTS_SM_CTRL_REG, data);
5029
5030                 tmp = si_halt_rlc(rdev);
5031
5032                 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5033                 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5034                 WREG32(RLC_SERDES_WR_CTRL, 0x00e000ff);
5035
5036                 si_update_rlc(rdev, tmp);
5037         }
5038 }
5039
5040 static void si_enable_uvd_mgcg(struct radeon_device *rdev,
5041                                bool enable)
5042 {
5043         u32 orig, data, tmp;
5044
5045         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
5046                 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5047                 tmp |= 0x3fff;
5048                 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5049
5050                 orig = data = RREG32(UVD_CGC_CTRL);
5051                 data |= DCM;
5052                 if (orig != data)
5053                         WREG32(UVD_CGC_CTRL, data);
5054
5055                 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0);
5056                 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0);
5057         } else {
5058                 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5059                 tmp &= ~0x3fff;
5060                 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5061
5062                 orig = data = RREG32(UVD_CGC_CTRL);
5063                 data &= ~DCM;
5064                 if (orig != data)
5065                         WREG32(UVD_CGC_CTRL, data);
5066
5067                 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0xffffffff);
5068                 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0xffffffff);
5069         }
5070 }
5071
5072 static const u32 mc_cg_registers[] =
5073 {
5074         MC_HUB_MISC_HUB_CG,
5075         MC_HUB_MISC_SIP_CG,
5076         MC_HUB_MISC_VM_CG,
5077         MC_XPB_CLK_GAT,
5078         ATC_MISC_CG,
5079         MC_CITF_MISC_WR_CG,
5080         MC_CITF_MISC_RD_CG,
5081         MC_CITF_MISC_VM_CG,
5082         VM_L2_CG,
5083 };
5084
5085 static void si_enable_mc_ls(struct radeon_device *rdev,
5086                             bool enable)
5087 {
5088         int i;
5089         u32 orig, data;
5090
5091         for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5092                 orig = data = RREG32(mc_cg_registers[i]);
5093                 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
5094                         data |= MC_LS_ENABLE;
5095                 else
5096                         data &= ~MC_LS_ENABLE;
5097                 if (data != orig)
5098                         WREG32(mc_cg_registers[i], data);
5099         }
5100 }
5101
5102 static void si_enable_mc_mgcg(struct radeon_device *rdev,
5103                                bool enable)
5104 {
5105         int i;
5106         u32 orig, data;
5107
5108         for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5109                 orig = data = RREG32(mc_cg_registers[i]);
5110                 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
5111                         data |= MC_CG_ENABLE;
5112                 else
5113                         data &= ~MC_CG_ENABLE;
5114                 if (data != orig)
5115                         WREG32(mc_cg_registers[i], data);
5116         }
5117 }
5118
5119 static void si_enable_dma_mgcg(struct radeon_device *rdev,
5120                                bool enable)
5121 {
5122         u32 orig, data, offset;
5123         int i;
5124
5125         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
5126                 for (i = 0; i < 2; i++) {
5127                         if (i == 0)
5128                                 offset = DMA0_REGISTER_OFFSET;
5129                         else
5130                                 offset = DMA1_REGISTER_OFFSET;
5131                         orig = data = RREG32(DMA_POWER_CNTL + offset);
5132                         data &= ~MEM_POWER_OVERRIDE;
5133                         if (data != orig)
5134                                 WREG32(DMA_POWER_CNTL + offset, data);
5135                         WREG32(DMA_CLK_CTRL + offset, 0x00000100);
5136                 }
5137         } else {
5138                 for (i = 0; i < 2; i++) {
5139                         if (i == 0)
5140                                 offset = DMA0_REGISTER_OFFSET;
5141                         else
5142                                 offset = DMA1_REGISTER_OFFSET;
5143                         orig = data = RREG32(DMA_POWER_CNTL + offset);
5144                         data |= MEM_POWER_OVERRIDE;
5145                         if (data != orig)
5146                                 WREG32(DMA_POWER_CNTL + offset, data);
5147
5148                         orig = data = RREG32(DMA_CLK_CTRL + offset);
5149                         data = 0xff000000;
5150                         if (data != orig)
5151                                 WREG32(DMA_CLK_CTRL + offset, data);
5152                 }
5153         }
5154 }
5155
5156 static void si_enable_bif_mgls(struct radeon_device *rdev,
5157                                bool enable)
5158 {
5159         u32 orig, data;
5160
5161         orig = data = RREG32_PCIE(PCIE_CNTL2);
5162
5163         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5164                 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
5165                         REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
5166         else
5167                 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
5168                           REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
5169
5170         if (orig != data)
5171                 WREG32_PCIE(PCIE_CNTL2, data);
5172 }
5173
5174 static void si_enable_hdp_mgcg(struct radeon_device *rdev,
5175                                bool enable)
5176 {
5177         u32 orig, data;
5178
5179         orig = data = RREG32(HDP_HOST_PATH_CNTL);
5180
5181         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
5182                 data &= ~CLOCK_GATING_DIS;
5183         else
5184                 data |= CLOCK_GATING_DIS;
5185
5186         if (orig != data)
5187                 WREG32(HDP_HOST_PATH_CNTL, data);
5188 }
5189
5190 static void si_enable_hdp_ls(struct radeon_device *rdev,
5191                              bool enable)
5192 {
5193         u32 orig, data;
5194
5195         orig = data = RREG32(HDP_MEM_POWER_LS);
5196
5197         if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
5198                 data |= HDP_LS_ENABLE;
5199         else
5200                 data &= ~HDP_LS_ENABLE;
5201
5202         if (orig != data)
5203                 WREG32(HDP_MEM_POWER_LS, data);
5204 }
5205
5206 void si_update_cg(struct radeon_device *rdev,
5207                   u32 block, bool enable)
5208 {
5209         if (block & RADEON_CG_BLOCK_GFX) {
5210                 si_enable_gui_idle_interrupt(rdev, false);
5211                 /* order matters! */
5212                 if (enable) {
5213                         si_enable_mgcg(rdev, true);
5214                         si_enable_cgcg(rdev, true);
5215                 } else {
5216                         si_enable_cgcg(rdev, false);
5217                         si_enable_mgcg(rdev, false);
5218                 }
5219                 si_enable_gui_idle_interrupt(rdev, true);
5220         }
5221
5222         if (block & RADEON_CG_BLOCK_MC) {
5223                 si_enable_mc_mgcg(rdev, enable);
5224                 si_enable_mc_ls(rdev, enable);
5225         }
5226
5227         if (block & RADEON_CG_BLOCK_SDMA) {
5228                 si_enable_dma_mgcg(rdev, enable);
5229         }
5230
5231         if (block & RADEON_CG_BLOCK_BIF) {
5232                 si_enable_bif_mgls(rdev, enable);
5233         }
5234
5235         if (block & RADEON_CG_BLOCK_UVD) {
5236                 if (rdev->has_uvd) {
5237                         si_enable_uvd_mgcg(rdev, enable);
5238                 }
5239         }
5240
5241         if (block & RADEON_CG_BLOCK_HDP) {
5242                 si_enable_hdp_mgcg(rdev, enable);
5243                 si_enable_hdp_ls(rdev, enable);
5244         }
5245 }
5246
5247 static void si_init_cg(struct radeon_device *rdev)
5248 {
5249         si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5250                             RADEON_CG_BLOCK_MC |
5251                             RADEON_CG_BLOCK_SDMA |
5252                             RADEON_CG_BLOCK_BIF |
5253                             RADEON_CG_BLOCK_HDP), true);
5254         if (rdev->has_uvd) {
5255                 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, true);
5256                 si_init_uvd_internal_cg(rdev);
5257         }
5258 }
5259
5260 static void si_fini_cg(struct radeon_device *rdev)
5261 {
5262         if (rdev->has_uvd) {
5263                 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, false);
5264         }
5265         si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5266                             RADEON_CG_BLOCK_MC |
5267                             RADEON_CG_BLOCK_SDMA |
5268                             RADEON_CG_BLOCK_BIF |
5269                             RADEON_CG_BLOCK_HDP), false);
5270 }
5271
5272 u32 si_get_csb_size(struct radeon_device *rdev)
5273 {
5274         u32 count = 0;
5275         const struct cs_section_def *sect = NULL;
5276         const struct cs_extent_def *ext = NULL;
5277
5278         if (rdev->rlc.cs_data == NULL)
5279                 return 0;
5280
5281         /* begin clear state */
5282         count += 2;
5283         /* context control state */
5284         count += 3;
5285
5286         for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5287                 for (ext = sect->section; ext->extent != NULL; ++ext) {
5288                         if (sect->id == SECT_CONTEXT)
5289                                 count += 2 + ext->reg_count;
5290                         else
5291                                 return 0;
5292                 }
5293         }
5294         /* pa_sc_raster_config */
5295         count += 3;
5296         /* end clear state */
5297         count += 2;
5298         /* clear state */
5299         count += 2;
5300
5301         return count;
5302 }
5303
5304 void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
5305 {
5306         u32 count = 0, i;
5307         const struct cs_section_def *sect = NULL;
5308         const struct cs_extent_def *ext = NULL;
5309
5310         if (rdev->rlc.cs_data == NULL)
5311                 return;
5312         if (buffer == NULL)
5313                 return;
5314
5315         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5316         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
5317
5318         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5319         buffer[count++] = cpu_to_le32(0x80000000);
5320         buffer[count++] = cpu_to_le32(0x80000000);
5321
5322         for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5323                 for (ext = sect->section; ext->extent != NULL; ++ext) {
5324                         if (sect->id == SECT_CONTEXT) {
5325                                 buffer[count++] =
5326                                         cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
5327                                 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
5328                                 for (i = 0; i < ext->reg_count; i++)
5329                                         buffer[count++] = cpu_to_le32(ext->extent[i]);
5330                         } else {
5331                                 return;
5332                         }
5333                 }
5334         }
5335
5336         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1));
5337         buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
5338         switch (rdev->family) {
5339         case CHIP_TAHITI:
5340         case CHIP_PITCAIRN:
5341                 buffer[count++] = cpu_to_le32(0x2a00126a);
5342                 break;
5343         case CHIP_VERDE:
5344                 buffer[count++] = cpu_to_le32(0x0000124a);
5345                 break;
5346         case CHIP_OLAND:
5347                 buffer[count++] = cpu_to_le32(0x00000082);
5348                 break;
5349         case CHIP_HAINAN:
5350                 buffer[count++] = cpu_to_le32(0x00000000);
5351                 break;
5352         default:
5353                 buffer[count++] = cpu_to_le32(0x00000000);
5354                 break;
5355         }
5356
5357         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5358         buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
5359
5360         buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
5361         buffer[count++] = cpu_to_le32(0);
5362 }
5363
5364 static void si_init_pg(struct radeon_device *rdev)
5365 {
5366         if (rdev->pg_flags) {
5367                 if (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA) {
5368                         si_init_dma_pg(rdev);
5369                 }
5370                 si_init_ao_cu_mask(rdev);
5371                 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
5372                         si_init_gfx_cgpg(rdev);
5373                 }
5374                 si_enable_dma_pg(rdev, true);
5375                 si_enable_gfx_cgpg(rdev, true);
5376         } else {
5377                 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5378                 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5379         }
5380 }
5381
5382 static void si_fini_pg(struct radeon_device *rdev)
5383 {
5384         if (rdev->pg_flags) {
5385                 si_enable_dma_pg(rdev, false);
5386                 si_enable_gfx_cgpg(rdev, false);
5387         }
5388 }
5389
5390 /*
5391  * RLC
5392  */
5393 void si_rlc_reset(struct radeon_device *rdev)
5394 {
5395         u32 tmp = RREG32(GRBM_SOFT_RESET);
5396
5397         tmp |= SOFT_RESET_RLC;
5398         WREG32(GRBM_SOFT_RESET, tmp);
5399         udelay(50);
5400         tmp &= ~SOFT_RESET_RLC;
5401         WREG32(GRBM_SOFT_RESET, tmp);
5402         udelay(50);
5403 }
5404
5405 static void si_rlc_stop(struct radeon_device *rdev)
5406 {
5407         WREG32(RLC_CNTL, 0);
5408
5409         si_enable_gui_idle_interrupt(rdev, false);
5410
5411         si_wait_for_rlc_serdes(rdev);
5412 }
5413
5414 static void si_rlc_start(struct radeon_device *rdev)
5415 {
5416         WREG32(RLC_CNTL, RLC_ENABLE);
5417
5418         si_enable_gui_idle_interrupt(rdev, true);
5419
5420         udelay(50);
5421 }
5422
5423 static bool si_lbpw_supported(struct radeon_device *rdev)
5424 {
5425         u32 tmp;
5426
5427         /* Enable LBPW only for DDR3 */
5428         tmp = RREG32(MC_SEQ_MISC0);
5429         if ((tmp & 0xF0000000) == 0xB0000000)
5430                 return true;
5431         return false;
5432 }
5433
5434 static void si_enable_lbpw(struct radeon_device *rdev, bool enable)
5435 {
5436         u32 tmp;
5437
5438         tmp = RREG32(RLC_LB_CNTL);
5439         if (enable)
5440                 tmp |= LOAD_BALANCE_ENABLE;
5441         else
5442                 tmp &= ~LOAD_BALANCE_ENABLE;
5443         WREG32(RLC_LB_CNTL, tmp);
5444
5445         if (!enable) {
5446                 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5447                 WREG32(SPI_LB_CU_MASK, 0x00ff);
5448         }
5449 }
5450
5451 static int si_rlc_resume(struct radeon_device *rdev)
5452 {
5453         u32 i;
5454         const __be32 *fw_data;
5455
5456         if (!rdev->rlc_fw)
5457                 return -EINVAL;
5458
5459         si_rlc_stop(rdev);
5460
5461         si_rlc_reset(rdev);
5462
5463         si_init_pg(rdev);
5464
5465         si_init_cg(rdev);
5466
5467         WREG32(RLC_RL_BASE, 0);
5468         WREG32(RLC_RL_SIZE, 0);
5469         WREG32(RLC_LB_CNTL, 0);
5470         WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
5471         WREG32(RLC_LB_CNTR_INIT, 0);
5472         WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5473
5474         WREG32(RLC_MC_CNTL, 0);
5475         WREG32(RLC_UCODE_CNTL, 0);
5476
5477         fw_data = (const __be32 *)rdev->rlc_fw->data;
5478         for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
5479                 WREG32(RLC_UCODE_ADDR, i);
5480                 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
5481         }
5482         WREG32(RLC_UCODE_ADDR, 0);
5483
5484         si_enable_lbpw(rdev, si_lbpw_supported(rdev));
5485
5486         si_rlc_start(rdev);
5487
5488         return 0;
5489 }
5490
5491 static void si_enable_interrupts(struct radeon_device *rdev)
5492 {
5493         u32 ih_cntl = RREG32(IH_CNTL);
5494         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5495
5496         ih_cntl |= ENABLE_INTR;
5497         ih_rb_cntl |= IH_RB_ENABLE;
5498         WREG32(IH_CNTL, ih_cntl);
5499         WREG32(IH_RB_CNTL, ih_rb_cntl);
5500         rdev->ih.enabled = true;
5501 }
5502
5503 static void si_disable_interrupts(struct radeon_device *rdev)
5504 {
5505         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5506         u32 ih_cntl = RREG32(IH_CNTL);
5507
5508         ih_rb_cntl &= ~IH_RB_ENABLE;
5509         ih_cntl &= ~ENABLE_INTR;
5510         WREG32(IH_RB_CNTL, ih_rb_cntl);
5511         WREG32(IH_CNTL, ih_cntl);
5512         /* set rptr, wptr to 0 */
5513         WREG32(IH_RB_RPTR, 0);
5514         WREG32(IH_RB_WPTR, 0);
5515         rdev->ih.enabled = false;
5516         rdev->ih.rptr = 0;
5517 }
5518
5519 static void si_disable_interrupt_state(struct radeon_device *rdev)
5520 {
5521         u32 tmp;
5522
5523         tmp = RREG32(CP_INT_CNTL_RING0) &
5524                 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5525         WREG32(CP_INT_CNTL_RING0, tmp);
5526         WREG32(CP_INT_CNTL_RING1, 0);
5527         WREG32(CP_INT_CNTL_RING2, 0);
5528         tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5529         WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
5530         tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5531         WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
5532         WREG32(GRBM_INT_CNTL, 0);
5533         if (rdev->num_crtc >= 2) {
5534                 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5535                 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5536         }
5537         if (rdev->num_crtc >= 4) {
5538                 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5539                 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5540         }
5541         if (rdev->num_crtc >= 6) {
5542                 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5543                 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5544         }
5545
5546         if (rdev->num_crtc >= 2) {
5547                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5548                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5549         }
5550         if (rdev->num_crtc >= 4) {
5551                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5552                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5553         }
5554         if (rdev->num_crtc >= 6) {
5555                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5556                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5557         }
5558
5559         if (!ASIC_IS_NODCE(rdev)) {
5560                 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
5561
5562                 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5563                 WREG32(DC_HPD1_INT_CONTROL, tmp);
5564                 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5565                 WREG32(DC_HPD2_INT_CONTROL, tmp);
5566                 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5567                 WREG32(DC_HPD3_INT_CONTROL, tmp);
5568                 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5569                 WREG32(DC_HPD4_INT_CONTROL, tmp);
5570                 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5571                 WREG32(DC_HPD5_INT_CONTROL, tmp);
5572                 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5573                 WREG32(DC_HPD6_INT_CONTROL, tmp);
5574         }
5575 }
5576
5577 static int si_irq_init(struct radeon_device *rdev)
5578 {
5579         int ret = 0;
5580         int rb_bufsz;
5581         u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
5582
5583         /* allocate ring */
5584         ret = r600_ih_ring_alloc(rdev);
5585         if (ret)
5586                 return ret;
5587
5588         /* disable irqs */
5589         si_disable_interrupts(rdev);
5590
5591         /* init rlc */
5592         ret = si_rlc_resume(rdev);
5593         if (ret) {
5594                 r600_ih_ring_fini(rdev);
5595                 return ret;
5596         }
5597
5598         /* setup interrupt control */
5599         /* set dummy read address to ring address */
5600         WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
5601         interrupt_cntl = RREG32(INTERRUPT_CNTL);
5602         /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
5603          * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
5604          */
5605         interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
5606         /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
5607         interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
5608         WREG32(INTERRUPT_CNTL, interrupt_cntl);
5609
5610         WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
5611         rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
5612
5613         ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
5614                       IH_WPTR_OVERFLOW_CLEAR |
5615                       (rb_bufsz << 1));
5616
5617         if (rdev->wb.enabled)
5618                 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
5619
5620         /* set the writeback address whether it's enabled or not */
5621         WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
5622         WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
5623
5624         WREG32(IH_RB_CNTL, ih_rb_cntl);
5625
5626         /* set rptr, wptr to 0 */
5627         WREG32(IH_RB_RPTR, 0);
5628         WREG32(IH_RB_WPTR, 0);
5629
5630         /* Default settings for IH_CNTL (disabled at first) */
5631         ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
5632         /* RPTR_REARM only works if msi's are enabled */
5633         if (rdev->msi_enabled)
5634                 ih_cntl |= RPTR_REARM;
5635         WREG32(IH_CNTL, ih_cntl);
5636
5637         /* force the active interrupt state to all disabled */
5638         si_disable_interrupt_state(rdev);
5639
5640         pci_set_master(rdev->pdev);
5641
5642         /* enable irqs */
5643         si_enable_interrupts(rdev);
5644
5645         return ret;
5646 }
5647
5648 int si_irq_set(struct radeon_device *rdev)
5649 {
5650         u32 cp_int_cntl;
5651         u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
5652         u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
5653         u32 hpd1 = 0, hpd2 = 0, hpd3 = 0, hpd4 = 0, hpd5 = 0, hpd6 = 0;
5654         u32 grbm_int_cntl = 0;
5655         u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
5656         u32 dma_cntl, dma_cntl1;
5657         u32 thermal_int = 0;
5658
5659         if (!rdev->irq.installed) {
5660                 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
5661                 return -EINVAL;
5662         }
5663         /* don't enable anything if the ih is disabled */
5664         if (!rdev->ih.enabled) {
5665                 si_disable_interrupts(rdev);
5666                 /* force the active interrupt state to all disabled */
5667                 si_disable_interrupt_state(rdev);
5668                 return 0;
5669         }
5670
5671         cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
5672                 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5673
5674         if (!ASIC_IS_NODCE(rdev)) {
5675                 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
5676                 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
5677                 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
5678                 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
5679                 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
5680                 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
5681         }
5682
5683         dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5684         dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5685
5686         thermal_int = RREG32(CG_THERMAL_INT) &
5687                 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
5688
5689         /* enable CP interrupts on all rings */
5690         if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
5691                 DRM_DEBUG("si_irq_set: sw int gfx\n");
5692                 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
5693         }
5694         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
5695                 DRM_DEBUG("si_irq_set: sw int cp1\n");
5696                 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
5697         }
5698         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
5699                 DRM_DEBUG("si_irq_set: sw int cp2\n");
5700                 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
5701         }
5702         if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
5703                 DRM_DEBUG("si_irq_set: sw int dma\n");
5704                 dma_cntl |= TRAP_ENABLE;
5705         }
5706
5707         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
5708                 DRM_DEBUG("si_irq_set: sw int dma1\n");
5709                 dma_cntl1 |= TRAP_ENABLE;
5710         }
5711         if (rdev->irq.crtc_vblank_int[0] ||
5712             atomic_read(&rdev->irq.pflip[0])) {
5713                 DRM_DEBUG("si_irq_set: vblank 0\n");
5714                 crtc1 |= VBLANK_INT_MASK;
5715         }
5716         if (rdev->irq.crtc_vblank_int[1] ||
5717             atomic_read(&rdev->irq.pflip[1])) {
5718                 DRM_DEBUG("si_irq_set: vblank 1\n");
5719                 crtc2 |= VBLANK_INT_MASK;
5720         }
5721         if (rdev->irq.crtc_vblank_int[2] ||
5722             atomic_read(&rdev->irq.pflip[2])) {
5723                 DRM_DEBUG("si_irq_set: vblank 2\n");
5724                 crtc3 |= VBLANK_INT_MASK;
5725         }
5726         if (rdev->irq.crtc_vblank_int[3] ||
5727             atomic_read(&rdev->irq.pflip[3])) {
5728                 DRM_DEBUG("si_irq_set: vblank 3\n");
5729                 crtc4 |= VBLANK_INT_MASK;
5730         }
5731         if (rdev->irq.crtc_vblank_int[4] ||
5732             atomic_read(&rdev->irq.pflip[4])) {
5733                 DRM_DEBUG("si_irq_set: vblank 4\n");
5734                 crtc5 |= VBLANK_INT_MASK;
5735         }
5736         if (rdev->irq.crtc_vblank_int[5] ||
5737             atomic_read(&rdev->irq.pflip[5])) {
5738                 DRM_DEBUG("si_irq_set: vblank 5\n");
5739                 crtc6 |= VBLANK_INT_MASK;
5740         }
5741         if (rdev->irq.hpd[0]) {
5742                 DRM_DEBUG("si_irq_set: hpd 1\n");
5743                 hpd1 |= DC_HPDx_INT_EN;
5744         }
5745         if (rdev->irq.hpd[1]) {
5746                 DRM_DEBUG("si_irq_set: hpd 2\n");
5747                 hpd2 |= DC_HPDx_INT_EN;
5748         }
5749         if (rdev->irq.hpd[2]) {
5750                 DRM_DEBUG("si_irq_set: hpd 3\n");
5751                 hpd3 |= DC_HPDx_INT_EN;
5752         }
5753         if (rdev->irq.hpd[3]) {
5754                 DRM_DEBUG("si_irq_set: hpd 4\n");
5755                 hpd4 |= DC_HPDx_INT_EN;
5756         }
5757         if (rdev->irq.hpd[4]) {
5758                 DRM_DEBUG("si_irq_set: hpd 5\n");
5759                 hpd5 |= DC_HPDx_INT_EN;
5760         }
5761         if (rdev->irq.hpd[5]) {
5762                 DRM_DEBUG("si_irq_set: hpd 6\n");
5763                 hpd6 |= DC_HPDx_INT_EN;
5764         }
5765
5766         WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
5767         WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
5768         WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
5769
5770         WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl);
5771         WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1);
5772
5773         WREG32(GRBM_INT_CNTL, grbm_int_cntl);
5774
5775         if (rdev->irq.dpm_thermal) {
5776                 DRM_DEBUG("dpm thermal\n");
5777                 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
5778         }
5779
5780         if (rdev->num_crtc >= 2) {
5781                 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
5782                 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
5783         }
5784         if (rdev->num_crtc >= 4) {
5785                 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
5786                 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
5787         }
5788         if (rdev->num_crtc >= 6) {
5789                 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
5790                 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
5791         }
5792
5793         if (rdev->num_crtc >= 2) {
5794                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
5795                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
5796         }
5797         if (rdev->num_crtc >= 4) {
5798                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
5799                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
5800         }
5801         if (rdev->num_crtc >= 6) {
5802                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
5803                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
5804         }
5805
5806         if (!ASIC_IS_NODCE(rdev)) {
5807                 WREG32(DC_HPD1_INT_CONTROL, hpd1);
5808                 WREG32(DC_HPD2_INT_CONTROL, hpd2);
5809                 WREG32(DC_HPD3_INT_CONTROL, hpd3);
5810                 WREG32(DC_HPD4_INT_CONTROL, hpd4);
5811                 WREG32(DC_HPD5_INT_CONTROL, hpd5);
5812                 WREG32(DC_HPD6_INT_CONTROL, hpd6);
5813         }
5814
5815         WREG32(CG_THERMAL_INT, thermal_int);
5816
5817         return 0;
5818 }
5819
5820 static inline void si_irq_ack(struct radeon_device *rdev)
5821 {
5822         u32 tmp;
5823
5824         if (ASIC_IS_NODCE(rdev))
5825                 return;
5826
5827         rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
5828         rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
5829         rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
5830         rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
5831         rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
5832         rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
5833         rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
5834         rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
5835         if (rdev->num_crtc >= 4) {
5836                 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
5837                 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
5838         }
5839         if (rdev->num_crtc >= 6) {
5840                 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
5841                 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
5842         }
5843
5844         if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
5845                 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5846         if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
5847                 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5848         if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
5849                 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
5850         if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
5851                 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
5852         if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
5853                 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
5854         if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
5855                 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
5856
5857         if (rdev->num_crtc >= 4) {
5858                 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
5859                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5860                 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
5861                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5862                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
5863                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
5864                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
5865                         WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
5866                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
5867                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
5868                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
5869                         WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
5870         }
5871
5872         if (rdev->num_crtc >= 6) {
5873                 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
5874                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5875                 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
5876                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5877                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
5878                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
5879                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
5880                         WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
5881                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
5882                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
5883                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
5884                         WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
5885         }
5886
5887         if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
5888                 tmp = RREG32(DC_HPD1_INT_CONTROL);
5889                 tmp |= DC_HPDx_INT_ACK;
5890                 WREG32(DC_HPD1_INT_CONTROL, tmp);
5891         }
5892         if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
5893                 tmp = RREG32(DC_HPD2_INT_CONTROL);
5894                 tmp |= DC_HPDx_INT_ACK;
5895                 WREG32(DC_HPD2_INT_CONTROL, tmp);
5896         }
5897         if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
5898                 tmp = RREG32(DC_HPD3_INT_CONTROL);
5899                 tmp |= DC_HPDx_INT_ACK;
5900                 WREG32(DC_HPD3_INT_CONTROL, tmp);
5901         }
5902         if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
5903                 tmp = RREG32(DC_HPD4_INT_CONTROL);
5904                 tmp |= DC_HPDx_INT_ACK;
5905                 WREG32(DC_HPD4_INT_CONTROL, tmp);
5906         }
5907         if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
5908                 tmp = RREG32(DC_HPD5_INT_CONTROL);
5909                 tmp |= DC_HPDx_INT_ACK;
5910                 WREG32(DC_HPD5_INT_CONTROL, tmp);
5911         }
5912         if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
5913                 tmp = RREG32(DC_HPD5_INT_CONTROL);
5914                 tmp |= DC_HPDx_INT_ACK;
5915                 WREG32(DC_HPD6_INT_CONTROL, tmp);
5916         }
5917 }
5918
5919 static void si_irq_disable(struct radeon_device *rdev)
5920 {
5921         si_disable_interrupts(rdev);
5922         /* Wait and acknowledge irq */
5923         mdelay(1);
5924         si_irq_ack(rdev);
5925         si_disable_interrupt_state(rdev);
5926 }
5927
5928 static void si_irq_suspend(struct radeon_device *rdev)
5929 {
5930         si_irq_disable(rdev);
5931         si_rlc_stop(rdev);
5932 }
5933
5934 static void si_irq_fini(struct radeon_device *rdev)
5935 {
5936         si_irq_suspend(rdev);
5937         r600_ih_ring_fini(rdev);
5938 }
5939
5940 static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
5941 {
5942         u32 wptr, tmp;
5943
5944         if (rdev->wb.enabled)
5945                 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
5946         else
5947                 wptr = RREG32(IH_RB_WPTR);
5948
5949         if (wptr & RB_OVERFLOW) {
5950                 /* When a ring buffer overflow happen start parsing interrupt
5951                  * from the last not overwritten vector (wptr + 16). Hopefully
5952                  * this should allow us to catchup.
5953                  */
5954                 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
5955                         wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
5956                 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
5957                 tmp = RREG32(IH_RB_CNTL);
5958                 tmp |= IH_WPTR_OVERFLOW_CLEAR;
5959                 WREG32(IH_RB_CNTL, tmp);
5960         }
5961         return (wptr & rdev->ih.ptr_mask);
5962 }
5963
5964 /*        SI IV Ring
5965  * Each IV ring entry is 128 bits:
5966  * [7:0]    - interrupt source id
5967  * [31:8]   - reserved
5968  * [59:32]  - interrupt source data
5969  * [63:60]  - reserved
5970  * [71:64]  - RINGID
5971  * [79:72]  - VMID
5972  * [127:80] - reserved
5973  */
5974 int si_irq_process(struct radeon_device *rdev)
5975 {
5976         u32 wptr;
5977         u32 rptr;
5978         u32 src_id, src_data, ring_id;
5979         u32 ring_index;
5980         bool queue_hotplug = false;
5981         bool queue_thermal = false;
5982         u32 status, addr;
5983
5984         if (!rdev->ih.enabled || rdev->shutdown)
5985                 return IRQ_NONE;
5986
5987         wptr = si_get_ih_wptr(rdev);
5988
5989 restart_ih:
5990         /* is somebody else already processing irqs? */
5991         if (atomic_xchg(&rdev->ih.lock, 1))
5992                 return IRQ_NONE;
5993
5994         rptr = rdev->ih.rptr;
5995         DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
5996
5997         /* Order reading of wptr vs. reading of IH ring data */
5998         rmb();
5999
6000         /* display interrupts */
6001         si_irq_ack(rdev);
6002
6003         while (rptr != wptr) {
6004                 /* wptr/rptr are in bytes! */
6005                 ring_index = rptr / 4;
6006                 src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
6007                 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
6008                 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
6009
6010                 switch (src_id) {
6011                 case 1: /* D1 vblank/vline */
6012                         switch (src_data) {
6013                         case 0: /* D1 vblank */
6014                                 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
6015                                         if (rdev->irq.crtc_vblank_int[0]) {
6016                                                 drm_handle_vblank(rdev->ddev, 0);
6017                                                 rdev->pm.vblank_sync = true;
6018                                                 wake_up(&rdev->irq.vblank_queue);
6019                                         }
6020                                         if (atomic_read(&rdev->irq.pflip[0]))
6021                                                 radeon_crtc_handle_flip(rdev, 0);
6022                                         rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
6023                                         DRM_DEBUG("IH: D1 vblank\n");
6024                                 }
6025                                 break;
6026                         case 1: /* D1 vline */
6027                                 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
6028                                         rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
6029                                         DRM_DEBUG("IH: D1 vline\n");
6030                                 }
6031                                 break;
6032                         default:
6033                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6034                                 break;
6035                         }
6036                         break;
6037                 case 2: /* D2 vblank/vline */
6038                         switch (src_data) {
6039                         case 0: /* D2 vblank */
6040                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
6041                                         if (rdev->irq.crtc_vblank_int[1]) {
6042                                                 drm_handle_vblank(rdev->ddev, 1);
6043                                                 rdev->pm.vblank_sync = true;
6044                                                 wake_up(&rdev->irq.vblank_queue);
6045                                         }
6046                                         if (atomic_read(&rdev->irq.pflip[1]))
6047                                                 radeon_crtc_handle_flip(rdev, 1);
6048                                         rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
6049                                         DRM_DEBUG("IH: D2 vblank\n");
6050                                 }
6051                                 break;
6052                         case 1: /* D2 vline */
6053                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
6054                                         rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
6055                                         DRM_DEBUG("IH: D2 vline\n");
6056                                 }
6057                                 break;
6058                         default:
6059                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6060                                 break;
6061                         }
6062                         break;
6063                 case 3: /* D3 vblank/vline */
6064                         switch (src_data) {
6065                         case 0: /* D3 vblank */
6066                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
6067                                         if (rdev->irq.crtc_vblank_int[2]) {
6068                                                 drm_handle_vblank(rdev->ddev, 2);
6069                                                 rdev->pm.vblank_sync = true;
6070                                                 wake_up(&rdev->irq.vblank_queue);
6071                                         }
6072                                         if (atomic_read(&rdev->irq.pflip[2]))
6073                                                 radeon_crtc_handle_flip(rdev, 2);
6074                                         rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
6075                                         DRM_DEBUG("IH: D3 vblank\n");
6076                                 }
6077                                 break;
6078                         case 1: /* D3 vline */
6079                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
6080                                         rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
6081                                         DRM_DEBUG("IH: D3 vline\n");
6082                                 }
6083                                 break;
6084                         default:
6085                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6086                                 break;
6087                         }
6088                         break;
6089                 case 4: /* D4 vblank/vline */
6090                         switch (src_data) {
6091                         case 0: /* D4 vblank */
6092                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
6093                                         if (rdev->irq.crtc_vblank_int[3]) {
6094                                                 drm_handle_vblank(rdev->ddev, 3);
6095                                                 rdev->pm.vblank_sync = true;
6096                                                 wake_up(&rdev->irq.vblank_queue);
6097                                         }
6098                                         if (atomic_read(&rdev->irq.pflip[3]))
6099                                                 radeon_crtc_handle_flip(rdev, 3);
6100                                         rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
6101                                         DRM_DEBUG("IH: D4 vblank\n");
6102                                 }
6103                                 break;
6104                         case 1: /* D4 vline */
6105                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
6106                                         rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
6107                                         DRM_DEBUG("IH: D4 vline\n");
6108                                 }
6109                                 break;
6110                         default:
6111                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6112                                 break;
6113                         }
6114                         break;
6115                 case 5: /* D5 vblank/vline */
6116                         switch (src_data) {
6117                         case 0: /* D5 vblank */
6118                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
6119                                         if (rdev->irq.crtc_vblank_int[4]) {
6120                                                 drm_handle_vblank(rdev->ddev, 4);
6121                                                 rdev->pm.vblank_sync = true;
6122                                                 wake_up(&rdev->irq.vblank_queue);
6123                                         }
6124                                         if (atomic_read(&rdev->irq.pflip[4]))
6125                                                 radeon_crtc_handle_flip(rdev, 4);
6126                                         rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
6127                                         DRM_DEBUG("IH: D5 vblank\n");
6128                                 }
6129                                 break;
6130                         case 1: /* D5 vline */
6131                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
6132                                         rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
6133                                         DRM_DEBUG("IH: D5 vline\n");
6134                                 }
6135                                 break;
6136                         default:
6137                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6138                                 break;
6139                         }
6140                         break;
6141                 case 6: /* D6 vblank/vline */
6142                         switch (src_data) {
6143                         case 0: /* D6 vblank */
6144                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
6145                                         if (rdev->irq.crtc_vblank_int[5]) {
6146                                                 drm_handle_vblank(rdev->ddev, 5);
6147                                                 rdev->pm.vblank_sync = true;
6148                                                 wake_up(&rdev->irq.vblank_queue);
6149                                         }
6150                                         if (atomic_read(&rdev->irq.pflip[5]))
6151                                                 radeon_crtc_handle_flip(rdev, 5);
6152                                         rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
6153                                         DRM_DEBUG("IH: D6 vblank\n");
6154                                 }
6155                                 break;
6156                         case 1: /* D6 vline */
6157                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
6158                                         rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
6159                                         DRM_DEBUG("IH: D6 vline\n");
6160                                 }
6161                                 break;
6162                         default:
6163                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6164                                 break;
6165                         }
6166                         break;
6167                 case 42: /* HPD hotplug */
6168                         switch (src_data) {
6169                         case 0:
6170                                 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
6171                                         rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
6172                                         queue_hotplug = true;
6173                                         DRM_DEBUG("IH: HPD1\n");
6174                                 }
6175                                 break;
6176                         case 1:
6177                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
6178                                         rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
6179                                         queue_hotplug = true;
6180                                         DRM_DEBUG("IH: HPD2\n");
6181                                 }
6182                                 break;
6183                         case 2:
6184                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6185                                         rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
6186                                         queue_hotplug = true;
6187                                         DRM_DEBUG("IH: HPD3\n");
6188                                 }
6189                                 break;
6190                         case 3:
6191                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6192                                         rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
6193                                         queue_hotplug = true;
6194                                         DRM_DEBUG("IH: HPD4\n");
6195                                 }
6196                                 break;
6197                         case 4:
6198                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6199                                         rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
6200                                         queue_hotplug = true;
6201                                         DRM_DEBUG("IH: HPD5\n");
6202                                 }
6203                                 break;
6204                         case 5:
6205                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6206                                         rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
6207                                         queue_hotplug = true;
6208                                         DRM_DEBUG("IH: HPD6\n");
6209                                 }
6210                                 break;
6211                         default:
6212                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6213                                 break;
6214                         }
6215                         break;
6216                 case 146:
6217                 case 147:
6218                         addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
6219                         status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
6220                         dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
6221                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
6222                                 addr);
6223                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
6224                                 status);
6225                         si_vm_decode_fault(rdev, status, addr);
6226                         /* reset addr and status */
6227                         WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
6228                         break;
6229                 case 176: /* RINGID0 CP_INT */
6230                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6231                         break;
6232                 case 177: /* RINGID1 CP_INT */
6233                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6234                         break;
6235                 case 178: /* RINGID2 CP_INT */
6236                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6237                         break;
6238                 case 181: /* CP EOP event */
6239                         DRM_DEBUG("IH: CP EOP\n");
6240                         switch (ring_id) {
6241                         case 0:
6242                                 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6243                                 break;
6244                         case 1:
6245                                 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6246                                 break;
6247                         case 2:
6248                                 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6249                                 break;
6250                         }
6251                         break;
6252                 case 224: /* DMA trap event */
6253                         DRM_DEBUG("IH: DMA trap\n");
6254                         radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
6255                         break;
6256                 case 230: /* thermal low to high */
6257                         DRM_DEBUG("IH: thermal low to high\n");
6258                         rdev->pm.dpm.thermal.high_to_low = false;
6259                         queue_thermal = true;
6260                         break;
6261                 case 231: /* thermal high to low */
6262                         DRM_DEBUG("IH: thermal high to low\n");
6263                         rdev->pm.dpm.thermal.high_to_low = true;
6264                         queue_thermal = true;
6265                         break;
6266                 case 233: /* GUI IDLE */
6267                         DRM_DEBUG("IH: GUI idle\n");
6268                         break;
6269                 case 244: /* DMA trap event */
6270                         DRM_DEBUG("IH: DMA1 trap\n");
6271                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6272                         break;
6273                 default:
6274                         DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6275                         break;
6276                 }
6277
6278                 /* wptr/rptr are in bytes! */
6279                 rptr += 16;
6280                 rptr &= rdev->ih.ptr_mask;
6281         }
6282         if (queue_hotplug)
6283                 schedule_work(&rdev->hotplug_work);
6284         if (queue_thermal && rdev->pm.dpm_enabled)
6285                 schedule_work(&rdev->pm.dpm.thermal.work);
6286         rdev->ih.rptr = rptr;
6287         WREG32(IH_RB_RPTR, rdev->ih.rptr);
6288         atomic_set(&rdev->ih.lock, 0);
6289
6290         /* make sure wptr hasn't changed while processing */
6291         wptr = si_get_ih_wptr(rdev);
6292         if (wptr != rptr)
6293                 goto restart_ih;
6294
6295         return IRQ_HANDLED;
6296 }
6297
6298 /*
6299  * startup/shutdown callbacks
6300  */
6301 static int si_startup(struct radeon_device *rdev)
6302 {
6303         struct radeon_ring *ring;
6304         int r;
6305
6306         /* enable pcie gen2/3 link */
6307         si_pcie_gen3_enable(rdev);
6308         /* enable aspm */
6309         si_program_aspm(rdev);
6310
6311         /* scratch needs to be initialized before MC */
6312         r = r600_vram_scratch_init(rdev);
6313         if (r)
6314                 return r;
6315
6316         si_mc_program(rdev);
6317
6318         if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
6319             !rdev->rlc_fw || !rdev->mc_fw) {
6320                 r = si_init_microcode(rdev);
6321                 if (r) {
6322                         DRM_ERROR("Failed to load firmware!\n");
6323                         return r;
6324                 }
6325         }
6326
6327         r = si_mc_load_microcode(rdev);
6328         if (r) {
6329                 DRM_ERROR("Failed to load MC firmware!\n");
6330                 return r;
6331         }
6332
6333         r = si_pcie_gart_enable(rdev);
6334         if (r)
6335                 return r;
6336         si_gpu_init(rdev);
6337
6338         /* allocate rlc buffers */
6339         if (rdev->family == CHIP_VERDE) {
6340                 rdev->rlc.reg_list = verde_rlc_save_restore_register_list;
6341                 rdev->rlc.reg_list_size =
6342                         (u32)ARRAY_SIZE(verde_rlc_save_restore_register_list);
6343         }
6344         rdev->rlc.cs_data = si_cs_data;
6345         r = sumo_rlc_init(rdev);
6346         if (r) {
6347                 DRM_ERROR("Failed to init rlc BOs!\n");
6348                 return r;
6349         }
6350
6351         /* allocate wb buffer */
6352         r = radeon_wb_init(rdev);
6353         if (r)
6354                 return r;
6355
6356         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
6357         if (r) {
6358                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6359                 return r;
6360         }
6361
6362         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6363         if (r) {
6364                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6365                 return r;
6366         }
6367
6368         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6369         if (r) {
6370                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6371                 return r;
6372         }
6373
6374         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
6375         if (r) {
6376                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6377                 return r;
6378         }
6379
6380         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6381         if (r) {
6382                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6383                 return r;
6384         }
6385
6386         if (rdev->has_uvd) {
6387                 r = uvd_v2_2_resume(rdev);
6388                 if (!r) {
6389                         r = radeon_fence_driver_start_ring(rdev,
6390                                                            R600_RING_TYPE_UVD_INDEX);
6391                         if (r)
6392                                 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
6393                 }
6394                 if (r)
6395                         rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
6396         }
6397
6398         /* Enable IRQ */
6399         if (!rdev->irq.installed) {
6400                 r = radeon_irq_kms_init(rdev);
6401                 if (r)
6402                         return r;
6403         }
6404
6405         r = si_irq_init(rdev);
6406         if (r) {
6407                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
6408                 radeon_irq_kms_fini(rdev);
6409                 return r;
6410         }
6411         si_irq_set(rdev);
6412
6413         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6414         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
6415                              CP_RB0_RPTR, CP_RB0_WPTR,
6416                              RADEON_CP_PACKET2);
6417         if (r)
6418                 return r;
6419
6420         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6421         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
6422                              CP_RB1_RPTR, CP_RB1_WPTR,
6423                              RADEON_CP_PACKET2);
6424         if (r)
6425                 return r;
6426
6427         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6428         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
6429                              CP_RB2_RPTR, CP_RB2_WPTR,
6430                              RADEON_CP_PACKET2);
6431         if (r)
6432                 return r;
6433
6434         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6435         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
6436                              DMA_RB_RPTR + DMA0_REGISTER_OFFSET,
6437                              DMA_RB_WPTR + DMA0_REGISTER_OFFSET,
6438                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
6439         if (r)
6440                 return r;
6441
6442         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6443         r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
6444                              DMA_RB_RPTR + DMA1_REGISTER_OFFSET,
6445                              DMA_RB_WPTR + DMA1_REGISTER_OFFSET,
6446                              DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
6447         if (r)
6448                 return r;
6449
6450         r = si_cp_load_microcode(rdev);
6451         if (r)
6452                 return r;
6453         r = si_cp_resume(rdev);
6454         if (r)
6455                 return r;
6456
6457         r = cayman_dma_resume(rdev);
6458         if (r)
6459                 return r;
6460
6461         if (rdev->has_uvd) {
6462                 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6463                 if (ring->ring_size) {
6464                         r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
6465                                              UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
6466                                              RADEON_CP_PACKET2);
6467                         if (!r)
6468                                 r = uvd_v1_0_init(rdev);
6469                         if (r)
6470                                 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
6471                 }
6472         }
6473
6474         r = radeon_ib_pool_init(rdev);
6475         if (r) {
6476                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
6477                 return r;
6478         }
6479
6480         r = radeon_vm_manager_init(rdev);
6481         if (r) {
6482                 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
6483                 return r;
6484         }
6485
6486         r = dce6_audio_init(rdev);
6487         if (r)
6488                 return r;
6489
6490         return 0;
6491 }
6492
6493 int si_resume(struct radeon_device *rdev)
6494 {
6495         int r;
6496
6497         /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
6498          * posting will perform necessary task to bring back GPU into good
6499          * shape.
6500          */
6501         /* post card */
6502         atom_asic_init(rdev->mode_info.atom_context);
6503
6504         /* init golden registers */
6505         si_init_golden_registers(rdev);
6506
6507         rdev->accel_working = true;
6508         r = si_startup(rdev);
6509         if (r) {
6510                 DRM_ERROR("si startup failed on resume\n");
6511                 rdev->accel_working = false;
6512                 return r;
6513         }
6514
6515         return r;
6516
6517 }
6518
6519 int si_suspend(struct radeon_device *rdev)
6520 {
6521         dce6_audio_fini(rdev);
6522         radeon_vm_manager_fini(rdev);
6523         si_cp_enable(rdev, false);
6524         cayman_dma_stop(rdev);
6525         if (rdev->has_uvd) {
6526                 uvd_v1_0_fini(rdev);
6527                 radeon_uvd_suspend(rdev);
6528         }
6529         si_fini_pg(rdev);
6530         si_fini_cg(rdev);
6531         si_irq_suspend(rdev);
6532         radeon_wb_disable(rdev);
6533         si_pcie_gart_disable(rdev);
6534         return 0;
6535 }
6536
6537 /* Plan is to move initialization in that function and use
6538  * helper function so that radeon_device_init pretty much
6539  * do nothing more than calling asic specific function. This
6540  * should also allow to remove a bunch of callback function
6541  * like vram_info.
6542  */
6543 int si_init(struct radeon_device *rdev)
6544 {
6545         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6546         int r;
6547
6548         /* Read BIOS */
6549         if (!radeon_get_bios(rdev)) {
6550                 if (ASIC_IS_AVIVO(rdev))
6551                         return -EINVAL;
6552         }
6553         /* Must be an ATOMBIOS */
6554         if (!rdev->is_atom_bios) {
6555                 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
6556                 return -EINVAL;
6557         }
6558         r = radeon_atombios_init(rdev);
6559         if (r)
6560                 return r;
6561
6562         /* Post card if necessary */
6563         if (!radeon_card_posted(rdev)) {
6564                 if (!rdev->bios) {
6565                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
6566                         return -EINVAL;
6567                 }
6568                 DRM_INFO("GPU not posted. posting now...\n");
6569                 atom_asic_init(rdev->mode_info.atom_context);
6570         }
6571         /* init golden registers */
6572         si_init_golden_registers(rdev);
6573         /* Initialize scratch registers */
6574         si_scratch_init(rdev);
6575         /* Initialize surface registers */
6576         radeon_surface_init(rdev);
6577         /* Initialize clocks */
6578         radeon_get_clock_info(rdev->ddev);
6579
6580         /* Fence driver */
6581         r = radeon_fence_driver_init(rdev);
6582         if (r)
6583                 return r;
6584
6585         /* initialize memory controller */
6586         r = si_mc_init(rdev);
6587         if (r)
6588                 return r;
6589         /* Memory manager */
6590         r = radeon_bo_init(rdev);
6591         if (r)
6592                 return r;
6593
6594         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6595         ring->ring_obj = NULL;
6596         r600_ring_init(rdev, ring, 1024 * 1024);
6597
6598         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6599         ring->ring_obj = NULL;
6600         r600_ring_init(rdev, ring, 1024 * 1024);
6601
6602         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6603         ring->ring_obj = NULL;
6604         r600_ring_init(rdev, ring, 1024 * 1024);
6605
6606         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6607         ring->ring_obj = NULL;
6608         r600_ring_init(rdev, ring, 64 * 1024);
6609
6610         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6611         ring->ring_obj = NULL;
6612         r600_ring_init(rdev, ring, 64 * 1024);
6613
6614         if (rdev->has_uvd) {
6615                 r = radeon_uvd_init(rdev);
6616                 if (!r) {
6617                         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6618                         ring->ring_obj = NULL;
6619                         r600_ring_init(rdev, ring, 4096);
6620                 }
6621         }
6622
6623         rdev->ih.ring_obj = NULL;
6624         r600_ih_ring_init(rdev, 64 * 1024);
6625
6626         r = r600_pcie_gart_init(rdev);
6627         if (r)
6628                 return r;
6629
6630         rdev->accel_working = true;
6631         r = si_startup(rdev);
6632         if (r) {
6633                 dev_err(rdev->dev, "disabling GPU acceleration\n");
6634                 si_cp_fini(rdev);
6635                 cayman_dma_fini(rdev);
6636                 si_irq_fini(rdev);
6637                 sumo_rlc_fini(rdev);
6638                 radeon_wb_fini(rdev);
6639                 radeon_ib_pool_fini(rdev);
6640                 radeon_vm_manager_fini(rdev);
6641                 radeon_irq_kms_fini(rdev);
6642                 si_pcie_gart_fini(rdev);
6643                 rdev->accel_working = false;
6644         }
6645
6646         /* Don't start up if the MC ucode is missing.
6647          * The default clocks and voltages before the MC ucode
6648          * is loaded are not suffient for advanced operations.
6649          */
6650         if (!rdev->mc_fw) {
6651                 DRM_ERROR("radeon: MC ucode required for NI+.\n");
6652                 return -EINVAL;
6653         }
6654
6655         return 0;
6656 }
6657
6658 void si_fini(struct radeon_device *rdev)
6659 {
6660         si_cp_fini(rdev);
6661         cayman_dma_fini(rdev);
6662         si_fini_pg(rdev);
6663         si_fini_cg(rdev);
6664         si_irq_fini(rdev);
6665         sumo_rlc_fini(rdev);
6666         radeon_wb_fini(rdev);
6667         radeon_vm_manager_fini(rdev);
6668         radeon_ib_pool_fini(rdev);
6669         radeon_irq_kms_fini(rdev);
6670         if (rdev->has_uvd) {
6671                 uvd_v1_0_fini(rdev);
6672                 radeon_uvd_fini(rdev);
6673         }
6674         si_pcie_gart_fini(rdev);
6675         r600_vram_scratch_fini(rdev);
6676         radeon_gem_fini(rdev);
6677         radeon_fence_driver_fini(rdev);
6678         radeon_bo_fini(rdev);
6679         radeon_atombios_fini(rdev);
6680         kfree(rdev->bios);
6681         rdev->bios = NULL;
6682 }
6683
6684 /**
6685  * si_get_gpu_clock_counter - return GPU clock counter snapshot
6686  *
6687  * @rdev: radeon_device pointer
6688  *
6689  * Fetches a GPU clock counter snapshot (SI).
6690  * Returns the 64 bit clock counter snapshot.
6691  */
6692 uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
6693 {
6694         uint64_t clock;
6695
6696         mutex_lock(&rdev->gpu_clock_mutex);
6697         WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
6698         clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
6699                 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
6700         mutex_unlock(&rdev->gpu_clock_mutex);
6701         return clock;
6702 }
6703
6704 int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
6705 {
6706         unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
6707         int r;
6708
6709         /* bypass vclk and dclk with bclk */
6710         WREG32_P(CG_UPLL_FUNC_CNTL_2,
6711                 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
6712                 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
6713
6714         /* put PLL in bypass mode */
6715         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
6716
6717         if (!vclk || !dclk) {
6718                 /* keep the Bypass mode, put PLL to sleep */
6719                 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
6720                 return 0;
6721         }
6722
6723         r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
6724                                           16384, 0x03FFFFFF, 0, 128, 5,
6725                                           &fb_div, &vclk_div, &dclk_div);
6726         if (r)
6727                 return r;
6728
6729         /* set RESET_ANTI_MUX to 0 */
6730         WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
6731
6732         /* set VCO_MODE to 1 */
6733         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
6734
6735         /* toggle UPLL_SLEEP to 1 then back to 0 */
6736         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
6737         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
6738
6739         /* deassert UPLL_RESET */
6740         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
6741
6742         mdelay(1);
6743
6744         r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
6745         if (r)
6746                 return r;
6747
6748         /* assert UPLL_RESET again */
6749         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
6750
6751         /* disable spread spectrum. */
6752         WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
6753
6754         /* set feedback divider */
6755         WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
6756
6757         /* set ref divider to 0 */
6758         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
6759
6760         if (fb_div < 307200)
6761                 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
6762         else
6763                 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
6764
6765         /* set PDIV_A and PDIV_B */
6766         WREG32_P(CG_UPLL_FUNC_CNTL_2,
6767                 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
6768                 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
6769
6770         /* give the PLL some time to settle */
6771         mdelay(15);
6772
6773         /* deassert PLL_RESET */
6774         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
6775
6776         mdelay(15);
6777
6778         /* switch from bypass mode to normal mode */
6779         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
6780
6781         r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
6782         if (r)
6783                 return r;
6784
6785         /* switch VCLK and DCLK selection */
6786         WREG32_P(CG_UPLL_FUNC_CNTL_2,
6787                 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
6788                 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
6789
6790         mdelay(100);
6791
6792         return 0;
6793 }
6794
6795 static void si_pcie_gen3_enable(struct radeon_device *rdev)
6796 {
6797         struct pci_dev *root = rdev->pdev->bus->self;
6798         int bridge_pos, gpu_pos;
6799         u32 speed_cntl, mask, current_data_rate;
6800         int ret, i;
6801         u16 tmp16;
6802
6803         if (radeon_pcie_gen2 == 0)
6804                 return;
6805
6806         if (rdev->flags & RADEON_IS_IGP)
6807                 return;
6808
6809         if (!(rdev->flags & RADEON_IS_PCIE))
6810                 return;
6811
6812         ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
6813         if (ret != 0)
6814                 return;
6815
6816         if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
6817                 return;
6818
6819         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
6820         current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
6821                 LC_CURRENT_DATA_RATE_SHIFT;
6822         if (mask & DRM_PCIE_SPEED_80) {
6823                 if (current_data_rate == 2) {
6824                         DRM_INFO("PCIE gen 3 link speeds already enabled\n");
6825                         return;
6826                 }
6827                 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
6828         } else if (mask & DRM_PCIE_SPEED_50) {
6829                 if (current_data_rate == 1) {
6830                         DRM_INFO("PCIE gen 2 link speeds already enabled\n");
6831                         return;
6832                 }
6833                 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
6834         }
6835
6836         bridge_pos = pci_pcie_cap(root);
6837         if (!bridge_pos)
6838                 return;
6839
6840         gpu_pos = pci_pcie_cap(rdev->pdev);
6841         if (!gpu_pos)
6842                 return;
6843
6844         if (mask & DRM_PCIE_SPEED_80) {
6845                 /* re-try equalization if gen3 is not already enabled */
6846                 if (current_data_rate != 2) {
6847                         u16 bridge_cfg, gpu_cfg;
6848                         u16 bridge_cfg2, gpu_cfg2;
6849                         u32 max_lw, current_lw, tmp;
6850
6851                         pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
6852                         pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
6853
6854                         tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
6855                         pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
6856
6857                         tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
6858                         pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
6859
6860                         tmp = RREG32_PCIE(PCIE_LC_STATUS1);
6861                         max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
6862                         current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
6863
6864                         if (current_lw < max_lw) {
6865                                 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
6866                                 if (tmp & LC_RENEGOTIATION_SUPPORT) {
6867                                         tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
6868                                         tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
6869                                         tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
6870                                         WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
6871                                 }
6872                         }
6873
6874                         for (i = 0; i < 10; i++) {
6875                                 /* check status */
6876                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
6877                                 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
6878                                         break;
6879
6880                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
6881                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
6882
6883                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
6884                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
6885
6886                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
6887                                 tmp |= LC_SET_QUIESCE;
6888                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
6889
6890                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
6891                                 tmp |= LC_REDO_EQ;
6892                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
6893
6894                                 mdelay(100);
6895
6896                                 /* linkctl */
6897                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
6898                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
6899                                 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
6900                                 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
6901
6902                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
6903                                 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
6904                                 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
6905                                 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
6906
6907                                 /* linkctl2 */
6908                                 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
6909                                 tmp16 &= ~((1 << 4) | (7 << 9));
6910                                 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
6911                                 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
6912
6913                                 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
6914                                 tmp16 &= ~((1 << 4) | (7 << 9));
6915                                 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
6916                                 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
6917
6918                                 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
6919                                 tmp &= ~LC_SET_QUIESCE;
6920                                 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
6921                         }
6922                 }
6923         }
6924
6925         /* set the link speed */
6926         speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
6927         speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
6928         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
6929
6930         pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
6931         tmp16 &= ~0xf;
6932         if (mask & DRM_PCIE_SPEED_80)
6933                 tmp16 |= 3; /* gen3 */
6934         else if (mask & DRM_PCIE_SPEED_50)
6935                 tmp16 |= 2; /* gen2 */
6936         else
6937                 tmp16 |= 1; /* gen1 */
6938         pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
6939
6940         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
6941         speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
6942         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
6943
6944         for (i = 0; i < rdev->usec_timeout; i++) {
6945                 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
6946                 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
6947                         break;
6948                 udelay(1);
6949         }
6950 }
6951
6952 static void si_program_aspm(struct radeon_device *rdev)
6953 {
6954         u32 data, orig;
6955         bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
6956         bool disable_clkreq = false;
6957
6958         if (radeon_aspm == 0)
6959                 return;
6960
6961         if (!(rdev->flags & RADEON_IS_PCIE))
6962                 return;
6963
6964         orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
6965         data &= ~LC_XMIT_N_FTS_MASK;
6966         data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
6967         if (orig != data)
6968                 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
6969
6970         orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
6971         data |= LC_GO_TO_RECOVERY;
6972         if (orig != data)
6973                 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
6974
6975         orig = data = RREG32_PCIE(PCIE_P_CNTL);
6976         data |= P_IGNORE_EDB_ERR;
6977         if (orig != data)
6978                 WREG32_PCIE(PCIE_P_CNTL, data);
6979
6980         orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
6981         data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
6982         data |= LC_PMI_TO_L1_DIS;
6983         if (!disable_l0s)
6984                 data |= LC_L0S_INACTIVITY(7);
6985
6986         if (!disable_l1) {
6987                 data |= LC_L1_INACTIVITY(7);
6988                 data &= ~LC_PMI_TO_L1_DIS;
6989                 if (orig != data)
6990                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
6991
6992                 if (!disable_plloff_in_l1) {
6993                         bool clk_req_support;
6994
6995                         orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
6996                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
6997                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
6998                         if (orig != data)
6999                                 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7000
7001                         orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7002                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7003                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7004                         if (orig != data)
7005                                 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7006
7007                         orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7008                         data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7009                         data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7010                         if (orig != data)
7011                                 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7012
7013                         orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7014                         data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7015                         data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7016                         if (orig != data)
7017                                 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7018
7019                         if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) {
7020                                 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7021                                 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7022                                 if (orig != data)
7023                                         WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7024
7025                                 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7026                                 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7027                                 if (orig != data)
7028                                         WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7029
7030                                 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2);
7031                                 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7032                                 if (orig != data)
7033                                         WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2, data);
7034
7035                                 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3);
7036                                 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7037                                 if (orig != data)
7038                                         WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3, data);
7039
7040                                 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7041                                 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7042                                 if (orig != data)
7043                                         WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7044
7045                                 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7046                                 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7047                                 if (orig != data)
7048                                         WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7049
7050                                 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2);
7051                                 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7052                                 if (orig != data)
7053                                         WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2, data);
7054
7055                                 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3);
7056                                 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7057                                 if (orig != data)
7058                                         WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3, data);
7059                         }
7060                         orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7061                         data &= ~LC_DYN_LANES_PWR_STATE_MASK;
7062                         data |= LC_DYN_LANES_PWR_STATE(3);
7063                         if (orig != data)
7064                                 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
7065
7066                         orig = data = RREG32_PIF_PHY0(PB0_PIF_CNTL);
7067                         data &= ~LS2_EXIT_TIME_MASK;
7068                         if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7069                                 data |= LS2_EXIT_TIME(5);
7070                         if (orig != data)
7071                                 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
7072
7073                         orig = data = RREG32_PIF_PHY1(PB1_PIF_CNTL);
7074                         data &= ~LS2_EXIT_TIME_MASK;
7075                         if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7076                                 data |= LS2_EXIT_TIME(5);
7077                         if (orig != data)
7078                                 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
7079
7080                         if (!disable_clkreq) {
7081                                 struct pci_dev *root = rdev->pdev->bus->self;
7082                                 u32 lnkcap;
7083
7084                                 clk_req_support = false;
7085                                 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
7086                                 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
7087                                         clk_req_support = true;
7088                         } else {
7089                                 clk_req_support = false;
7090                         }
7091
7092                         if (clk_req_support) {
7093                                 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
7094                                 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
7095                                 if (orig != data)
7096                                         WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
7097
7098                                 orig = data = RREG32(THM_CLK_CNTL);
7099                                 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
7100                                 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
7101                                 if (orig != data)
7102                                         WREG32(THM_CLK_CNTL, data);
7103
7104                                 orig = data = RREG32(MISC_CLK_CNTL);
7105                                 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
7106                                 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
7107                                 if (orig != data)
7108                                         WREG32(MISC_CLK_CNTL, data);
7109
7110                                 orig = data = RREG32(CG_CLKPIN_CNTL);
7111                                 data &= ~BCLK_AS_XCLK;
7112                                 if (orig != data)
7113                                         WREG32(CG_CLKPIN_CNTL, data);
7114
7115                                 orig = data = RREG32(CG_CLKPIN_CNTL_2);
7116                                 data &= ~FORCE_BIF_REFCLK_EN;
7117                                 if (orig != data)
7118                                         WREG32(CG_CLKPIN_CNTL_2, data);
7119
7120                                 orig = data = RREG32(MPLL_BYPASSCLK_SEL);
7121                                 data &= ~MPLL_CLKOUT_SEL_MASK;
7122                                 data |= MPLL_CLKOUT_SEL(4);
7123                                 if (orig != data)
7124                                         WREG32(MPLL_BYPASSCLK_SEL, data);
7125
7126                                 orig = data = RREG32(SPLL_CNTL_MODE);
7127                                 data &= ~SPLL_REFCLK_SEL_MASK;
7128                                 if (orig != data)
7129                                         WREG32(SPLL_CNTL_MODE, data);
7130                         }
7131                 }
7132         } else {
7133                 if (orig != data)
7134                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7135         }
7136
7137         orig = data = RREG32_PCIE(PCIE_CNTL2);
7138         data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
7139         if (orig != data)
7140                 WREG32_PCIE(PCIE_CNTL2, data);
7141
7142         if (!disable_l0s) {
7143                 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7144                 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
7145                         data = RREG32_PCIE(PCIE_LC_STATUS1);
7146                         if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
7147                                 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7148                                 data &= ~LC_L0S_INACTIVITY_MASK;
7149                                 if (orig != data)
7150                                         WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7151                         }
7152                 }
7153         }
7154 }