drivers/gpu/drm/radeon/r600.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors: Dave Airlie
  25  *          Alex Deucher
  26  *          Jerome Glisse
  27  */
  28 #include <linux/slab.h>
  29 #include <linux/seq_file.h>
  30 #include <linux/firmware.h>
  31 #include <linux/platform_device.h>
  32 #include <linux/module.h>
  33 #include <drm/drmP.h>
  34 #include <drm/radeon_drm.h>
  35 #include "radeon.h"
  36 #include "radeon_asic.h"
  37 #include "radeon_mode.h"
  38 #include "r600d.h"
  39 #include "atom.h"
  40 #include "avivod.h"
  41
  42 #define PFP_UCODE_SIZE 576
  43 #define PM4_UCODE_SIZE 1792
  44 #define RLC_UCODE_SIZE 768
  45 #define R700_PFP_UCODE_SIZE 848
  46 #define R700_PM4_UCODE_SIZE 1360
  47 #define R700_RLC_UCODE_SIZE 1024
  48 #define EVERGREEN_PFP_UCODE_SIZE 1120
  49 #define EVERGREEN_PM4_UCODE_SIZE 1376
  50 #define EVERGREEN_RLC_UCODE_SIZE 768
  51 #define CAYMAN_RLC_UCODE_SIZE 1024
  52 #define ARUBA_RLC_UCODE_SIZE 1536
  53
  54 /* Firmware Names */
  55 MODULE_FIRMWARE("radeon/R600_pfp.bin");
  56 MODULE_FIRMWARE("radeon/R600_me.bin");
  57 MODULE_FIRMWARE("radeon/RV610_pfp.bin");
  58 MODULE_FIRMWARE("radeon/RV610_me.bin");
  59 MODULE_FIRMWARE("radeon/RV630_pfp.bin");
  60 MODULE_FIRMWARE("radeon/RV630_me.bin");
  61 MODULE_FIRMWARE("radeon/RV620_pfp.bin");
  62 MODULE_FIRMWARE("radeon/RV620_me.bin");
  63 MODULE_FIRMWARE("radeon/RV635_pfp.bin");
  64 MODULE_FIRMWARE("radeon/RV635_me.bin");
  65 MODULE_FIRMWARE("radeon/RV670_pfp.bin");
  66 MODULE_FIRMWARE("radeon/RV670_me.bin");
  67 MODULE_FIRMWARE("radeon/RS780_pfp.bin");
  68 MODULE_FIRMWARE("radeon/RS780_me.bin");
  69 MODULE_FIRMWARE("radeon/RV770_pfp.bin");
  70 MODULE_FIRMWARE("radeon/RV770_me.bin");
  71 MODULE_FIRMWARE("radeon/RV730_pfp.bin");
  72 MODULE_FIRMWARE("radeon/RV730_me.bin");
  73 MODULE_FIRMWARE("radeon/RV710_pfp.bin");
  74 MODULE_FIRMWARE("radeon/RV710_me.bin");
  75 MODULE_FIRMWARE("radeon/R600_rlc.bin");
  76 MODULE_FIRMWARE("radeon/R700_rlc.bin");
  77 MODULE_FIRMWARE("radeon/CEDAR_pfp.bin");
  78 MODULE_FIRMWARE("radeon/CEDAR_me.bin");
  79 MODULE_FIRMWARE("radeon/CEDAR_rlc.bin");
  80 MODULE_FIRMWARE("radeon/REDWOOD_pfp.bin");
  81 MODULE_FIRMWARE("radeon/REDWOOD_me.bin");
  82 MODULE_FIRMWARE("radeon/REDWOOD_rlc.bin");
  83 MODULE_FIRMWARE("radeon/JUNIPER_pfp.bin");
  84 MODULE_FIRMWARE("radeon/JUNIPER_me.bin");
  85 MODULE_FIRMWARE("radeon/JUNIPER_rlc.bin");
  86 MODULE_FIRMWARE("radeon/CYPRESS_pfp.bin");
  87 MODULE_FIRMWARE("radeon/CYPRESS_me.bin");
  88 MODULE_FIRMWARE("radeon/CYPRESS_rlc.bin");
  89 MODULE_FIRMWARE("radeon/PALM_pfp.bin");
  90 MODULE_FIRMWARE("radeon/PALM_me.bin");
  91 MODULE_FIRMWARE("radeon/SUMO_rlc.bin");
  92 MODULE_FIRMWARE("radeon/SUMO_pfp.bin");
  93 MODULE_FIRMWARE("radeon/SUMO_me.bin");
  94 MODULE_FIRMWARE("radeon/SUMO2_pfp.bin");
  95 MODULE_FIRMWARE("radeon/SUMO2_me.bin");
  96
  97 static const u32 crtc_offsets[2] =
  98 {
  99         0,
 100         AVIVO_D2CRTC_H_TOTAL - AVIVO_D1CRTC_H_TOTAL
 101 };
 102
 103 int r600_debugfs_mc_info_init(struct radeon_device *rdev);
 104
 105 /* r600,rv610,rv630,rv620,rv635,rv670 */
 106 int r600_mc_wait_for_idle(struct radeon_device *rdev);
 107 static void r600_gpu_init(struct radeon_device *rdev);
 108 void r600_fini(struct radeon_device *rdev);
 109 void r600_irq_disable(struct radeon_device *rdev);
 110 static void r600_pcie_gen2_enable(struct radeon_device *rdev);
 111
 112 /**
 113  * r600_get_xclk - get the xclk
 114  *
 115  * @rdev: radeon_device pointer
 116  *
 117  * Returns the reference clock used by the gfx engine
 118  * (r6xx, IGPs, APUs).
 119  */
 120 u32 r600_get_xclk(struct radeon_device *rdev)
 121 {
 122         return rdev->clock.spll.reference_freq;
 123 }
 124
 125 /* get temperature in millidegrees */
 126 int rv6xx_get_temp(struct radeon_device *rdev)
 127 {
 128         u32 temp = (RREG32(CG_THERMAL_STATUS) & ASIC_T_MASK) >>
 129                 ASIC_T_SHIFT;
 130         int actual_temp = temp & 0xff;
 131
 132         if (temp & 0x100)
 133                 actual_temp -= 256;
 134
 135         return actual_temp * 1000;
 136 }
 137
 138 void r600_pm_get_dynpm_state(struct radeon_device *rdev)
 139 {
 140         int i;
 141
 142         rdev->pm.dynpm_can_upclock = true;
 143         rdev->pm.dynpm_can_downclock = true;
 144
 145         /* power state array is low to high, default is first */
 146         if ((rdev->flags & RADEON_IS_IGP) || (rdev->family == CHIP_R600)) {
 147                 int min_power_state_index = 0;
 148
 149                 if (rdev->pm.num_power_states > 2)
 150                         min_power_state_index = 1;
 151
 152                 switch (rdev->pm.dynpm_planned_action) {
 153                 case DYNPM_ACTION_MINIMUM:
 154                         rdev->pm.requested_power_state_index = min_power_state_index;
 155                         rdev->pm.requested_clock_mode_index = 0;
 156                         rdev->pm.dynpm_can_downclock = false;
 157                         break;
 158                 case DYNPM_ACTION_DOWNCLOCK:
 159                         if (rdev->pm.current_power_state_index == min_power_state_index) {
 160                                 rdev->pm.requested_power_state_index = rdev->pm.current_power_state_index;
 161                                 rdev->pm.dynpm_can_downclock = false;
 162                         } else {
 163                                 if (rdev->pm.active_crtc_count > 1) {
 164                                         for (i = 0; i < rdev->pm.num_power_states; i++) {
 165                                                 if (rdev->pm.power_state[i].flags & RADEON_PM_STATE_SINGLE_DISPLAY_ONLY)
 166                                                         continue;
 167                                                 else if (i >= rdev->pm.current_power_state_index) {
 168                                                         rdev->pm.requested_power_state_index =
 169                                                                 rdev->pm.current_power_state_index;
 170                                                         break;
 171                                                 } else {
 172                                                         rdev->pm.requested_power_state_index = i;
 173                                                         break;
 174                                                 }
 175                                         }
 176                                 } else {
 177                                         if (rdev->pm.current_power_state_index == 0)
 178                                                 rdev->pm.requested_power_state_index =
 179                                                         rdev->pm.num_power_states - 1;
 180                                         else
 181                                                 rdev->pm.requested_power_state_index =
 182                                                         rdev->pm.current_power_state_index - 1;
 183                                 }
 184                         }
 185                         rdev->pm.requested_clock_mode_index = 0;
 186                         /* don't use the power state if crtcs are active and no display flag is set */
 187                         if ((rdev->pm.active_crtc_count > 0) &&
 188                             (rdev->pm.power_state[rdev->pm.requested_power_state_index].
 189                              clock_info[rdev->pm.requested_clock_mode_index].flags &
 190                              RADEON_PM_MODE_NO_DISPLAY)) {
 191                                 rdev->pm.requested_power_state_index++;
 192                         }
 193                         break;
 194                 case DYNPM_ACTION_UPCLOCK:
 195                         if (rdev->pm.current_power_state_index == (rdev->pm.num_power_states - 1)) {
 196                                 rdev->pm.requested_power_state_index = rdev->pm.current_power_state_index;
 197                                 rdev->pm.dynpm_can_upclock = false;
 198                         } else {
 199                                 if (rdev->pm.active_crtc_count > 1) {
 200                                         for (i = (rdev->pm.num_power_states - 1); i >= 0; i--) {
 201                                                 if (rdev->pm.power_state[i].flags & RADEON_PM_STATE_SINGLE_DISPLAY_ONLY)
 202                                                         continue;
 203                                                 else if (i <= rdev->pm.current_power_state_index) {
 204                                                         rdev->pm.requested_power_state_index =
 205                                                                 rdev->pm.current_power_state_index;
 206                                                         break;
 207                                                 } else {
 208                                                         rdev->pm.requested_power_state_index = i;
 209                                                         break;
 210                                                 }
 211                                         }
 212                                 } else
 213                                         rdev->pm.requested_power_state_index =
 214                                                 rdev->pm.current_power_state_index + 1;
 215                         }
 216                         rdev->pm.requested_clock_mode_index = 0;
 217                         break;
 218                 case DYNPM_ACTION_DEFAULT:
 219                         rdev->pm.requested_power_state_index = rdev->pm.default_power_state_index;
 220                         rdev->pm.requested_clock_mode_index = 0;
 221                         rdev->pm.dynpm_can_upclock = false;
 222                         break;
 223                 case DYNPM_ACTION_NONE:
 224                 default:
 225                         DRM_ERROR("Requested mode for not defined action\n");
 226                         return;
 227                 }
 228         } else {
 229                 /* XXX select a power state based on AC/DC, single/dualhead, etc. */
 230                 /* for now just select the first power state and switch between clock modes */
 231                 /* power state array is low to high, default is first (0) */
 232                 if (rdev->pm.active_crtc_count > 1) {
 233                         rdev->pm.requested_power_state_index = -1;
 234                         /* start at 1 as we don't want the default mode */
 235                         for (i = 1; i < rdev->pm.num_power_states; i++) {
 236                                 if (rdev->pm.power_state[i].flags & RADEON_PM_STATE_SINGLE_DISPLAY_ONLY)
 237                                         continue;
 238                                 else if ((rdev->pm.power_state[i].type == POWER_STATE_TYPE_PERFORMANCE) ||
 239                                          (rdev->pm.power_state[i].type == POWER_STATE_TYPE_BATTERY)) {
 240                                         rdev->pm.requested_power_state_index = i;
 241                                         break;
 242                                 }
 243                         }
 244                         /* if nothing selected, grab the default state. */
 245                         if (rdev->pm.requested_power_state_index == -1)
 246                                 rdev->pm.requested_power_state_index = 0;
 247                 } else
 248                         rdev->pm.requested_power_state_index = 1;
 249
 250                 switch (rdev->pm.dynpm_planned_action) {
 251                 case DYNPM_ACTION_MINIMUM:
 252                         rdev->pm.requested_clock_mode_index = 0;
 253                         rdev->pm.dynpm_can_downclock = false;
 254                         break;
 255                 case DYNPM_ACTION_DOWNCLOCK:
 256                         if (rdev->pm.requested_power_state_index == rdev->pm.current_power_state_index) {
 257                                 if (rdev->pm.current_clock_mode_index == 0) {
 258                                         rdev->pm.requested_clock_mode_index = 0;
 259                                         rdev->pm.dynpm_can_downclock = false;
 260                                 } else
 261                                         rdev->pm.requested_clock_mode_index =
 262                                                 rdev->pm.current_clock_mode_index - 1;
 263                         } else {
 264                                 rdev->pm.requested_clock_mode_index = 0;
 265                                 rdev->pm.dynpm_can_downclock = false;
 266                         }
 267                         /* don't use the power state if crtcs are active and no display flag is set */
 268                         if ((rdev->pm.active_crtc_count > 0) &&
 269                             (rdev->pm.power_state[rdev->pm.requested_power_state_index].
 270                              clock_info[rdev->pm.requested_clock_mode_index].flags &
 271                              RADEON_PM_MODE_NO_DISPLAY)) {
 272                                 rdev->pm.requested_clock_mode_index++;
 273                         }
 274                         break;
 275                 case DYNPM_ACTION_UPCLOCK:
 276                         if (rdev->pm.requested_power_state_index == rdev->pm.current_power_state_index) {
 277                                 if (rdev->pm.current_clock_mode_index ==
 278                                     (rdev->pm.power_state[rdev->pm.requested_power_state_index].num_clock_modes - 1)) {
 279                                         rdev->pm.requested_clock_mode_index = rdev->pm.current_clock_mode_index;
 280                                         rdev->pm.dynpm_can_upclock = false;
 281                                 } else
 282                                         rdev->pm.requested_clock_mode_index =
 283                                                 rdev->pm.current_clock_mode_index + 1;
 284                         } else {
 285                                 rdev->pm.requested_clock_mode_index =
 286                                         rdev->pm.power_state[rdev->pm.requested_power_state_index].num_clock_modes - 1;
 287                                 rdev->pm.dynpm_can_upclock = false;
 288                         }
 289                         break;
 290                 case DYNPM_ACTION_DEFAULT:
 291                         rdev->pm.requested_power_state_index = rdev->pm.default_power_state_index;
 292                         rdev->pm.requested_clock_mode_index = 0;
 293                         rdev->pm.dynpm_can_upclock = false;
 294                         break;
 295                 case DYNPM_ACTION_NONE:
 296                 default:
 297                         DRM_ERROR("Requested mode for not defined action\n");
 298                         return;
 299                 }
 300         }
 301
 302         DRM_DEBUG_DRIVER("Requested: e: %d m: %d p: %d\n",
 303                   rdev->pm.power_state[rdev->pm.requested_power_state_index].
 304                   clock_info[rdev->pm.requested_clock_mode_index].sclk,
 305                   rdev->pm.power_state[rdev->pm.requested_power_state_index].
 306                   clock_info[rdev->pm.requested_clock_mode_index].mclk,
 307                   rdev->pm.power_state[rdev->pm.requested_power_state_index].
 308                   pcie_lanes);
 309 }
 310
 311 void rs780_pm_init_profile(struct radeon_device *rdev)
 312 {
 313         if (rdev->pm.num_power_states == 2) {
 314                 /* default */
 315                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 316                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 317                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
 318                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
 319                 /* low sh */
 320                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = 0;
 321                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = 0;
 322                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
 323                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
 324                 /* mid sh */
 325                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = 0;
 326                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = 0;
 327                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
 328                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
 329                 /* high sh */
 330                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = 0;
 331                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = 1;
 332                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
 333                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 0;
 334                 /* low mh */
 335                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = 0;
 336                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = 0;
 337                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
 338                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
 339                 /* mid mh */
 340                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = 0;
 341                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = 0;
 342                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
 343                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
 344                 /* high mh */
 345                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = 0;
 346                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = 1;
 347                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
 348                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 0;
 349         } else if (rdev->pm.num_power_states == 3) {
 350                 /* default */
 351                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 352                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 353                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
 354                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
 355                 /* low sh */
 356                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = 1;
 357                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = 1;
 358                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
 359                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
 360                 /* mid sh */
 361                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = 1;
 362                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = 1;
 363                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
 364                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
 365                 /* high sh */
 366                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = 1;
 367                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = 2;
 368                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
 369                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 0;
 370                 /* low mh */
 371                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = 1;
 372                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = 1;
 373                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
 374                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
 375                 /* mid mh */
 376                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = 1;
 377                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = 1;
 378                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
 379                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
 380                 /* high mh */
 381                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = 1;
 382                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = 2;
 383                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
 384                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 0;
 385         } else {
 386                 /* default */
 387                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 388                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 389                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
 390                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
 391                 /* low sh */
 392                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = 2;
 393                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = 2;
 394                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
 395                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
 396                 /* mid sh */
 397                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = 2;
 398                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = 2;
 399                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
 400                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
 401                 /* high sh */
 402                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = 2;
 403                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = 3;
 404                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
 405                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 0;
 406                 /* low mh */
 407                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = 2;
 408                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = 0;
 409                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
 410                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
 411                 /* mid mh */
 412                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = 2;
 413                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = 0;
 414                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
 415                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
 416                 /* high mh */
 417                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = 2;
 418                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = 3;
 419                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
 420                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 0;
 421         }
 422 }
 423
 424 void r600_pm_init_profile(struct radeon_device *rdev)
 425 {
 426         int idx;
 427
 428         if (rdev->family == CHIP_R600) {
 429                 /* XXX */
 430                 /* default */
 431                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 432                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 433                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
 434                 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
 435                 /* low sh */
 436                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 437                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 438                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
 439                 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
 440                 /* mid sh */
 441                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 442                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 443                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
 444                 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
 445                 /* high sh */
 446                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 447                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 448                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
 449                 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 0;
 450                 /* low mh */
 451                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 452                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 453                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
 454                 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
 455                 /* mid mh */
 456                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 457                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 458                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
 459                 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
 460                 /* high mh */
 461                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 462                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 463                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
 464                 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 0;
 465         } else {
 466                 if (rdev->pm.num_power_states < 4) {
 467                         /* default */
 468                         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 469                         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 470                         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
 471                         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
 472                         /* low sh */
 473                         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = 1;
 474                         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = 1;
 475                         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
 476                         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
 477                         /* mid sh */
 478                         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = 1;
 479                         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = 1;
 480                         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
 481                         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
 482                         /* high sh */
 483                         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = 1;
 484                         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = 1;
 485                         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
 486                         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
 487                         /* low mh */
 488                         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = 2;
 489                         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = 2;
 490                         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
 491                         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
 492                         /* low mh */
 493                         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = 2;
 494                         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = 2;
 495                         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
 496                         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
 497                         /* high mh */
 498                         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = 2;
 499                         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = 2;
 500                         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
 501                         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
 502                 } else {
 503                         /* default */
 504                         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
 505                         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
 506                         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
 507                         rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
 508                         /* low sh */
 509                         if (rdev->flags & RADEON_IS_MOBILITY)
 510                                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
 511                         else
 512                                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
 513                         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
 514                         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
 515                         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
 516                         rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
 517                         /* mid sh */
 518                         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
 519                         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
 520                         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
 521                         rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
 522                         /* high sh */
 523                         idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
 524                         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
 525                         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
 526                         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
 527                         rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
 528                         /* low mh */
 529                         if (rdev->flags & RADEON_IS_MOBILITY)
 530                                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 1);
 531                         else
 532                                 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 1);
 533                         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
 534                         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
 535                         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
 536                         rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
 537                         /* mid mh */
 538                         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
 539                         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
 540                         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
 541                         rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
 542                         /* high mh */
 543                         idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 1);
 544                         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
 545                         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
 546                         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
 547                         rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
 548                 }
 549         }
 550 }
 551
 552 void r600_pm_misc(struct radeon_device *rdev)
 553 {
 554         int req_ps_idx = rdev->pm.requested_power_state_index;
 555         int req_cm_idx = rdev->pm.requested_clock_mode_index;
 556         struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
 557         struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
 558
 559         if ((voltage->type == VOLTAGE_SW) && voltage->voltage) {
 560                 /* 0xff01 is a flag rather then an actual voltage */
 561                 if (voltage->voltage == 0xff01)
 562                         return;
 563                 if (voltage->voltage != rdev->pm.current_vddc) {
 564                         radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
 565                         rdev->pm.current_vddc = voltage->voltage;
 566                         DRM_DEBUG_DRIVER("Setting: v: %d\n", voltage->voltage);
 567                 }
 568         }
 569 }
 570
 571 bool r600_gui_idle(struct radeon_device *rdev)
 572 {
 573         if (RREG32(GRBM_STATUS) & GUI_ACTIVE)
 574                 return false;
 575         else
 576                 return true;
 577 }
 578
 579 /* hpd for digital panel detect/disconnect */
 580 bool r600_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
 581 {
 582         bool connected = false;
 583
 584         if (ASIC_IS_DCE3(rdev)) {
 585                 switch (hpd) {
 586                 case RADEON_HPD_1:
 587                         if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
 588                                 connected = true;
 589                         break;
 590                 case RADEON_HPD_2:
 591                         if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
 592                                 connected = true;
 593                         break;
 594                 case RADEON_HPD_3:
 595                         if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
 596                                 connected = true;
 597                         break;
 598                 case RADEON_HPD_4:
 599                         if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
 600                                 connected = true;
 601                         break;
 602                         /* DCE 3.2 */
 603                 case RADEON_HPD_5:
 604                         if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
 605                                 connected = true;
 606                         break;
 607                 case RADEON_HPD_6:
 608                         if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
 609                                 connected = true;
 610                         break;
 611                 default:
 612                         break;
 613                 }
 614         } else {
 615                 switch (hpd) {
 616                 case RADEON_HPD_1:
 617                         if (RREG32(DC_HOT_PLUG_DETECT1_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
 618                                 connected = true;
 619                         break;
 620                 case RADEON_HPD_2:
 621                         if (RREG32(DC_HOT_PLUG_DETECT2_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
 622                                 connected = true;
 623                         break;
 624                 case RADEON_HPD_3:
 625                         if (RREG32(DC_HOT_PLUG_DETECT3_INT_STATUS) & DC_HOT_PLUG_DETECTx_SENSE)
 626                                 connected = true;
 627                         break;
 628                 default:
 629                         break;
 630                 }
 631         }
 632         return connected;
 633 }
 634
 635 void r600_hpd_set_polarity(struct radeon_device *rdev,
 636                            enum radeon_hpd_id hpd)
 637 {
 638         u32 tmp;
 639         bool connected = r600_hpd_sense(rdev, hpd);
 640
 641         if (ASIC_IS_DCE3(rdev)) {
 642                 switch (hpd) {
 643                 case RADEON_HPD_1:
 644                         tmp = RREG32(DC_HPD1_INT_CONTROL);
 645                         if (connected)
 646                                 tmp &= ~DC_HPDx_INT_POLARITY;
 647                         else
 648                                 tmp |= DC_HPDx_INT_POLARITY;
 649                         WREG32(DC_HPD1_INT_CONTROL, tmp);
 650                         break;
 651                 case RADEON_HPD_2:
 652                         tmp = RREG32(DC_HPD2_INT_CONTROL);
 653                         if (connected)
 654                                 tmp &= ~DC_HPDx_INT_POLARITY;
 655                         else
 656                                 tmp |= DC_HPDx_INT_POLARITY;
 657                         WREG32(DC_HPD2_INT_CONTROL, tmp);
 658                         break;
 659                 case RADEON_HPD_3:
 660                         tmp = RREG32(DC_HPD3_INT_CONTROL);
 661                         if (connected)
 662                                 tmp &= ~DC_HPDx_INT_POLARITY;
 663                         else
 664                                 tmp |= DC_HPDx_INT_POLARITY;
 665                         WREG32(DC_HPD3_INT_CONTROL, tmp);
 666                         break;
 667                 case RADEON_HPD_4:
 668                         tmp = RREG32(DC_HPD4_INT_CONTROL);
 669                         if (connected)
 670                                 tmp &= ~DC_HPDx_INT_POLARITY;
 671                         else
 672                                 tmp |= DC_HPDx_INT_POLARITY;
 673                         WREG32(DC_HPD4_INT_CONTROL, tmp);
 674                         break;
 675                 case RADEON_HPD_5:
 676                         tmp = RREG32(DC_HPD5_INT_CONTROL);
 677                         if (connected)
 678                                 tmp &= ~DC_HPDx_INT_POLARITY;
 679                         else
 680                                 tmp |= DC_HPDx_INT_POLARITY;
 681                         WREG32(DC_HPD5_INT_CONTROL, tmp);
 682                         break;
 683                         /* DCE 3.2 */
 684                 case RADEON_HPD_6:
 685                         tmp = RREG32(DC_HPD6_INT_CONTROL);
 686                         if (connected)
 687                                 tmp &= ~DC_HPDx_INT_POLARITY;
 688                         else
 689                                 tmp |= DC_HPDx_INT_POLARITY;
 690                         WREG32(DC_HPD6_INT_CONTROL, tmp);
 691                         break;
 692                 default:
 693                         break;
 694                 }
 695         } else {
 696                 switch (hpd) {
 697                 case RADEON_HPD_1:
 698                         tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL);
 699                         if (connected)
 700                                 tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
 701                         else
 702                                 tmp |= DC_HOT_PLUG_DETECTx_INT_POLARITY;
 703                         WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
 704                         break;
 705                 case RADEON_HPD_2:
 706                         tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL);
 707                         if (connected)
 708                                 tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
 709                         else
 710                                 tmp |= DC_HOT_PLUG_DETECTx_INT_POLARITY;
 711                         WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
 712                         break;
 713                 case RADEON_HPD_3:
 714                         tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL);
 715                         if (connected)
 716                                 tmp &= ~DC_HOT_PLUG_DETECTx_INT_POLARITY;
 717                         else
 718                                 tmp |= DC_HOT_PLUG_DETECTx_INT_POLARITY;
 719                         WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
 720                         break;
 721                 default:
 722                         break;
 723                 }
 724         }
 725 }
 726
 727 void r600_hpd_init(struct radeon_device *rdev)
 728 {
 729         struct drm_device *dev = rdev->ddev;
 730         struct drm_connector *connector;
 731         unsigned enable = 0;
 732
 733         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 734                 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
 735
 736                 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
 737                     connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
 738                         /* don't try to enable hpd on eDP or LVDS avoid breaking the
 739                          * aux dp channel on imac and help (but not completely fix)
 740                          * https://bugzilla.redhat.com/show_bug.cgi?id=726143
 741                          */
 742                         continue;
 743                 }
 744                 if (ASIC_IS_DCE3(rdev)) {
 745                         u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) | DC_HPDx_RX_INT_TIMER(0xfa);
 746                         if (ASIC_IS_DCE32(rdev))
 747                                 tmp |= DC_HPDx_EN;
 748
 749                         switch (radeon_connector->hpd.hpd) {
 750                         case RADEON_HPD_1:
 751                                 WREG32(DC_HPD1_CONTROL, tmp);
 752                                 break;
 753                         case RADEON_HPD_2:
 754                                 WREG32(DC_HPD2_CONTROL, tmp);
 755                                 break;
 756                         case RADEON_HPD_3:
 757                                 WREG32(DC_HPD3_CONTROL, tmp);
 758                                 break;
 759                         case RADEON_HPD_4:
 760                                 WREG32(DC_HPD4_CONTROL, tmp);
 761                                 break;
 762                                 /* DCE 3.2 */
 763                         case RADEON_HPD_5:
 764                                 WREG32(DC_HPD5_CONTROL, tmp);
 765                                 break;
 766                         case RADEON_HPD_6:
 767                                 WREG32(DC_HPD6_CONTROL, tmp);
 768                                 break;
 769                         default:
 770                                 break;
 771                         }
 772                 } else {
 773                         switch (radeon_connector->hpd.hpd) {
 774                         case RADEON_HPD_1:
 775                                 WREG32(DC_HOT_PLUG_DETECT1_CONTROL, DC_HOT_PLUG_DETECTx_EN);
 776                                 break;
 777                         case RADEON_HPD_2:
 778                                 WREG32(DC_HOT_PLUG_DETECT2_CONTROL, DC_HOT_PLUG_DETECTx_EN);
 779                                 break;
 780                         case RADEON_HPD_3:
 781                                 WREG32(DC_HOT_PLUG_DETECT3_CONTROL, DC_HOT_PLUG_DETECTx_EN);
 782                                 break;
 783                         default:
 784                                 break;
 785                         }
 786                 }
 787                 enable |= 1 << radeon_connector->hpd.hpd;
 788                 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
 789         }
 790         radeon_irq_kms_enable_hpd(rdev, enable);
 791 }
 792
 793 void r600_hpd_fini(struct radeon_device *rdev)
 794 {
 795         struct drm_device *dev = rdev->ddev;
 796         struct drm_connector *connector;
 797         unsigned disable = 0;
 798
 799         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 800                 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
 801                 if (ASIC_IS_DCE3(rdev)) {
 802                         switch (radeon_connector->hpd.hpd) {
 803                         case RADEON_HPD_1:
 804                                 WREG32(DC_HPD1_CONTROL, 0);
 805                                 break;
 806                         case RADEON_HPD_2:
 807                                 WREG32(DC_HPD2_CONTROL, 0);
 808                                 break;
 809                         case RADEON_HPD_3:
 810                                 WREG32(DC_HPD3_CONTROL, 0);
 811                                 break;
 812                         case RADEON_HPD_4:
 813                                 WREG32(DC_HPD4_CONTROL, 0);
 814                                 break;
 815                                 /* DCE 3.2 */
 816                         case RADEON_HPD_5:
 817                                 WREG32(DC_HPD5_CONTROL, 0);
 818                                 break;
 819                         case RADEON_HPD_6:
 820                                 WREG32(DC_HPD6_CONTROL, 0);
 821                                 break;
 822                         default:
 823                                 break;
 824                         }
 825                 } else {
 826                         switch (radeon_connector->hpd.hpd) {
 827                         case RADEON_HPD_1:
 828                                 WREG32(DC_HOT_PLUG_DETECT1_CONTROL, 0);
 829                                 break;
 830                         case RADEON_HPD_2:
 831                                 WREG32(DC_HOT_PLUG_DETECT2_CONTROL, 0);
 832                                 break;
 833                         case RADEON_HPD_3:
 834                                 WREG32(DC_HOT_PLUG_DETECT3_CONTROL, 0);
 835                                 break;
 836                         default:
 837                                 break;
 838                         }
 839                 }
 840                 disable |= 1 << radeon_connector->hpd.hpd;
 841         }
 842         radeon_irq_kms_disable_hpd(rdev, disable);
 843 }
 844
 845 /*
 846  * R600 PCIE GART
 847  */
 848 void r600_pcie_gart_tlb_flush(struct radeon_device *rdev)
 849 {
 850         unsigned i;
 851         u32 tmp;
 852
 853         /* flush hdp cache so updates hit vram */
 854         if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_RV740) &&
 855             !(rdev->flags & RADEON_IS_AGP)) {
 856                 void __iomem *ptr = (void *)rdev->gart.ptr;
 857                 u32 tmp;
 858
 859                 /* r7xx hw bug.  write to HDP_DEBUG1 followed by fb read
 860                  * rather than write to HDP_REG_COHERENCY_FLUSH_CNTL
 861                  * This seems to cause problems on some AGP cards. Just use the old
 862                  * method for them.
 863                  */
 864                 WREG32(HDP_DEBUG1, 0);
 865                 tmp = readl((void __iomem *)ptr);
 866         } else
 867                 WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
 868
 869         WREG32(VM_CONTEXT0_INVALIDATION_LOW_ADDR, rdev->mc.gtt_start >> 12);
 870         WREG32(VM_CONTEXT0_INVALIDATION_HIGH_ADDR, (rdev->mc.gtt_end - 1) >> 12);
 871         WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
 872         for (i = 0; i < rdev->usec_timeout; i++) {
 873                 /* read MC_STATUS */
 874                 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
 875                 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
 876                 if (tmp == 2) {
 877                         printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
 878                         return;
 879                 }
 880                 if (tmp) {
 881                         return;
 882                 }
 883                 udelay(1);
 884         }
 885 }
 886
 887 int r600_pcie_gart_init(struct radeon_device *rdev)
 888 {
 889         int r;
 890
 891         if (rdev->gart.robj) {
 892                 WARN(1, "R600 PCIE GART already initialized\n");
 893                 return 0;
 894         }
 895         /* Initialize common gart structure */
 896         r = radeon_gart_init(rdev);
 897         if (r)
 898                 return r;
 899         rdev->gart.table_size = rdev->gart.num_gpu_pages * 8;
 900         return radeon_gart_table_vram_alloc(rdev);
 901 }
 902
 903 static int r600_pcie_gart_enable(struct radeon_device *rdev)
 904 {
 905         u32 tmp;
 906         int r, i;
 907
 908         if (rdev->gart.robj == NULL) {
 909                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
 910                 return -EINVAL;
 911         }
 912         r = radeon_gart_table_vram_pin(rdev);
 913         if (r)
 914                 return r;
 915         radeon_gart_restore(rdev);
 916
 917         /* Setup L2 cache */
 918         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
 919                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
 920                                 EFFECTIVE_L2_QUEUE_SIZE(7));
 921         WREG32(VM_L2_CNTL2, 0);
 922         WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) | BANK_SELECT_1(1));
 923         /* Setup TLB control */
 924         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
 925                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
 926                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5) |
 927                 ENABLE_WAIT_L2_QUERY;
 928         WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
 929         WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
 930         WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp | ENABLE_L1_STRICT_ORDERING);
 931         WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
 932         WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
 933         WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
 934         WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
 935         WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
 936         WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
 937         WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
 938         WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
 939         WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
 940         WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
 941         WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
 942         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
 943         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
 944         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
 945         WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
 946                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
 947         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
 948                         (u32)(rdev->dummy_page.addr >> 12));
 949         for (i = 1; i < 7; i++)
 950                 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
 951
 952         r600_pcie_gart_tlb_flush(rdev);
 953         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
 954                  (unsigned)(rdev->mc.gtt_size >> 20),
 955                  (unsigned long long)rdev->gart.table_addr);
 956         rdev->gart.ready = true;
 957         return 0;
 958 }
 959
 960 static void r600_pcie_gart_disable(struct radeon_device *rdev)
 961 {
 962         u32 tmp;
 963         int i;
 964
 965         /* Disable all tables */
 966         for (i = 0; i < 7; i++)
 967                 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
 968
 969         /* Disable L2 cache */
 970         WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
 971                                 EFFECTIVE_L2_QUEUE_SIZE(7));
 972         WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) | BANK_SELECT_1(1));
 973         /* Setup L1 TLB control */
 974         tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5) |
 975                 ENABLE_WAIT_L2_QUERY;
 976         WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
 977         WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
 978         WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
 979         WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
 980         WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
 981         WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
 982         WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
 983         WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
 984         WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp);
 985         WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp);
 986         WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
 987         WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
 988         WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp);
 989         WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
 990         radeon_gart_table_vram_unpin(rdev);
 991 }
 992
 993 static void r600_pcie_gart_fini(struct radeon_device *rdev)
 994 {
 995         radeon_gart_fini(rdev);
 996         r600_pcie_gart_disable(rdev);
 997         radeon_gart_table_vram_free(rdev);
 998 }
 999
1000 static void r600_agp_enable(struct radeon_device *rdev)
1001 {
1002         u32 tmp;
1003         int i;
1004
1005         /* Setup L2 cache */
1006         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
1007                                 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1008                                 EFFECTIVE_L2_QUEUE_SIZE(7));
1009         WREG32(VM_L2_CNTL2, 0);
1010         WREG32(VM_L2_CNTL3, BANK_SELECT_0(0) | BANK_SELECT_1(1));
1011         /* Setup TLB control */
1012         tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
1013                 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1014                 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5) |
1015                 ENABLE_WAIT_L2_QUERY;
1016         WREG32(MC_VM_L1_TLB_MCB_RD_SYS_CNTL, tmp);
1017         WREG32(MC_VM_L1_TLB_MCB_WR_SYS_CNTL, tmp);
1018         WREG32(MC_VM_L1_TLB_MCB_RD_HDP_CNTL, tmp | ENABLE_L1_STRICT_ORDERING);
1019         WREG32(MC_VM_L1_TLB_MCB_WR_HDP_CNTL, tmp);
1020         WREG32(MC_VM_L1_TLB_MCD_RD_A_CNTL, tmp);
1021         WREG32(MC_VM_L1_TLB_MCD_WR_A_CNTL, tmp);
1022         WREG32(MC_VM_L1_TLB_MCD_RD_B_CNTL, tmp);
1023         WREG32(MC_VM_L1_TLB_MCD_WR_B_CNTL, tmp);
1024         WREG32(MC_VM_L1_TLB_MCB_RD_GFX_CNTL, tmp);
1025         WREG32(MC_VM_L1_TLB_MCB_WR_GFX_CNTL, tmp);
1026         WREG32(MC_VM_L1_TLB_MCB_RD_PDMA_CNTL, tmp);
1027         WREG32(MC_VM_L1_TLB_MCB_WR_PDMA_CNTL, tmp);
1028         WREG32(MC_VM_L1_TLB_MCB_RD_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
1029         WREG32(MC_VM_L1_TLB_MCB_WR_SEM_CNTL, tmp | ENABLE_SEMAPHORE_MODE);
1030         for (i = 0; i < 7; i++)
1031                 WREG32(VM_CONTEXT0_CNTL + (i * 4), 0);
1032 }
1033
1034 int r600_mc_wait_for_idle(struct radeon_device *rdev)
1035 {
1036         unsigned i;
1037         u32 tmp;
1038
1039         for (i = 0; i < rdev->usec_timeout; i++) {
1040                 /* read MC_STATUS */
1041                 tmp = RREG32(R_000E50_SRBM_STATUS) & 0x3F00;
1042                 if (!tmp)
1043                         return 0;
1044                 udelay(1);
1045         }
1046         return -1;
1047 }
1048
1049 static void r600_mc_program(struct radeon_device *rdev)
1050 {
1051         struct rv515_mc_save save;
1052         u32 tmp;
1053         int i, j;
1054
1055         /* Initialize HDP */
1056         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1057                 WREG32((0x2c14 + j), 0x00000000);
1058                 WREG32((0x2c18 + j), 0x00000000);
1059                 WREG32((0x2c1c + j), 0x00000000);
1060                 WREG32((0x2c20 + j), 0x00000000);
1061                 WREG32((0x2c24 + j), 0x00000000);
1062         }
1063         WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
1064
1065         rv515_mc_stop(rdev, &save);
1066         if (r600_mc_wait_for_idle(rdev)) {
1067                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1068         }
1069         /* Lockout access through VGA aperture (doesn't exist before R600) */
1070         WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
1071         /* Update configuration */
1072         if (rdev->flags & RADEON_IS_AGP) {
1073                 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
1074                         /* VRAM before AGP */
1075                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1076                                 rdev->mc.vram_start >> 12);
1077                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1078                                 rdev->mc.gtt_end >> 12);
1079                 } else {
1080                         /* VRAM after AGP */
1081                         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
1082                                 rdev->mc.gtt_start >> 12);
1083                         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1084                                 rdev->mc.vram_end >> 12);
1085                 }
1086         } else {
1087                 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR, rdev->mc.vram_start >> 12);
1088                 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR, rdev->mc.vram_end >> 12);
1089         }
1090         WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
1091         tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
1092         tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
1093         WREG32(MC_VM_FB_LOCATION, tmp);
1094         WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
1095         WREG32(HDP_NONSURFACE_INFO, (2 << 7));
1096         WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
1097         if (rdev->flags & RADEON_IS_AGP) {
1098                 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 22);
1099                 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 22);
1100                 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
1101         } else {
1102                 WREG32(MC_VM_AGP_BASE, 0);
1103                 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
1104                 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
1105         }
1106         if (r600_mc_wait_for_idle(rdev)) {
1107                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1108         }
1109         rv515_mc_resume(rdev, &save);
1110         /* we need to own VRAM, so turn off the VGA renderer here
1111          * to stop it overwriting our objects */
1112         rv515_vga_render_disable(rdev);
1113 }
1114
1115 /**
1116  * r600_vram_gtt_location - try to find VRAM & GTT location
1117  * @rdev: radeon device structure holding all necessary informations
1118  * @mc: memory controller structure holding memory informations
1119  *
1120  * Function will place try to place VRAM at same place as in CPU (PCI)
1121  * address space as some GPU seems to have issue when we reprogram at
1122  * different address space.
1123  *
1124  * If there is not enough space to fit the unvisible VRAM after the
1125  * aperture then we limit the VRAM size to the aperture.
1126  *
1127  * If we are using AGP then place VRAM adjacent to AGP aperture are we need
1128  * them to be in one from GPU point of view so that we can program GPU to
1129  * catch access outside them (weird GPU policy see ??).
1130  *
1131  * This function will never fails, worst case are limiting VRAM or GTT.
1132  *
1133  * Note: GTT start, end, size should be initialized before calling this
1134  * function on AGP platform.
1135  */
1136 static void r600_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc)
1137 {
1138         u64 size_bf, size_af;
1139
1140         if (mc->mc_vram_size > 0xE0000000) {
1141                 /* leave room for at least 512M GTT */
1142                 dev_warn(rdev->dev, "limiting VRAM\n");
1143                 mc->real_vram_size = 0xE0000000;
1144                 mc->mc_vram_size = 0xE0000000;
1145         }
1146         if (rdev->flags & RADEON_IS_AGP) {
1147                 size_bf = mc->gtt_start;
1148                 size_af = 0xFFFFFFFF - mc->gtt_end;
1149                 if (size_bf > size_af) {
1150                         if (mc->mc_vram_size > size_bf) {
1151                                 dev_warn(rdev->dev, "limiting VRAM\n");
1152                                 mc->real_vram_size = size_bf;
1153                                 mc->mc_vram_size = size_bf;
1154                         }
1155                         mc->vram_start = mc->gtt_start - mc->mc_vram_size;
1156                 } else {
1157                         if (mc->mc_vram_size > size_af) {
1158                                 dev_warn(rdev->dev, "limiting VRAM\n");
1159                                 mc->real_vram_size = size_af;
1160                                 mc->mc_vram_size = size_af;
1161                         }
1162                         mc->vram_start = mc->gtt_end + 1;
1163                 }
1164                 mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
1165                 dev_info(rdev->dev, "VRAM: %lluM 0x%08llX - 0x%08llX (%lluM used)\n",
1166                                 mc->mc_vram_size >> 20, mc->vram_start,
1167                                 mc->vram_end, mc->real_vram_size >> 20);
1168         } else {
1169                 u64 base = 0;
1170                 if (rdev->flags & RADEON_IS_IGP) {
1171                         base = RREG32(MC_VM_FB_LOCATION) & 0xFFFF;
1172                         base <<= 24;
1173                 }
1174                 radeon_vram_location(rdev, &rdev->mc, base);
1175                 rdev->mc.gtt_base_align = 0;
1176                 radeon_gtt_location(rdev, mc);
1177         }
1178 }
1179
1180 static int r600_mc_init(struct radeon_device *rdev)
1181 {
1182         u32 tmp;
1183         int chansize, numchan;
1184
1185         /* Get VRAM informations */
1186         rdev->mc.vram_is_ddr = true;
1187         tmp = RREG32(RAMCFG);
1188         if (tmp & CHANSIZE_OVERRIDE) {
1189                 chansize = 16;
1190         } else if (tmp & CHANSIZE_MASK) {
1191                 chansize = 64;
1192         } else {
1193                 chansize = 32;
1194         }
1195         tmp = RREG32(CHMAP);
1196         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1197         case 0:
1198         default:
1199                 numchan = 1;
1200                 break;
1201         case 1:
1202                 numchan = 2;
1203                 break;
1204         case 2:
1205                 numchan = 4;
1206                 break;
1207         case 3:
1208                 numchan = 8;
1209                 break;
1210         }
1211         rdev->mc.vram_width = numchan * chansize;
1212         /* Could aper size report 0 ? */
1213         rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
1214         rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
1215         /* Setup GPU memory space */
1216         rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
1217         rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
1218         rdev->mc.visible_vram_size = rdev->mc.aper_size;
1219         r600_vram_gtt_location(rdev, &rdev->mc);
1220
1221         if (rdev->flags & RADEON_IS_IGP) {
1222                 rs690_pm_info(rdev);
1223                 rdev->mc.igp_sideport_enabled = radeon_atombios_sideport_present(rdev);
1224         }
1225         radeon_update_bandwidth_info(rdev);
1226         return 0;
1227 }
1228
1229 int r600_vram_scratch_init(struct radeon_device *rdev)
1230 {
1231         int r;
1232
1233         if (rdev->vram_scratch.robj == NULL) {
1234                 r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE,
1235                                      PAGE_SIZE, true, RADEON_GEM_DOMAIN_VRAM,
1236                                      NULL, &rdev->vram_scratch.robj);
1237                 if (r) {
1238                         return r;
1239                 }
1240         }
1241
1242         r = radeon_bo_reserve(rdev->vram_scratch.robj, false);
1243         if (unlikely(r != 0))
1244                 return r;
1245         r = radeon_bo_pin(rdev->vram_scratch.robj,
1246                           RADEON_GEM_DOMAIN_VRAM, &rdev->vram_scratch.gpu_addr);
1247         if (r) {
1248                 radeon_bo_unreserve(rdev->vram_scratch.robj);
1249                 return r;
1250         }
1251         r = radeon_bo_kmap(rdev->vram_scratch.robj,
1252                                 (void **)&rdev->vram_scratch.ptr);
1253         if (r)
1254                 radeon_bo_unpin(rdev->vram_scratch.robj);
1255         radeon_bo_unreserve(rdev->vram_scratch.robj);
1256
1257         return r;
1258 }
1259
1260 void r600_vram_scratch_fini(struct radeon_device *rdev)
1261 {
1262         int r;
1263
1264         if (rdev->vram_scratch.robj == NULL) {
1265                 return;
1266         }
1267         r = radeon_bo_reserve(rdev->vram_scratch.robj, false);
1268         if (likely(r == 0)) {
1269                 radeon_bo_kunmap(rdev->vram_scratch.robj);
1270                 radeon_bo_unpin(rdev->vram_scratch.robj);
1271                 radeon_bo_unreserve(rdev->vram_scratch.robj);
1272         }
1273         radeon_bo_unref(&rdev->vram_scratch.robj);
1274 }
1275
1276 void r600_set_bios_scratch_engine_hung(struct radeon_device *rdev, bool hung)
1277 {
1278         u32 tmp = RREG32(R600_BIOS_3_SCRATCH);
1279
1280         if (hung)
1281                 tmp |= ATOM_S3_ASIC_GUI_ENGINE_HUNG;
1282         else
1283                 tmp &= ~ATOM_S3_ASIC_GUI_ENGINE_HUNG;
1284
1285         WREG32(R600_BIOS_3_SCRATCH, tmp);
1286 }
1287
1288 static void r600_print_gpu_status_regs(struct radeon_device *rdev)
1289 {
1290         dev_info(rdev->dev, "  R_008010_GRBM_STATUS      = 0x%08X\n",
1291                  RREG32(R_008010_GRBM_STATUS));
1292         dev_info(rdev->dev, "  R_008014_GRBM_STATUS2     = 0x%08X\n",
1293                  RREG32(R_008014_GRBM_STATUS2));
1294         dev_info(rdev->dev, "  R_000E50_SRBM_STATUS      = 0x%08X\n",
1295                  RREG32(R_000E50_SRBM_STATUS));
1296         dev_info(rdev->dev, "  R_008674_CP_STALLED_STAT1 = 0x%08X\n",
1297                  RREG32(CP_STALLED_STAT1));
1298         dev_info(rdev->dev, "  R_008678_CP_STALLED_STAT2 = 0x%08X\n",
1299                  RREG32(CP_STALLED_STAT2));
1300         dev_info(rdev->dev, "  R_00867C_CP_BUSY_STAT     = 0x%08X\n",
1301                  RREG32(CP_BUSY_STAT));
1302         dev_info(rdev->dev, "  R_008680_CP_STAT          = 0x%08X\n",
1303                  RREG32(CP_STAT));
1304         dev_info(rdev->dev, "  R_00D034_DMA_STATUS_REG   = 0x%08X\n",
1305                 RREG32(DMA_STATUS_REG));
1306 }
1307
1308 static bool r600_is_display_hung(struct radeon_device *rdev)
1309 {
1310         u32 crtc_hung = 0;
1311         u32 crtc_status[2];
1312         u32 i, j, tmp;
1313
1314         for (i = 0; i < rdev->num_crtc; i++) {
1315                 if (RREG32(AVIVO_D1CRTC_CONTROL + crtc_offsets[i]) & AVIVO_CRTC_EN) {
1316                         crtc_status[i] = RREG32(AVIVO_D1CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
1317                         crtc_hung |= (1 << i);
1318                 }
1319         }
1320
1321         for (j = 0; j < 10; j++) {
1322                 for (i = 0; i < rdev->num_crtc; i++) {
1323                         if (crtc_hung & (1 << i)) {
1324                                 tmp = RREG32(AVIVO_D1CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
1325                                 if (tmp != crtc_status[i])
1326                                         crtc_hung &= ~(1 << i);
1327                         }
1328                 }
1329                 if (crtc_hung == 0)
1330                         return false;
1331                 udelay(100);
1332         }
1333
1334         return true;
1335 }
1336
1337 static u32 r600_gpu_check_soft_reset(struct radeon_device *rdev)
1338 {
1339         u32 reset_mask = 0;
1340         u32 tmp;
1341
1342         /* GRBM_STATUS */
1343         tmp = RREG32(R_008010_GRBM_STATUS);
1344         if (rdev->family >= CHIP_RV770) {
1345                 if (G_008010_PA_BUSY(tmp) | G_008010_SC_BUSY(tmp) |
1346                     G_008010_SH_BUSY(tmp) | G_008010_SX_BUSY(tmp) |
1347                     G_008010_TA_BUSY(tmp) | G_008010_VGT_BUSY(tmp) |
1348                     G_008010_DB03_BUSY(tmp) | G_008010_CB03_BUSY(tmp) |
1349                     G_008010_SPI03_BUSY(tmp) | G_008010_VGT_BUSY_NO_DMA(tmp))
1350                         reset_mask |= RADEON_RESET_GFX;
1351         } else {
1352                 if (G_008010_PA_BUSY(tmp) | G_008010_SC_BUSY(tmp) |
1353                     G_008010_SH_BUSY(tmp) | G_008010_SX_BUSY(tmp) |
1354                     G_008010_TA03_BUSY(tmp) | G_008010_VGT_BUSY(tmp) |
1355                     G_008010_DB03_BUSY(tmp) | G_008010_CB03_BUSY(tmp) |
1356                     G_008010_SPI03_BUSY(tmp) | G_008010_VGT_BUSY_NO_DMA(tmp))
1357                         reset_mask |= RADEON_RESET_GFX;
1358         }
1359
1360         if (G_008010_CF_RQ_PENDING(tmp) | G_008010_PF_RQ_PENDING(tmp) |
1361             G_008010_CP_BUSY(tmp) | G_008010_CP_COHERENCY_BUSY(tmp))
1362                 reset_mask |= RADEON_RESET_CP;
1363
1364         if (G_008010_GRBM_EE_BUSY(tmp))
1365                 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
1366
1367         /* DMA_STATUS_REG */
1368         tmp = RREG32(DMA_STATUS_REG);
1369         if (!(tmp & DMA_IDLE))
1370                 reset_mask |= RADEON_RESET_DMA;
1371
1372         /* SRBM_STATUS */
1373         tmp = RREG32(R_000E50_SRBM_STATUS);
1374         if (G_000E50_RLC_RQ_PENDING(tmp) | G_000E50_RLC_BUSY(tmp))
1375                 reset_mask |= RADEON_RESET_RLC;
1376
1377         if (G_000E50_IH_BUSY(tmp))
1378                 reset_mask |= RADEON_RESET_IH;
1379
1380         if (G_000E50_SEM_BUSY(tmp))
1381                 reset_mask |= RADEON_RESET_SEM;
1382
1383         if (G_000E50_GRBM_RQ_PENDING(tmp))
1384                 reset_mask |= RADEON_RESET_GRBM;
1385
1386         if (G_000E50_VMC_BUSY(tmp))
1387                 reset_mask |= RADEON_RESET_VMC;
1388
1389         if (G_000E50_MCB_BUSY(tmp) | G_000E50_MCDZ_BUSY(tmp) |
1390             G_000E50_MCDY_BUSY(tmp) | G_000E50_MCDX_BUSY(tmp) |
1391             G_000E50_MCDW_BUSY(tmp))
1392                 reset_mask |= RADEON_RESET_MC;
1393
1394         if (r600_is_display_hung(rdev))
1395                 reset_mask |= RADEON_RESET_DISPLAY;
1396
1397         return reset_mask;
1398 }
1399
1400 static void r600_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
1401 {
1402         struct rv515_mc_save save;
1403         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
1404         u32 tmp;
1405
1406         if (reset_mask == 0)
1407                 return;
1408
1409         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
1410
1411         r600_print_gpu_status_regs(rdev);
1412
1413         /* Disable CP parsing/prefetching */
1414         if (rdev->family >= CHIP_RV770)
1415                 WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1) | S_0086D8_CP_PFP_HALT(1));
1416         else
1417                 WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1));
1418
1419         /* disable the RLC */
1420         WREG32(RLC_CNTL, 0);
1421
1422         if (reset_mask & RADEON_RESET_DMA) {
1423                 /* Disable DMA */
1424                 tmp = RREG32(DMA_RB_CNTL);
1425                 tmp &= ~DMA_RB_ENABLE;
1426                 WREG32(DMA_RB_CNTL, tmp);
1427         }
1428
1429         mdelay(50);
1430
1431         rv515_mc_stop(rdev, &save);
1432         if (r600_mc_wait_for_idle(rdev)) {
1433                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1434         }
1435
1436         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
1437                 if (rdev->family >= CHIP_RV770)
1438                         grbm_soft_reset |= S_008020_SOFT_RESET_DB(1) |
1439                                 S_008020_SOFT_RESET_CB(1) |
1440                                 S_008020_SOFT_RESET_PA(1) |
1441                                 S_008020_SOFT_RESET_SC(1) |
1442                                 S_008020_SOFT_RESET_SPI(1) |
1443                                 S_008020_SOFT_RESET_SX(1) |
1444                                 S_008020_SOFT_RESET_SH(1) |
1445                                 S_008020_SOFT_RESET_TC(1) |
1446                                 S_008020_SOFT_RESET_TA(1) |
1447                                 S_008020_SOFT_RESET_VC(1) |
1448                                 S_008020_SOFT_RESET_VGT(1);
1449                 else
1450                         grbm_soft_reset |= S_008020_SOFT_RESET_CR(1) |
1451                                 S_008020_SOFT_RESET_DB(1) |
1452                                 S_008020_SOFT_RESET_CB(1) |
1453                                 S_008020_SOFT_RESET_PA(1) |
1454                                 S_008020_SOFT_RESET_SC(1) |
1455                                 S_008020_SOFT_RESET_SMX(1) |
1456                                 S_008020_SOFT_RESET_SPI(1) |
1457                                 S_008020_SOFT_RESET_SX(1) |
1458                                 S_008020_SOFT_RESET_SH(1) |
1459                                 S_008020_SOFT_RESET_TC(1) |
1460                                 S_008020_SOFT_RESET_TA(1) |
1461                                 S_008020_SOFT_RESET_VC(1) |
1462                                 S_008020_SOFT_RESET_VGT(1);
1463         }
1464
1465         if (reset_mask & RADEON_RESET_CP) {
1466                 grbm_soft_reset |= S_008020_SOFT_RESET_CP(1) |
1467                         S_008020_SOFT_RESET_VGT(1);
1468
1469                 srbm_soft_reset |= S_000E60_SOFT_RESET_GRBM(1);
1470         }
1471
1472         if (reset_mask & RADEON_RESET_DMA) {
1473                 if (rdev->family >= CHIP_RV770)
1474                         srbm_soft_reset |= RV770_SOFT_RESET_DMA;
1475                 else
1476                         srbm_soft_reset |= SOFT_RESET_DMA;
1477         }
1478
1479         if (reset_mask & RADEON_RESET_RLC)
1480                 srbm_soft_reset |= S_000E60_SOFT_RESET_RLC(1);
1481
1482         if (reset_mask & RADEON_RESET_SEM)
1483                 srbm_soft_reset |= S_000E60_SOFT_RESET_SEM(1);
1484
1485         if (reset_mask & RADEON_RESET_IH)
1486                 srbm_soft_reset |= S_000E60_SOFT_RESET_IH(1);
1487
1488         if (reset_mask & RADEON_RESET_GRBM)
1489                 srbm_soft_reset |= S_000E60_SOFT_RESET_GRBM(1);
1490
1491         if (!(rdev->flags & RADEON_IS_IGP)) {
1492                 if (reset_mask & RADEON_RESET_MC)
1493                         srbm_soft_reset |= S_000E60_SOFT_RESET_MC(1);
1494         }
1495
1496         if (reset_mask & RADEON_RESET_VMC)
1497                 srbm_soft_reset |= S_000E60_SOFT_RESET_VMC(1);
1498
1499         if (grbm_soft_reset) {
1500                 tmp = RREG32(R_008020_GRBM_SOFT_RESET);
1501                 tmp |= grbm_soft_reset;
1502                 dev_info(rdev->dev, "R_008020_GRBM_SOFT_RESET=0x%08X\n", tmp);
1503                 WREG32(R_008020_GRBM_SOFT_RESET, tmp);
1504                 tmp = RREG32(R_008020_GRBM_SOFT_RESET);
1505
1506                 udelay(50);
1507
1508                 tmp &= ~grbm_soft_reset;
1509                 WREG32(R_008020_GRBM_SOFT_RESET, tmp);
1510                 tmp = RREG32(R_008020_GRBM_SOFT_RESET);
1511         }
1512
1513         if (srbm_soft_reset) {
1514                 tmp = RREG32(SRBM_SOFT_RESET);
1515                 tmp |= srbm_soft_reset;
1516                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1517                 WREG32(SRBM_SOFT_RESET, tmp);
1518                 tmp = RREG32(SRBM_SOFT_RESET);
1519
1520                 udelay(50);
1521
1522                 tmp &= ~srbm_soft_reset;
1523                 WREG32(SRBM_SOFT_RESET, tmp);
1524                 tmp = RREG32(SRBM_SOFT_RESET);
1525         }
1526
1527         /* Wait a little for things to settle down */
1528         mdelay(1);
1529
1530         rv515_mc_resume(rdev, &save);
1531         udelay(50);
1532
1533         r600_print_gpu_status_regs(rdev);
1534 }
1535
1536 int r600_asic_reset(struct radeon_device *rdev)
1537 {
1538         u32 reset_mask;
1539
1540         reset_mask = r600_gpu_check_soft_reset(rdev);
1541
1542         if (reset_mask)
1543                 r600_set_bios_scratch_engine_hung(rdev, true);
1544
1545         r600_gpu_soft_reset(rdev, reset_mask);
1546
1547         reset_mask = r600_gpu_check_soft_reset(rdev);
1548
1549         if (!reset_mask)
1550                 r600_set_bios_scratch_engine_hung(rdev, false);
1551
1552         return 0;
1553 }
1554
1555 /**
1556  * r600_gfx_is_lockup - Check if the GFX engine is locked up
1557  *
1558  * @rdev: radeon_device pointer
1559  * @ring: radeon_ring structure holding ring information
1560  *
1561  * Check if the GFX engine is locked up.
1562  * Returns true if the engine appears to be locked up, false if not.
1563  */
1564 bool r600_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
1565 {
1566         u32 reset_mask = r600_gpu_check_soft_reset(rdev);
1567
1568         if (!(reset_mask & (RADEON_RESET_GFX |
1569                             RADEON_RESET_COMPUTE |
1570                             RADEON_RESET_CP))) {
1571                 radeon_ring_lockup_update(ring);
1572                 return false;
1573         }
1574         /* force CP activities */
1575         radeon_ring_force_activity(rdev, ring);
1576         return radeon_ring_test_lockup(rdev, ring);
1577 }
1578
1579 /**
1580  * r600_dma_is_lockup - Check if the DMA engine is locked up
1581  *
1582  * @rdev: radeon_device pointer
1583  * @ring: radeon_ring structure holding ring information
1584  *
1585  * Check if the async DMA engine is locked up.
1586  * Returns true if the engine appears to be locked up, false if not.
1587  */
1588 bool r600_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
1589 {
1590         u32 reset_mask = r600_gpu_check_soft_reset(rdev);
1591
1592         if (!(reset_mask & RADEON_RESET_DMA)) {
1593                 radeon_ring_lockup_update(ring);
1594                 return false;
1595         }
1596         /* force ring activities */
1597         radeon_ring_force_activity(rdev, ring);
1598         return radeon_ring_test_lockup(rdev, ring);
1599 }
1600
1601 u32 r6xx_remap_render_backend(struct radeon_device *rdev,
1602                               u32 tiling_pipe_num,
1603                               u32 max_rb_num,
1604                               u32 total_max_rb_num,
1605                               u32 disabled_rb_mask)
1606 {
1607         u32 rendering_pipe_num, rb_num_width, req_rb_num;
1608         u32 pipe_rb_ratio, pipe_rb_remain, tmp;
1609         u32 data = 0, mask = 1 << (max_rb_num - 1);
1610         unsigned i, j;
1611
1612         /* mask out the RBs that don't exist on that asic */
1613         tmp = disabled_rb_mask | ((0xff << max_rb_num) & 0xff);
1614         /* make sure at least one RB is available */
1615         if ((tmp & 0xff) != 0xff)
1616                 disabled_rb_mask = tmp;
1617
1618         rendering_pipe_num = 1 << tiling_pipe_num;
1619         req_rb_num = total_max_rb_num - r600_count_pipe_bits(disabled_rb_mask);
1620         BUG_ON(rendering_pipe_num < req_rb_num);
1621
1622         pipe_rb_ratio = rendering_pipe_num / req_rb_num;
1623         pipe_rb_remain = rendering_pipe_num - pipe_rb_ratio * req_rb_num;
1624
1625         if (rdev->family <= CHIP_RV740) {
1626                 /* r6xx/r7xx */
1627                 rb_num_width = 2;
1628         } else {
1629                 /* eg+ */
1630                 rb_num_width = 4;
1631         }
1632
1633         for (i = 0; i < max_rb_num; i++) {
1634                 if (!(mask & disabled_rb_mask)) {
1635                         for (j = 0; j < pipe_rb_ratio; j++) {
1636                                 data <<= rb_num_width;
1637                                 data |= max_rb_num - i - 1;
1638                         }
1639                         if (pipe_rb_remain) {
1640                                 data <<= rb_num_width;
1641                                 data |= max_rb_num - i - 1;
1642                                 pipe_rb_remain--;
1643                         }
1644                 }
1645                 mask >>= 1;
1646         }
1647
1648         return data;
1649 }
1650
1651 int r600_count_pipe_bits(uint32_t val)
1652 {
1653         return hweight32(val);
1654 }
1655
1656 static void r600_gpu_init(struct radeon_device *rdev)
1657 {
1658         u32 tiling_config;
1659         u32 ramcfg;
1660         u32 cc_rb_backend_disable;
1661         u32 cc_gc_shader_pipe_config;
1662         u32 tmp;
1663         int i, j;
1664         u32 sq_config;
1665         u32 sq_gpr_resource_mgmt_1 = 0;
1666         u32 sq_gpr_resource_mgmt_2 = 0;
1667         u32 sq_thread_resource_mgmt = 0;
1668         u32 sq_stack_resource_mgmt_1 = 0;
1669         u32 sq_stack_resource_mgmt_2 = 0;
1670         u32 disabled_rb_mask;
1671
1672         rdev->config.r600.tiling_group_size = 256;
1673         switch (rdev->family) {
1674         case CHIP_R600:
1675                 rdev->config.r600.max_pipes = 4;
1676                 rdev->config.r600.max_tile_pipes = 8;
1677                 rdev->config.r600.max_simds = 4;
1678                 rdev->config.r600.max_backends = 4;
1679                 rdev->config.r600.max_gprs = 256;
1680                 rdev->config.r600.max_threads = 192;
1681                 rdev->config.r600.max_stack_entries = 256;
1682                 rdev->config.r600.max_hw_contexts = 8;
1683                 rdev->config.r600.max_gs_threads = 16;
1684                 rdev->config.r600.sx_max_export_size = 128;
1685                 rdev->config.r600.sx_max_export_pos_size = 16;
1686                 rdev->config.r600.sx_max_export_smx_size = 128;
1687                 rdev->config.r600.sq_num_cf_insts = 2;
1688                 break;
1689         case CHIP_RV630:
1690         case CHIP_RV635:
1691                 rdev->config.r600.max_pipes = 2;
1692                 rdev->config.r600.max_tile_pipes = 2;
1693                 rdev->config.r600.max_simds = 3;
1694                 rdev->config.r600.max_backends = 1;
1695                 rdev->config.r600.max_gprs = 128;
1696                 rdev->config.r600.max_threads = 192;
1697                 rdev->config.r600.max_stack_entries = 128;
1698                 rdev->config.r600.max_hw_contexts = 8;
1699                 rdev->config.r600.max_gs_threads = 4;
1700                 rdev->config.r600.sx_max_export_size = 128;
1701                 rdev->config.r600.sx_max_export_pos_size = 16;
1702                 rdev->config.r600.sx_max_export_smx_size = 128;
1703                 rdev->config.r600.sq_num_cf_insts = 2;
1704                 break;
1705         case CHIP_RV610:
1706         case CHIP_RV620:
1707         case CHIP_RS780:
1708         case CHIP_RS880:
1709                 rdev->config.r600.max_pipes = 1;
1710                 rdev->config.r600.max_tile_pipes = 1;
1711                 rdev->config.r600.max_simds = 2;
1712                 rdev->config.r600.max_backends = 1;
1713                 rdev->config.r600.max_gprs = 128;
1714                 rdev->config.r600.max_threads = 192;
1715                 rdev->config.r600.max_stack_entries = 128;
1716                 rdev->config.r600.max_hw_contexts = 4;
1717                 rdev->config.r600.max_gs_threads = 4;
1718                 rdev->config.r600.sx_max_export_size = 128;
1719                 rdev->config.r600.sx_max_export_pos_size = 16;
1720                 rdev->config.r600.sx_max_export_smx_size = 128;
1721                 rdev->config.r600.sq_num_cf_insts = 1;
1722                 break;
1723         case CHIP_RV670:
1724                 rdev->config.r600.max_pipes = 4;
1725                 rdev->config.r600.max_tile_pipes = 4;
1726                 rdev->config.r600.max_simds = 4;
1727                 rdev->config.r600.max_backends = 4;
1728                 rdev->config.r600.max_gprs = 192;
1729                 rdev->config.r600.max_threads = 192;
1730                 rdev->config.r600.max_stack_entries = 256;
1731                 rdev->config.r600.max_hw_contexts = 8;
1732                 rdev->config.r600.max_gs_threads = 16;
1733                 rdev->config.r600.sx_max_export_size = 128;
1734                 rdev->config.r600.sx_max_export_pos_size = 16;
1735                 rdev->config.r600.sx_max_export_smx_size = 128;
1736                 rdev->config.r600.sq_num_cf_insts = 2;
1737                 break;
1738         default:
1739                 break;
1740         }
1741
1742         /* Initialize HDP */
1743         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1744                 WREG32((0x2c14 + j), 0x00000000);
1745                 WREG32((0x2c18 + j), 0x00000000);
1746                 WREG32((0x2c1c + j), 0x00000000);
1747                 WREG32((0x2c20 + j), 0x00000000);
1748                 WREG32((0x2c24 + j), 0x00000000);
1749         }
1750
1751         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
1752
1753         /* Setup tiling */
1754         tiling_config = 0;
1755         ramcfg = RREG32(RAMCFG);
1756         switch (rdev->config.r600.max_tile_pipes) {
1757         case 1:
1758                 tiling_config |= PIPE_TILING(0);
1759                 break;
1760         case 2:
1761                 tiling_config |= PIPE_TILING(1);
1762                 break;
1763         case 4:
1764                 tiling_config |= PIPE_TILING(2);
1765                 break;
1766         case 8:
1767                 tiling_config |= PIPE_TILING(3);
1768                 break;
1769         default:
1770                 break;
1771         }
1772         rdev->config.r600.tiling_npipes = rdev->config.r600.max_tile_pipes;
1773         rdev->config.r600.tiling_nbanks = 4 << ((ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
1774         tiling_config |= BANK_TILING((ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT);
1775         tiling_config |= GROUP_SIZE((ramcfg & BURSTLENGTH_MASK) >> BURSTLENGTH_SHIFT);
1776
1777         tmp = (ramcfg & NOOFROWS_MASK) >> NOOFROWS_SHIFT;
1778         if (tmp > 3) {
1779                 tiling_config |= ROW_TILING(3);
1780                 tiling_config |= SAMPLE_SPLIT(3);
1781         } else {
1782                 tiling_config |= ROW_TILING(tmp);
1783                 tiling_config |= SAMPLE_SPLIT(tmp);
1784         }
1785         tiling_config |= BANK_SWAPS(1);
1786
1787         cc_rb_backend_disable = RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000;
1788         tmp = R6XX_MAX_BACKENDS -
1789                 r600_count_pipe_bits((cc_rb_backend_disable >> 16) & R6XX_MAX_BACKENDS_MASK);
1790         if (tmp < rdev->config.r600.max_backends) {
1791                 rdev->config.r600.max_backends = tmp;
1792         }
1793
1794         cc_gc_shader_pipe_config = RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0x00ffff00;
1795         tmp = R6XX_MAX_PIPES -
1796                 r600_count_pipe_bits((cc_gc_shader_pipe_config >> 8) & R6XX_MAX_PIPES_MASK);
1797         if (tmp < rdev->config.r600.max_pipes) {
1798                 rdev->config.r600.max_pipes = tmp;
1799         }
1800         tmp = R6XX_MAX_SIMDS -
1801                 r600_count_pipe_bits((cc_gc_shader_pipe_config >> 16) & R6XX_MAX_SIMDS_MASK);
1802         if (tmp < rdev->config.r600.max_simds) {
1803                 rdev->config.r600.max_simds = tmp;
1804         }
1805
1806         disabled_rb_mask = (RREG32(CC_RB_BACKEND_DISABLE) >> 16) & R6XX_MAX_BACKENDS_MASK;
1807         tmp = (tiling_config & PIPE_TILING__MASK) >> PIPE_TILING__SHIFT;
1808         tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.r600.max_backends,
1809                                         R6XX_MAX_BACKENDS, disabled_rb_mask);
1810         tiling_config |= tmp << 16;
1811         rdev->config.r600.backend_map = tmp;
1812
1813         rdev->config.r600.tile_config = tiling_config;
1814         WREG32(GB_TILING_CONFIG, tiling_config);
1815         WREG32(DCP_TILING_CONFIG, tiling_config & 0xffff);
1816         WREG32(HDP_TILING_CONFIG, tiling_config & 0xffff);
1817         WREG32(DMA_TILING_CONFIG, tiling_config & 0xffff);
1818
1819         tmp = R6XX_MAX_PIPES - r600_count_pipe_bits((cc_gc_shader_pipe_config & INACTIVE_QD_PIPES_MASK) >> 8);
1820         WREG32(VGT_OUT_DEALLOC_CNTL, (tmp * 4) & DEALLOC_DIST_MASK);
1821         WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, ((tmp * 4) - 2) & VTX_REUSE_DEPTH_MASK);
1822
1823         /* Setup some CP states */
1824         WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) | ROQ_IB2_START(0x2b)));
1825         WREG32(CP_MEQ_THRESHOLDS, (MEQ_END(0x40) | ROQ_END(0x40)));
1826
1827         WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO | SYNC_GRADIENT |
1828                              SYNC_WALKER | SYNC_ALIGNER));
1829         /* Setup various GPU states */
1830         if (rdev->family == CHIP_RV670)
1831                 WREG32(ARB_GDEC_RD_CNTL, 0x00000021);
1832
1833         tmp = RREG32(SX_DEBUG_1);
1834         tmp |= SMX_EVENT_RELEASE;
1835         if ((rdev->family > CHIP_R600))
1836                 tmp |= ENABLE_NEW_SMX_ADDRESS;
1837         WREG32(SX_DEBUG_1, tmp);
1838
1839         if (((rdev->family) == CHIP_R600) ||
1840             ((rdev->family) == CHIP_RV630) ||
1841             ((rdev->family) == CHIP_RV610) ||
1842             ((rdev->family) == CHIP_RV620) ||
1843             ((rdev->family) == CHIP_RS780) ||
1844             ((rdev->family) == CHIP_RS880)) {
1845                 WREG32(DB_DEBUG, PREZ_MUST_WAIT_FOR_POSTZ_DONE);
1846         } else {
1847                 WREG32(DB_DEBUG, 0);
1848         }
1849         WREG32(DB_WATERMARKS, (DEPTH_FREE(4) | DEPTH_CACHELINE_FREE(16) |
1850                                DEPTH_FLUSH(16) | DEPTH_PENDING_FREE(4)));
1851
1852         WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
1853         WREG32(VGT_NUM_INSTANCES, 0);
1854
1855         WREG32(SPI_CONFIG_CNTL, GPR_WRITE_PRIORITY(0));
1856         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(0));
1857
1858         tmp = RREG32(SQ_MS_FIFO_SIZES);
1859         if (((rdev->family) == CHIP_RV610) ||
1860             ((rdev->family) == CHIP_RV620) ||
1861             ((rdev->family) == CHIP_RS780) ||
1862             ((rdev->family) == CHIP_RS880)) {
1863                 tmp = (CACHE_FIFO_SIZE(0xa) |
1864                        FETCH_FIFO_HIWATER(0xa) |
1865                        DONE_FIFO_HIWATER(0xe0) |
1866                        ALU_UPDATE_FIFO_HIWATER(0x8));
1867         } else if (((rdev->family) == CHIP_R600) ||
1868                    ((rdev->family) == CHIP_RV630)) {
1869                 tmp &= ~DONE_FIFO_HIWATER(0xff);
1870                 tmp |= DONE_FIFO_HIWATER(0x4);
1871         }
1872         WREG32(SQ_MS_FIFO_SIZES, tmp);
1873
1874         /* SQ_CONFIG, SQ_GPR_RESOURCE_MGMT, SQ_THREAD_RESOURCE_MGMT, SQ_STACK_RESOURCE_MGMT
1875          * should be adjusted as needed by the 2D/3D drivers.  This just sets default values
1876          */
1877         sq_config = RREG32(SQ_CONFIG);
1878         sq_config &= ~(PS_PRIO(3) |
1879                        VS_PRIO(3) |
1880                        GS_PRIO(3) |
1881                        ES_PRIO(3));
1882         sq_config |= (DX9_CONSTS |
1883                       VC_ENABLE |
1884                       PS_PRIO(0) |
1885                       VS_PRIO(1) |
1886                       GS_PRIO(2) |
1887                       ES_PRIO(3));
1888
1889         if ((rdev->family) == CHIP_R600) {
1890                 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(124) |
1891                                           NUM_VS_GPRS(124) |
1892                                           NUM_CLAUSE_TEMP_GPRS(4));
1893                 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(0) |
1894                                           NUM_ES_GPRS(0));
1895                 sq_thread_resource_mgmt = (NUM_PS_THREADS(136) |
1896                                            NUM_VS_THREADS(48) |
1897                                            NUM_GS_THREADS(4) |
1898                                            NUM_ES_THREADS(4));
1899                 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(128) |
1900                                             NUM_VS_STACK_ENTRIES(128));
1901                 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(0) |
1902                                             NUM_ES_STACK_ENTRIES(0));
1903         } else if (((rdev->family) == CHIP_RV610) ||
1904                    ((rdev->family) == CHIP_RV620) ||
1905                    ((rdev->family) == CHIP_RS780) ||
1906                    ((rdev->family) == CHIP_RS880)) {
1907                 /* no vertex cache */
1908                 sq_config &= ~VC_ENABLE;
1909
1910                 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) |
1911                                           NUM_VS_GPRS(44) |
1912                                           NUM_CLAUSE_TEMP_GPRS(2));
1913                 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(17) |
1914                                           NUM_ES_GPRS(17));
1915                 sq_thread_resource_mgmt = (NUM_PS_THREADS(79) |
1916                                            NUM_VS_THREADS(78) |
1917                                            NUM_GS_THREADS(4) |
1918                                            NUM_ES_THREADS(31));
1919                 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(40) |
1920                                             NUM_VS_STACK_ENTRIES(40));
1921                 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(32) |
1922                                             NUM_ES_STACK_ENTRIES(16));
1923         } else if (((rdev->family) == CHIP_RV630) ||
1924                    ((rdev->family) == CHIP_RV635)) {
1925                 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) |
1926                                           NUM_VS_GPRS(44) |
1927                                           NUM_CLAUSE_TEMP_GPRS(2));
1928                 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(18) |
1929                                           NUM_ES_GPRS(18));
1930                 sq_thread_resource_mgmt = (NUM_PS_THREADS(79) |
1931                                            NUM_VS_THREADS(78) |
1932                                            NUM_GS_THREADS(4) |
1933                                            NUM_ES_THREADS(31));
1934                 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(40) |
1935                                             NUM_VS_STACK_ENTRIES(40));
1936                 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(32) |
1937                                             NUM_ES_STACK_ENTRIES(16));
1938         } else if ((rdev->family) == CHIP_RV670) {
1939                 sq_gpr_resource_mgmt_1 = (NUM_PS_GPRS(44) |
1940                                           NUM_VS_GPRS(44) |
1941                                           NUM_CLAUSE_TEMP_GPRS(2));
1942                 sq_gpr_resource_mgmt_2 = (NUM_GS_GPRS(17) |
1943                                           NUM_ES_GPRS(17));
1944                 sq_thread_resource_mgmt = (NUM_PS_THREADS(79) |
1945                                            NUM_VS_THREADS(78) |
1946                                            NUM_GS_THREADS(4) |
1947                                            NUM_ES_THREADS(31));
1948                 sq_stack_resource_mgmt_1 = (NUM_PS_STACK_ENTRIES(64) |
1949                                             NUM_VS_STACK_ENTRIES(64));
1950                 sq_stack_resource_mgmt_2 = (NUM_GS_STACK_ENTRIES(64) |
1951                                             NUM_ES_STACK_ENTRIES(64));
1952         }
1953
1954         WREG32(SQ_CONFIG, sq_config);
1955         WREG32(SQ_GPR_RESOURCE_MGMT_1,  sq_gpr_resource_mgmt_1);
1956         WREG32(SQ_GPR_RESOURCE_MGMT_2,  sq_gpr_resource_mgmt_2);
1957         WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
1958         WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
1959         WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
1960
1961         if (((rdev->family) == CHIP_RV610) ||
1962             ((rdev->family) == CHIP_RV620) ||
1963             ((rdev->family) == CHIP_RS780) ||
1964             ((rdev->family) == CHIP_RS880)) {
1965                 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(TC_ONLY));
1966         } else {
1967                 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC));
1968         }
1969
1970         /* More default values. 2D/3D driver should adjust as needed */
1971         WREG32(PA_SC_AA_SAMPLE_LOCS_2S, (S0_X(0xc) | S0_Y(0x4) |
1972                                          S1_X(0x4) | S1_Y(0xc)));
1973         WREG32(PA_SC_AA_SAMPLE_LOCS_4S, (S0_X(0xe) | S0_Y(0xe) |
1974                                          S1_X(0x2) | S1_Y(0x2) |
1975                                          S2_X(0xa) | S2_Y(0x6) |
1976                                          S3_X(0x6) | S3_Y(0xa)));
1977         WREG32(PA_SC_AA_SAMPLE_LOCS_8S_WD0, (S0_X(0xe) | S0_Y(0xb) |
1978                                              S1_X(0x4) | S1_Y(0xc) |
1979                                              S2_X(0x1) | S2_Y(0x6) |
1980                                              S3_X(0xa) | S3_Y(0xe)));
1981         WREG32(PA_SC_AA_SAMPLE_LOCS_8S_WD1, (S4_X(0x6) | S4_Y(0x1) |
1982                                              S5_X(0x0) | S5_Y(0x0) |
1983                                              S6_X(0xb) | S6_Y(0x4) |
1984                                              S7_X(0x7) | S7_Y(0x8)));
1985
1986         WREG32(VGT_STRMOUT_EN, 0);
1987         tmp = rdev->config.r600.max_pipes * 16;
1988         switch (rdev->family) {
1989         case CHIP_RV610:
1990         case CHIP_RV620:
1991         case CHIP_RS780:
1992         case CHIP_RS880:
1993                 tmp += 32;
1994                 break;
1995         case CHIP_RV670:
1996                 tmp += 128;
1997                 break;
1998         default:
1999                 break;
2000         }
2001         if (tmp > 256) {
2002                 tmp = 256;
2003         }
2004         WREG32(VGT_ES_PER_GS, 128);
2005         WREG32(VGT_GS_PER_ES, tmp);
2006         WREG32(VGT_GS_PER_VS, 2);
2007         WREG32(VGT_GS_VERTEX_REUSE, 16);
2008
2009         /* more default values. 2D/3D driver should adjust as needed */
2010         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
2011         WREG32(VGT_STRMOUT_EN, 0);
2012         WREG32(SX_MISC, 0);
2013         WREG32(PA_SC_MODE_CNTL, 0);
2014         WREG32(PA_SC_AA_CONFIG, 0);
2015         WREG32(PA_SC_LINE_STIPPLE, 0);
2016         WREG32(SPI_INPUT_Z, 0);
2017         WREG32(SPI_PS_IN_CONTROL_0, NUM_INTERP(2));
2018         WREG32(CB_COLOR7_FRAG, 0);
2019
2020         /* Clear render buffer base addresses */
2021         WREG32(CB_COLOR0_BASE, 0);
2022         WREG32(CB_COLOR1_BASE, 0);
2023         WREG32(CB_COLOR2_BASE, 0);
2024         WREG32(CB_COLOR3_BASE, 0);
2025         WREG32(CB_COLOR4_BASE, 0);
2026         WREG32(CB_COLOR5_BASE, 0);
2027         WREG32(CB_COLOR6_BASE, 0);
2028         WREG32(CB_COLOR7_BASE, 0);
2029         WREG32(CB_COLOR7_FRAG, 0);
2030
2031         switch (rdev->family) {
2032         case CHIP_RV610:
2033         case CHIP_RV620:
2034         case CHIP_RS780:
2035         case CHIP_RS880:
2036                 tmp = TC_L2_SIZE(8);
2037                 break;
2038         case CHIP_RV630:
2039         case CHIP_RV635:
2040                 tmp = TC_L2_SIZE(4);
2041                 break;
2042         case CHIP_R600:
2043                 tmp = TC_L2_SIZE(0) | L2_DISABLE_LATE_HIT;
2044                 break;
2045         default:
2046                 tmp = TC_L2_SIZE(0);
2047                 break;
2048         }
2049         WREG32(TC_CNTL, tmp);
2050
2051         tmp = RREG32(HDP_HOST_PATH_CNTL);
2052         WREG32(HDP_HOST_PATH_CNTL, tmp);
2053
2054         tmp = RREG32(ARB_POP);
2055         tmp |= ENABLE_TC128;
2056         WREG32(ARB_POP, tmp);
2057
2058         WREG32(PA_SC_MULTI_CHIP_CNTL, 0);
2059         WREG32(PA_CL_ENHANCE, (CLIP_VTX_REORDER_ENA |
2060                                NUM_CLIP_SEQ(3)));
2061         WREG32(PA_SC_ENHANCE, FORCE_EOV_MAX_CLK_CNT(4095));
2062         WREG32(VC_ENHANCE, 0);
2063 }
2064
2065
2066 /*
2067  * Indirect registers accessor
2068  */
2069 u32 r600_pciep_rreg(struct radeon_device *rdev, u32 reg)
2070 {
2071         u32 r;
2072
2073         WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
2074         (void)RREG32(PCIE_PORT_INDEX);
2075         r = RREG32(PCIE_PORT_DATA);
2076         return r;
2077 }
2078
2079 void r600_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
2080 {
2081         WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
2082         (void)RREG32(PCIE_PORT_INDEX);
2083         WREG32(PCIE_PORT_DATA, (v));
2084         (void)RREG32(PCIE_PORT_DATA);
2085 }
2086
2087 /*
2088  * CP & Ring
2089  */
2090 void r600_cp_stop(struct radeon_device *rdev)
2091 {
2092         radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
2093         WREG32(R_0086D8_CP_ME_CNTL, S_0086D8_CP_ME_HALT(1));
2094         WREG32(SCRATCH_UMSK, 0);
2095         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
2096 }
2097
2098 int r600_init_microcode(struct radeon_device *rdev)
2099 {
2100         struct platform_device *pdev;
2101         const char *chip_name;
2102         const char *rlc_chip_name;
2103         size_t pfp_req_size, me_req_size, rlc_req_size;
2104         char fw_name[30];
2105         int err;
2106
2107         DRM_DEBUG("\n");
2108
2109         pdev = platform_device_register_simple("radeon_cp", 0, NULL, 0);
2110         err = IS_ERR(pdev);
2111         if (err) {
2112                 printk(KERN_ERR "radeon_cp: Failed to register firmware\n");
2113                 return -EINVAL;
2114         }
2115
2116         switch (rdev->family) {
2117         case CHIP_R600:
2118                 chip_name = "R600";
2119                 rlc_chip_name = "R600";
2120                 break;
2121         case CHIP_RV610:
2122                 chip_name = "RV610";
2123                 rlc_chip_name = "R600";
2124                 break;
2125         case CHIP_RV630:
2126                 chip_name = "RV630";
2127                 rlc_chip_name = "R600";
2128                 break;
2129         case CHIP_RV620:
2130                 chip_name = "RV620";
2131                 rlc_chip_name = "R600";
2132                 break;
2133         case CHIP_RV635:
2134                 chip_name = "RV635";
2135                 rlc_chip_name = "R600";
2136                 break;
2137         case CHIP_RV670:
2138                 chip_name = "RV670";
2139                 rlc_chip_name = "R600";
2140                 break;
2141         case CHIP_RS780:
2142         case CHIP_RS880:
2143                 chip_name = "RS780";
2144                 rlc_chip_name = "R600";
2145                 break;
2146         case CHIP_RV770:
2147                 chip_name = "RV770";
2148                 rlc_chip_name = "R700";
2149                 break;
2150         case CHIP_RV730:
2151         case CHIP_RV740:
2152                 chip_name = "RV730";
2153                 rlc_chip_name = "R700";
2154                 break;
2155         case CHIP_RV710:
2156                 chip_name = "RV710";
2157                 rlc_chip_name = "R700";
2158                 break;
2159         case CHIP_CEDAR:
2160                 chip_name = "CEDAR";
2161                 rlc_chip_name = "CEDAR";
2162                 break;
2163         case CHIP_REDWOOD:
2164                 chip_name = "REDWOOD";
2165                 rlc_chip_name = "REDWOOD";
2166                 break;
2167         case CHIP_JUNIPER:
2168                 chip_name = "JUNIPER";
2169                 rlc_chip_name = "JUNIPER";
2170                 break;
2171         case CHIP_CYPRESS:
2172         case CHIP_HEMLOCK:
2173                 chip_name = "CYPRESS";
2174                 rlc_chip_name = "CYPRESS";
2175                 break;
2176         case CHIP_PALM:
2177                 chip_name = "PALM";
2178                 rlc_chip_name = "SUMO";
2179                 break;
2180         case CHIP_SUMO:
2181                 chip_name = "SUMO";
2182                 rlc_chip_name = "SUMO";
2183                 break;
2184         case CHIP_SUMO2:
2185                 chip_name = "SUMO2";
2186                 rlc_chip_name = "SUMO";
2187                 break;
2188         default: BUG();
2189         }
2190
2191         if (rdev->family >= CHIP_CEDAR) {
2192                 pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
2193                 me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
2194                 rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
2195         } else if (rdev->family >= CHIP_RV770) {
2196                 pfp_req_size = R700_PFP_UCODE_SIZE * 4;
2197                 me_req_size = R700_PM4_UCODE_SIZE * 4;
2198                 rlc_req_size = R700_RLC_UCODE_SIZE * 4;
2199         } else {
2200                 pfp_req_size = PFP_UCODE_SIZE * 4;
2201                 me_req_size = PM4_UCODE_SIZE * 12;
2202                 rlc_req_size = RLC_UCODE_SIZE * 4;
2203         }
2204
2205         DRM_INFO("Loading %s Microcode\n", chip_name);
2206
2207         snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
2208         err = request_firmware(&rdev->pfp_fw, fw_name, &pdev->dev);
2209         if (err)
2210                 goto out;
2211         if (rdev->pfp_fw->size != pfp_req_size) {
2212                 printk(KERN_ERR
2213                        "r600_cp: Bogus length %zu in firmware \"%s\"\n",
2214                        rdev->pfp_fw->size, fw_name);
2215                 err = -EINVAL;
2216                 goto out;
2217         }
2218
2219         snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
2220         err = request_firmware(&rdev->me_fw, fw_name, &pdev->dev);
2221         if (err)
2222                 goto out;
2223         if (rdev->me_fw->size != me_req_size) {
2224                 printk(KERN_ERR
2225                        "r600_cp: Bogus length %zu in firmware \"%s\"\n",
2226                        rdev->me_fw->size, fw_name);
2227                 err = -EINVAL;
2228         }
2229
2230         snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
2231         err = request_firmware(&rdev->rlc_fw, fw_name, &pdev->dev);
2232         if (err)
2233                 goto out;
2234         if (rdev->rlc_fw->size != rlc_req_size) {
2235                 printk(KERN_ERR
2236                        "r600_rlc: Bogus length %zu in firmware \"%s\"\n",
2237                        rdev->rlc_fw->size, fw_name);
2238                 err = -EINVAL;
2239         }
2240
2241 out:
2242         platform_device_unregister(pdev);
2243
2244         if (err) {
2245                 if (err != -EINVAL)
2246                         printk(KERN_ERR
2247                                "r600_cp: Failed to load firmware \"%s\"\n",
2248                                fw_name);
2249                 release_firmware(rdev->pfp_fw);
2250                 rdev->pfp_fw = NULL;
2251                 release_firmware(rdev->me_fw);
2252                 rdev->me_fw = NULL;
2253                 release_firmware(rdev->rlc_fw);
2254                 rdev->rlc_fw = NULL;
2255         }
2256         return err;
2257 }
2258
2259 static int r600_cp_load_microcode(struct radeon_device *rdev)
2260 {
2261         const __be32 *fw_data;
2262         int i;
2263
2264         if (!rdev->me_fw || !rdev->pfp_fw)
2265                 return -EINVAL;
2266
2267         r600_cp_stop(rdev);
2268
2269         WREG32(CP_RB_CNTL,
2270 #ifdef __BIG_ENDIAN
2271                BUF_SWAP_32BIT |
2272 #endif
2273                RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
2274
2275         /* Reset cp */
2276         WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
2277         RREG32(GRBM_SOFT_RESET);
2278         mdelay(15);
2279         WREG32(GRBM_SOFT_RESET, 0);
2280
2281         WREG32(CP_ME_RAM_WADDR, 0);
2282
2283         fw_data = (const __be32 *)rdev->me_fw->data;
2284         WREG32(CP_ME_RAM_WADDR, 0);
2285         for (i = 0; i < PM4_UCODE_SIZE * 3; i++)
2286                 WREG32(CP_ME_RAM_DATA,
2287                        be32_to_cpup(fw_data++));
2288
2289         fw_data = (const __be32 *)rdev->pfp_fw->data;
2290         WREG32(CP_PFP_UCODE_ADDR, 0);
2291         for (i = 0; i < PFP_UCODE_SIZE; i++)
2292                 WREG32(CP_PFP_UCODE_DATA,
2293                        be32_to_cpup(fw_data++));
2294
2295         WREG32(CP_PFP_UCODE_ADDR, 0);
2296         WREG32(CP_ME_RAM_WADDR, 0);
2297         WREG32(CP_ME_RAM_RADDR, 0);
2298         return 0;
2299 }
2300
2301 int r600_cp_start(struct radeon_device *rdev)
2302 {
2303         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2304         int r;
2305         uint32_t cp_me;
2306
2307         r = radeon_ring_lock(rdev, ring, 7);
2308         if (r) {
2309                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2310                 return r;
2311         }
2312         radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2313         radeon_ring_write(ring, 0x1);
2314         if (rdev->family >= CHIP_RV770) {
2315                 radeon_ring_write(ring, 0x0);
2316                 radeon_ring_write(ring, rdev->config.rv770.max_hw_contexts - 1);
2317         } else {
2318                 radeon_ring_write(ring, 0x3);
2319                 radeon_ring_write(ring, rdev->config.r600.max_hw_contexts - 1);
2320         }
2321         radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2322         radeon_ring_write(ring, 0);
2323         radeon_ring_write(ring, 0);
2324         radeon_ring_unlock_commit(rdev, ring);
2325
2326         cp_me = 0xff;
2327         WREG32(R_0086D8_CP_ME_CNTL, cp_me);
2328         return 0;
2329 }
2330
2331 int r600_cp_resume(struct radeon_device *rdev)
2332 {
2333         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2334         u32 tmp;
2335         u32 rb_bufsz;
2336         int r;
2337
2338         /* Reset cp */
2339         WREG32(GRBM_SOFT_RESET, SOFT_RESET_CP);
2340         RREG32(GRBM_SOFT_RESET);
2341         mdelay(15);
2342         WREG32(GRBM_SOFT_RESET, 0);
2343
2344         /* Set ring buffer size */
2345         rb_bufsz = drm_order(ring->ring_size / 8);
2346         tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
2347 #ifdef __BIG_ENDIAN
2348         tmp |= BUF_SWAP_32BIT;
2349 #endif
2350         WREG32(CP_RB_CNTL, tmp);
2351         WREG32(CP_SEM_WAIT_TIMER, 0x0);
2352
2353         /* Set the write pointer delay */
2354         WREG32(CP_RB_WPTR_DELAY, 0);
2355
2356         /* Initialize the ring buffer's read and write pointers */
2357         WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
2358         WREG32(CP_RB_RPTR_WR, 0);
2359         ring->wptr = 0;
2360         WREG32(CP_RB_WPTR, ring->wptr);
2361
2362         /* set the wb address whether it's enabled or not */
2363         WREG32(CP_RB_RPTR_ADDR,
2364                ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
2365         WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2366         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2367
2368         if (rdev->wb.enabled)
2369                 WREG32(SCRATCH_UMSK, 0xff);
2370         else {
2371                 tmp |= RB_NO_UPDATE;
2372                 WREG32(SCRATCH_UMSK, 0);
2373         }
2374
2375         mdelay(1);
2376         WREG32(CP_RB_CNTL, tmp);
2377
2378         WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
2379         WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
2380
2381         ring->rptr = RREG32(CP_RB_RPTR);
2382
2383         r600_cp_start(rdev);
2384         ring->ready = true;
2385         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
2386         if (r) {
2387                 ring->ready = false;
2388                 return r;
2389         }
2390         return 0;
2391 }
2392
2393 void r600_ring_init(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ring_size)
2394 {
2395         u32 rb_bufsz;
2396         int r;
2397
2398         /* Align ring size */
2399         rb_bufsz = drm_order(ring_size / 8);
2400         ring_size = (1 << (rb_bufsz + 1)) * 4;
2401         ring->ring_size = ring_size;
2402         ring->align_mask = 16 - 1;
2403
2404         if (radeon_ring_supports_scratch_reg(rdev, ring)) {
2405                 r = radeon_scratch_get(rdev, &ring->rptr_save_reg);
2406                 if (r) {
2407                         DRM_ERROR("failed to get scratch reg for rptr save (%d).\n", r);
2408                         ring->rptr_save_reg = 0;
2409                 }
2410         }
2411 }
2412
2413 void r600_cp_fini(struct radeon_device *rdev)
2414 {
2415         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2416         r600_cp_stop(rdev);
2417         radeon_ring_fini(rdev, ring);
2418         radeon_scratch_free(rdev, ring->rptr_save_reg);
2419 }
2420
2421 /*
2422  * DMA
2423  * Starting with R600, the GPU has an asynchronous
2424  * DMA engine.  The programming model is very similar
2425  * to the 3D engine (ring buffer, IBs, etc.), but the
2426  * DMA controller has it's own packet format that is
2427  * different form the PM4 format used by the 3D engine.
2428  * It supports copying data, writing embedded data,
2429  * solid fills, and a number of other things.  It also
2430  * has support for tiling/detiling of buffers.
2431  */
2432 /**
2433  * r600_dma_stop - stop the async dma engine
2434  *
2435  * @rdev: radeon_device pointer
2436  *
2437  * Stop the async dma engine (r6xx-evergreen).
2438  */
2439 void r600_dma_stop(struct radeon_device *rdev)
2440 {
2441         u32 rb_cntl = RREG32(DMA_RB_CNTL);
2442
2443         radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
2444
2445         rb_cntl &= ~DMA_RB_ENABLE;
2446         WREG32(DMA_RB_CNTL, rb_cntl);
2447
2448         rdev->ring[R600_RING_TYPE_DMA_INDEX].ready = false;
2449 }
2450
2451 /**
2452  * r600_dma_resume - setup and start the async dma engine
2453  *
2454  * @rdev: radeon_device pointer
2455  *
2456  * Set up the DMA ring buffer and enable it. (r6xx-evergreen).
2457  * Returns 0 for success, error for failure.
2458  */
2459 int r600_dma_resume(struct radeon_device *rdev)
2460 {
2461         struct radeon_ring *ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2462         u32 rb_cntl, dma_cntl, ib_cntl;
2463         u32 rb_bufsz;
2464         int r;
2465
2466         /* Reset dma */
2467         if (rdev->family >= CHIP_RV770)
2468                 WREG32(SRBM_SOFT_RESET, RV770_SOFT_RESET_DMA);
2469         else
2470                 WREG32(SRBM_SOFT_RESET, SOFT_RESET_DMA);
2471         RREG32(SRBM_SOFT_RESET);
2472         udelay(50);
2473         WREG32(SRBM_SOFT_RESET, 0);
2474
2475         WREG32(DMA_SEM_INCOMPLETE_TIMER_CNTL, 0);
2476         WREG32(DMA_SEM_WAIT_FAIL_TIMER_CNTL, 0);
2477
2478         /* Set ring buffer size in dwords */
2479         rb_bufsz = drm_order(ring->ring_size / 4);
2480         rb_cntl = rb_bufsz << 1;
2481 #ifdef __BIG_ENDIAN
2482         rb_cntl |= DMA_RB_SWAP_ENABLE | DMA_RPTR_WRITEBACK_SWAP_ENABLE;
2483 #endif
2484         WREG32(DMA_RB_CNTL, rb_cntl);
2485
2486         /* Initialize the ring buffer's read and write pointers */
2487         WREG32(DMA_RB_RPTR, 0);
2488         WREG32(DMA_RB_WPTR, 0);
2489
2490         /* set the wb address whether it's enabled or not */
2491         WREG32(DMA_RB_RPTR_ADDR_HI,
2492                upper_32_bits(rdev->wb.gpu_addr + R600_WB_DMA_RPTR_OFFSET) & 0xFF);
2493         WREG32(DMA_RB_RPTR_ADDR_LO,
2494                ((rdev->wb.gpu_addr + R600_WB_DMA_RPTR_OFFSET) & 0xFFFFFFFC));
2495
2496         if (rdev->wb.enabled)
2497                 rb_cntl |= DMA_RPTR_WRITEBACK_ENABLE;
2498
2499         WREG32(DMA_RB_BASE, ring->gpu_addr >> 8);
2500
2501         /* enable DMA IBs */
2502         ib_cntl = DMA_IB_ENABLE;
2503 #ifdef __BIG_ENDIAN
2504         ib_cntl |= DMA_IB_SWAP_ENABLE;
2505 #endif
2506         WREG32(DMA_IB_CNTL, ib_cntl);
2507
2508         dma_cntl = RREG32(DMA_CNTL);
2509         dma_cntl &= ~CTXEMPTY_INT_ENABLE;
2510         WREG32(DMA_CNTL, dma_cntl);
2511
2512         if (rdev->family >= CHIP_RV770)
2513                 WREG32(DMA_MODE, 1);
2514
2515         ring->wptr = 0;
2516         WREG32(DMA_RB_WPTR, ring->wptr << 2);
2517
2518         ring->rptr = RREG32(DMA_RB_RPTR) >> 2;
2519
2520         WREG32(DMA_RB_CNTL, rb_cntl | DMA_RB_ENABLE);
2521
2522         ring->ready = true;
2523
2524         r = radeon_ring_test(rdev, R600_RING_TYPE_DMA_INDEX, ring);
2525         if (r) {
2526                 ring->ready = false;
2527                 return r;
2528         }
2529
2530         radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
2531
2532         return 0;
2533 }
2534
2535 /**
2536  * r600_dma_fini - tear down the async dma engine
2537  *
2538  * @rdev: radeon_device pointer
2539  *
2540  * Stop the async dma engine and free the ring (r6xx-evergreen).
2541  */
2542 void r600_dma_fini(struct radeon_device *rdev)
2543 {
2544         r600_dma_stop(rdev);
2545         radeon_ring_fini(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX]);
2546 }
2547
2548 /*
2549  * GPU scratch registers helpers function.
2550  */
2551 void r600_scratch_init(struct radeon_device *rdev)
2552 {
2553         int i;
2554
2555         rdev->scratch.num_reg = 7;
2556         rdev->scratch.reg_base = SCRATCH_REG0;
2557         for (i = 0; i < rdev->scratch.num_reg; i++) {
2558                 rdev->scratch.free[i] = true;
2559                 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
2560         }
2561 }
2562
2563 int r600_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
2564 {
2565         uint32_t scratch;
2566         uint32_t tmp = 0;
2567         unsigned i;
2568         int r;
2569
2570         r = radeon_scratch_get(rdev, &scratch);
2571         if (r) {
2572                 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
2573                 return r;
2574         }
2575         WREG32(scratch, 0xCAFEDEAD);
2576         r = radeon_ring_lock(rdev, ring, 3);
2577         if (r) {
2578                 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
2579                 radeon_scratch_free(rdev, scratch);
2580                 return r;
2581         }
2582         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2583         radeon_ring_write(ring, ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
2584         radeon_ring_write(ring, 0xDEADBEEF);
2585         radeon_ring_unlock_commit(rdev, ring);
2586         for (i = 0; i < rdev->usec_timeout; i++) {
2587                 tmp = RREG32(scratch);
2588                 if (tmp == 0xDEADBEEF)
2589                         break;
2590                 DRM_UDELAY(1);
2591         }
2592         if (i < rdev->usec_timeout) {
2593                 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
2594         } else {
2595                 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
2596                           ring->idx, scratch, tmp);
2597                 r = -EINVAL;
2598         }
2599         radeon_scratch_free(rdev, scratch);
2600         return r;
2601 }
2602
2603 /**
2604  * r600_dma_ring_test - simple async dma engine test
2605  *
2606  * @rdev: radeon_device pointer
2607  * @ring: radeon_ring structure holding ring information
2608  *
2609  * Test the DMA engine by writing using it to write an
2610  * value to memory. (r6xx-SI).
2611  * Returns 0 for success, error for failure.
2612  */
2613 int r600_dma_ring_test(struct radeon_device *rdev,
2614                        struct radeon_ring *ring)
2615 {
2616         unsigned i;
2617         int r;
2618         void __iomem *ptr = (void *)rdev->vram_scratch.ptr;
2619         u32 tmp;
2620
2621         if (!ptr) {
2622                 DRM_ERROR("invalid vram scratch pointer\n");
2623                 return -EINVAL;
2624         }
2625
2626         tmp = 0xCAFEDEAD;
2627         writel(tmp, ptr);
2628
2629         r = radeon_ring_lock(rdev, ring, 4);
2630         if (r) {
2631                 DRM_ERROR("radeon: dma failed to lock ring %d (%d).\n", ring->idx, r);
2632                 return r;
2633         }
2634         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 1));
2635         radeon_ring_write(ring, rdev->vram_scratch.gpu_addr & 0xfffffffc);
2636         radeon_ring_write(ring, upper_32_bits(rdev->vram_scratch.gpu_addr) & 0xff);
2637         radeon_ring_write(ring, 0xDEADBEEF);
2638         radeon_ring_unlock_commit(rdev, ring);
2639
2640         for (i = 0; i < rdev->usec_timeout; i++) {
2641                 tmp = readl(ptr);
2642                 if (tmp == 0xDEADBEEF)
2643                         break;
2644                 DRM_UDELAY(1);
2645         }
2646
2647         if (i < rdev->usec_timeout) {
2648                 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
2649         } else {
2650                 DRM_ERROR("radeon: ring %d test failed (0x%08X)\n",
2651                           ring->idx, tmp);
2652                 r = -EINVAL;
2653         }
2654         return r;
2655 }
2656
2657 /*
2658  * CP fences/semaphores
2659  */
2660
2661 void r600_fence_ring_emit(struct radeon_device *rdev,
2662                           struct radeon_fence *fence)
2663 {
2664         struct radeon_ring *ring = &rdev->ring[fence->ring];
2665
2666         if (rdev->wb.use_event) {
2667                 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
2668                 /* flush read cache over gart */
2669                 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
2670                 radeon_ring_write(ring, PACKET3_TC_ACTION_ENA |
2671                                         PACKET3_VC_ACTION_ENA |
2672                                         PACKET3_SH_ACTION_ENA);
2673                 radeon_ring_write(ring, 0xFFFFFFFF);
2674                 radeon_ring_write(ring, 0);
2675                 radeon_ring_write(ring, 10); /* poll interval */
2676                 /* EVENT_WRITE_EOP - flush caches, send int */
2677                 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
2678                 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) | EVENT_INDEX(5));
2679                 radeon_ring_write(ring, addr & 0xffffffff);
2680                 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
2681                 radeon_ring_write(ring, fence->seq);
2682                 radeon_ring_write(ring, 0);
2683         } else {
2684                 /* flush read cache over gart */
2685                 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
2686                 radeon_ring_write(ring, PACKET3_TC_ACTION_ENA |
2687                                         PACKET3_VC_ACTION_ENA |
2688                                         PACKET3_SH_ACTION_ENA);
2689                 radeon_ring_write(ring, 0xFFFFFFFF);
2690                 radeon_ring_write(ring, 0);
2691                 radeon_ring_write(ring, 10); /* poll interval */
2692                 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0));
2693                 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT) | EVENT_INDEX(0));
2694                 /* wait for 3D idle clean */
2695                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2696                 radeon_ring_write(ring, (WAIT_UNTIL - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
2697                 radeon_ring_write(ring, WAIT_3D_IDLE_bit | WAIT_3D_IDLECLEAN_bit);
2698                 /* Emit fence sequence & fire IRQ */
2699                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2700                 radeon_ring_write(ring, ((rdev->fence_drv[fence->ring].scratch_reg - PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
2701                 radeon_ring_write(ring, fence->seq);
2702                 /* CP_INTERRUPT packet 3 no longer exists, use packet 0 */
2703                 radeon_ring_write(ring, PACKET0(CP_INT_STATUS, 0));
2704                 radeon_ring_write(ring, RB_INT_STAT);
2705         }
2706 }
2707
2708 void r600_semaphore_ring_emit(struct radeon_device *rdev,
2709                               struct radeon_ring *ring,
2710                               struct radeon_semaphore *semaphore,
2711                               bool emit_wait)
2712 {
2713         uint64_t addr = semaphore->gpu_addr;
2714         unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
2715
2716         if (rdev->family < CHIP_CAYMAN)
2717                 sel |= PACKET3_SEM_WAIT_ON_SIGNAL;
2718
2719         radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
2720         radeon_ring_write(ring, addr & 0xffffffff);
2721         radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | sel);
2722 }
2723
2724 /*
2725  * DMA fences/semaphores
2726  */
2727
2728 /**
2729  * r600_dma_fence_ring_emit - emit a fence on the DMA ring
2730  *
2731  * @rdev: radeon_device pointer
2732  * @fence: radeon fence object
2733  *
2734  * Add a DMA fence packet to the ring to write
2735  * the fence seq number and DMA trap packet to generate
2736  * an interrupt if needed (r6xx-r7xx).
2737  */
2738 void r600_dma_fence_ring_emit(struct radeon_device *rdev,
2739                               struct radeon_fence *fence)
2740 {
2741         struct radeon_ring *ring = &rdev->ring[fence->ring];
2742         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
2743
2744         /* write the fence */
2745         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_FENCE, 0, 0, 0));
2746         radeon_ring_write(ring, addr & 0xfffffffc);
2747         radeon_ring_write(ring, (upper_32_bits(addr) & 0xff));
2748         radeon_ring_write(ring, lower_32_bits(fence->seq));
2749         /* generate an interrupt */
2750         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_TRAP, 0, 0, 0));
2751 }
2752
2753 /**
2754  * r600_dma_semaphore_ring_emit - emit a semaphore on the dma ring
2755  *
2756  * @rdev: radeon_device pointer
2757  * @ring: radeon_ring structure holding ring information
2758  * @semaphore: radeon semaphore object
2759  * @emit_wait: wait or signal semaphore
2760  *
2761  * Add a DMA semaphore packet to the ring wait on or signal
2762  * other rings (r6xx-SI).
2763  */
2764 void r600_dma_semaphore_ring_emit(struct radeon_device *rdev,
2765                                   struct radeon_ring *ring,
2766                                   struct radeon_semaphore *semaphore,
2767                                   bool emit_wait)
2768 {
2769         u64 addr = semaphore->gpu_addr;
2770         u32 s = emit_wait ? 0 : 1;
2771
2772         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SEMAPHORE, 0, s, 0));
2773         radeon_ring_write(ring, addr & 0xfffffffc);
2774         radeon_ring_write(ring, upper_32_bits(addr) & 0xff);
2775 }
2776
2777 int r600_copy_blit(struct radeon_device *rdev,
2778                    uint64_t src_offset,
2779                    uint64_t dst_offset,
2780                    unsigned num_gpu_pages,
2781                    struct radeon_fence **fence)
2782 {
2783         struct radeon_semaphore *sem = NULL;
2784         struct radeon_sa_bo *vb = NULL;
2785         int r;
2786
2787         r = r600_blit_prepare_copy(rdev, num_gpu_pages, fence, &vb, &sem);
2788         if (r) {
2789                 return r;
2790         }
2791         r600_kms_blit_copy(rdev, src_offset, dst_offset, num_gpu_pages, vb);
2792         r600_blit_done_copy(rdev, fence, vb, sem);
2793         return 0;
2794 }
2795
2796 /**
2797  * r600_copy_dma - copy pages using the DMA engine
2798  *
2799  * @rdev: radeon_device pointer
2800  * @src_offset: src GPU address
2801  * @dst_offset: dst GPU address
2802  * @num_gpu_pages: number of GPU pages to xfer
2803  * @fence: radeon fence object
2804  *
2805  * Copy GPU paging using the DMA engine (r6xx).
2806  * Used by the radeon ttm implementation to move pages if
2807  * registered as the asic copy callback.
2808  */
2809 int r600_copy_dma(struct radeon_device *rdev,
2810                   uint64_t src_offset, uint64_t dst_offset,
2811                   unsigned num_gpu_pages,
2812                   struct radeon_fence **fence)
2813 {
2814         struct radeon_semaphore *sem = NULL;
2815         int ring_index = rdev->asic->copy.dma_ring_index;
2816         struct radeon_ring *ring = &rdev->ring[ring_index];
2817         u32 size_in_dw, cur_size_in_dw;
2818         int i, num_loops;
2819         int r = 0;
2820
2821         r = radeon_semaphore_create(rdev, &sem);
2822         if (r) {
2823                 DRM_ERROR("radeon: moving bo (%d).\n", r);
2824                 return r;
2825         }
2826
2827         size_in_dw = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT) / 4;
2828         num_loops = DIV_ROUND_UP(size_in_dw, 0xFFFE);
2829         r = radeon_ring_lock(rdev, ring, num_loops * 4 + 8);
2830         if (r) {
2831                 DRM_ERROR("radeon: moving bo (%d).\n", r);
2832                 radeon_semaphore_free(rdev, &sem, NULL);
2833                 return r;
2834         }
2835
2836         if (radeon_fence_need_sync(*fence, ring->idx)) {
2837                 radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
2838                                             ring->idx);
2839                 radeon_fence_note_sync(*fence, ring->idx);
2840         } else {
2841                 radeon_semaphore_free(rdev, &sem, NULL);
2842         }
2843
2844         for (i = 0; i < num_loops; i++) {
2845                 cur_size_in_dw = size_in_dw;
2846                 if (cur_size_in_dw > 0xFFFE)
2847                         cur_size_in_dw = 0xFFFE;
2848                 size_in_dw -= cur_size_in_dw;
2849                 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 0, 0, cur_size_in_dw));
2850                 radeon_ring_write(ring, dst_offset & 0xfffffffc);
2851                 radeon_ring_write(ring, src_offset & 0xfffffffc);
2852                 radeon_ring_write(ring, (((upper_32_bits(dst_offset) & 0xff) << 16) |
2853                                          (upper_32_bits(src_offset) & 0xff)));
2854                 src_offset += cur_size_in_dw * 4;
2855                 dst_offset += cur_size_in_dw * 4;
2856         }
2857
2858         r = radeon_fence_emit(rdev, fence, ring->idx);
2859         if (r) {
2860                 radeon_ring_unlock_undo(rdev, ring);
2861                 return r;
2862         }
2863
2864         radeon_ring_unlock_commit(rdev, ring);
2865         radeon_semaphore_free(rdev, &sem, *fence);
2866
2867         return r;
2868 }
2869
2870 int r600_set_surface_reg(struct radeon_device *rdev, int reg,
2871                          uint32_t tiling_flags, uint32_t pitch,
2872                          uint32_t offset, uint32_t obj_size)
2873 {
2874         /* FIXME: implement */
2875         return 0;
2876 }
2877
2878 void r600_clear_surface_reg(struct radeon_device *rdev, int reg)
2879 {
2880         /* FIXME: implement */
2881 }
2882
2883 static int r600_startup(struct radeon_device *rdev)
2884 {
2885         struct radeon_ring *ring;
2886         int r;
2887
2888         /* enable pcie gen2 link */
2889         r600_pcie_gen2_enable(rdev);
2890
2891         if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
2892                 r = r600_init_microcode(rdev);
2893                 if (r) {
2894                         DRM_ERROR("Failed to load firmware!\n");
2895                         return r;
2896                 }
2897         }
2898
2899         r = r600_vram_scratch_init(rdev);
2900         if (r)
2901                 return r;
2902
2903         r600_mc_program(rdev);
2904         if (rdev->flags & RADEON_IS_AGP) {
2905                 r600_agp_enable(rdev);
2906         } else {
2907                 r = r600_pcie_gart_enable(rdev);
2908                 if (r)
2909                         return r;
2910         }
2911         r600_gpu_init(rdev);
2912         r = r600_blit_init(rdev);
2913         if (r) {
2914                 r600_blit_fini(rdev);
2915                 rdev->asic->copy.copy = NULL;
2916                 dev_warn(rdev->dev, "failed blitter (%d) falling back to memcpy\n", r);
2917         }
2918
2919         /* allocate wb buffer */
2920         r = radeon_wb_init(rdev);
2921         if (r)
2922                 return r;
2923
2924         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
2925         if (r) {
2926                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2927                 return r;
2928         }
2929
2930         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
2931         if (r) {
2932                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2933                 return r;
2934         }
2935
2936         /* Enable IRQ */
2937         r = r600_irq_init(rdev);
2938         if (r) {
2939                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
2940                 radeon_irq_kms_fini(rdev);
2941                 return r;
2942         }
2943         r600_irq_set(rdev);
2944
2945         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2946         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
2947                              R600_CP_RB_RPTR, R600_CP_RB_WPTR,
2948                              0, 0xfffff, RADEON_CP_PACKET2);
2949         if (r)
2950                 return r;
2951
2952         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2953         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2954                              DMA_RB_RPTR, DMA_RB_WPTR,
2955                              2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2956         if (r)
2957                 return r;
2958
2959         r = r600_cp_load_microcode(rdev);
2960         if (r)
2961                 return r;
2962         r = r600_cp_resume(rdev);
2963         if (r)
2964                 return r;
2965
2966         r = r600_dma_resume(rdev);
2967         if (r)
2968                 return r;
2969
2970         r = radeon_ib_pool_init(rdev);
2971         if (r) {
2972                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
2973                 return r;
2974         }
2975
2976         r = r600_audio_init(rdev);
2977         if (r) {
2978                 DRM_ERROR("radeon: audio init failed\n");
2979                 return r;
2980         }
2981
2982         return 0;
2983 }
2984
2985 void r600_vga_set_state(struct radeon_device *rdev, bool state)
2986 {
2987         uint32_t temp;
2988
2989         temp = RREG32(CONFIG_CNTL);
2990         if (state == false) {
2991                 temp &= ~(1<<0);
2992                 temp |= (1<<1);
2993         } else {
2994                 temp &= ~(1<<1);
2995         }
2996         WREG32(CONFIG_CNTL, temp);
2997 }
2998
2999 int r600_resume(struct radeon_device *rdev)
3000 {
3001         int r;
3002
3003         /* Do not reset GPU before posting, on r600 hw unlike on r500 hw,
3004          * posting will perform necessary task to bring back GPU into good
3005          * shape.
3006          */
3007         /* post card */
3008         atom_asic_init(rdev->mode_info.atom_context);
3009
3010         rdev->accel_working = true;
3011         r = r600_startup(rdev);
3012         if (r) {
3013                 DRM_ERROR("r600 startup failed on resume\n");
3014                 rdev->accel_working = false;
3015                 return r;
3016         }
3017
3018         return r;
3019 }
3020
3021 int r600_suspend(struct radeon_device *rdev)
3022 {
3023         r600_audio_fini(rdev);
3024         r600_cp_stop(rdev);
3025         r600_dma_stop(rdev);
3026         r600_irq_suspend(rdev);
3027         radeon_wb_disable(rdev);
3028         r600_pcie_gart_disable(rdev);
3029
3030         return 0;
3031 }
3032
3033 /* Plan is to move initialization in that function and use
3034  * helper function so that radeon_device_init pretty much
3035  * do nothing more than calling asic specific function. This
3036  * should also allow to remove a bunch of callback function
3037  * like vram_info.
3038  */
3039 int r600_init(struct radeon_device *rdev)
3040 {
3041         int r;
3042
3043         if (r600_debugfs_mc_info_init(rdev)) {
3044                 DRM_ERROR("Failed to register debugfs file for mc !\n");
3045         }
3046         /* Read BIOS */
3047         if (!radeon_get_bios(rdev)) {
3048                 if (ASIC_IS_AVIVO(rdev))
3049                         return -EINVAL;
3050         }
3051         /* Must be an ATOMBIOS */
3052         if (!rdev->is_atom_bios) {
3053                 dev_err(rdev->dev, "Expecting atombios for R600 GPU\n");
3054                 return -EINVAL;
3055         }
3056         r = radeon_atombios_init(rdev);
3057         if (r)
3058                 return r;
3059         /* Post card if necessary */
3060         if (!radeon_card_posted(rdev)) {
3061                 if (!rdev->bios) {
3062                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
3063                         return -EINVAL;
3064                 }
3065                 DRM_INFO("GPU not posted. posting now...\n");
3066                 atom_asic_init(rdev->mode_info.atom_context);
3067         }
3068         /* Initialize scratch registers */
3069         r600_scratch_init(rdev);
3070         /* Initialize surface registers */
3071         radeon_surface_init(rdev);
3072         /* Initialize clocks */
3073         radeon_get_clock_info(rdev->ddev);
3074         /* Fence driver */
3075         r = radeon_fence_driver_init(rdev);
3076         if (r)
3077                 return r;
3078         if (rdev->flags & RADEON_IS_AGP) {
3079                 r = radeon_agp_init(rdev);
3080                 if (r)
3081                         radeon_agp_disable(rdev);
3082         }
3083         r = r600_mc_init(rdev);
3084         if (r)
3085                 return r;
3086         /* Memory manager */
3087         r = radeon_bo_init(rdev);
3088         if (r)
3089                 return r;
3090
3091         r = radeon_irq_kms_init(rdev);
3092         if (r)
3093                 return r;
3094
3095         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
3096         r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
3097
3098         rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
3099         r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
3100
3101         rdev->ih.ring_obj = NULL;
3102         r600_ih_ring_init(rdev, 64 * 1024);
3103
3104         r = r600_pcie_gart_init(rdev);
3105         if (r)
3106                 return r;
3107
3108         rdev->accel_working = true;
3109         r = r600_startup(rdev);
3110         if (r) {
3111                 dev_err(rdev->dev, "disabling GPU acceleration\n");
3112                 r600_cp_fini(rdev);
3113                 r600_dma_fini(rdev);
3114                 r600_irq_fini(rdev);
3115                 radeon_wb_fini(rdev);
3116                 radeon_ib_pool_fini(rdev);
3117                 radeon_irq_kms_fini(rdev);
3118                 r600_pcie_gart_fini(rdev);
3119                 rdev->accel_working = false;
3120         }
3121
3122         return 0;
3123 }
3124
3125 void r600_fini(struct radeon_device *rdev)
3126 {
3127         r600_audio_fini(rdev);
3128         r600_blit_fini(rdev);
3129         r600_cp_fini(rdev);
3130         r600_dma_fini(rdev);
3131         r600_irq_fini(rdev);
3132         radeon_wb_fini(rdev);
3133         radeon_ib_pool_fini(rdev);
3134         radeon_irq_kms_fini(rdev);
3135         r600_pcie_gart_fini(rdev);
3136         r600_vram_scratch_fini(rdev);
3137         radeon_agp_fini(rdev);
3138         radeon_gem_fini(rdev);
3139         radeon_fence_driver_fini(rdev);
3140         radeon_bo_fini(rdev);
3141         radeon_atombios_fini(rdev);
3142         kfree(rdev->bios);
3143         rdev->bios = NULL;
3144 }
3145
3146
3147 /*
3148  * CS stuff
3149  */
3150 void r600_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3151 {
3152         struct radeon_ring *ring = &rdev->ring[ib->ring];
3153         u32 next_rptr;
3154
3155         if (ring->rptr_save_reg) {
3156                 next_rptr = ring->wptr + 3 + 4;
3157                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3158                 radeon_ring_write(ring, ((ring->rptr_save_reg -
3159                                          PACKET3_SET_CONFIG_REG_OFFSET) >> 2));
3160                 radeon_ring_write(ring, next_rptr);
3161         } else if (rdev->wb.enabled) {
3162                 next_rptr = ring->wptr + 5 + 4;
3163                 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
3164                 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3165                 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
3166                 radeon_ring_write(ring, next_rptr);
3167                 radeon_ring_write(ring, 0);
3168         }
3169
3170         radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
3171         radeon_ring_write(ring,
3172 #ifdef __BIG_ENDIAN
3173                           (2 << 0) |
3174 #endif
3175                           (ib->gpu_addr & 0xFFFFFFFC));
3176         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
3177         radeon_ring_write(ring, ib->length_dw);
3178 }
3179
3180 int r600_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3181 {
3182         struct radeon_ib ib;
3183         uint32_t scratch;
3184         uint32_t tmp = 0;
3185         unsigned i;
3186         int r;
3187
3188         r = radeon_scratch_get(rdev, &scratch);
3189         if (r) {
3190                 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3191                 return r;
3192         }
3193         WREG32(scratch, 0xCAFEDEAD);
3194         r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3195         if (r) {
3196                 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
3197                 goto free_scratch;
3198         }
3199         ib.ptr[0] = PACKET3(PACKET3_SET_CONFIG_REG, 1);
3200         ib.ptr[1] = ((scratch - PACKET3_SET_CONFIG_REG_OFFSET) >> 2);
3201         ib.ptr[2] = 0xDEADBEEF;
3202         ib.length_dw = 3;
3203         r = radeon_ib_schedule(rdev, &ib, NULL);
3204         if (r) {
3205                 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3206                 goto free_ib;
3207         }
3208         r = radeon_fence_wait(ib.fence, false);
3209         if (r) {
3210                 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
3211                 goto free_ib;
3212         }
3213         for (i = 0; i < rdev->usec_timeout; i++) {
3214                 tmp = RREG32(scratch);
3215                 if (tmp == 0xDEADBEEF)
3216                         break;
3217                 DRM_UDELAY(1);
3218         }
3219         if (i < rdev->usec_timeout) {
3220                 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3221         } else {
3222                 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3223                           scratch, tmp);
3224                 r = -EINVAL;
3225         }
3226 free_ib:
3227         radeon_ib_free(rdev, &ib);
3228 free_scratch:
3229         radeon_scratch_free(rdev, scratch);
3230         return r;
3231 }
3232
3233 /**
3234  * r600_dma_ib_test - test an IB on the DMA engine
3235  *
3236  * @rdev: radeon_device pointer
3237  * @ring: radeon_ring structure holding ring information
3238  *
3239  * Test a simple IB in the DMA ring (r6xx-SI).
3240  * Returns 0 on success, error on failure.
3241  */
3242 int r600_dma_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3243 {
3244         struct radeon_ib ib;
3245         unsigned i;
3246         int r;
3247         void __iomem *ptr = (void *)rdev->vram_scratch.ptr;
3248         u32 tmp = 0;
3249
3250         if (!ptr) {
3251                 DRM_ERROR("invalid vram scratch pointer\n");
3252                 return -EINVAL;
3253         }
3254
3255         tmp = 0xCAFEDEAD;
3256         writel(tmp, ptr);
3257
3258         r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3259         if (r) {
3260                 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
3261                 return r;
3262         }
3263
3264         ib.ptr[0] = DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 1);
3265         ib.ptr[1] = rdev->vram_scratch.gpu_addr & 0xfffffffc;
3266         ib.ptr[2] = upper_32_bits(rdev->vram_scratch.gpu_addr) & 0xff;
3267         ib.ptr[3] = 0xDEADBEEF;
3268         ib.length_dw = 4;
3269
3270         r = radeon_ib_schedule(rdev, &ib, NULL);
3271         if (r) {
3272                 radeon_ib_free(rdev, &ib);
3273                 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3274                 return r;
3275         }
3276         r = radeon_fence_wait(ib.fence, false);
3277         if (r) {
3278                 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
3279                 return r;
3280         }
3281         for (i = 0; i < rdev->usec_timeout; i++) {
3282                 tmp = readl(ptr);
3283                 if (tmp == 0xDEADBEEF)
3284                         break;
3285                 DRM_UDELAY(1);
3286         }
3287         if (i < rdev->usec_timeout) {
3288                 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3289         } else {
3290                 DRM_ERROR("radeon: ib test failed (0x%08X)\n", tmp);
3291                 r = -EINVAL;
3292         }
3293         radeon_ib_free(rdev, &ib);
3294         return r;
3295 }
3296
3297 /**
3298  * r600_dma_ring_ib_execute - Schedule an IB on the DMA engine
3299  *
3300  * @rdev: radeon_device pointer
3301  * @ib: IB object to schedule
3302  *
3303  * Schedule an IB in the DMA ring (r6xx-r7xx).
3304  */
3305 void r600_dma_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3306 {
3307         struct radeon_ring *ring = &rdev->ring[ib->ring];
3308
3309         if (rdev->wb.enabled) {
3310                 u32 next_rptr = ring->wptr + 4;
3311                 while ((next_rptr & 7) != 5)
3312                         next_rptr++;
3313                 next_rptr += 3;
3314                 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 1));
3315                 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3316                 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xff);
3317                 radeon_ring_write(ring, next_rptr);
3318         }
3319
3320         /* The indirect buffer packet must end on an 8 DW boundary in the DMA ring.
3321          * Pad as necessary with NOPs.
3322          */
3323         while ((ring->wptr & 7) != 5)
3324                 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
3325         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_INDIRECT_BUFFER, 0, 0, 0));
3326         radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFE0));
3327         radeon_ring_write(ring, (ib->length_dw << 16) | (upper_32_bits(ib->gpu_addr) & 0xFF));
3328
3329 }
3330
3331 /*
3332  * Interrupts
3333  *
3334  * Interrupts use a ring buffer on r6xx/r7xx hardware.  It works pretty
3335  * the same as the CP ring buffer, but in reverse.  Rather than the CPU
3336  * writing to the ring and the GPU consuming, the GPU writes to the ring
3337  * and host consumes.  As the host irq handler processes interrupts, it
3338  * increments the rptr.  When the rptr catches up with the wptr, all the
3339  * current interrupts have been processed.
3340  */
3341
3342 void r600_ih_ring_init(struct radeon_device *rdev, unsigned ring_size)
3343 {
3344         u32 rb_bufsz;
3345
3346         /* Align ring size */
3347         rb_bufsz = drm_order(ring_size / 4);
3348         ring_size = (1 << rb_bufsz) * 4;
3349         rdev->ih.ring_size = ring_size;
3350         rdev->ih.ptr_mask = rdev->ih.ring_size - 1;
3351         rdev->ih.rptr = 0;
3352 }
3353
3354 int r600_ih_ring_alloc(struct radeon_device *rdev)
3355 {
3356         int r;
3357
3358         /* Allocate ring buffer */
3359         if (rdev->ih.ring_obj == NULL) {
3360                 r = radeon_bo_create(rdev, rdev->ih.ring_size,
3361                                      PAGE_SIZE, true,
3362                                      RADEON_GEM_DOMAIN_GTT,
3363                                      NULL, &rdev->ih.ring_obj);
3364                 if (r) {
3365                         DRM_ERROR("radeon: failed to create ih ring buffer (%d).\n", r);
3366                         return r;
3367                 }
3368                 r = radeon_bo_reserve(rdev->ih.ring_obj, false);
3369                 if (unlikely(r != 0))
3370                         return r;
3371                 r = radeon_bo_pin(rdev->ih.ring_obj,
3372                                   RADEON_GEM_DOMAIN_GTT,
3373                                   &rdev->ih.gpu_addr);
3374                 if (r) {
3375                         radeon_bo_unreserve(rdev->ih.ring_obj);
3376                         DRM_ERROR("radeon: failed to pin ih ring buffer (%d).\n", r);
3377                         return r;
3378                 }
3379                 r = radeon_bo_kmap(rdev->ih.ring_obj,
3380                                    (void **)&rdev->ih.ring);
3381                 radeon_bo_unreserve(rdev->ih.ring_obj);
3382                 if (r) {
3383                         DRM_ERROR("radeon: failed to map ih ring buffer (%d).\n", r);
3384                         return r;
3385                 }
3386         }
3387         return 0;
3388 }
3389
3390 void r600_ih_ring_fini(struct radeon_device *rdev)
3391 {
3392         int r;
3393         if (rdev->ih.ring_obj) {
3394                 r = radeon_bo_reserve(rdev->ih.ring_obj, false);
3395                 if (likely(r == 0)) {
3396                         radeon_bo_kunmap(rdev->ih.ring_obj);
3397                         radeon_bo_unpin(rdev->ih.ring_obj);
3398                         radeon_bo_unreserve(rdev->ih.ring_obj);
3399                 }
3400                 radeon_bo_unref(&rdev->ih.ring_obj);
3401                 rdev->ih.ring = NULL;
3402                 rdev->ih.ring_obj = NULL;
3403         }
3404 }
3405
3406 void r600_rlc_stop(struct radeon_device *rdev)
3407 {
3408
3409         if ((rdev->family >= CHIP_RV770) &&
3410             (rdev->family <= CHIP_RV740)) {
3411                 /* r7xx asics need to soft reset RLC before halting */
3412                 WREG32(SRBM_SOFT_RESET, SOFT_RESET_RLC);
3413                 RREG32(SRBM_SOFT_RESET);
3414                 mdelay(15);
3415                 WREG32(SRBM_SOFT_RESET, 0);
3416                 RREG32(SRBM_SOFT_RESET);
3417         }
3418
3419         WREG32(RLC_CNTL, 0);
3420 }
3421
3422 static void r600_rlc_start(struct radeon_device *rdev)
3423 {
3424         WREG32(RLC_CNTL, RLC_ENABLE);
3425 }
3426
3427 static int r600_rlc_init(struct radeon_device *rdev)
3428 {
3429         u32 i;
3430         const __be32 *fw_data;
3431
3432         if (!rdev->rlc_fw)
3433                 return -EINVAL;
3434
3435         r600_rlc_stop(rdev);
3436
3437         WREG32(RLC_HB_CNTL, 0);
3438
3439         if (rdev->family == CHIP_ARUBA) {
3440                 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
3441                 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
3442         }
3443         if (rdev->family <= CHIP_CAYMAN) {
3444                 WREG32(RLC_HB_BASE, 0);
3445                 WREG32(RLC_HB_RPTR, 0);
3446                 WREG32(RLC_HB_WPTR, 0);
3447         }
3448         if (rdev->family <= CHIP_CAICOS) {
3449                 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
3450                 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
3451         }
3452         WREG32(RLC_MC_CNTL, 0);
3453         WREG32(RLC_UCODE_CNTL, 0);
3454
3455         fw_data = (const __be32 *)rdev->rlc_fw->data;
3456         if (rdev->family >= CHIP_ARUBA) {
3457                 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
3458                         WREG32(RLC_UCODE_ADDR, i);
3459                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
3460                 }
3461         } else if (rdev->family >= CHIP_CAYMAN) {
3462                 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
3463                         WREG32(RLC_UCODE_ADDR, i);
3464                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
3465                 }
3466         } else if (rdev->family >= CHIP_CEDAR) {
3467                 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
3468                         WREG32(RLC_UCODE_ADDR, i);
3469                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
3470                 }
3471         } else if (rdev->family >= CHIP_RV770) {
3472                 for (i = 0; i < R700_RLC_UCODE_SIZE; i++) {
3473                         WREG32(RLC_UCODE_ADDR, i);
3474                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
3475                 }
3476         } else {
3477                 for (i = 0; i < RLC_UCODE_SIZE; i++) {
3478                         WREG32(RLC_UCODE_ADDR, i);
3479                         WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
3480                 }
3481         }
3482         WREG32(RLC_UCODE_ADDR, 0);
3483
3484         r600_rlc_start(rdev);
3485
3486         return 0;
3487 }
3488
3489 static void r600_enable_interrupts(struct radeon_device *rdev)
3490 {
3491         u32 ih_cntl = RREG32(IH_CNTL);
3492         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
3493
3494         ih_cntl |= ENABLE_INTR;
3495         ih_rb_cntl |= IH_RB_ENABLE;
3496         WREG32(IH_CNTL, ih_cntl);
3497         WREG32(IH_RB_CNTL, ih_rb_cntl);
3498         rdev->ih.enabled = true;
3499 }
3500
3501 void r600_disable_interrupts(struct radeon_device *rdev)
3502 {
3503         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
3504         u32 ih_cntl = RREG32(IH_CNTL);
3505
3506         ih_rb_cntl &= ~IH_RB_ENABLE;
3507         ih_cntl &= ~ENABLE_INTR;
3508         WREG32(IH_RB_CNTL, ih_rb_cntl);
3509         WREG32(IH_CNTL, ih_cntl);
3510         /* set rptr, wptr to 0 */
3511         WREG32(IH_RB_RPTR, 0);
3512         WREG32(IH_RB_WPTR, 0);
3513         rdev->ih.enabled = false;
3514         rdev->ih.rptr = 0;
3515 }
3516
3517 static void r600_disable_interrupt_state(struct radeon_device *rdev)
3518 {
3519         u32 tmp;
3520
3521         WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
3522         tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
3523         WREG32(DMA_CNTL, tmp);
3524         WREG32(GRBM_INT_CNTL, 0);
3525         WREG32(DxMODE_INT_MASK, 0);
3526         WREG32(D1GRPH_INTERRUPT_CONTROL, 0);
3527         WREG32(D2GRPH_INTERRUPT_CONTROL, 0);
3528         if (ASIC_IS_DCE3(rdev)) {
3529                 WREG32(DCE3_DACA_AUTODETECT_INT_CONTROL, 0);
3530                 WREG32(DCE3_DACB_AUTODETECT_INT_CONTROL, 0);
3531                 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3532                 WREG32(DC_HPD1_INT_CONTROL, tmp);
3533                 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3534                 WREG32(DC_HPD2_INT_CONTROL, tmp);
3535                 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3536                 WREG32(DC_HPD3_INT_CONTROL, tmp);
3537                 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3538                 WREG32(DC_HPD4_INT_CONTROL, tmp);
3539                 if (ASIC_IS_DCE32(rdev)) {
3540                         tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3541                         WREG32(DC_HPD5_INT_CONTROL, tmp);
3542                         tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
3543                         WREG32(DC_HPD6_INT_CONTROL, tmp);
3544                         tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3545                         WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0, tmp);
3546                         tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3547                         WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1, tmp);
3548                 } else {
3549                         tmp = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3550                         WREG32(HDMI0_AUDIO_PACKET_CONTROL, tmp);
3551                         tmp = RREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3552                         WREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL, tmp);
3553                 }
3554         } else {
3555                 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
3556                 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
3557                 tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
3558                 WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
3559                 tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
3560                 WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
3561                 tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL) & DC_HOT_PLUG_DETECTx_INT_POLARITY;
3562                 WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
3563                 tmp = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3564                 WREG32(HDMI0_AUDIO_PACKET_CONTROL, tmp);
3565                 tmp = RREG32(HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3566                 WREG32(HDMI1_AUDIO_PACKET_CONTROL, tmp);
3567         }
3568 }
3569
3570 int r600_irq_init(struct radeon_device *rdev)
3571 {
3572         int ret = 0;
3573         int rb_bufsz;
3574         u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
3575
3576         /* allocate ring */
3577         ret = r600_ih_ring_alloc(rdev);
3578         if (ret)
3579                 return ret;
3580
3581         /* disable irqs */
3582         r600_disable_interrupts(rdev);
3583
3584         /* init rlc */
3585         ret = r600_rlc_init(rdev);
3586         if (ret) {
3587                 r600_ih_ring_fini(rdev);
3588                 return ret;
3589         }
3590
3591         /* setup interrupt control */
3592         /* set dummy read address to ring address */
3593         WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
3594         interrupt_cntl = RREG32(INTERRUPT_CNTL);
3595         /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
3596          * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
3597          */
3598         interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
3599         /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
3600         interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
3601         WREG32(INTERRUPT_CNTL, interrupt_cntl);
3602
3603         WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
3604         rb_bufsz = drm_order(rdev->ih.ring_size / 4);
3605
3606         ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
3607                       IH_WPTR_OVERFLOW_CLEAR |
3608                       (rb_bufsz << 1));
3609
3610         if (rdev->wb.enabled)
3611                 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
3612
3613         /* set the writeback address whether it's enabled or not */
3614         WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
3615         WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
3616
3617         WREG32(IH_RB_CNTL, ih_rb_cntl);
3618
3619         /* set rptr, wptr to 0 */
3620         WREG32(IH_RB_RPTR, 0);
3621         WREG32(IH_RB_WPTR, 0);
3622
3623         /* Default settings for IH_CNTL (disabled at first) */
3624         ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10);
3625         /* RPTR_REARM only works if msi's are enabled */
3626         if (rdev->msi_enabled)
3627                 ih_cntl |= RPTR_REARM;
3628         WREG32(IH_CNTL, ih_cntl);
3629
3630         /* force the active interrupt state to all disabled */
3631         if (rdev->family >= CHIP_CEDAR)
3632                 evergreen_disable_interrupt_state(rdev);
3633         else
3634                 r600_disable_interrupt_state(rdev);
3635
3636         /* at this point everything should be setup correctly to enable master */
3637         pci_set_master(rdev->pdev);
3638
3639         /* enable irqs */
3640         r600_enable_interrupts(rdev);
3641
3642         return ret;
3643 }
3644
3645 void r600_irq_suspend(struct radeon_device *rdev)
3646 {
3647         r600_irq_disable(rdev);
3648         r600_rlc_stop(rdev);
3649 }
3650
3651 void r600_irq_fini(struct radeon_device *rdev)
3652 {
3653         r600_irq_suspend(rdev);
3654         r600_ih_ring_fini(rdev);
3655 }
3656
3657 int r600_irq_set(struct radeon_device *rdev)
3658 {
3659         u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
3660         u32 mode_int = 0;
3661         u32 hpd1, hpd2, hpd3, hpd4 = 0, hpd5 = 0, hpd6 = 0;
3662         u32 grbm_int_cntl = 0;
3663         u32 hdmi0, hdmi1;
3664         u32 d1grph = 0, d2grph = 0;
3665         u32 dma_cntl;
3666
3667         if (!rdev->irq.installed) {
3668                 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
3669                 return -EINVAL;
3670         }
3671         /* don't enable anything if the ih is disabled */
3672         if (!rdev->ih.enabled) {
3673                 r600_disable_interrupts(rdev);
3674                 /* force the active interrupt state to all disabled */
3675                 r600_disable_interrupt_state(rdev);
3676                 return 0;
3677         }
3678
3679         if (ASIC_IS_DCE3(rdev)) {
3680                 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
3681                 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
3682                 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
3683                 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
3684                 if (ASIC_IS_DCE32(rdev)) {
3685                         hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
3686                         hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
3687                         hdmi0 = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
3688                         hdmi1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
3689                 } else {
3690                         hdmi0 = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3691                         hdmi1 = RREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3692                 }
3693         } else {
3694                 hpd1 = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL) & ~DC_HPDx_INT_EN;
3695                 hpd2 = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL) & ~DC_HPDx_INT_EN;
3696                 hpd3 = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL) & ~DC_HPDx_INT_EN;
3697                 hdmi0 = RREG32(HDMI0_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3698                 hdmi1 = RREG32(HDMI1_AUDIO_PACKET_CONTROL) & ~HDMI0_AZ_FORMAT_WTRIG_MASK;
3699         }
3700         dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
3701
3702         if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
3703                 DRM_DEBUG("r600_irq_set: sw int\n");
3704                 cp_int_cntl |= RB_INT_ENABLE;
3705                 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
3706         }
3707
3708         if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
3709                 DRM_DEBUG("r600_irq_set: sw int dma\n");
3710                 dma_cntl |= TRAP_ENABLE;
3711         }
3712
3713         if (rdev->irq.crtc_vblank_int[0] ||
3714             atomic_read(&rdev->irq.pflip[0])) {
3715                 DRM_DEBUG("r600_irq_set: vblank 0\n");
3716                 mode_int |= D1MODE_VBLANK_INT_MASK;
3717         }
3718         if (rdev->irq.crtc_vblank_int[1] ||
3719             atomic_read(&rdev->irq.pflip[1])) {
3720                 DRM_DEBUG("r600_irq_set: vblank 1\n");
3721                 mode_int |= D2MODE_VBLANK_INT_MASK;
3722         }
3723         if (rdev->irq.hpd[0]) {
3724                 DRM_DEBUG("r600_irq_set: hpd 1\n");
3725                 hpd1 |= DC_HPDx_INT_EN;
3726         }
3727         if (rdev->irq.hpd[1]) {
3728                 DRM_DEBUG("r600_irq_set: hpd 2\n");
3729                 hpd2 |= DC_HPDx_INT_EN;
3730         }
3731         if (rdev->irq.hpd[2]) {
3732                 DRM_DEBUG("r600_irq_set: hpd 3\n");
3733                 hpd3 |= DC_HPDx_INT_EN;
3734         }
3735         if (rdev->irq.hpd[3]) {
3736                 DRM_DEBUG("r600_irq_set: hpd 4\n");
3737                 hpd4 |= DC_HPDx_INT_EN;
3738         }
3739         if (rdev->irq.hpd[4]) {
3740                 DRM_DEBUG("r600_irq_set: hpd 5\n");
3741                 hpd5 |= DC_HPDx_INT_EN;
3742         }
3743         if (rdev->irq.hpd[5]) {
3744                 DRM_DEBUG("r600_irq_set: hpd 6\n");
3745                 hpd6 |= DC_HPDx_INT_EN;
3746         }
3747         if (rdev->irq.afmt[0]) {
3748                 DRM_DEBUG("r600_irq_set: hdmi 0\n");
3749                 hdmi0 |= HDMI0_AZ_FORMAT_WTRIG_MASK;
3750         }
3751         if (rdev->irq.afmt[1]) {
3752                 DRM_DEBUG("r600_irq_set: hdmi 0\n");
3753                 hdmi1 |= HDMI0_AZ_FORMAT_WTRIG_MASK;
3754         }
3755
3756         WREG32(CP_INT_CNTL, cp_int_cntl);
3757         WREG32(DMA_CNTL, dma_cntl);
3758         WREG32(DxMODE_INT_MASK, mode_int);
3759         WREG32(D1GRPH_INTERRUPT_CONTROL, d1grph);
3760         WREG32(D2GRPH_INTERRUPT_CONTROL, d2grph);
3761         WREG32(GRBM_INT_CNTL, grbm_int_cntl);
3762         if (ASIC_IS_DCE3(rdev)) {
3763                 WREG32(DC_HPD1_INT_CONTROL, hpd1);
3764                 WREG32(DC_HPD2_INT_CONTROL, hpd2);
3765                 WREG32(DC_HPD3_INT_CONTROL, hpd3);
3766                 WREG32(DC_HPD4_INT_CONTROL, hpd4);
3767                 if (ASIC_IS_DCE32(rdev)) {
3768                         WREG32(DC_HPD5_INT_CONTROL, hpd5);
3769                         WREG32(DC_HPD6_INT_CONTROL, hpd6);
3770                         WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0, hdmi0);
3771                         WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1, hdmi1);
3772                 } else {
3773                         WREG32(HDMI0_AUDIO_PACKET_CONTROL, hdmi0);
3774                         WREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL, hdmi1);
3775                 }
3776         } else {
3777                 WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, hpd1);
3778                 WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, hpd2);
3779                 WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, hpd3);
3780                 WREG32(HDMI0_AUDIO_PACKET_CONTROL, hdmi0);
3781                 WREG32(HDMI1_AUDIO_PACKET_CONTROL, hdmi1);
3782         }
3783
3784         return 0;
3785 }
3786
3787 static void r600_irq_ack(struct radeon_device *rdev)
3788 {
3789         u32 tmp;
3790
3791         if (ASIC_IS_DCE3(rdev)) {
3792                 rdev->irq.stat_regs.r600.disp_int = RREG32(DCE3_DISP_INTERRUPT_STATUS);
3793                 rdev->irq.stat_regs.r600.disp_int_cont = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE);
3794                 rdev->irq.stat_regs.r600.disp_int_cont2 = RREG32(DCE3_DISP_INTERRUPT_STATUS_CONTINUE2);
3795                 if (ASIC_IS_DCE32(rdev)) {
3796                         rdev->irq.stat_regs.r600.hdmi0_status = RREG32(AFMT_STATUS + DCE3_HDMI_OFFSET0);
3797                         rdev->irq.stat_regs.r600.hdmi1_status = RREG32(AFMT_STATUS + DCE3_HDMI_OFFSET1);
3798                 } else {
3799                         rdev->irq.stat_regs.r600.hdmi0_status = RREG32(HDMI0_STATUS);
3800                         rdev->irq.stat_regs.r600.hdmi1_status = RREG32(DCE3_HDMI1_STATUS);
3801                 }
3802         } else {
3803                 rdev->irq.stat_regs.r600.disp_int = RREG32(DISP_INTERRUPT_STATUS);
3804                 rdev->irq.stat_regs.r600.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
3805                 rdev->irq.stat_regs.r600.disp_int_cont2 = 0;
3806                 rdev->irq.stat_regs.r600.hdmi0_status = RREG32(HDMI0_STATUS);
3807                 rdev->irq.stat_regs.r600.hdmi1_status = RREG32(HDMI1_STATUS);
3808         }
3809         rdev->irq.stat_regs.r600.d1grph_int = RREG32(D1GRPH_INTERRUPT_STATUS);
3810         rdev->irq.stat_regs.r600.d2grph_int = RREG32(D2GRPH_INTERRUPT_STATUS);
3811
3812         if (rdev->irq.stat_regs.r600.d1grph_int & DxGRPH_PFLIP_INT_OCCURRED)
3813                 WREG32(D1GRPH_INTERRUPT_STATUS, DxGRPH_PFLIP_INT_CLEAR);
3814         if (rdev->irq.stat_regs.r600.d2grph_int & DxGRPH_PFLIP_INT_OCCURRED)
3815                 WREG32(D2GRPH_INTERRUPT_STATUS, DxGRPH_PFLIP_INT_CLEAR);
3816         if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VBLANK_INTERRUPT)
3817                 WREG32(D1MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
3818         if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VLINE_INTERRUPT)
3819                 WREG32(D1MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
3820         if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VBLANK_INTERRUPT)
3821                 WREG32(D2MODE_VBLANK_STATUS, DxMODE_VBLANK_ACK);
3822         if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VLINE_INTERRUPT)
3823                 WREG32(D2MODE_VLINE_STATUS, DxMODE_VLINE_ACK);
3824         if (rdev->irq.stat_regs.r600.disp_int & DC_HPD1_INTERRUPT) {
3825                 if (ASIC_IS_DCE3(rdev)) {
3826                         tmp = RREG32(DC_HPD1_INT_CONTROL);
3827                         tmp |= DC_HPDx_INT_ACK;
3828                         WREG32(DC_HPD1_INT_CONTROL, tmp);
3829                 } else {
3830                         tmp = RREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL);
3831                         tmp |= DC_HPDx_INT_ACK;
3832                         WREG32(DC_HOT_PLUG_DETECT1_INT_CONTROL, tmp);
3833                 }
3834         }
3835         if (rdev->irq.stat_regs.r600.disp_int & DC_HPD2_INTERRUPT) {
3836                 if (ASIC_IS_DCE3(rdev)) {
3837                         tmp = RREG32(DC_HPD2_INT_CONTROL);
3838                         tmp |= DC_HPDx_INT_ACK;
3839                         WREG32(DC_HPD2_INT_CONTROL, tmp);
3840                 } else {
3841                         tmp = RREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL);
3842                         tmp |= DC_HPDx_INT_ACK;
3843                         WREG32(DC_HOT_PLUG_DETECT2_INT_CONTROL, tmp);
3844                 }
3845         }
3846         if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD3_INTERRUPT) {
3847                 if (ASIC_IS_DCE3(rdev)) {
3848                         tmp = RREG32(DC_HPD3_INT_CONTROL);
3849                         tmp |= DC_HPDx_INT_ACK;
3850                         WREG32(DC_HPD3_INT_CONTROL, tmp);
3851                 } else {
3852                         tmp = RREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL);
3853                         tmp |= DC_HPDx_INT_ACK;
3854                         WREG32(DC_HOT_PLUG_DETECT3_INT_CONTROL, tmp);
3855                 }
3856         }
3857         if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD4_INTERRUPT) {
3858                 tmp = RREG32(DC_HPD4_INT_CONTROL);
3859                 tmp |= DC_HPDx_INT_ACK;
3860                 WREG32(DC_HPD4_INT_CONTROL, tmp);
3861         }
3862         if (ASIC_IS_DCE32(rdev)) {
3863                 if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD5_INTERRUPT) {
3864                         tmp = RREG32(DC_HPD5_INT_CONTROL);
3865                         tmp |= DC_HPDx_INT_ACK;
3866                         WREG32(DC_HPD5_INT_CONTROL, tmp);
3867                 }
3868                 if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD6_INTERRUPT) {
3869                         tmp = RREG32(DC_HPD5_INT_CONTROL);
3870                         tmp |= DC_HPDx_INT_ACK;
3871                         WREG32(DC_HPD6_INT_CONTROL, tmp);
3872                 }
3873                 if (rdev->irq.stat_regs.r600.hdmi0_status & AFMT_AZ_FORMAT_WTRIG) {
3874                         tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0);
3875                         tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
3876                         WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET0, tmp);
3877                 }
3878                 if (rdev->irq.stat_regs.r600.hdmi1_status & AFMT_AZ_FORMAT_WTRIG) {
3879                         tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1);
3880                         tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
3881                         WREG32(AFMT_AUDIO_PACKET_CONTROL + DCE3_HDMI_OFFSET1, tmp);
3882                 }
3883         } else {
3884                 if (rdev->irq.stat_regs.r600.hdmi0_status & HDMI0_AZ_FORMAT_WTRIG) {
3885                         tmp = RREG32(HDMI0_AUDIO_PACKET_CONTROL);
3886                         tmp |= HDMI0_AZ_FORMAT_WTRIG_ACK;
3887                         WREG32(HDMI0_AUDIO_PACKET_CONTROL, tmp);
3888                 }
3889                 if (rdev->irq.stat_regs.r600.hdmi1_status & HDMI0_AZ_FORMAT_WTRIG) {
3890                         if (ASIC_IS_DCE3(rdev)) {
3891                                 tmp = RREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL);
3892                                 tmp |= HDMI0_AZ_FORMAT_WTRIG_ACK;
3893                                 WREG32(DCE3_HDMI1_AUDIO_PACKET_CONTROL, tmp);
3894                         } else {
3895                                 tmp = RREG32(HDMI1_AUDIO_PACKET_CONTROL);
3896                                 tmp |= HDMI0_AZ_FORMAT_WTRIG_ACK;
3897                                 WREG32(HDMI1_AUDIO_PACKET_CONTROL, tmp);
3898                         }
3899                 }
3900         }
3901 }
3902
3903 void r600_irq_disable(struct radeon_device *rdev)
3904 {
3905         r600_disable_interrupts(rdev);
3906         /* Wait and acknowledge irq */
3907         mdelay(1);
3908         r600_irq_ack(rdev);
3909         r600_disable_interrupt_state(rdev);
3910 }
3911
3912 static u32 r600_get_ih_wptr(struct radeon_device *rdev)
3913 {
3914         u32 wptr, tmp;
3915
3916         if (rdev->wb.enabled)
3917                 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
3918         else
3919                 wptr = RREG32(IH_RB_WPTR);
3920
3921         if (wptr & RB_OVERFLOW) {
3922                 /* When a ring buffer overflow happen start parsing interrupt
3923                  * from the last not overwritten vector (wptr + 16). Hopefully
3924                  * this should allow us to catchup.
3925                  */
3926                 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
3927                         wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
3928                 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
3929                 tmp = RREG32(IH_RB_CNTL);
3930                 tmp |= IH_WPTR_OVERFLOW_CLEAR;
3931                 WREG32(IH_RB_CNTL, tmp);
3932         }
3933         return (wptr & rdev->ih.ptr_mask);
3934 }
3935
3936 /*        r600 IV Ring
3937  * Each IV ring entry is 128 bits:
3938  * [7:0]    - interrupt source id
3939  * [31:8]   - reserved
3940  * [59:32]  - interrupt source data
3941  * [127:60]  - reserved
3942  *
3943  * The basic interrupt vector entries
3944  * are decoded as follows:
3945  * src_id  src_data  description
3946  *      1         0  D1 Vblank
3947  *      1         1  D1 Vline
3948  *      5         0  D2 Vblank
3949  *      5         1  D2 Vline
3950  *     19         0  FP Hot plug detection A
3951  *     19         1  FP Hot plug detection B
3952  *     19         2  DAC A auto-detection
3953  *     19         3  DAC B auto-detection
3954  *     21         4  HDMI block A
3955  *     21         5  HDMI block B
3956  *    176         -  CP_INT RB
3957  *    177         -  CP_INT IB1
3958  *    178         -  CP_INT IB2
3959  *    181         -  EOP Interrupt
3960  *    233         -  GUI Idle
3961  *
3962  * Note, these are based on r600 and may need to be
3963  * adjusted or added to on newer asics
3964  */
3965
3966 int r600_irq_process(struct radeon_device *rdev)
3967 {
3968         u32 wptr;
3969         u32 rptr;
3970         u32 src_id, src_data;
3971         u32 ring_index;
3972         bool queue_hotplug = false;
3973         bool queue_hdmi = false;
3974
3975         if (!rdev->ih.enabled || rdev->shutdown)
3976                 return IRQ_NONE;
3977
3978         /* No MSIs, need a dummy read to flush PCI DMAs */
3979         if (!rdev->msi_enabled)
3980                 RREG32(IH_RB_WPTR);
3981
3982         wptr = r600_get_ih_wptr(rdev);
3983
3984 restart_ih:
3985         /* is somebody else already processing irqs? */
3986         if (atomic_xchg(&rdev->ih.lock, 1))
3987                 return IRQ_NONE;
3988
3989         rptr = rdev->ih.rptr;
3990         DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
3991
3992         /* Order reading of wptr vs. reading of IH ring data */
3993         rmb();
3994
3995         /* display interrupts */
3996         r600_irq_ack(rdev);
3997
3998         while (rptr != wptr) {
3999                 /* wptr/rptr are in bytes! */
4000                 ring_index = rptr / 4;
4001                 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4002                 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4003
4004                 switch (src_id) {
4005                 case 1: /* D1 vblank/vline */
4006                         switch (src_data) {
4007                         case 0: /* D1 vblank */
4008                                 if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VBLANK_INTERRUPT) {
4009                                         if (rdev->irq.crtc_vblank_int[0]) {
4010                                                 drm_handle_vblank(rdev->ddev, 0);
4011                                                 rdev->pm.vblank_sync = true;
4012                                                 wake_up(&rdev->irq.vblank_queue);
4013                                         }
4014                                         if (atomic_read(&rdev->irq.pflip[0]))
4015                                                 radeon_crtc_handle_flip(rdev, 0);
4016                                         rdev->irq.stat_regs.r600.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
4017                                         DRM_DEBUG("IH: D1 vblank\n");
4018                                 }
4019                                 break;
4020                         case 1: /* D1 vline */
4021                                 if (rdev->irq.stat_regs.r600.disp_int & LB_D1_VLINE_INTERRUPT) {
4022                                         rdev->irq.stat_regs.r600.disp_int &= ~LB_D1_VLINE_INTERRUPT;
4023                                         DRM_DEBUG("IH: D1 vline\n");
4024                                 }
4025                                 break;
4026                         default:
4027                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4028                                 break;
4029                         }
4030                         break;
4031                 case 5: /* D2 vblank/vline */
4032                         switch (src_data) {
4033                         case 0: /* D2 vblank */
4034                                 if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VBLANK_INTERRUPT) {
4035                                         if (rdev->irq.crtc_vblank_int[1]) {
4036                                                 drm_handle_vblank(rdev->ddev, 1);
4037                                                 rdev->pm.vblank_sync = true;
4038                                                 wake_up(&rdev->irq.vblank_queue);
4039                                         }
4040                                         if (atomic_read(&rdev->irq.pflip[1]))
4041                                                 radeon_crtc_handle_flip(rdev, 1);
4042                                         rdev->irq.stat_regs.r600.disp_int &= ~LB_D2_VBLANK_INTERRUPT;
4043                                         DRM_DEBUG("IH: D2 vblank\n");
4044                                 }
4045                                 break;
4046                         case 1: /* D1 vline */
4047                                 if (rdev->irq.stat_regs.r600.disp_int & LB_D2_VLINE_INTERRUPT) {
4048                                         rdev->irq.stat_regs.r600.disp_int &= ~LB_D2_VLINE_INTERRUPT;
4049                                         DRM_DEBUG("IH: D2 vline\n");
4050                                 }
4051                                 break;
4052                         default:
4053                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4054                                 break;
4055                         }
4056                         break;
4057                 case 19: /* HPD/DAC hotplug */
4058                         switch (src_data) {
4059                         case 0:
4060                                 if (rdev->irq.stat_regs.r600.disp_int & DC_HPD1_INTERRUPT) {
4061                                         rdev->irq.stat_regs.r600.disp_int &= ~DC_HPD1_INTERRUPT;
4062                                         queue_hotplug = true;
4063                                         DRM_DEBUG("IH: HPD1\n");
4064                                 }
4065                                 break;
4066                         case 1:
4067                                 if (rdev->irq.stat_regs.r600.disp_int & DC_HPD2_INTERRUPT) {
4068                                         rdev->irq.stat_regs.r600.disp_int &= ~DC_HPD2_INTERRUPT;
4069                                         queue_hotplug = true;
4070                                         DRM_DEBUG("IH: HPD2\n");
4071                                 }
4072                                 break;
4073                         case 4:
4074                                 if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD3_INTERRUPT) {
4075                                         rdev->irq.stat_regs.r600.disp_int_cont &= ~DC_HPD3_INTERRUPT;
4076                                         queue_hotplug = true;
4077                                         DRM_DEBUG("IH: HPD3\n");
4078                                 }
4079                                 break;
4080                         case 5:
4081                                 if (rdev->irq.stat_regs.r600.disp_int_cont & DC_HPD4_INTERRUPT) {
4082                                         rdev->irq.stat_regs.r600.disp_int_cont &= ~DC_HPD4_INTERRUPT;
4083                                         queue_hotplug = true;
4084                                         DRM_DEBUG("IH: HPD4\n");
4085                                 }
4086                                 break;
4087                         case 10:
4088                                 if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD5_INTERRUPT) {
4089                                         rdev->irq.stat_regs.r600.disp_int_cont2 &= ~DC_HPD5_INTERRUPT;
4090                                         queue_hotplug = true;
4091                                         DRM_DEBUG("IH: HPD5\n");
4092                                 }
4093                                 break;
4094                         case 12:
4095                                 if (rdev->irq.stat_regs.r600.disp_int_cont2 & DC_HPD6_INTERRUPT) {
4096                                         rdev->irq.stat_regs.r600.disp_int_cont2 &= ~DC_HPD6_INTERRUPT;
4097                                         queue_hotplug = true;
4098                                         DRM_DEBUG("IH: HPD6\n");
4099                                 }
4100                                 break;
4101                         default:
4102                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4103                                 break;
4104                         }
4105                         break;
4106                 case 21: /* hdmi */
4107                         switch (src_data) {
4108                         case 4:
4109                                 if (rdev->irq.stat_regs.r600.hdmi0_status & HDMI0_AZ_FORMAT_WTRIG) {
4110                                         rdev->irq.stat_regs.r600.hdmi0_status &= ~HDMI0_AZ_FORMAT_WTRIG;
4111                                         queue_hdmi = true;
4112                                         DRM_DEBUG("IH: HDMI0\n");
4113                                 }
4114                                 break;
4115                         case 5:
4116                                 if (rdev->irq.stat_regs.r600.hdmi1_status & HDMI0_AZ_FORMAT_WTRIG) {
4117                                         rdev->irq.stat_regs.r600.hdmi1_status &= ~HDMI0_AZ_FORMAT_WTRIG;
4118                                         queue_hdmi = true;
4119                                         DRM_DEBUG("IH: HDMI1\n");
4120                                 }
4121                                 break;
4122                         default:
4123                                 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
4124                                 break;
4125                         }
4126                         break;
4127                 case 176: /* CP_INT in ring buffer */
4128                 case 177: /* CP_INT in IB1 */
4129                 case 178: /* CP_INT in IB2 */
4130                         DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
4131                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4132                         break;
4133                 case 181: /* CP EOP event */
4134                         DRM_DEBUG("IH: CP EOP\n");
4135                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4136                         break;
4137                 case 224: /* DMA trap event */
4138                         DRM_DEBUG("IH: DMA trap\n");
4139                         radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
4140                         break;
4141                 case 233: /* GUI IDLE */
4142                         DRM_DEBUG("IH: GUI idle\n");
4143                         break;
4144                 default:
4145                         DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4146                         break;
4147                 }
4148
4149                 /* wptr/rptr are in bytes! */
4150                 rptr += 16;
4151                 rptr &= rdev->ih.ptr_mask;
4152         }
4153         if (queue_hotplug)
4154                 schedule_work(&rdev->hotplug_work);
4155         if (queue_hdmi)
4156                 schedule_work(&rdev->audio_work);
4157         rdev->ih.rptr = rptr;
4158         WREG32(IH_RB_RPTR, rdev->ih.rptr);
4159         atomic_set(&rdev->ih.lock, 0);
4160
4161         /* make sure wptr hasn't changed while processing */
4162         wptr = r600_get_ih_wptr(rdev);
4163         if (wptr != rptr)
4164                 goto restart_ih;
4165
4166         return IRQ_HANDLED;
4167 }
4168
4169 /*
4170  * Debugfs info
4171  */
4172 #if defined(CONFIG_DEBUG_FS)
4173
4174 static int r600_debugfs_mc_info(struct seq_file *m, void *data)
4175 {
4176         struct drm_info_node *node = (struct drm_info_node *) m->private;
4177         struct drm_device *dev = node->minor->dev;
4178         struct radeon_device *rdev = dev->dev_private;
4179
4180         DREG32_SYS(m, rdev, R_000E50_SRBM_STATUS);
4181         DREG32_SYS(m, rdev, VM_L2_STATUS);
4182         return 0;
4183 }
4184
4185 static struct drm_info_list r600_mc_info_list[] = {
4186         {"r600_mc_info", r600_debugfs_mc_info, 0, NULL},
4187 };
4188 #endif
4189
4190 int r600_debugfs_mc_info_init(struct radeon_device *rdev)
4191 {
4192 #if defined(CONFIG_DEBUG_FS)
4193         return radeon_debugfs_add_files(rdev, r600_mc_info_list, ARRAY_SIZE(r600_mc_info_list));
4194 #else
4195         return 0;
4196 #endif
4197 }
4198
4199 /**
4200  * r600_ioctl_wait_idle - flush host path cache on wait idle ioctl
4201  * rdev: radeon device structure
4202  * bo: buffer object struct which userspace is waiting for idle
4203  *
4204  * Some R6XX/R7XX doesn't seems to take into account HDP flush performed
4205  * through ring buffer, this leads to corruption in rendering, see
4206  * http://bugzilla.kernel.org/show_bug.cgi?id=15186 to avoid this we
4207  * directly perform HDP flush by writing register through MMIO.
4208  */
4209 void r600_ioctl_wait_idle(struct radeon_device *rdev, struct radeon_bo *bo)
4210 {
4211         /* r7xx hw bug.  write to HDP_DEBUG1 followed by fb read
4212          * rather than write to HDP_REG_COHERENCY_FLUSH_CNTL.
4213          * This seems to cause problems on some AGP cards. Just use the old
4214          * method for them.
4215          */
4216         if ((rdev->family >= CHIP_RV770) && (rdev->family <= CHIP_RV740) &&
4217             rdev->vram_scratch.ptr && !(rdev->flags & RADEON_IS_AGP)) {
4218                 void __iomem *ptr = (void *)rdev->vram_scratch.ptr;
4219                 u32 tmp;
4220
4221                 WREG32(HDP_DEBUG1, 0);
4222                 tmp = readl((void __iomem *)ptr);
4223         } else
4224                 WREG32(R_005480_HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
4225 }
4226
4227 void r600_set_pcie_lanes(struct radeon_device *rdev, int lanes)
4228 {
4229         u32 link_width_cntl, mask, target_reg;
4230
4231         if (rdev->flags & RADEON_IS_IGP)
4232                 return;
4233
4234         if (!(rdev->flags & RADEON_IS_PCIE))
4235                 return;
4236
4237         /* x2 cards have a special sequence */
4238         if (ASIC_IS_X2(rdev))
4239                 return;
4240
4241         /* FIXME wait for idle */
4242
4243         switch (lanes) {
4244         case 0:
4245                 mask = RADEON_PCIE_LC_LINK_WIDTH_X0;
4246                 break;
4247         case 1:
4248                 mask = RADEON_PCIE_LC_LINK_WIDTH_X1;
4249                 break;
4250         case 2:
4251                 mask = RADEON_PCIE_LC_LINK_WIDTH_X2;
4252                 break;
4253         case 4:
4254                 mask = RADEON_PCIE_LC_LINK_WIDTH_X4;
4255                 break;
4256         case 8:
4257                 mask = RADEON_PCIE_LC_LINK_WIDTH_X8;
4258                 break;
4259         case 12:
4260                 mask = RADEON_PCIE_LC_LINK_WIDTH_X12;
4261                 break;
4262         case 16:
4263         default:
4264                 mask = RADEON_PCIE_LC_LINK_WIDTH_X16;
4265                 break;
4266         }
4267
4268         link_width_cntl = RREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
4269
4270         if ((link_width_cntl & RADEON_PCIE_LC_LINK_WIDTH_RD_MASK) ==
4271             (mask << RADEON_PCIE_LC_LINK_WIDTH_RD_SHIFT))
4272                 return;
4273
4274         if (link_width_cntl & R600_PCIE_LC_UPCONFIGURE_DIS)
4275                 return;
4276
4277         link_width_cntl &= ~(RADEON_PCIE_LC_LINK_WIDTH_MASK |
4278                              RADEON_PCIE_LC_RECONFIG_NOW |
4279                              R600_PCIE_LC_RENEGOTIATE_EN |
4280                              R600_PCIE_LC_RECONFIG_ARC_MISSING_ESCAPE);
4281         link_width_cntl |= mask;
4282
4283         WREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
4284
4285         /* some northbridges can renegotiate the link rather than requiring
4286          * a complete re-config.
4287          * e.g., AMD 780/790 northbridges (pci ids: 0x5956, 0x5957, 0x5958, etc.)
4288          */
4289         if (link_width_cntl & R600_PCIE_LC_RENEGOTIATION_SUPPORT)
4290                 link_width_cntl |= R600_PCIE_LC_RENEGOTIATE_EN | R600_PCIE_LC_UPCONFIGURE_SUPPORT;
4291         else
4292                 link_width_cntl |= R600_PCIE_LC_RECONFIG_ARC_MISSING_ESCAPE;
4293
4294         WREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL, (link_width_cntl |
4295                                                        RADEON_PCIE_LC_RECONFIG_NOW));
4296
4297         if (rdev->family >= CHIP_RV770)
4298                 target_reg = R700_TARGET_AND_CURRENT_PROFILE_INDEX;
4299         else
4300                 target_reg = R600_TARGET_AND_CURRENT_PROFILE_INDEX;
4301
4302         /* wait for lane set to complete */
4303         link_width_cntl = RREG32(target_reg);
4304         while (link_width_cntl == 0xffffffff)
4305                 link_width_cntl = RREG32(target_reg);
4306
4307 }
4308
4309 int r600_get_pcie_lanes(struct radeon_device *rdev)
4310 {
4311         u32 link_width_cntl;
4312
4313         if (rdev->flags & RADEON_IS_IGP)
4314                 return 0;
4315
4316         if (!(rdev->flags & RADEON_IS_PCIE))
4317                 return 0;
4318
4319         /* x2 cards have a special sequence */
4320         if (ASIC_IS_X2(rdev))
4321                 return 0;
4322
4323         /* FIXME wait for idle */
4324
4325         link_width_cntl = RREG32_PCIE_P(RADEON_PCIE_LC_LINK_WIDTH_CNTL);
4326
4327         switch ((link_width_cntl & RADEON_PCIE_LC_LINK_WIDTH_RD_MASK) >> RADEON_PCIE_LC_LINK_WIDTH_RD_SHIFT) {
4328         case RADEON_PCIE_LC_LINK_WIDTH_X0:
4329                 return 0;
4330         case RADEON_PCIE_LC_LINK_WIDTH_X1:
4331                 return 1;
4332         case RADEON_PCIE_LC_LINK_WIDTH_X2:
4333                 return 2;
4334         case RADEON_PCIE_LC_LINK_WIDTH_X4:
4335                 return 4;
4336         case RADEON_PCIE_LC_LINK_WIDTH_X8:
4337                 return 8;
4338         case RADEON_PCIE_LC_LINK_WIDTH_X16:
4339         default:
4340                 return 16;
4341         }
4342 }
4343
4344 static void r600_pcie_gen2_enable(struct radeon_device *rdev)
4345 {
4346         u32 link_width_cntl, lanes, speed_cntl, training_cntl, tmp;
4347         u16 link_cntl2;
4348         u32 mask;
4349         int ret;
4350
4351         if (radeon_pcie_gen2 == 0)
4352                 return;
4353
4354         if (rdev->flags & RADEON_IS_IGP)
4355                 return;
4356
4357         if (!(rdev->flags & RADEON_IS_PCIE))
4358                 return;
4359
4360         /* x2 cards have a special sequence */
4361         if (ASIC_IS_X2(rdev))
4362                 return;
4363
4364         /* only RV6xx+ chips are supported */
4365         if (rdev->family <= CHIP_R600)
4366                 return;
4367
4368         ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
4369         if (ret != 0)
4370                 return;
4371
4372         if (!(mask & DRM_PCIE_SPEED_50))
4373                 return;
4374
4375         speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
4376         if (speed_cntl & LC_CURRENT_DATA_RATE) {
4377                 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
4378                 return;
4379         }
4380
4381         DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
4382
4383         /* 55 nm r6xx asics */
4384         if ((rdev->family == CHIP_RV670) ||
4385             (rdev->family == CHIP_RV620) ||
4386             (rdev->family == CHIP_RV635)) {
4387                 /* advertise upconfig capability */
4388                 link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
4389                 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
4390                 WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
4391                 link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
4392                 if (link_width_cntl & LC_RENEGOTIATION_SUPPORT) {
4393                         lanes = (link_width_cntl & LC_LINK_WIDTH_RD_MASK) >> LC_LINK_WIDTH_RD_SHIFT;
4394                         link_width_cntl &= ~(LC_LINK_WIDTH_MASK |
4395                                              LC_RECONFIG_ARC_MISSING_ESCAPE);
4396                         link_width_cntl |= lanes | LC_RECONFIG_NOW | LC_RENEGOTIATE_EN;
4397                         WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
4398                 } else {
4399                         link_width_cntl |= LC_UPCONFIGURE_DIS;
4400                         WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
4401                 }
4402         }
4403
4404         speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
4405         if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
4406             (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
4407
4408                 /* 55 nm r6xx asics */
4409                 if ((rdev->family == CHIP_RV670) ||
4410                     (rdev->family == CHIP_RV620) ||
4411                     (rdev->family == CHIP_RV635)) {
4412                         WREG32(MM_CFGREGS_CNTL, 0x8);
4413                         link_cntl2 = RREG32(0x4088);
4414                         WREG32(MM_CFGREGS_CNTL, 0);
4415                         /* not supported yet */
4416                         if (link_cntl2 & SELECTABLE_DEEMPHASIS)
4417                                 return;
4418                 }
4419
4420                 speed_cntl &= ~LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_MASK;
4421                 speed_cntl |= (0x3 << LC_SPEED_CHANGE_ATTEMPTS_ALLOWED_SHIFT);
4422                 speed_cntl &= ~LC_VOLTAGE_TIMER_SEL_MASK;
4423                 speed_cntl &= ~LC_FORCE_DIS_HW_SPEED_CHANGE;
4424                 speed_cntl |= LC_FORCE_EN_HW_SPEED_CHANGE;
4425                 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
4426
4427                 tmp = RREG32(0x541c);
4428                 WREG32(0x541c, tmp | 0x8);
4429                 WREG32(MM_CFGREGS_CNTL, MM_WR_TO_CFG_EN);
4430                 link_cntl2 = RREG16(0x4088);
4431                 link_cntl2 &= ~TARGET_LINK_SPEED_MASK;
4432                 link_cntl2 |= 0x2;
4433                 WREG16(0x4088, link_cntl2);
4434                 WREG32(MM_CFGREGS_CNTL, 0);
4435
4436                 if ((rdev->family == CHIP_RV670) ||
4437                     (rdev->family == CHIP_RV620) ||
4438                     (rdev->family == CHIP_RV635)) {
4439                         training_cntl = RREG32_PCIE_P(PCIE_LC_TRAINING_CNTL);
4440                         training_cntl &= ~LC_POINT_7_PLUS_EN;
4441                         WREG32_PCIE_P(PCIE_LC_TRAINING_CNTL, training_cntl);
4442                 } else {
4443                         speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
4444                         speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
4445                         WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
4446                 }
4447
4448                 speed_cntl = RREG32_PCIE_P(PCIE_LC_SPEED_CNTL);
4449                 speed_cntl |= LC_GEN2_EN_STRAP;
4450                 WREG32_PCIE_P(PCIE_LC_SPEED_CNTL, speed_cntl);
4451
4452         } else {
4453                 link_width_cntl = RREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL);
4454                 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
4455                 if (1)
4456                         link_width_cntl |= LC_UPCONFIGURE_DIS;
4457                 else
4458                         link_width_cntl &= ~LC_UPCONFIGURE_DIS;
4459                 WREG32_PCIE_P(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
4460         }
4461 }
4462
4463 /**
4464  * r600_get_gpu_clock_counter - return GPU clock counter snapshot
4465  *
4466  * @rdev: radeon_device pointer
4467  *
4468  * Fetches a GPU clock counter snapshot (R6xx-cayman).
4469  * Returns the 64 bit clock counter snapshot.
4470  */
4471 uint64_t r600_get_gpu_clock_counter(struct radeon_device *rdev)
4472 {
4473         uint64_t clock;
4474
4475         mutex_lock(&rdev->gpu_clock_mutex);
4476         WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
4477         clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
4478                 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
4479         mutex_unlock(&rdev->gpu_clock_mutex);
4480         return clock;
4481 }