drivers/gpu/drm/i915/intel_runtime_pm.c

   1 /*
   2  * Copyright © 2012-2014 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eugeni Dodonov <eugeni.dodonov@intel.com>
  25  *    Daniel Vetter <daniel.vetter@ffwll.ch>
  26  *
  27  */
  28
  29 #include <linux/pm_runtime.h>
  30 #include <linux/vgaarb.h>
  31
  32 #include "i915_drv.h"
  33 #include "intel_drv.h"
  34
  35 /**
  36  * DOC: runtime pm
  37  *
  38  * The i915 driver supports dynamic enabling and disabling of entire hardware
  39  * blocks at runtime. This is especially important on the display side where
  40  * software is supposed to control many power gates manually on recent hardware,
  41  * since on the GT side a lot of the power management is done by the hardware.
  42  * But even there some manual control at the device level is required.
  43  *
  44  * Since i915 supports a diverse set of platforms with a unified codebase and
  45  * hardware engineers just love to shuffle functionality around between power
  46  * domains there's a sizeable amount of indirection required. This file provides
  47  * generic functions to the driver for grabbing and releasing references for
  48  * abstract power domains. It then maps those to the actual power wells
  49  * present for a given platform.
  50  */
  51
  52 bool intel_display_power_well_is_enabled(struct drm_i915_private *dev_priv,
  53                                     int power_well_id);
  54
  55 static struct i915_power_well *
  56 lookup_power_well(struct drm_i915_private *dev_priv, int power_well_id);
  57
  58 const char *
  59 intel_display_power_domain_str(enum intel_display_power_domain domain)
  60 {
  61         switch (domain) {
  62         case POWER_DOMAIN_PIPE_A:
  63                 return "PIPE_A";
  64         case POWER_DOMAIN_PIPE_B:
  65                 return "PIPE_B";
  66         case POWER_DOMAIN_PIPE_C:
  67                 return "PIPE_C";
  68         case POWER_DOMAIN_PIPE_A_PANEL_FITTER:
  69                 return "PIPE_A_PANEL_FITTER";
  70         case POWER_DOMAIN_PIPE_B_PANEL_FITTER:
  71                 return "PIPE_B_PANEL_FITTER";
  72         case POWER_DOMAIN_PIPE_C_PANEL_FITTER:
  73                 return "PIPE_C_PANEL_FITTER";
  74         case POWER_DOMAIN_TRANSCODER_A:
  75                 return "TRANSCODER_A";
  76         case POWER_DOMAIN_TRANSCODER_B:
  77                 return "TRANSCODER_B";
  78         case POWER_DOMAIN_TRANSCODER_C:
  79                 return "TRANSCODER_C";
  80         case POWER_DOMAIN_TRANSCODER_EDP:
  81                 return "TRANSCODER_EDP";
  82         case POWER_DOMAIN_TRANSCODER_DSI_A:
  83                 return "TRANSCODER_DSI_A";
  84         case POWER_DOMAIN_TRANSCODER_DSI_C:
  85                 return "TRANSCODER_DSI_C";
  86         case POWER_DOMAIN_PORT_DDI_A_LANES:
  87                 return "PORT_DDI_A_LANES";
  88         case POWER_DOMAIN_PORT_DDI_B_LANES:
  89                 return "PORT_DDI_B_LANES";
  90         case POWER_DOMAIN_PORT_DDI_C_LANES:
  91                 return "PORT_DDI_C_LANES";
  92         case POWER_DOMAIN_PORT_DDI_D_LANES:
  93                 return "PORT_DDI_D_LANES";
  94         case POWER_DOMAIN_PORT_DDI_E_LANES:
  95                 return "PORT_DDI_E_LANES";
  96         case POWER_DOMAIN_PORT_DSI:
  97                 return "PORT_DSI";
  98         case POWER_DOMAIN_PORT_CRT:
  99                 return "PORT_CRT";
 100         case POWER_DOMAIN_PORT_OTHER:
 101                 return "PORT_OTHER";
 102         case POWER_DOMAIN_VGA:
 103                 return "VGA";
 104         case POWER_DOMAIN_AUDIO:
 105                 return "AUDIO";
 106         case POWER_DOMAIN_PLLS:
 107                 return "PLLS";
 108         case POWER_DOMAIN_AUX_A:
 109                 return "AUX_A";
 110         case POWER_DOMAIN_AUX_B:
 111                 return "AUX_B";
 112         case POWER_DOMAIN_AUX_C:
 113                 return "AUX_C";
 114         case POWER_DOMAIN_AUX_D:
 115                 return "AUX_D";
 116         case POWER_DOMAIN_GMBUS:
 117                 return "GMBUS";
 118         case POWER_DOMAIN_INIT:
 119                 return "INIT";
 120         case POWER_DOMAIN_MODESET:
 121                 return "MODESET";
 122         default:
 123                 MISSING_CASE(domain);
 124                 return "?";
 125         }
 126 }
 127
 128 static void intel_power_well_enable(struct drm_i915_private *dev_priv,
 129                                     struct i915_power_well *power_well)
 130 {
 131         DRM_DEBUG_KMS("enabling %s\n", power_well->name);
 132         power_well->ops->enable(dev_priv, power_well);
 133         power_well->hw_enabled = true;
 134 }
 135
 136 static void intel_power_well_disable(struct drm_i915_private *dev_priv,
 137                                      struct i915_power_well *power_well)
 138 {
 139         DRM_DEBUG_KMS("disabling %s\n", power_well->name);
 140         power_well->hw_enabled = false;
 141         power_well->ops->disable(dev_priv, power_well);
 142 }
 143
 144 static void intel_power_well_get(struct drm_i915_private *dev_priv,
 145                                  struct i915_power_well *power_well)
 146 {
 147         if (!power_well->count++)
 148                 intel_power_well_enable(dev_priv, power_well);
 149 }
 150
 151 static void intel_power_well_put(struct drm_i915_private *dev_priv,
 152                                  struct i915_power_well *power_well)
 153 {
 154         WARN(!power_well->count, "Use count on power well %s is already zero",
 155              power_well->name);
 156
 157         if (!--power_well->count)
 158                 intel_power_well_disable(dev_priv, power_well);
 159 }
 160
 161 /*
 162  * We should only use the power well if we explicitly asked the hardware to
 163  * enable it, so check if it's enabled and also check if we've requested it to
 164  * be enabled.
 165  */
 166 static bool hsw_power_well_enabled(struct drm_i915_private *dev_priv,
 167                                    struct i915_power_well *power_well)
 168 {
 169         return I915_READ(HSW_PWR_WELL_DRIVER) ==
 170                      (HSW_PWR_WELL_ENABLE_REQUEST | HSW_PWR_WELL_STATE_ENABLED);
 171 }
 172
 173 /**
 174  * __intel_display_power_is_enabled - unlocked check for a power domain
 175  * @dev_priv: i915 device instance
 176  * @domain: power domain to check
 177  *
 178  * This is the unlocked version of intel_display_power_is_enabled() and should
 179  * only be used from error capture and recovery code where deadlocks are
 180  * possible.
 181  *
 182  * Returns:
 183  * True when the power domain is enabled, false otherwise.
 184  */
 185 bool __intel_display_power_is_enabled(struct drm_i915_private *dev_priv,
 186                                       enum intel_display_power_domain domain)
 187 {
 188         struct i915_power_well *power_well;
 189         bool is_enabled;
 190
 191         if (dev_priv->pm.suspended)
 192                 return false;
 193
 194         is_enabled = true;
 195
 196         for_each_power_domain_well_rev(dev_priv, power_well, BIT_ULL(domain)) {
 197                 if (power_well->always_on)
 198                         continue;
 199
 200                 if (!power_well->hw_enabled) {
 201                         is_enabled = false;
 202                         break;
 203                 }
 204         }
 205
 206         return is_enabled;
 207 }
 208
 209 /**
 210  * intel_display_power_is_enabled - check for a power domain
 211  * @dev_priv: i915 device instance
 212  * @domain: power domain to check
 213  *
 214  * This function can be used to check the hw power domain state. It is mostly
 215  * used in hardware state readout functions. Everywhere else code should rely
 216  * upon explicit power domain reference counting to ensure that the hardware
 217  * block is powered up before accessing it.
 218  *
 219  * Callers must hold the relevant modesetting locks to ensure that concurrent
 220  * threads can't disable the power well while the caller tries to read a few
 221  * registers.
 222  *
 223  * Returns:
 224  * True when the power domain is enabled, false otherwise.
 225  */
 226 bool intel_display_power_is_enabled(struct drm_i915_private *dev_priv,
 227                                     enum intel_display_power_domain domain)
 228 {
 229         struct i915_power_domains *power_domains;
 230         bool ret;
 231
 232         power_domains = &dev_priv->power_domains;
 233
 234         mutex_lock(&power_domains->lock);
 235         ret = __intel_display_power_is_enabled(dev_priv, domain);
 236         mutex_unlock(&power_domains->lock);
 237
 238         return ret;
 239 }
 240
 241 /**
 242  * intel_display_set_init_power - set the initial power domain state
 243  * @dev_priv: i915 device instance
 244  * @enable: whether to enable or disable the initial power domain state
 245  *
 246  * For simplicity our driver load/unload and system suspend/resume code assumes
 247  * that all power domains are always enabled. This functions controls the state
 248  * of this little hack. While the initial power domain state is enabled runtime
 249  * pm is effectively disabled.
 250  */
 251 void intel_display_set_init_power(struct drm_i915_private *dev_priv,
 252                                   bool enable)
 253 {
 254         if (dev_priv->power_domains.init_power_on == enable)
 255                 return;
 256
 257         if (enable)
 258                 intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
 259         else
 260                 intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
 261
 262         dev_priv->power_domains.init_power_on = enable;
 263 }
 264
 265 /*
 266  * Starting with Haswell, we have a "Power Down Well" that can be turned off
 267  * when not needed anymore. We have 4 registers that can request the power well
 268  * to be enabled, and it will only be disabled if none of the registers is
 269  * requesting it to be enabled.
 270  */
 271 static void hsw_power_well_post_enable(struct drm_i915_private *dev_priv)
 272 {
 273         struct pci_dev *pdev = dev_priv->drm.pdev;
 274
 275         /*
 276          * After we re-enable the power well, if we touch VGA register 0x3d5
 277          * we'll get unclaimed register interrupts. This stops after we write
 278          * anything to the VGA MSR register. The vgacon module uses this
 279          * register all the time, so if we unbind our driver and, as a
 280          * consequence, bind vgacon, we'll get stuck in an infinite loop at
 281          * console_unlock(). So make here we touch the VGA MSR register, making
 282          * sure vgacon can keep working normally without triggering interrupts
 283          * and error messages.
 284          */
 285         vga_get_uninterruptible(pdev, VGA_RSRC_LEGACY_IO);
 286         outb(inb(VGA_MSR_READ), VGA_MSR_WRITE);
 287         vga_put(pdev, VGA_RSRC_LEGACY_IO);
 288
 289         if (IS_BROADWELL(dev_priv))
 290                 gen8_irq_power_well_post_enable(dev_priv,
 291                                                 1 << PIPE_C | 1 << PIPE_B);
 292 }
 293
 294 static void hsw_power_well_pre_disable(struct drm_i915_private *dev_priv)
 295 {
 296         if (IS_BROADWELL(dev_priv))
 297                 gen8_irq_power_well_pre_disable(dev_priv,
 298                                                 1 << PIPE_C | 1 << PIPE_B);
 299 }
 300
 301 static void skl_power_well_post_enable(struct drm_i915_private *dev_priv,
 302                                        struct i915_power_well *power_well)
 303 {
 304         struct pci_dev *pdev = dev_priv->drm.pdev;
 305
 306         /*
 307          * After we re-enable the power well, if we touch VGA register 0x3d5
 308          * we'll get unclaimed register interrupts. This stops after we write
 309          * anything to the VGA MSR register. The vgacon module uses this
 310          * register all the time, so if we unbind our driver and, as a
 311          * consequence, bind vgacon, we'll get stuck in an infinite loop at
 312          * console_unlock(). So make here we touch the VGA MSR register, making
 313          * sure vgacon can keep working normally without triggering interrupts
 314          * and error messages.
 315          */
 316         if (power_well->id == SKL_DISP_PW_2) {
 317                 vga_get_uninterruptible(pdev, VGA_RSRC_LEGACY_IO);
 318                 outb(inb(VGA_MSR_READ), VGA_MSR_WRITE);
 319                 vga_put(pdev, VGA_RSRC_LEGACY_IO);
 320
 321                 gen8_irq_power_well_post_enable(dev_priv,
 322                                                 1 << PIPE_C | 1 << PIPE_B);
 323         }
 324 }
 325
 326 static void skl_power_well_pre_disable(struct drm_i915_private *dev_priv,
 327                                        struct i915_power_well *power_well)
 328 {
 329         if (power_well->id == SKL_DISP_PW_2)
 330                 gen8_irq_power_well_pre_disable(dev_priv,
 331                                                 1 << PIPE_C | 1 << PIPE_B);
 332 }
 333
 334 static void hsw_set_power_well(struct drm_i915_private *dev_priv,
 335                                struct i915_power_well *power_well, bool enable)
 336 {
 337         bool is_enabled, enable_requested;
 338         uint32_t tmp;
 339
 340         tmp = I915_READ(HSW_PWR_WELL_DRIVER);
 341         is_enabled = tmp & HSW_PWR_WELL_STATE_ENABLED;
 342         enable_requested = tmp & HSW_PWR_WELL_ENABLE_REQUEST;
 343
 344         if (enable) {
 345                 if (!enable_requested)
 346                         I915_WRITE(HSW_PWR_WELL_DRIVER,
 347                                    HSW_PWR_WELL_ENABLE_REQUEST);
 348
 349                 if (!is_enabled) {
 350                         DRM_DEBUG_KMS("Enabling power well\n");
 351                         if (intel_wait_for_register(dev_priv,
 352                                                     HSW_PWR_WELL_DRIVER,
 353                                                     HSW_PWR_WELL_STATE_ENABLED,
 354                                                     HSW_PWR_WELL_STATE_ENABLED,
 355                                                     20))
 356                                 DRM_ERROR("Timeout enabling power well\n");
 357                         hsw_power_well_post_enable(dev_priv);
 358                 }
 359
 360         } else {
 361                 if (enable_requested) {
 362                         hsw_power_well_pre_disable(dev_priv);
 363                         I915_WRITE(HSW_PWR_WELL_DRIVER, 0);
 364                         POSTING_READ(HSW_PWR_WELL_DRIVER);
 365                         DRM_DEBUG_KMS("Requesting to disable the power well\n");
 366                 }
 367         }
 368 }
 369
 370 #define SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS (         \
 371         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
 372         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
 373         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
 374         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
 375         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
 376         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
 377         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
 378         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
 379         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
 380         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |                \
 381         BIT_ULL(POWER_DOMAIN_PORT_DDI_E_LANES) |                \
 382         BIT_ULL(POWER_DOMAIN_AUX_B) |                       \
 383         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
 384         BIT_ULL(POWER_DOMAIN_AUX_D) |                   \
 385         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
 386         BIT_ULL(POWER_DOMAIN_VGA) |                             \
 387         BIT_ULL(POWER_DOMAIN_INIT))
 388 #define SKL_DISPLAY_DDI_A_E_POWER_DOMAINS (             \
 389         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_LANES) |                \
 390         BIT_ULL(POWER_DOMAIN_PORT_DDI_E_LANES) |                \
 391         BIT_ULL(POWER_DOMAIN_INIT))
 392 #define SKL_DISPLAY_DDI_B_POWER_DOMAINS (               \
 393         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
 394         BIT_ULL(POWER_DOMAIN_INIT))
 395 #define SKL_DISPLAY_DDI_C_POWER_DOMAINS (               \
 396         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
 397         BIT_ULL(POWER_DOMAIN_INIT))
 398 #define SKL_DISPLAY_DDI_D_POWER_DOMAINS (               \
 399         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |                \
 400         BIT_ULL(POWER_DOMAIN_INIT))
 401 #define SKL_DISPLAY_DC_OFF_POWER_DOMAINS (              \
 402         SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS |         \
 403         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
 404         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
 405         BIT_ULL(POWER_DOMAIN_INIT))
 406
 407 #define BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS (         \
 408         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
 409         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
 410         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
 411         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
 412         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
 413         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
 414         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
 415         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
 416         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
 417         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
 418         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
 419         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
 420         BIT_ULL(POWER_DOMAIN_VGA) |                             \
 421         BIT_ULL(POWER_DOMAIN_GMBUS) |                   \
 422         BIT_ULL(POWER_DOMAIN_INIT))
 423 #define BXT_DISPLAY_DC_OFF_POWER_DOMAINS (              \
 424         BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS |         \
 425         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
 426         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
 427         BIT_ULL(POWER_DOMAIN_INIT))
 428 #define BXT_DPIO_CMN_A_POWER_DOMAINS (                  \
 429         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_LANES) |                \
 430         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
 431         BIT_ULL(POWER_DOMAIN_INIT))
 432 #define BXT_DPIO_CMN_BC_POWER_DOMAINS (                 \
 433         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
 434         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
 435         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
 436         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
 437         BIT_ULL(POWER_DOMAIN_INIT))
 438
 439 #define GLK_DISPLAY_POWERWELL_2_POWER_DOMAINS (         \
 440         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
 441         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
 442         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
 443         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
 444         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
 445         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
 446         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
 447         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
 448         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
 449         BIT_ULL(POWER_DOMAIN_AUX_B) |                       \
 450         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
 451         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
 452         BIT_ULL(POWER_DOMAIN_VGA) |                             \
 453         BIT_ULL(POWER_DOMAIN_INIT))
 454 #define GLK_DISPLAY_DDI_A_POWER_DOMAINS (               \
 455         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_LANES) |                \
 456         BIT_ULL(POWER_DOMAIN_INIT))
 457 #define GLK_DISPLAY_DDI_B_POWER_DOMAINS (               \
 458         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
 459         BIT_ULL(POWER_DOMAIN_INIT))
 460 #define GLK_DISPLAY_DDI_C_POWER_DOMAINS (               \
 461         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
 462         BIT_ULL(POWER_DOMAIN_INIT))
 463 #define GLK_DPIO_CMN_A_POWER_DOMAINS (                  \
 464         BIT_ULL(POWER_DOMAIN_PORT_DDI_A_LANES) |                \
 465         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
 466         BIT_ULL(POWER_DOMAIN_INIT))
 467 #define GLK_DPIO_CMN_B_POWER_DOMAINS (                  \
 468         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
 469         BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
 470         BIT_ULL(POWER_DOMAIN_INIT))
 471 #define GLK_DPIO_CMN_C_POWER_DOMAINS (                  \
 472         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
 473         BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
 474         BIT_ULL(POWER_DOMAIN_INIT))
 475 #define GLK_DISPLAY_AUX_A_POWER_DOMAINS (               \
 476         BIT_ULL(POWER_DOMAIN_AUX_A) |           \
 477         BIT_ULL(POWER_DOMAIN_INIT))
 478 #define GLK_DISPLAY_AUX_B_POWER_DOMAINS (               \
 479         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
 480         BIT_ULL(POWER_DOMAIN_INIT))
 481 #define GLK_DISPLAY_AUX_C_POWER_DOMAINS (               \
 482         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
 483         BIT_ULL(POWER_DOMAIN_INIT))
 484 #define GLK_DISPLAY_DC_OFF_POWER_DOMAINS (              \
 485         GLK_DISPLAY_POWERWELL_2_POWER_DOMAINS |         \
 486         BIT_ULL(POWER_DOMAIN_MODESET) |                 \
 487         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
 488         BIT_ULL(POWER_DOMAIN_INIT))
 489
 490 static void assert_can_enable_dc9(struct drm_i915_private *dev_priv)
 491 {
 492         WARN_ONCE((I915_READ(DC_STATE_EN) & DC_STATE_EN_DC9),
 493                   "DC9 already programmed to be enabled.\n");
 494         WARN_ONCE(I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5,
 495                   "DC5 still not disabled to enable DC9.\n");
 496         WARN_ONCE(I915_READ(HSW_PWR_WELL_DRIVER), "Power well on.\n");
 497         WARN_ONCE(intel_irqs_enabled(dev_priv),
 498                   "Interrupts not disabled yet.\n");
 499
 500          /*
 501           * TODO: check for the following to verify the conditions to enter DC9
 502           * state are satisfied:
 503           * 1] Check relevant display engine registers to verify if mode set
 504           * disable sequence was followed.
 505           * 2] Check if display uninitialize sequence is initialized.
 506           */
 507 }
 508
 509 static void assert_can_disable_dc9(struct drm_i915_private *dev_priv)
 510 {
 511         WARN_ONCE(intel_irqs_enabled(dev_priv),
 512                   "Interrupts not disabled yet.\n");
 513         WARN_ONCE(I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5,
 514                   "DC5 still not disabled.\n");
 515
 516          /*
 517           * TODO: check for the following to verify DC9 state was indeed
 518           * entered before programming to disable it:
 519           * 1] Check relevant display engine registers to verify if mode
 520           *  set disable sequence was followed.
 521           * 2] Check if display uninitialize sequence is initialized.
 522           */
 523 }
 524
 525 static void gen9_write_dc_state(struct drm_i915_private *dev_priv,
 526                                 u32 state)
 527 {
 528         int rewrites = 0;
 529         int rereads = 0;
 530         u32 v;
 531
 532         I915_WRITE(DC_STATE_EN, state);
 533
 534         /* It has been observed that disabling the dc6 state sometimes
 535          * doesn't stick and dmc keeps returning old value. Make sure
 536          * the write really sticks enough times and also force rewrite until
 537          * we are confident that state is exactly what we want.
 538          */
 539         do  {
 540                 v = I915_READ(DC_STATE_EN);
 541
 542                 if (v != state) {
 543                         I915_WRITE(DC_STATE_EN, state);
 544                         rewrites++;
 545                         rereads = 0;
 546                 } else if (rereads++ > 5) {
 547                         break;
 548                 }
 549
 550         } while (rewrites < 100);
 551
 552         if (v != state)
 553                 DRM_ERROR("Writing dc state to 0x%x failed, now 0x%x\n",
 554                           state, v);
 555
 556         /* Most of the times we need one retry, avoid spam */
 557         if (rewrites > 1)
 558                 DRM_DEBUG_KMS("Rewrote dc state to 0x%x %d times\n",
 559                               state, rewrites);
 560 }
 561
 562 static u32 gen9_dc_mask(struct drm_i915_private *dev_priv)
 563 {
 564         u32 mask;
 565
 566         mask = DC_STATE_EN_UPTO_DC5;
 567         if (IS_GEN9_LP(dev_priv))
 568                 mask |= DC_STATE_EN_DC9;
 569         else
 570                 mask |= DC_STATE_EN_UPTO_DC6;
 571
 572         return mask;
 573 }
 574
 575 void gen9_sanitize_dc_state(struct drm_i915_private *dev_priv)
 576 {
 577         u32 val;
 578
 579         val = I915_READ(DC_STATE_EN) & gen9_dc_mask(dev_priv);
 580
 581         DRM_DEBUG_KMS("Resetting DC state tracking from %02x to %02x\n",
 582                       dev_priv->csr.dc_state, val);
 583         dev_priv->csr.dc_state = val;
 584 }
 585
 586 static void gen9_set_dc_state(struct drm_i915_private *dev_priv, uint32_t state)
 587 {
 588         uint32_t val;
 589         uint32_t mask;
 590
 591         if (WARN_ON_ONCE(state & ~dev_priv->csr.allowed_dc_mask))
 592                 state &= dev_priv->csr.allowed_dc_mask;
 593
 594         val = I915_READ(DC_STATE_EN);
 595         mask = gen9_dc_mask(dev_priv);
 596         DRM_DEBUG_KMS("Setting DC state from %02x to %02x\n",
 597                       val & mask, state);
 598
 599         /* Check if DMC is ignoring our DC state requests */
 600         if ((val & mask) != dev_priv->csr.dc_state)
 601                 DRM_ERROR("DC state mismatch (0x%x -> 0x%x)\n",
 602                           dev_priv->csr.dc_state, val & mask);
 603
 604         val &= ~mask;
 605         val |= state;
 606
 607         gen9_write_dc_state(dev_priv, val);
 608
 609         dev_priv->csr.dc_state = val & mask;
 610 }
 611
 612 void bxt_enable_dc9(struct drm_i915_private *dev_priv)
 613 {
 614         assert_can_enable_dc9(dev_priv);
 615
 616         DRM_DEBUG_KMS("Enabling DC9\n");
 617
 618         intel_power_sequencer_reset(dev_priv);
 619         gen9_set_dc_state(dev_priv, DC_STATE_EN_DC9);
 620 }
 621
 622 void bxt_disable_dc9(struct drm_i915_private *dev_priv)
 623 {
 624         assert_can_disable_dc9(dev_priv);
 625
 626         DRM_DEBUG_KMS("Disabling DC9\n");
 627
 628         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
 629
 630         intel_pps_unlock_regs_wa(dev_priv);
 631 }
 632
 633 static void assert_csr_loaded(struct drm_i915_private *dev_priv)
 634 {
 635         WARN_ONCE(!I915_READ(CSR_PROGRAM(0)),
 636                   "CSR program storage start is NULL\n");
 637         WARN_ONCE(!I915_READ(CSR_SSP_BASE), "CSR SSP Base Not fine\n");
 638         WARN_ONCE(!I915_READ(CSR_HTP_SKL), "CSR HTP Not fine\n");
 639 }
 640
 641 static void assert_can_enable_dc5(struct drm_i915_private *dev_priv)
 642 {
 643         bool pg2_enabled = intel_display_power_well_is_enabled(dev_priv,
 644                                         SKL_DISP_PW_2);
 645
 646         WARN_ONCE(pg2_enabled, "PG2 not disabled to enable DC5.\n");
 647
 648         WARN_ONCE((I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5),
 649                   "DC5 already programmed to be enabled.\n");
 650         assert_rpm_wakelock_held(dev_priv);
 651
 652         assert_csr_loaded(dev_priv);
 653 }
 654
 655 void gen9_enable_dc5(struct drm_i915_private *dev_priv)
 656 {
 657         assert_can_enable_dc5(dev_priv);
 658
 659         DRM_DEBUG_KMS("Enabling DC5\n");
 660
 661         gen9_set_dc_state(dev_priv, DC_STATE_EN_UPTO_DC5);
 662 }
 663
 664 static void assert_can_enable_dc6(struct drm_i915_private *dev_priv)
 665 {
 666         WARN_ONCE(I915_READ(UTIL_PIN_CTL) & UTIL_PIN_ENABLE,
 667                   "Backlight is not disabled.\n");
 668         WARN_ONCE((I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC6),
 669                   "DC6 already programmed to be enabled.\n");
 670
 671         assert_csr_loaded(dev_priv);
 672 }
 673
 674 void skl_enable_dc6(struct drm_i915_private *dev_priv)
 675 {
 676         assert_can_enable_dc6(dev_priv);
 677
 678         DRM_DEBUG_KMS("Enabling DC6\n");
 679
 680         gen9_set_dc_state(dev_priv, DC_STATE_EN_UPTO_DC6);
 681
 682 }
 683
 684 void skl_disable_dc6(struct drm_i915_private *dev_priv)
 685 {
 686         DRM_DEBUG_KMS("Disabling DC6\n");
 687
 688         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
 689 }
 690
 691 static void
 692 gen9_sanitize_power_well_requests(struct drm_i915_private *dev_priv,
 693                                   struct i915_power_well *power_well)
 694 {
 695         enum skl_disp_power_wells power_well_id = power_well->id;
 696         u32 val;
 697         u32 mask;
 698
 699         mask = SKL_POWER_WELL_REQ(power_well_id);
 700
 701         val = I915_READ(HSW_PWR_WELL_KVMR);
 702         if (WARN_ONCE(val & mask, "Clearing unexpected KVMR request for %s\n",
 703                       power_well->name))
 704                 I915_WRITE(HSW_PWR_WELL_KVMR, val & ~mask);
 705
 706         val = I915_READ(HSW_PWR_WELL_BIOS);
 707         val |= I915_READ(HSW_PWR_WELL_DEBUG);
 708
 709         if (!(val & mask))
 710                 return;
 711
 712         /*
 713          * DMC is known to force on the request bits for power well 1 on SKL
 714          * and BXT and the misc IO power well on SKL but we don't expect any
 715          * other request bits to be set, so WARN for those.
 716          */
 717         if (power_well_id == SKL_DISP_PW_1 ||
 718             (IS_GEN9_BC(dev_priv) &&
 719              power_well_id == SKL_DISP_PW_MISC_IO))
 720                 DRM_DEBUG_DRIVER("Clearing auxiliary requests for %s forced on "
 721                                  "by DMC\n", power_well->name);
 722         else
 723                 WARN_ONCE(1, "Clearing unexpected auxiliary requests for %s\n",
 724                           power_well->name);
 725
 726         I915_WRITE(HSW_PWR_WELL_BIOS, val & ~mask);
 727         I915_WRITE(HSW_PWR_WELL_DEBUG, val & ~mask);
 728 }
 729
 730 static void skl_set_power_well(struct drm_i915_private *dev_priv,
 731                                struct i915_power_well *power_well, bool enable)
 732 {
 733         uint32_t tmp, fuse_status;
 734         uint32_t req_mask, state_mask;
 735         bool is_enabled, enable_requested, check_fuse_status = false;
 736
 737         tmp = I915_READ(HSW_PWR_WELL_DRIVER);
 738         fuse_status = I915_READ(SKL_FUSE_STATUS);
 739
 740         switch (power_well->id) {
 741         case SKL_DISP_PW_1:
 742                 if (intel_wait_for_register(dev_priv,
 743                                             SKL_FUSE_STATUS,
 744                                             SKL_FUSE_PG0_DIST_STATUS,
 745                                             SKL_FUSE_PG0_DIST_STATUS,
 746                                             1)) {
 747                         DRM_ERROR("PG0 not enabled\n");
 748                         return;
 749                 }
 750                 break;
 751         case SKL_DISP_PW_2:
 752                 if (!(fuse_status & SKL_FUSE_PG1_DIST_STATUS)) {
 753                         DRM_ERROR("PG1 in disabled state\n");
 754                         return;
 755                 }
 756                 break;
 757         case SKL_DISP_PW_MISC_IO:
 758         case SKL_DISP_PW_DDI_A_E: /* GLK_DISP_PW_DDI_A */
 759         case SKL_DISP_PW_DDI_B:
 760         case SKL_DISP_PW_DDI_C:
 761         case SKL_DISP_PW_DDI_D:
 762         case GLK_DISP_PW_AUX_A:
 763         case GLK_DISP_PW_AUX_B:
 764         case GLK_DISP_PW_AUX_C:
 765                 break;
 766         default:
 767                 WARN(1, "Unknown power well %lu\n", power_well->id);
 768                 return;
 769         }
 770
 771         req_mask = SKL_POWER_WELL_REQ(power_well->id);
 772         enable_requested = tmp & req_mask;
 773         state_mask = SKL_POWER_WELL_STATE(power_well->id);
 774         is_enabled = tmp & state_mask;
 775
 776         if (!enable && enable_requested)
 777                 skl_power_well_pre_disable(dev_priv, power_well);
 778
 779         if (enable) {
 780                 if (!enable_requested) {
 781                         WARN((tmp & state_mask) &&
 782                                 !I915_READ(HSW_PWR_WELL_BIOS),
 783                                 "Invalid for power well status to be enabled, unless done by the BIOS, \
 784                                 when request is to disable!\n");
 785                         I915_WRITE(HSW_PWR_WELL_DRIVER, tmp | req_mask);
 786                 }
 787
 788                 if (!is_enabled) {
 789                         DRM_DEBUG_KMS("Enabling %s\n", power_well->name);
 790                         check_fuse_status = true;
 791                 }
 792         } else {
 793                 if (enable_requested) {
 794                         I915_WRITE(HSW_PWR_WELL_DRIVER, tmp & ~req_mask);
 795                         POSTING_READ(HSW_PWR_WELL_DRIVER);
 796                         DRM_DEBUG_KMS("Disabling %s\n", power_well->name);
 797                 }
 798
 799                 if (IS_GEN9(dev_priv))
 800                         gen9_sanitize_power_well_requests(dev_priv, power_well);
 801         }
 802
 803         if (wait_for(!!(I915_READ(HSW_PWR_WELL_DRIVER) & state_mask) == enable,
 804                      1))
 805                 DRM_ERROR("%s %s timeout\n",
 806                           power_well->name, enable ? "enable" : "disable");
 807
 808         if (check_fuse_status) {
 809                 if (power_well->id == SKL_DISP_PW_1) {
 810                         if (intel_wait_for_register(dev_priv,
 811                                                     SKL_FUSE_STATUS,
 812                                                     SKL_FUSE_PG1_DIST_STATUS,
 813                                                     SKL_FUSE_PG1_DIST_STATUS,
 814                                                     1))
 815                                 DRM_ERROR("PG1 distributing status timeout\n");
 816                 } else if (power_well->id == SKL_DISP_PW_2) {
 817                         if (intel_wait_for_register(dev_priv,
 818                                                     SKL_FUSE_STATUS,
 819                                                     SKL_FUSE_PG2_DIST_STATUS,
 820                                                     SKL_FUSE_PG2_DIST_STATUS,
 821                                                     1))
 822                                 DRM_ERROR("PG2 distributing status timeout\n");
 823                 }
 824         }
 825
 826         if (enable && !is_enabled)
 827                 skl_power_well_post_enable(dev_priv, power_well);
 828 }
 829
 830 static void hsw_power_well_sync_hw(struct drm_i915_private *dev_priv,
 831                                    struct i915_power_well *power_well)
 832 {
 833         /*
 834          * We're taking over the BIOS, so clear any requests made by it since
 835          * the driver is in charge now.
 836          */
 837         if (I915_READ(HSW_PWR_WELL_BIOS) & HSW_PWR_WELL_ENABLE_REQUEST)
 838                 I915_WRITE(HSW_PWR_WELL_BIOS, 0);
 839 }
 840
 841 static void hsw_power_well_enable(struct drm_i915_private *dev_priv,
 842                                   struct i915_power_well *power_well)
 843 {
 844         hsw_set_power_well(dev_priv, power_well, true);
 845 }
 846
 847 static void hsw_power_well_disable(struct drm_i915_private *dev_priv,
 848                                    struct i915_power_well *power_well)
 849 {
 850         hsw_set_power_well(dev_priv, power_well, false);
 851 }
 852
 853 static bool skl_power_well_enabled(struct drm_i915_private *dev_priv,
 854                                         struct i915_power_well *power_well)
 855 {
 856         uint32_t mask = SKL_POWER_WELL_REQ(power_well->id) |
 857                 SKL_POWER_WELL_STATE(power_well->id);
 858
 859         return (I915_READ(HSW_PWR_WELL_DRIVER) & mask) == mask;
 860 }
 861
 862 static void skl_power_well_sync_hw(struct drm_i915_private *dev_priv,
 863                                 struct i915_power_well *power_well)
 864 {
 865         uint32_t mask = SKL_POWER_WELL_REQ(power_well->id);
 866         uint32_t bios_req = I915_READ(HSW_PWR_WELL_BIOS);
 867
 868         /* Clear any request made by BIOS as driver is taking over */
 869         if (bios_req & mask) {
 870                 I915_WRITE(HSW_PWR_WELL_BIOS, bios_req & ~mask);
 871         }
 872 }
 873
 874 static void skl_power_well_enable(struct drm_i915_private *dev_priv,
 875                                 struct i915_power_well *power_well)
 876 {
 877         skl_set_power_well(dev_priv, power_well, true);
 878 }
 879
 880 static void skl_power_well_disable(struct drm_i915_private *dev_priv,
 881                                 struct i915_power_well *power_well)
 882 {
 883         skl_set_power_well(dev_priv, power_well, false);
 884 }
 885
 886 static void bxt_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
 887                                            struct i915_power_well *power_well)
 888 {
 889         bxt_ddi_phy_init(dev_priv, power_well->data);
 890 }
 891
 892 static void bxt_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
 893                                             struct i915_power_well *power_well)
 894 {
 895         bxt_ddi_phy_uninit(dev_priv, power_well->data);
 896 }
 897
 898 static bool bxt_dpio_cmn_power_well_enabled(struct drm_i915_private *dev_priv,
 899                                             struct i915_power_well *power_well)
 900 {
 901         return bxt_ddi_phy_is_enabled(dev_priv, power_well->data);
 902 }
 903
 904 static void bxt_verify_ddi_phy_power_wells(struct drm_i915_private *dev_priv)
 905 {
 906         struct i915_power_well *power_well;
 907
 908         power_well = lookup_power_well(dev_priv, BXT_DPIO_CMN_A);
 909         if (power_well->count > 0)
 910                 bxt_ddi_phy_verify_state(dev_priv, power_well->data);
 911
 912         power_well = lookup_power_well(dev_priv, BXT_DPIO_CMN_BC);
 913         if (power_well->count > 0)
 914                 bxt_ddi_phy_verify_state(dev_priv, power_well->data);
 915
 916         if (IS_GEMINILAKE(dev_priv)) {
 917                 power_well = lookup_power_well(dev_priv, GLK_DPIO_CMN_C);
 918                 if (power_well->count > 0)
 919                         bxt_ddi_phy_verify_state(dev_priv, power_well->data);
 920         }
 921 }
 922
 923 static bool gen9_dc_off_power_well_enabled(struct drm_i915_private *dev_priv,
 924                                            struct i915_power_well *power_well)
 925 {
 926         return (I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5_DC6_MASK) == 0;
 927 }
 928
 929 static void gen9_assert_dbuf_enabled(struct drm_i915_private *dev_priv)
 930 {
 931         u32 tmp = I915_READ(DBUF_CTL);
 932
 933         WARN((tmp & (DBUF_POWER_STATE | DBUF_POWER_REQUEST)) !=
 934              (DBUF_POWER_STATE | DBUF_POWER_REQUEST),
 935              "Unexpected DBuf power power state (0x%08x)\n", tmp);
 936 }
 937
 938 static void gen9_dc_off_power_well_enable(struct drm_i915_private *dev_priv,
 939                                           struct i915_power_well *power_well)
 940 {
 941         struct intel_cdclk_state cdclk_state = {};
 942
 943         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
 944
 945         dev_priv->display.get_cdclk(dev_priv, &cdclk_state);
 946         WARN_ON(!intel_cdclk_state_compare(&dev_priv->cdclk.hw, &cdclk_state));
 947
 948         gen9_assert_dbuf_enabled(dev_priv);
 949
 950         if (IS_GEN9_LP(dev_priv))
 951                 bxt_verify_ddi_phy_power_wells(dev_priv);
 952 }
 953
 954 static void gen9_dc_off_power_well_disable(struct drm_i915_private *dev_priv,
 955                                            struct i915_power_well *power_well)
 956 {
 957         if (!dev_priv->csr.dmc_payload)
 958                 return;
 959
 960         if (dev_priv->csr.allowed_dc_mask & DC_STATE_EN_UPTO_DC6)
 961                 skl_enable_dc6(dev_priv);
 962         else if (dev_priv->csr.allowed_dc_mask & DC_STATE_EN_UPTO_DC5)
 963                 gen9_enable_dc5(dev_priv);
 964 }
 965
 966 static void i9xx_power_well_sync_hw_noop(struct drm_i915_private *dev_priv,
 967                                          struct i915_power_well *power_well)
 968 {
 969 }
 970
 971 static void i9xx_always_on_power_well_noop(struct drm_i915_private *dev_priv,
 972                                            struct i915_power_well *power_well)
 973 {
 974 }
 975
 976 static bool i9xx_always_on_power_well_enabled(struct drm_i915_private *dev_priv,
 977                                              struct i915_power_well *power_well)
 978 {
 979         return true;
 980 }
 981
 982 static void vlv_set_power_well(struct drm_i915_private *dev_priv,
 983                                struct i915_power_well *power_well, bool enable)
 984 {
 985         enum punit_power_well power_well_id = power_well->id;
 986         u32 mask;
 987         u32 state;
 988         u32 ctrl;
 989
 990         mask = PUNIT_PWRGT_MASK(power_well_id);
 991         state = enable ? PUNIT_PWRGT_PWR_ON(power_well_id) :
 992                          PUNIT_PWRGT_PWR_GATE(power_well_id);
 993
 994         mutex_lock(&dev_priv->rps.hw_lock);
 995
 996 #define COND \
 997         ((vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask) == state)
 998
 999         if (COND)
1000                 goto out;
1001
1002         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL);
1003         ctrl &= ~mask;
1004         ctrl |= state;
1005         vlv_punit_write(dev_priv, PUNIT_REG_PWRGT_CTRL, ctrl);
1006
1007         if (wait_for(COND, 100))
1008                 DRM_ERROR("timeout setting power well state %08x (%08x)\n",
1009                           state,
1010                           vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL));
1011
1012 #undef COND
1013
1014 out:
1015         mutex_unlock(&dev_priv->rps.hw_lock);
1016 }
1017
1018 static void vlv_power_well_enable(struct drm_i915_private *dev_priv,
1019                                   struct i915_power_well *power_well)
1020 {
1021         vlv_set_power_well(dev_priv, power_well, true);
1022 }
1023
1024 static void vlv_power_well_disable(struct drm_i915_private *dev_priv,
1025                                    struct i915_power_well *power_well)
1026 {
1027         vlv_set_power_well(dev_priv, power_well, false);
1028 }
1029
1030 static bool vlv_power_well_enabled(struct drm_i915_private *dev_priv,
1031                                    struct i915_power_well *power_well)
1032 {
1033         int power_well_id = power_well->id;
1034         bool enabled = false;
1035         u32 mask;
1036         u32 state;
1037         u32 ctrl;
1038
1039         mask = PUNIT_PWRGT_MASK(power_well_id);
1040         ctrl = PUNIT_PWRGT_PWR_ON(power_well_id);
1041
1042         mutex_lock(&dev_priv->rps.hw_lock);
1043
1044         state = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask;
1045         /*
1046          * We only ever set the power-on and power-gate states, anything
1047          * else is unexpected.
1048          */
1049         WARN_ON(state != PUNIT_PWRGT_PWR_ON(power_well_id) &&
1050                 state != PUNIT_PWRGT_PWR_GATE(power_well_id));
1051         if (state == ctrl)
1052                 enabled = true;
1053
1054         /*
1055          * A transient state at this point would mean some unexpected party
1056          * is poking at the power controls too.
1057          */
1058         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL) & mask;
1059         WARN_ON(ctrl != state);
1060
1061         mutex_unlock(&dev_priv->rps.hw_lock);
1062
1063         return enabled;
1064 }
1065
1066 static void vlv_init_display_clock_gating(struct drm_i915_private *dev_priv)
1067 {
1068         u32 val;
1069
1070         /*
1071          * On driver load, a pipe may be active and driving a DSI display.
1072          * Preserve DPOUNIT_CLOCK_GATE_DISABLE to avoid the pipe getting stuck
1073          * (and never recovering) in this case. intel_dsi_post_disable() will
1074          * clear it when we turn off the display.
1075          */
1076         val = I915_READ(DSPCLK_GATE_D);
1077         val &= DPOUNIT_CLOCK_GATE_DISABLE;
1078         val |= VRHUNIT_CLOCK_GATE_DISABLE;
1079         I915_WRITE(DSPCLK_GATE_D, val);
1080
1081         /*
1082          * Disable trickle feed and enable pnd deadline calculation
1083          */
1084         I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE);
1085         I915_WRITE(CBR1_VLV, 0);
1086
1087         WARN_ON(dev_priv->rawclk_freq == 0);
1088
1089         I915_WRITE(RAWCLK_FREQ_VLV,
1090                    DIV_ROUND_CLOSEST(dev_priv->rawclk_freq, 1000));
1091 }
1092
1093 static void vlv_display_power_well_init(struct drm_i915_private *dev_priv)
1094 {
1095         struct intel_encoder *encoder;
1096         enum pipe pipe;
1097
1098         /*
1099          * Enable the CRI clock source so we can get at the
1100          * display and the reference clock for VGA
1101          * hotplug / manual detection. Supposedly DSI also
1102          * needs the ref clock up and running.
1103          *
1104          * CHV DPLL B/C have some issues if VGA mode is enabled.
1105          */
1106         for_each_pipe(dev_priv, pipe) {
1107                 u32 val = I915_READ(DPLL(pipe));
1108
1109                 val |= DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
1110                 if (pipe != PIPE_A)
1111                         val |= DPLL_INTEGRATED_CRI_CLK_VLV;
1112
1113                 I915_WRITE(DPLL(pipe), val);
1114         }
1115
1116         vlv_init_display_clock_gating(dev_priv);
1117
1118         spin_lock_irq(&dev_priv->irq_lock);
1119         valleyview_enable_display_irqs(dev_priv);
1120         spin_unlock_irq(&dev_priv->irq_lock);
1121
1122         /*
1123          * During driver initialization/resume we can avoid restoring the
1124          * part of the HW/SW state that will be inited anyway explicitly.
1125          */
1126         if (dev_priv->power_domains.initializing)
1127                 return;
1128
1129         intel_hpd_init(dev_priv);
1130
1131         /* Re-enable the ADPA, if we have one */
1132         for_each_intel_encoder(&dev_priv->drm, encoder) {
1133                 if (encoder->type == INTEL_OUTPUT_ANALOG)
1134                         intel_crt_reset(&encoder->base);
1135         }
1136
1137         i915_redisable_vga_power_on(dev_priv);
1138
1139         intel_pps_unlock_regs_wa(dev_priv);
1140 }
1141
1142 static void vlv_display_power_well_deinit(struct drm_i915_private *dev_priv)
1143 {
1144         spin_lock_irq(&dev_priv->irq_lock);
1145         valleyview_disable_display_irqs(dev_priv);
1146         spin_unlock_irq(&dev_priv->irq_lock);
1147
1148         /* make sure we're done processing display irqs */
1149         synchronize_irq(dev_priv->drm.irq);
1150
1151         intel_power_sequencer_reset(dev_priv);
1152
1153         /* Prevent us from re-enabling polling on accident in late suspend */
1154         if (!dev_priv->drm.dev->power.is_suspended)
1155                 intel_hpd_poll_init(dev_priv);
1156 }
1157
1158 static void vlv_display_power_well_enable(struct drm_i915_private *dev_priv,
1159                                           struct i915_power_well *power_well)
1160 {
1161         WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DISP2D);
1162
1163         vlv_set_power_well(dev_priv, power_well, true);
1164
1165         vlv_display_power_well_init(dev_priv);
1166 }
1167
1168 static void vlv_display_power_well_disable(struct drm_i915_private *dev_priv,
1169                                            struct i915_power_well *power_well)
1170 {
1171         WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DISP2D);
1172
1173         vlv_display_power_well_deinit(dev_priv);
1174
1175         vlv_set_power_well(dev_priv, power_well, false);
1176 }
1177
1178 static void vlv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
1179                                            struct i915_power_well *power_well)
1180 {
1181         WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DPIO_CMN_BC);
1182
1183         /* since ref/cri clock was enabled */
1184         udelay(1); /* >10ns for cmnreset, >0ns for sidereset */
1185
1186         vlv_set_power_well(dev_priv, power_well, true);
1187
1188         /*
1189          * From VLV2A0_DP_eDP_DPIO_driver_vbios_notes_10.docx -
1190          *  6.  De-assert cmn_reset/side_reset. Same as VLV X0.
1191          *   a. GUnit 0x2110 bit[0] set to 1 (def 0)
1192          *   b. The other bits such as sfr settings / modesel may all
1193          *      be set to 0.
1194          *
1195          * This should only be done on init and resume from S3 with
1196          * both PLLs disabled, or we risk losing DPIO and PLL
1197          * synchronization.
1198          */
1199         I915_WRITE(DPIO_CTL, I915_READ(DPIO_CTL) | DPIO_CMNRST);
1200 }
1201
1202 static void vlv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
1203                                             struct i915_power_well *power_well)
1204 {
1205         enum pipe pipe;
1206
1207         WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DPIO_CMN_BC);
1208
1209         for_each_pipe(dev_priv, pipe)
1210                 assert_pll_disabled(dev_priv, pipe);
1211
1212         /* Assert common reset */
1213         I915_WRITE(DPIO_CTL, I915_READ(DPIO_CTL) & ~DPIO_CMNRST);
1214
1215         vlv_set_power_well(dev_priv, power_well, false);
1216 }
1217
1218 #define POWER_DOMAIN_MASK (GENMASK_ULL(POWER_DOMAIN_NUM - 1, 0))
1219
1220 static struct i915_power_well *lookup_power_well(struct drm_i915_private *dev_priv,
1221                                                  int power_well_id)
1222 {
1223         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1224         int i;
1225
1226         for (i = 0; i < power_domains->power_well_count; i++) {
1227                 struct i915_power_well *power_well;
1228
1229                 power_well = &power_domains->power_wells[i];
1230                 if (power_well->id == power_well_id)
1231                         return power_well;
1232         }
1233
1234         return NULL;
1235 }
1236
1237 #define BITS_SET(val, bits) (((val) & (bits)) == (bits))
1238
1239 static void assert_chv_phy_status(struct drm_i915_private *dev_priv)
1240 {
1241         struct i915_power_well *cmn_bc =
1242                 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_BC);
1243         struct i915_power_well *cmn_d =
1244                 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_D);
1245         u32 phy_control = dev_priv->chv_phy_control;
1246         u32 phy_status = 0;
1247         u32 phy_status_mask = 0xffffffff;
1248
1249         /*
1250          * The BIOS can leave the PHY is some weird state
1251          * where it doesn't fully power down some parts.
1252          * Disable the asserts until the PHY has been fully
1253          * reset (ie. the power well has been disabled at
1254          * least once).
1255          */
1256         if (!dev_priv->chv_phy_assert[DPIO_PHY0])
1257                 phy_status_mask &= ~(PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH0) |
1258                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 0) |
1259                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 1) |
1260                                      PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH1) |
1261                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 0) |
1262                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 1));
1263
1264         if (!dev_priv->chv_phy_assert[DPIO_PHY1])
1265                 phy_status_mask &= ~(PHY_STATUS_CMN_LDO(DPIO_PHY1, DPIO_CH0) |
1266                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 0) |
1267                                      PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 1));
1268
1269         if (cmn_bc->ops->is_enabled(dev_priv, cmn_bc)) {
1270                 phy_status |= PHY_POWERGOOD(DPIO_PHY0);
1271
1272                 /* this assumes override is only used to enable lanes */
1273                 if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH0)) == 0)
1274                         phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH0);
1275
1276                 if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH1)) == 0)
1277                         phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1);
1278
1279                 /* CL1 is on whenever anything is on in either channel */
1280                 if (BITS_SET(phy_control,
1281                              PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH0) |
1282                              PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1)))
1283                         phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH0);
1284
1285                 /*
1286                  * The DPLLB check accounts for the pipe B + port A usage
1287                  * with CL2 powered up but all the lanes in the second channel
1288                  * powered down.
1289                  */
1290                 if (BITS_SET(phy_control,
1291                              PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY0, DPIO_CH1)) &&
1292                     (I915_READ(DPLL(PIPE_B)) & DPLL_VCO_ENABLE) == 0)
1293                         phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH1);
1294
1295                 if (BITS_SET(phy_control,
1296                              PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY0, DPIO_CH0)))
1297                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 0);
1298                 if (BITS_SET(phy_control,
1299                              PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY0, DPIO_CH0)))
1300                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 1);
1301
1302                 if (BITS_SET(phy_control,
1303                              PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY0, DPIO_CH1)))
1304                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 0);
1305                 if (BITS_SET(phy_control,
1306                              PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY0, DPIO_CH1)))
1307                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 1);
1308         }
1309
1310         if (cmn_d->ops->is_enabled(dev_priv, cmn_d)) {
1311                 phy_status |= PHY_POWERGOOD(DPIO_PHY1);
1312
1313                 /* this assumes override is only used to enable lanes */
1314                 if ((phy_control & PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY1, DPIO_CH0)) == 0)
1315                         phy_control |= PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY1, DPIO_CH0);
1316
1317                 if (BITS_SET(phy_control,
1318                              PHY_CH_POWER_DOWN_OVRD(0xf, DPIO_PHY1, DPIO_CH0)))
1319                         phy_status |= PHY_STATUS_CMN_LDO(DPIO_PHY1, DPIO_CH0);
1320
1321                 if (BITS_SET(phy_control,
1322                              PHY_CH_POWER_DOWN_OVRD(0x3, DPIO_PHY1, DPIO_CH0)))
1323                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 0);
1324                 if (BITS_SET(phy_control,
1325                              PHY_CH_POWER_DOWN_OVRD(0xc, DPIO_PHY1, DPIO_CH0)))
1326                         phy_status |= PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 1);
1327         }
1328
1329         phy_status &= phy_status_mask;
1330
1331         /*
1332          * The PHY may be busy with some initial calibration and whatnot,
1333          * so the power state can take a while to actually change.
1334          */
1335         if (intel_wait_for_register(dev_priv,
1336                                     DISPLAY_PHY_STATUS,
1337                                     phy_status_mask,
1338                                     phy_status,
1339                                     10))
1340                 DRM_ERROR("Unexpected PHY_STATUS 0x%08x, expected 0x%08x (PHY_CONTROL=0x%08x)\n",
1341                           I915_READ(DISPLAY_PHY_STATUS) & phy_status_mask,
1342                            phy_status, dev_priv->chv_phy_control);
1343 }
1344
1345 #undef BITS_SET
1346
1347 static void chv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
1348                                            struct i915_power_well *power_well)
1349 {
1350         enum dpio_phy phy;
1351         enum pipe pipe;
1352         uint32_t tmp;
1353
1354         WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DPIO_CMN_BC &&
1355                      power_well->id != PUNIT_POWER_WELL_DPIO_CMN_D);
1356
1357         if (power_well->id == PUNIT_POWER_WELL_DPIO_CMN_BC) {
1358                 pipe = PIPE_A;
1359                 phy = DPIO_PHY0;
1360         } else {
1361                 pipe = PIPE_C;
1362                 phy = DPIO_PHY1;
1363         }
1364
1365         /* since ref/cri clock was enabled */
1366         udelay(1); /* >10ns for cmnreset, >0ns for sidereset */
1367         vlv_set_power_well(dev_priv, power_well, true);
1368
1369         /* Poll for phypwrgood signal */
1370         if (intel_wait_for_register(dev_priv,
1371                                     DISPLAY_PHY_STATUS,
1372                                     PHY_POWERGOOD(phy),
1373                                     PHY_POWERGOOD(phy),
1374                                     1))
1375                 DRM_ERROR("Display PHY %d is not power up\n", phy);
1376
1377         mutex_lock(&dev_priv->sb_lock);
1378
1379         /* Enable dynamic power down */
1380         tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW28);
1381         tmp |= DPIO_DYNPWRDOWNEN_CH0 | DPIO_CL1POWERDOWNEN |
1382                 DPIO_SUS_CLK_CONFIG_GATE_CLKREQ;
1383         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW28, tmp);
1384
1385         if (power_well->id == PUNIT_POWER_WELL_DPIO_CMN_BC) {
1386                 tmp = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW6_CH1);
1387                 tmp |= DPIO_DYNPWRDOWNEN_CH1;
1388                 vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW6_CH1, tmp);
1389         } else {
1390                 /*
1391                  * Force the non-existing CL2 off. BXT does this
1392                  * too, so maybe it saves some power even though
1393                  * CL2 doesn't exist?
1394                  */
1395                 tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW30);
1396                 tmp |= DPIO_CL2_LDOFUSE_PWRENB;
1397                 vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW30, tmp);
1398         }
1399
1400         mutex_unlock(&dev_priv->sb_lock);
1401
1402         dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(phy);
1403         I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
1404
1405         DRM_DEBUG_KMS("Enabled DPIO PHY%d (PHY_CONTROL=0x%08x)\n",
1406                       phy, dev_priv->chv_phy_control);
1407
1408         assert_chv_phy_status(dev_priv);
1409 }
1410
1411 static void chv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
1412                                             struct i915_power_well *power_well)
1413 {
1414         enum dpio_phy phy;
1415
1416         WARN_ON_ONCE(power_well->id != PUNIT_POWER_WELL_DPIO_CMN_BC &&
1417                      power_well->id != PUNIT_POWER_WELL_DPIO_CMN_D);
1418
1419         if (power_well->id == PUNIT_POWER_WELL_DPIO_CMN_BC) {
1420                 phy = DPIO_PHY0;
1421                 assert_pll_disabled(dev_priv, PIPE_A);
1422                 assert_pll_disabled(dev_priv, PIPE_B);
1423         } else {
1424                 phy = DPIO_PHY1;
1425                 assert_pll_disabled(dev_priv, PIPE_C);
1426         }
1427
1428         dev_priv->chv_phy_control &= ~PHY_COM_LANE_RESET_DEASSERT(phy);
1429         I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
1430
1431         vlv_set_power_well(dev_priv, power_well, false);
1432
1433         DRM_DEBUG_KMS("Disabled DPIO PHY%d (PHY_CONTROL=0x%08x)\n",
1434                       phy, dev_priv->chv_phy_control);
1435
1436         /* PHY is fully reset now, so we can enable the PHY state asserts */
1437         dev_priv->chv_phy_assert[phy] = true;
1438
1439         assert_chv_phy_status(dev_priv);
1440 }
1441
1442 static void assert_chv_phy_powergate(struct drm_i915_private *dev_priv, enum dpio_phy phy,
1443                                      enum dpio_channel ch, bool override, unsigned int mask)
1444 {
1445         enum pipe pipe = phy == DPIO_PHY0 ? PIPE_A : PIPE_C;
1446         u32 reg, val, expected, actual;
1447
1448         /*
1449          * The BIOS can leave the PHY is some weird state
1450          * where it doesn't fully power down some parts.
1451          * Disable the asserts until the PHY has been fully
1452          * reset (ie. the power well has been disabled at
1453          * least once).
1454          */
1455         if (!dev_priv->chv_phy_assert[phy])
1456                 return;
1457
1458         if (ch == DPIO_CH0)
1459                 reg = _CHV_CMN_DW0_CH0;
1460         else
1461                 reg = _CHV_CMN_DW6_CH1;
1462
1463         mutex_lock(&dev_priv->sb_lock);
1464         val = vlv_dpio_read(dev_priv, pipe, reg);
1465         mutex_unlock(&dev_priv->sb_lock);
1466
1467         /*
1468          * This assumes !override is only used when the port is disabled.
1469          * All lanes should power down even without the override when
1470          * the port is disabled.
1471          */
1472         if (!override || mask == 0xf) {
1473                 expected = DPIO_ALLDL_POWERDOWN | DPIO_ANYDL_POWERDOWN;
1474                 /*
1475                  * If CH1 common lane is not active anymore
1476                  * (eg. for pipe B DPLL) the entire channel will
1477                  * shut down, which causes the common lane registers
1478                  * to read as 0. That means we can't actually check
1479                  * the lane power down status bits, but as the entire
1480                  * register reads as 0 it's a good indication that the
1481                  * channel is indeed entirely powered down.
1482                  */
1483                 if (ch == DPIO_CH1 && val == 0)
1484                         expected = 0;
1485         } else if (mask != 0x0) {
1486                 expected = DPIO_ANYDL_POWERDOWN;
1487         } else {
1488                 expected = 0;
1489         }
1490
1491         if (ch == DPIO_CH0)
1492                 actual = val >> DPIO_ANYDL_POWERDOWN_SHIFT_CH0;
1493         else
1494                 actual = val >> DPIO_ANYDL_POWERDOWN_SHIFT_CH1;
1495         actual &= DPIO_ALLDL_POWERDOWN | DPIO_ANYDL_POWERDOWN;
1496
1497         WARN(actual != expected,
1498              "Unexpected DPIO lane power down: all %d, any %d. Expected: all %d, any %d. (0x%x = 0x%08x)\n",
1499              !!(actual & DPIO_ALLDL_POWERDOWN), !!(actual & DPIO_ANYDL_POWERDOWN),
1500              !!(expected & DPIO_ALLDL_POWERDOWN), !!(expected & DPIO_ANYDL_POWERDOWN),
1501              reg, val);
1502 }
1503
1504 bool chv_phy_powergate_ch(struct drm_i915_private *dev_priv, enum dpio_phy phy,
1505                           enum dpio_channel ch, bool override)
1506 {
1507         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1508         bool was_override;
1509
1510         mutex_lock(&power_domains->lock);
1511
1512         was_override = dev_priv->chv_phy_control & PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1513
1514         if (override == was_override)
1515                 goto out;
1516
1517         if (override)
1518                 dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1519         else
1520                 dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1521
1522         I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
1523
1524         DRM_DEBUG_KMS("Power gating DPIO PHY%d CH%d (DPIO_PHY_CONTROL=0x%08x)\n",
1525                       phy, ch, dev_priv->chv_phy_control);
1526
1527         assert_chv_phy_status(dev_priv);
1528
1529 out:
1530         mutex_unlock(&power_domains->lock);
1531
1532         return was_override;
1533 }
1534
1535 void chv_phy_powergate_lanes(struct intel_encoder *encoder,
1536                              bool override, unsigned int mask)
1537 {
1538         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
1539         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1540         enum dpio_phy phy = vlv_dport_to_phy(enc_to_dig_port(&encoder->base));
1541         enum dpio_channel ch = vlv_dport_to_channel(enc_to_dig_port(&encoder->base));
1542
1543         mutex_lock(&power_domains->lock);
1544
1545         dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD(0xf, phy, ch);
1546         dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD(mask, phy, ch);
1547
1548         if (override)
1549                 dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1550         else
1551                 dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
1552
1553         I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
1554
1555         DRM_DEBUG_KMS("Power gating DPIO PHY%d CH%d lanes 0x%x (PHY_CONTROL=0x%08x)\n",
1556                       phy, ch, mask, dev_priv->chv_phy_control);
1557
1558         assert_chv_phy_status(dev_priv);
1559
1560         assert_chv_phy_powergate(dev_priv, phy, ch, override, mask);
1561
1562         mutex_unlock(&power_domains->lock);
1563 }
1564
1565 static bool chv_pipe_power_well_enabled(struct drm_i915_private *dev_priv,
1566                                         struct i915_power_well *power_well)
1567 {
1568         enum pipe pipe = power_well->id;
1569         bool enabled;
1570         u32 state, ctrl;
1571
1572         mutex_lock(&dev_priv->rps.hw_lock);
1573
1574         state = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSS_MASK(pipe);
1575         /*
1576          * We only ever set the power-on and power-gate states, anything
1577          * else is unexpected.
1578          */
1579         WARN_ON(state != DP_SSS_PWR_ON(pipe) && state != DP_SSS_PWR_GATE(pipe));
1580         enabled = state == DP_SSS_PWR_ON(pipe);
1581
1582         /*
1583          * A transient state at this point would mean some unexpected party
1584          * is poking at the power controls too.
1585          */
1586         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSC_MASK(pipe);
1587         WARN_ON(ctrl << 16 != state);
1588
1589         mutex_unlock(&dev_priv->rps.hw_lock);
1590
1591         return enabled;
1592 }
1593
1594 static void chv_set_pipe_power_well(struct drm_i915_private *dev_priv,
1595                                     struct i915_power_well *power_well,
1596                                     bool enable)
1597 {
1598         enum pipe pipe = power_well->id;
1599         u32 state;
1600         u32 ctrl;
1601
1602         state = enable ? DP_SSS_PWR_ON(pipe) : DP_SSS_PWR_GATE(pipe);
1603
1604         mutex_lock(&dev_priv->rps.hw_lock);
1605
1606 #define COND \
1607         ((vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSS_MASK(pipe)) == state)
1608
1609         if (COND)
1610                 goto out;
1611
1612         ctrl = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
1613         ctrl &= ~DP_SSC_MASK(pipe);
1614         ctrl |= enable ? DP_SSC_PWR_ON(pipe) : DP_SSC_PWR_GATE(pipe);
1615         vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, ctrl);
1616
1617         if (wait_for(COND, 100))
1618                 DRM_ERROR("timeout setting power well state %08x (%08x)\n",
1619                           state,
1620                           vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ));
1621
1622 #undef COND
1623
1624 out:
1625         mutex_unlock(&dev_priv->rps.hw_lock);
1626 }
1627
1628 static void chv_pipe_power_well_enable(struct drm_i915_private *dev_priv,
1629                                        struct i915_power_well *power_well)
1630 {
1631         WARN_ON_ONCE(power_well->id != PIPE_A);
1632
1633         chv_set_pipe_power_well(dev_priv, power_well, true);
1634
1635         vlv_display_power_well_init(dev_priv);
1636 }
1637
1638 static void chv_pipe_power_well_disable(struct drm_i915_private *dev_priv,
1639                                         struct i915_power_well *power_well)
1640 {
1641         WARN_ON_ONCE(power_well->id != PIPE_A);
1642
1643         vlv_display_power_well_deinit(dev_priv);
1644
1645         chv_set_pipe_power_well(dev_priv, power_well, false);
1646 }
1647
1648 static void
1649 __intel_display_power_get_domain(struct drm_i915_private *dev_priv,
1650                                  enum intel_display_power_domain domain)
1651 {
1652         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1653         struct i915_power_well *power_well;
1654
1655         for_each_power_domain_well(dev_priv, power_well, BIT_ULL(domain))
1656                 intel_power_well_get(dev_priv, power_well);
1657
1658         power_domains->domain_use_count[domain]++;
1659 }
1660
1661 /**
1662  * intel_display_power_get - grab a power domain reference
1663  * @dev_priv: i915 device instance
1664  * @domain: power domain to reference
1665  *
1666  * This function grabs a power domain reference for @domain and ensures that the
1667  * power domain and all its parents are powered up. Therefore users should only
1668  * grab a reference to the innermost power domain they need.
1669  *
1670  * Any power domain reference obtained by this function must have a symmetric
1671  * call to intel_display_power_put() to release the reference again.
1672  */
1673 void intel_display_power_get(struct drm_i915_private *dev_priv,
1674                              enum intel_display_power_domain domain)
1675 {
1676         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1677
1678         intel_runtime_pm_get(dev_priv);
1679
1680         mutex_lock(&power_domains->lock);
1681
1682         __intel_display_power_get_domain(dev_priv, domain);
1683
1684         mutex_unlock(&power_domains->lock);
1685 }
1686
1687 /**
1688  * intel_display_power_get_if_enabled - grab a reference for an enabled display power domain
1689  * @dev_priv: i915 device instance
1690  * @domain: power domain to reference
1691  *
1692  * This function grabs a power domain reference for @domain and ensures that the
1693  * power domain and all its parents are powered up. Therefore users should only
1694  * grab a reference to the innermost power domain they need.
1695  *
1696  * Any power domain reference obtained by this function must have a symmetric
1697  * call to intel_display_power_put() to release the reference again.
1698  */
1699 bool intel_display_power_get_if_enabled(struct drm_i915_private *dev_priv,
1700                                         enum intel_display_power_domain domain)
1701 {
1702         struct i915_power_domains *power_domains = &dev_priv->power_domains;
1703         bool is_enabled;
1704
1705         if (!intel_runtime_pm_get_if_in_use(dev_priv))
1706                 return false;
1707
1708         mutex_lock(&power_domains->lock);
1709
1710         if (__intel_display_power_is_enabled(dev_priv, domain)) {
1711                 __intel_display_power_get_domain(dev_priv, domain);
1712                 is_enabled = true;
1713         } else {
1714                 is_enabled = false;
1715         }
1716
1717         mutex_unlock(&power_domains->lock);
1718
1719         if (!is_enabled)
1720                 intel_runtime_pm_put(dev_priv);
1721
1722         return is_enabled;
1723 }
1724
1725 /**
1726  * intel_display_power_put - release a power domain reference
1727  * @dev_priv: i915 device instance
1728  * @domain: power domain to reference
1729  *
1730  * This function drops the power domain reference obtained by
1731  * intel_display_power_get() and might power down the corresponding hardware
1732  * block right away if this is the last reference.
1733  */
1734 void intel_display_power_put(struct drm_i915_private *dev_priv,
1735                              enum intel_display_power_domain domain)
1736 {
1737         struct i915_power_domains *power_domains;
1738         struct i915_power_well *power_well;
1739
1740         power_domains = &dev_priv->power_domains;
1741
1742         mutex_lock(&power_domains->lock);
1743
1744         WARN(!power_domains->domain_use_count[domain],
1745              "Use count on domain %s is already zero\n",
1746              intel_display_power_domain_str(domain));
1747         power_domains->domain_use_count[domain]--;
1748
1749         for_each_power_domain_well_rev(dev_priv, power_well, BIT_ULL(domain))
1750                 intel_power_well_put(dev_priv, power_well);
1751
1752         mutex_unlock(&power_domains->lock);
1753
1754         intel_runtime_pm_put(dev_priv);
1755 }
1756
1757 #define HSW_DISPLAY_POWER_DOMAINS (                     \
1758         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
1759         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
1760         BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |             \
1761         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
1762         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
1763         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
1764         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
1765         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
1766         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1767         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1768         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |                \
1769         BIT_ULL(POWER_DOMAIN_PORT_CRT) | /* DDI E */    \
1770         BIT_ULL(POWER_DOMAIN_VGA) |                             \
1771         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
1772         BIT_ULL(POWER_DOMAIN_INIT))
1773
1774 #define BDW_DISPLAY_POWER_DOMAINS (                     \
1775         BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
1776         BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
1777         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |             \
1778         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |             \
1779         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
1780         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
1781         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
1782         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |                \
1783         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |                \
1784         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |                \
1785         BIT_ULL(POWER_DOMAIN_PORT_CRT) | /* DDI E */    \
1786         BIT_ULL(POWER_DOMAIN_VGA) |                             \
1787         BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
1788         BIT_ULL(POWER_DOMAIN_INIT))
1789
1790 #define VLV_DISPLAY_POWER_DOMAINS (             \
1791         BIT_ULL(POWER_DOMAIN_PIPE_A) |          \
1792         BIT_ULL(POWER_DOMAIN_PIPE_B) |          \
1793         BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |     \
1794         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |     \
1795         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |    \
1796         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |    \
1797         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
1798         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
1799         BIT_ULL(POWER_DOMAIN_PORT_DSI) |                \
1800         BIT_ULL(POWER_DOMAIN_PORT_CRT) |                \
1801         BIT_ULL(POWER_DOMAIN_VGA) |                     \
1802         BIT_ULL(POWER_DOMAIN_AUDIO) |           \
1803         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1804         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1805         BIT_ULL(POWER_DOMAIN_GMBUS) |           \
1806         BIT_ULL(POWER_DOMAIN_INIT))
1807
1808 #define VLV_DPIO_CMN_BC_POWER_DOMAINS (         \
1809         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
1810         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
1811         BIT_ULL(POWER_DOMAIN_PORT_CRT) |                \
1812         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1813         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1814         BIT_ULL(POWER_DOMAIN_INIT))
1815
1816 #define VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS (  \
1817         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
1818         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1819         BIT_ULL(POWER_DOMAIN_INIT))
1820
1821 #define VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS (  \
1822         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
1823         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1824         BIT_ULL(POWER_DOMAIN_INIT))
1825
1826 #define VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS (  \
1827         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
1828         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1829         BIT_ULL(POWER_DOMAIN_INIT))
1830
1831 #define VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS (  \
1832         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
1833         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1834         BIT_ULL(POWER_DOMAIN_INIT))
1835
1836 #define CHV_DISPLAY_POWER_DOMAINS (             \
1837         BIT_ULL(POWER_DOMAIN_PIPE_A) |          \
1838         BIT_ULL(POWER_DOMAIN_PIPE_B) |          \
1839         BIT_ULL(POWER_DOMAIN_PIPE_C) |          \
1840         BIT_ULL(POWER_DOMAIN_PIPE_A_PANEL_FITTER) |     \
1841         BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |     \
1842         BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |     \
1843         BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |    \
1844         BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |    \
1845         BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |    \
1846         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
1847         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
1848         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |        \
1849         BIT_ULL(POWER_DOMAIN_PORT_DSI) |                \
1850         BIT_ULL(POWER_DOMAIN_VGA) |                     \
1851         BIT_ULL(POWER_DOMAIN_AUDIO) |           \
1852         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1853         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1854         BIT_ULL(POWER_DOMAIN_AUX_D) |           \
1855         BIT_ULL(POWER_DOMAIN_GMBUS) |           \
1856         BIT_ULL(POWER_DOMAIN_INIT))
1857
1858 #define CHV_DPIO_CMN_BC_POWER_DOMAINS (         \
1859         BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
1860         BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
1861         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
1862         BIT_ULL(POWER_DOMAIN_AUX_C) |           \
1863         BIT_ULL(POWER_DOMAIN_INIT))
1864
1865 #define CHV_DPIO_CMN_D_POWER_DOMAINS (          \
1866         BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |        \
1867         BIT_ULL(POWER_DOMAIN_AUX_D) |           \
1868         BIT_ULL(POWER_DOMAIN_INIT))
1869
1870 static const struct i915_power_well_ops i9xx_always_on_power_well_ops = {
1871         .sync_hw = i9xx_power_well_sync_hw_noop,
1872         .enable = i9xx_always_on_power_well_noop,
1873         .disable = i9xx_always_on_power_well_noop,
1874         .is_enabled = i9xx_always_on_power_well_enabled,
1875 };
1876
1877 static const struct i915_power_well_ops chv_pipe_power_well_ops = {
1878         .sync_hw = i9xx_power_well_sync_hw_noop,
1879         .enable = chv_pipe_power_well_enable,
1880         .disable = chv_pipe_power_well_disable,
1881         .is_enabled = chv_pipe_power_well_enabled,
1882 };
1883
1884 static const struct i915_power_well_ops chv_dpio_cmn_power_well_ops = {
1885         .sync_hw = i9xx_power_well_sync_hw_noop,
1886         .enable = chv_dpio_cmn_power_well_enable,
1887         .disable = chv_dpio_cmn_power_well_disable,
1888         .is_enabled = vlv_power_well_enabled,
1889 };
1890
1891 static struct i915_power_well i9xx_always_on_power_well[] = {
1892         {
1893                 .name = "always-on",
1894                 .always_on = 1,
1895                 .domains = POWER_DOMAIN_MASK,
1896                 .ops = &i9xx_always_on_power_well_ops,
1897         },
1898 };
1899
1900 static const struct i915_power_well_ops hsw_power_well_ops = {
1901         .sync_hw = hsw_power_well_sync_hw,
1902         .enable = hsw_power_well_enable,
1903         .disable = hsw_power_well_disable,
1904         .is_enabled = hsw_power_well_enabled,
1905 };
1906
1907 static const struct i915_power_well_ops skl_power_well_ops = {
1908         .sync_hw = skl_power_well_sync_hw,
1909         .enable = skl_power_well_enable,
1910         .disable = skl_power_well_disable,
1911         .is_enabled = skl_power_well_enabled,
1912 };
1913
1914 static const struct i915_power_well_ops gen9_dc_off_power_well_ops = {
1915         .sync_hw = i9xx_power_well_sync_hw_noop,
1916         .enable = gen9_dc_off_power_well_enable,
1917         .disable = gen9_dc_off_power_well_disable,
1918         .is_enabled = gen9_dc_off_power_well_enabled,
1919 };
1920
1921 static const struct i915_power_well_ops bxt_dpio_cmn_power_well_ops = {
1922         .sync_hw = i9xx_power_well_sync_hw_noop,
1923         .enable = bxt_dpio_cmn_power_well_enable,
1924         .disable = bxt_dpio_cmn_power_well_disable,
1925         .is_enabled = bxt_dpio_cmn_power_well_enabled,
1926 };
1927
1928 static struct i915_power_well hsw_power_wells[] = {
1929         {
1930                 .name = "always-on",
1931                 .always_on = 1,
1932                 .domains = POWER_DOMAIN_MASK,
1933                 .ops = &i9xx_always_on_power_well_ops,
1934         },
1935         {
1936                 .name = "display",
1937                 .domains = HSW_DISPLAY_POWER_DOMAINS,
1938                 .ops = &hsw_power_well_ops,
1939         },
1940 };
1941
1942 static struct i915_power_well bdw_power_wells[] = {
1943         {
1944                 .name = "always-on",
1945                 .always_on = 1,
1946                 .domains = POWER_DOMAIN_MASK,
1947                 .ops = &i9xx_always_on_power_well_ops,
1948         },
1949         {
1950                 .name = "display",
1951                 .domains = BDW_DISPLAY_POWER_DOMAINS,
1952                 .ops = &hsw_power_well_ops,
1953         },
1954 };
1955
1956 static const struct i915_power_well_ops vlv_display_power_well_ops = {
1957         .sync_hw = i9xx_power_well_sync_hw_noop,
1958         .enable = vlv_display_power_well_enable,
1959         .disable = vlv_display_power_well_disable,
1960         .is_enabled = vlv_power_well_enabled,
1961 };
1962
1963 static const struct i915_power_well_ops vlv_dpio_cmn_power_well_ops = {
1964         .sync_hw = i9xx_power_well_sync_hw_noop,
1965         .enable = vlv_dpio_cmn_power_well_enable,
1966         .disable = vlv_dpio_cmn_power_well_disable,
1967         .is_enabled = vlv_power_well_enabled,
1968 };
1969
1970 static const struct i915_power_well_ops vlv_dpio_power_well_ops = {
1971         .sync_hw = i9xx_power_well_sync_hw_noop,
1972         .enable = vlv_power_well_enable,
1973         .disable = vlv_power_well_disable,
1974         .is_enabled = vlv_power_well_enabled,
1975 };
1976
1977 static struct i915_power_well vlv_power_wells[] = {
1978         {
1979                 .name = "always-on",
1980                 .always_on = 1,
1981                 .domains = POWER_DOMAIN_MASK,
1982                 .ops = &i9xx_always_on_power_well_ops,
1983                 .id = PUNIT_POWER_WELL_ALWAYS_ON,
1984         },
1985         {
1986                 .name = "display",
1987                 .domains = VLV_DISPLAY_POWER_DOMAINS,
1988                 .id = PUNIT_POWER_WELL_DISP2D,
1989                 .ops = &vlv_display_power_well_ops,
1990         },
1991         {
1992                 .name = "dpio-tx-b-01",
1993                 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
1994                            VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
1995                            VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
1996                            VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
1997                 .ops = &vlv_dpio_power_well_ops,
1998                 .id = PUNIT_POWER_WELL_DPIO_TX_B_LANES_01,
1999         },
2000         {
2001                 .name = "dpio-tx-b-23",
2002                 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
2003                            VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
2004                            VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
2005                            VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
2006                 .ops = &vlv_dpio_power_well_ops,
2007                 .id = PUNIT_POWER_WELL_DPIO_TX_B_LANES_23,
2008         },
2009         {
2010                 .name = "dpio-tx-c-01",
2011                 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
2012                            VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
2013                            VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
2014                            VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
2015                 .ops = &vlv_dpio_power_well_ops,
2016                 .id = PUNIT_POWER_WELL_DPIO_TX_C_LANES_01,
2017         },
2018         {
2019                 .name = "dpio-tx-c-23",
2020                 .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
2021                            VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
2022                            VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
2023                            VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
2024                 .ops = &vlv_dpio_power_well_ops,
2025                 .id = PUNIT_POWER_WELL_DPIO_TX_C_LANES_23,
2026         },
2027         {
2028                 .name = "dpio-common",
2029                 .domains = VLV_DPIO_CMN_BC_POWER_DOMAINS,
2030                 .id = PUNIT_POWER_WELL_DPIO_CMN_BC,
2031                 .ops = &vlv_dpio_cmn_power_well_ops,
2032         },
2033 };
2034
2035 static struct i915_power_well chv_power_wells[] = {
2036         {
2037                 .name = "always-on",
2038                 .always_on = 1,
2039                 .domains = POWER_DOMAIN_MASK,
2040                 .ops = &i9xx_always_on_power_well_ops,
2041         },
2042         {
2043                 .name = "display",
2044                 /*
2045                  * Pipe A power well is the new disp2d well. Pipe B and C
2046                  * power wells don't actually exist. Pipe A power well is
2047                  * required for any pipe to work.
2048                  */
2049                 .domains = CHV_DISPLAY_POWER_DOMAINS,
2050                 .id = PIPE_A,
2051                 .ops = &chv_pipe_power_well_ops,
2052         },
2053         {
2054                 .name = "dpio-common-bc",
2055                 .domains = CHV_DPIO_CMN_BC_POWER_DOMAINS,
2056                 .id = PUNIT_POWER_WELL_DPIO_CMN_BC,
2057                 .ops = &chv_dpio_cmn_power_well_ops,
2058         },
2059         {
2060                 .name = "dpio-common-d",
2061                 .domains = CHV_DPIO_CMN_D_POWER_DOMAINS,
2062                 .id = PUNIT_POWER_WELL_DPIO_CMN_D,
2063                 .ops = &chv_dpio_cmn_power_well_ops,
2064         },
2065 };
2066
2067 bool intel_display_power_well_is_enabled(struct drm_i915_private *dev_priv,
2068                                     int power_well_id)
2069 {
2070         struct i915_power_well *power_well;
2071         bool ret;
2072
2073         power_well = lookup_power_well(dev_priv, power_well_id);
2074         ret = power_well->ops->is_enabled(dev_priv, power_well);
2075
2076         return ret;
2077 }
2078
2079 static struct i915_power_well skl_power_wells[] = {
2080         {
2081                 .name = "always-on",
2082                 .always_on = 1,
2083                 .domains = POWER_DOMAIN_MASK,
2084                 .ops = &i9xx_always_on_power_well_ops,
2085                 .id = SKL_DISP_PW_ALWAYS_ON,
2086         },
2087         {
2088                 .name = "power well 1",
2089                 /* Handled by the DMC firmware */
2090                 .domains = 0,
2091                 .ops = &skl_power_well_ops,
2092                 .id = SKL_DISP_PW_1,
2093         },
2094         {
2095                 .name = "MISC IO power well",
2096                 /* Handled by the DMC firmware */
2097                 .domains = 0,
2098                 .ops = &skl_power_well_ops,
2099                 .id = SKL_DISP_PW_MISC_IO,
2100         },
2101         {
2102                 .name = "DC off",
2103                 .domains = SKL_DISPLAY_DC_OFF_POWER_DOMAINS,
2104                 .ops = &gen9_dc_off_power_well_ops,
2105                 .id = SKL_DISP_PW_DC_OFF,
2106         },
2107         {
2108                 .name = "power well 2",
2109                 .domains = SKL_DISPLAY_POWERWELL_2_POWER_DOMAINS,
2110                 .ops = &skl_power_well_ops,
2111                 .id = SKL_DISP_PW_2,
2112         },
2113         {
2114                 .name = "DDI A/E power well",
2115                 .domains = SKL_DISPLAY_DDI_A_E_POWER_DOMAINS,
2116                 .ops = &skl_power_well_ops,
2117                 .id = SKL_DISP_PW_DDI_A_E,
2118         },
2119         {
2120                 .name = "DDI B power well",
2121                 .domains = SKL_DISPLAY_DDI_B_POWER_DOMAINS,
2122                 .ops = &skl_power_well_ops,
2123                 .id = SKL_DISP_PW_DDI_B,
2124         },
2125         {
2126                 .name = "DDI C power well",
2127                 .domains = SKL_DISPLAY_DDI_C_POWER_DOMAINS,
2128                 .ops = &skl_power_well_ops,
2129                 .id = SKL_DISP_PW_DDI_C,
2130         },
2131         {
2132                 .name = "DDI D power well",
2133                 .domains = SKL_DISPLAY_DDI_D_POWER_DOMAINS,
2134                 .ops = &skl_power_well_ops,
2135                 .id = SKL_DISP_PW_DDI_D,
2136         },
2137 };
2138
2139 static struct i915_power_well bxt_power_wells[] = {
2140         {
2141                 .name = "always-on",
2142                 .always_on = 1,
2143                 .domains = POWER_DOMAIN_MASK,
2144                 .ops = &i9xx_always_on_power_well_ops,
2145         },
2146         {
2147                 .name = "power well 1",
2148                 .domains = 0,
2149                 .ops = &skl_power_well_ops,
2150                 .id = SKL_DISP_PW_1,
2151         },
2152         {
2153                 .name = "DC off",
2154                 .domains = BXT_DISPLAY_DC_OFF_POWER_DOMAINS,
2155                 .ops = &gen9_dc_off_power_well_ops,
2156                 .id = SKL_DISP_PW_DC_OFF,
2157         },
2158         {
2159                 .name = "power well 2",
2160                 .domains = BXT_DISPLAY_POWERWELL_2_POWER_DOMAINS,
2161                 .ops = &skl_power_well_ops,
2162                 .id = SKL_DISP_PW_2,
2163         },
2164         {
2165                 .name = "dpio-common-a",
2166                 .domains = BXT_DPIO_CMN_A_POWER_DOMAINS,
2167                 .ops = &bxt_dpio_cmn_power_well_ops,
2168                 .id = BXT_DPIO_CMN_A,
2169                 .data = DPIO_PHY1,
2170         },
2171         {
2172                 .name = "dpio-common-bc",
2173                 .domains = BXT_DPIO_CMN_BC_POWER_DOMAINS,
2174                 .ops = &bxt_dpio_cmn_power_well_ops,
2175                 .id = BXT_DPIO_CMN_BC,
2176                 .data = DPIO_PHY0,
2177         },
2178 };
2179
2180 static struct i915_power_well glk_power_wells[] = {
2181         {
2182                 .name = "always-on",
2183                 .always_on = 1,
2184                 .domains = POWER_DOMAIN_MASK,
2185                 .ops = &i9xx_always_on_power_well_ops,
2186         },
2187         {
2188                 .name = "power well 1",
2189                 /* Handled by the DMC firmware */
2190                 .domains = 0,
2191                 .ops = &skl_power_well_ops,
2192                 .id = SKL_DISP_PW_1,
2193         },
2194         {
2195                 .name = "DC off",
2196                 .domains = GLK_DISPLAY_DC_OFF_POWER_DOMAINS,
2197                 .ops = &gen9_dc_off_power_well_ops,
2198                 .id = SKL_DISP_PW_DC_OFF,
2199         },
2200         {
2201                 .name = "power well 2",
2202                 .domains = GLK_DISPLAY_POWERWELL_2_POWER_DOMAINS,
2203                 .ops = &skl_power_well_ops,
2204                 .id = SKL_DISP_PW_2,
2205         },
2206         {
2207                 .name = "dpio-common-a",
2208                 .domains = GLK_DPIO_CMN_A_POWER_DOMAINS,
2209                 .ops = &bxt_dpio_cmn_power_well_ops,
2210                 .id = BXT_DPIO_CMN_A,
2211                 .data = DPIO_PHY1,
2212         },
2213         {
2214                 .name = "dpio-common-b",
2215                 .domains = GLK_DPIO_CMN_B_POWER_DOMAINS,
2216                 .ops = &bxt_dpio_cmn_power_well_ops,
2217                 .id = BXT_DPIO_CMN_BC,
2218                 .data = DPIO_PHY0,
2219         },
2220         {
2221                 .name = "dpio-common-c",
2222                 .domains = GLK_DPIO_CMN_C_POWER_DOMAINS,
2223                 .ops = &bxt_dpio_cmn_power_well_ops,
2224                 .id = GLK_DPIO_CMN_C,
2225                 .data = DPIO_PHY2,
2226         },
2227         {
2228                 .name = "AUX A",
2229                 .domains = GLK_DISPLAY_AUX_A_POWER_DOMAINS,
2230                 .ops = &skl_power_well_ops,
2231                 .id = GLK_DISP_PW_AUX_A,
2232         },
2233         {
2234                 .name = "AUX B",
2235                 .domains = GLK_DISPLAY_AUX_B_POWER_DOMAINS,
2236                 .ops = &skl_power_well_ops,
2237                 .id = GLK_DISP_PW_AUX_B,
2238         },
2239         {
2240                 .name = "AUX C",
2241                 .domains = GLK_DISPLAY_AUX_C_POWER_DOMAINS,
2242                 .ops = &skl_power_well_ops,
2243                 .id = GLK_DISP_PW_AUX_C,
2244         },
2245         {
2246                 .name = "DDI A power well",
2247                 .domains = GLK_DISPLAY_DDI_A_POWER_DOMAINS,
2248                 .ops = &skl_power_well_ops,
2249                 .id = GLK_DISP_PW_DDI_A,
2250         },
2251         {
2252                 .name = "DDI B power well",
2253                 .domains = GLK_DISPLAY_DDI_B_POWER_DOMAINS,
2254                 .ops = &skl_power_well_ops,
2255                 .id = SKL_DISP_PW_DDI_B,
2256         },
2257         {
2258                 .name = "DDI C power well",
2259                 .domains = GLK_DISPLAY_DDI_C_POWER_DOMAINS,
2260                 .ops = &skl_power_well_ops,
2261                 .id = SKL_DISP_PW_DDI_C,
2262         },
2263 };
2264
2265 static int
2266 sanitize_disable_power_well_option(const struct drm_i915_private *dev_priv,
2267                                    int disable_power_well)
2268 {
2269         if (disable_power_well >= 0)
2270                 return !!disable_power_well;
2271
2272         return 1;
2273 }
2274
2275 static uint32_t get_allowed_dc_mask(const struct drm_i915_private *dev_priv,
2276                                     int enable_dc)
2277 {
2278         uint32_t mask;
2279         int requested_dc;
2280         int max_dc;
2281
2282         if (IS_GEN9_BC(dev_priv)) {
2283                 max_dc = 2;
2284                 mask = 0;
2285         } else if (IS_GEN9_LP(dev_priv)) {
2286                 max_dc = 1;
2287                 /*
2288                  * DC9 has a separate HW flow from the rest of the DC states,
2289                  * not depending on the DMC firmware. It's needed by system
2290                  * suspend/resume, so allow it unconditionally.
2291                  */
2292                 mask = DC_STATE_EN_DC9;
2293         } else {
2294                 max_dc = 0;
2295                 mask = 0;
2296         }
2297
2298         if (!i915.disable_power_well)
2299                 max_dc = 0;
2300
2301         if (enable_dc >= 0 && enable_dc <= max_dc) {
2302                 requested_dc = enable_dc;
2303         } else if (enable_dc == -1) {
2304                 requested_dc = max_dc;
2305         } else if (enable_dc > max_dc && enable_dc <= 2) {
2306                 DRM_DEBUG_KMS("Adjusting requested max DC state (%d->%d)\n",
2307                               enable_dc, max_dc);
2308                 requested_dc = max_dc;
2309         } else {
2310                 DRM_ERROR("Unexpected value for enable_dc (%d)\n", enable_dc);
2311                 requested_dc = max_dc;
2312         }
2313
2314         if (requested_dc > 1)
2315                 mask |= DC_STATE_EN_UPTO_DC6;
2316         if (requested_dc > 0)
2317                 mask |= DC_STATE_EN_UPTO_DC5;
2318
2319         DRM_DEBUG_KMS("Allowed DC state mask %02x\n", mask);
2320
2321         return mask;
2322 }
2323
2324 #define set_power_wells(power_domains, __power_wells) ({                \
2325         (power_domains)->power_wells = (__power_wells);                 \
2326         (power_domains)->power_well_count = ARRAY_SIZE(__power_wells);  \
2327 })
2328
2329 /**
2330  * intel_power_domains_init - initializes the power domain structures
2331  * @dev_priv: i915 device instance
2332  *
2333  * Initializes the power domain structures for @dev_priv depending upon the
2334  * supported platform.
2335  */
2336 int intel_power_domains_init(struct drm_i915_private *dev_priv)
2337 {
2338         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2339
2340         i915.disable_power_well = sanitize_disable_power_well_option(dev_priv,
2341                                                      i915.disable_power_well);
2342         dev_priv->csr.allowed_dc_mask = get_allowed_dc_mask(dev_priv,
2343                                                             i915.enable_dc);
2344
2345         BUILD_BUG_ON(POWER_DOMAIN_NUM > 64);
2346
2347         mutex_init(&power_domains->lock);
2348
2349         /*
2350          * The enabling order will be from lower to higher indexed wells,
2351          * the disabling order is reversed.
2352          */
2353         if (IS_HASWELL(dev_priv)) {
2354                 set_power_wells(power_domains, hsw_power_wells);
2355         } else if (IS_BROADWELL(dev_priv)) {
2356                 set_power_wells(power_domains, bdw_power_wells);
2357         } else if (IS_GEN9_BC(dev_priv)) {
2358                 set_power_wells(power_domains, skl_power_wells);
2359         } else if (IS_BROXTON(dev_priv)) {
2360                 set_power_wells(power_domains, bxt_power_wells);
2361         } else if (IS_GEMINILAKE(dev_priv)) {
2362                 set_power_wells(power_domains, glk_power_wells);
2363         } else if (IS_CHERRYVIEW(dev_priv)) {
2364                 set_power_wells(power_domains, chv_power_wells);
2365         } else if (IS_VALLEYVIEW(dev_priv)) {
2366                 set_power_wells(power_domains, vlv_power_wells);
2367         } else {
2368                 set_power_wells(power_domains, i9xx_always_on_power_well);
2369         }
2370
2371         return 0;
2372 }
2373
2374 /**
2375  * intel_power_domains_fini - finalizes the power domain structures
2376  * @dev_priv: i915 device instance
2377  *
2378  * Finalizes the power domain structures for @dev_priv depending upon the
2379  * supported platform. This function also disables runtime pm and ensures that
2380  * the device stays powered up so that the driver can be reloaded.
2381  */
2382 void intel_power_domains_fini(struct drm_i915_private *dev_priv)
2383 {
2384         struct device *kdev = &dev_priv->drm.pdev->dev;
2385
2386         /*
2387          * The i915.ko module is still not prepared to be loaded when
2388          * the power well is not enabled, so just enable it in case
2389          * we're going to unload/reload.
2390          * The following also reacquires the RPM reference the core passed
2391          * to the driver during loading, which is dropped in
2392          * intel_runtime_pm_enable(). We have to hand back the control of the
2393          * device to the core with this reference held.
2394          */
2395         intel_display_set_init_power(dev_priv, true);
2396
2397         /* Remove the refcount we took to keep power well support disabled. */
2398         if (!i915.disable_power_well)
2399                 intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
2400
2401         /*
2402          * Remove the refcount we took in intel_runtime_pm_enable() in case
2403          * the platform doesn't support runtime PM.
2404          */
2405         if (!HAS_RUNTIME_PM(dev_priv))
2406                 pm_runtime_put(kdev);
2407 }
2408
2409 static void intel_power_domains_sync_hw(struct drm_i915_private *dev_priv)
2410 {
2411         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2412         struct i915_power_well *power_well;
2413
2414         mutex_lock(&power_domains->lock);
2415         for_each_power_well(dev_priv, power_well) {
2416                 power_well->ops->sync_hw(dev_priv, power_well);
2417                 power_well->hw_enabled = power_well->ops->is_enabled(dev_priv,
2418                                                                      power_well);
2419         }
2420         mutex_unlock(&power_domains->lock);
2421 }
2422
2423 static void gen9_dbuf_enable(struct drm_i915_private *dev_priv)
2424 {
2425         I915_WRITE(DBUF_CTL, I915_READ(DBUF_CTL) | DBUF_POWER_REQUEST);
2426         POSTING_READ(DBUF_CTL);
2427
2428         udelay(10);
2429
2430         if (!(I915_READ(DBUF_CTL) & DBUF_POWER_STATE))
2431                 DRM_ERROR("DBuf power enable timeout\n");
2432 }
2433
2434 static void gen9_dbuf_disable(struct drm_i915_private *dev_priv)
2435 {
2436         I915_WRITE(DBUF_CTL, I915_READ(DBUF_CTL) & ~DBUF_POWER_REQUEST);
2437         POSTING_READ(DBUF_CTL);
2438
2439         udelay(10);
2440
2441         if (I915_READ(DBUF_CTL) & DBUF_POWER_STATE)
2442                 DRM_ERROR("DBuf power disable timeout!\n");
2443 }
2444
2445 static void skl_display_core_init(struct drm_i915_private *dev_priv,
2446                                    bool resume)
2447 {
2448         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2449         struct i915_power_well *well;
2450         uint32_t val;
2451
2452         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
2453
2454         /* enable PCH reset handshake */
2455         val = I915_READ(HSW_NDE_RSTWRN_OPT);
2456         I915_WRITE(HSW_NDE_RSTWRN_OPT, val | RESET_PCH_HANDSHAKE_ENABLE);
2457
2458         /* enable PG1 and Misc I/O */
2459         mutex_lock(&power_domains->lock);
2460
2461         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
2462         intel_power_well_enable(dev_priv, well);
2463
2464         well = lookup_power_well(dev_priv, SKL_DISP_PW_MISC_IO);
2465         intel_power_well_enable(dev_priv, well);
2466
2467         mutex_unlock(&power_domains->lock);
2468
2469         skl_init_cdclk(dev_priv);
2470
2471         gen9_dbuf_enable(dev_priv);
2472
2473         if (resume && dev_priv->csr.dmc_payload)
2474                 intel_csr_load_program(dev_priv);
2475 }
2476
2477 static void skl_display_core_uninit(struct drm_i915_private *dev_priv)
2478 {
2479         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2480         struct i915_power_well *well;
2481
2482         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
2483
2484         gen9_dbuf_disable(dev_priv);
2485
2486         skl_uninit_cdclk(dev_priv);
2487
2488         /* The spec doesn't call for removing the reset handshake flag */
2489         /* disable PG1 and Misc I/O */
2490
2491         mutex_lock(&power_domains->lock);
2492
2493         well = lookup_power_well(dev_priv, SKL_DISP_PW_MISC_IO);
2494         intel_power_well_disable(dev_priv, well);
2495
2496         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
2497         intel_power_well_disable(dev_priv, well);
2498
2499         mutex_unlock(&power_domains->lock);
2500 }
2501
2502 void bxt_display_core_init(struct drm_i915_private *dev_priv,
2503                            bool resume)
2504 {
2505         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2506         struct i915_power_well *well;
2507         uint32_t val;
2508
2509         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
2510
2511         /*
2512          * NDE_RSTWRN_OPT RST PCH Handshake En must always be 0b on BXT
2513          * or else the reset will hang because there is no PCH to respond.
2514          * Move the handshake programming to initialization sequence.
2515          * Previously was left up to BIOS.
2516          */
2517         val = I915_READ(HSW_NDE_RSTWRN_OPT);
2518         val &= ~RESET_PCH_HANDSHAKE_ENABLE;
2519         I915_WRITE(HSW_NDE_RSTWRN_OPT, val);
2520
2521         /* Enable PG1 */
2522         mutex_lock(&power_domains->lock);
2523
2524         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
2525         intel_power_well_enable(dev_priv, well);
2526
2527         mutex_unlock(&power_domains->lock);
2528
2529         bxt_init_cdclk(dev_priv);
2530
2531         gen9_dbuf_enable(dev_priv);
2532
2533         if (resume && dev_priv->csr.dmc_payload)
2534                 intel_csr_load_program(dev_priv);
2535 }
2536
2537 void bxt_display_core_uninit(struct drm_i915_private *dev_priv)
2538 {
2539         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2540         struct i915_power_well *well;
2541
2542         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
2543
2544         gen9_dbuf_disable(dev_priv);
2545
2546         bxt_uninit_cdclk(dev_priv);
2547
2548         /* The spec doesn't call for removing the reset handshake flag */
2549
2550         /* Disable PG1 */
2551         mutex_lock(&power_domains->lock);
2552
2553         well = lookup_power_well(dev_priv, SKL_DISP_PW_1);
2554         intel_power_well_disable(dev_priv, well);
2555
2556         mutex_unlock(&power_domains->lock);
2557 }
2558
2559 static void chv_phy_control_init(struct drm_i915_private *dev_priv)
2560 {
2561         struct i915_power_well *cmn_bc =
2562                 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_BC);
2563         struct i915_power_well *cmn_d =
2564                 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_D);
2565
2566         /*
2567          * DISPLAY_PHY_CONTROL can get corrupted if read. As a
2568          * workaround never ever read DISPLAY_PHY_CONTROL, and
2569          * instead maintain a shadow copy ourselves. Use the actual
2570          * power well state and lane status to reconstruct the
2571          * expected initial value.
2572          */
2573         dev_priv->chv_phy_control =
2574                 PHY_LDO_SEQ_DELAY(PHY_LDO_DELAY_600NS, DPIO_PHY0) |
2575                 PHY_LDO_SEQ_DELAY(PHY_LDO_DELAY_600NS, DPIO_PHY1) |
2576                 PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY0, DPIO_CH0) |
2577                 PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY0, DPIO_CH1) |
2578                 PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY1, DPIO_CH0);
2579
2580         /*
2581          * If all lanes are disabled we leave the override disabled
2582          * with all power down bits cleared to match the state we
2583          * would use after disabling the port. Otherwise enable the
2584          * override and set the lane powerdown bits accding to the
2585          * current lane status.
2586          */
2587         if (cmn_bc->ops->is_enabled(dev_priv, cmn_bc)) {
2588                 uint32_t status = I915_READ(DPLL(PIPE_A));
2589                 unsigned int mask;
2590
2591                 mask = status & DPLL_PORTB_READY_MASK;
2592                 if (mask == 0xf)
2593                         mask = 0x0;
2594                 else
2595                         dev_priv->chv_phy_control |=
2596                                 PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH0);
2597
2598                 dev_priv->chv_phy_control |=
2599                         PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY0, DPIO_CH0);
2600
2601                 mask = (status & DPLL_PORTC_READY_MASK) >> 4;
2602                 if (mask == 0xf)
2603                         mask = 0x0;
2604                 else
2605                         dev_priv->chv_phy_control |=
2606                                 PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH1);
2607
2608                 dev_priv->chv_phy_control |=
2609                         PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY0, DPIO_CH1);
2610
2611                 dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY0);
2612
2613                 dev_priv->chv_phy_assert[DPIO_PHY0] = false;
2614         } else {
2615                 dev_priv->chv_phy_assert[DPIO_PHY0] = true;
2616         }
2617
2618         if (cmn_d->ops->is_enabled(dev_priv, cmn_d)) {
2619                 uint32_t status = I915_READ(DPIO_PHY_STATUS);
2620                 unsigned int mask;
2621
2622                 mask = status & DPLL_PORTD_READY_MASK;
2623
2624                 if (mask == 0xf)
2625                         mask = 0x0;
2626                 else
2627                         dev_priv->chv_phy_control |=
2628                                 PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY1, DPIO_CH0);
2629
2630                 dev_priv->chv_phy_control |=
2631                         PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY1, DPIO_CH0);
2632
2633                 dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY1);
2634
2635                 dev_priv->chv_phy_assert[DPIO_PHY1] = false;
2636         } else {
2637                 dev_priv->chv_phy_assert[DPIO_PHY1] = true;
2638         }
2639
2640         I915_WRITE(DISPLAY_PHY_CONTROL, dev_priv->chv_phy_control);
2641
2642         DRM_DEBUG_KMS("Initial PHY_CONTROL=0x%08x\n",
2643                       dev_priv->chv_phy_control);
2644 }
2645
2646 static void vlv_cmnlane_wa(struct drm_i915_private *dev_priv)
2647 {
2648         struct i915_power_well *cmn =
2649                 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DPIO_CMN_BC);
2650         struct i915_power_well *disp2d =
2651                 lookup_power_well(dev_priv, PUNIT_POWER_WELL_DISP2D);
2652
2653         /* If the display might be already active skip this */
2654         if (cmn->ops->is_enabled(dev_priv, cmn) &&
2655             disp2d->ops->is_enabled(dev_priv, disp2d) &&
2656             I915_READ(DPIO_CTL) & DPIO_CMNRST)
2657                 return;
2658
2659         DRM_DEBUG_KMS("toggling display PHY side reset\n");
2660
2661         /* cmnlane needs DPLL registers */
2662         disp2d->ops->enable(dev_priv, disp2d);
2663
2664         /*
2665          * From VLV2A0_DP_eDP_HDMI_DPIO_driver_vbios_notes_11.docx:
2666          * Need to assert and de-assert PHY SB reset by gating the
2667          * common lane power, then un-gating it.
2668          * Simply ungating isn't enough to reset the PHY enough to get
2669          * ports and lanes running.
2670          */
2671         cmn->ops->disable(dev_priv, cmn);
2672 }
2673
2674 /**
2675  * intel_power_domains_init_hw - initialize hardware power domain state
2676  * @dev_priv: i915 device instance
2677  * @resume: Called from resume code paths or not
2678  *
2679  * This function initializes the hardware power domain state and enables all
2680  * power domains using intel_display_set_init_power().
2681  */
2682 void intel_power_domains_init_hw(struct drm_i915_private *dev_priv, bool resume)
2683 {
2684         struct i915_power_domains *power_domains = &dev_priv->power_domains;
2685
2686         power_domains->initializing = true;
2687
2688         if (IS_GEN9_BC(dev_priv)) {
2689                 skl_display_core_init(dev_priv, resume);
2690         } else if (IS_GEN9_LP(dev_priv)) {
2691                 bxt_display_core_init(dev_priv, resume);
2692         } else if (IS_CHERRYVIEW(dev_priv)) {
2693                 mutex_lock(&power_domains->lock);
2694                 chv_phy_control_init(dev_priv);
2695                 mutex_unlock(&power_domains->lock);
2696         } else if (IS_VALLEYVIEW(dev_priv)) {
2697                 mutex_lock(&power_domains->lock);
2698                 vlv_cmnlane_wa(dev_priv);
2699                 mutex_unlock(&power_domains->lock);
2700         }
2701
2702         /* For now, we need the power well to be always enabled. */
2703         intel_display_set_init_power(dev_priv, true);
2704         /* Disable power support if the user asked so. */
2705         if (!i915.disable_power_well)
2706                 intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
2707         intel_power_domains_sync_hw(dev_priv);
2708         power_domains->initializing = false;
2709 }
2710
2711 /**
2712  * intel_power_domains_suspend - suspend power domain state
2713  * @dev_priv: i915 device instance
2714  *
2715  * This function prepares the hardware power domain state before entering
2716  * system suspend. It must be paired with intel_power_domains_init_hw().
2717  */
2718 void intel_power_domains_suspend(struct drm_i915_private *dev_priv)
2719 {
2720         /*
2721          * Even if power well support was disabled we still want to disable
2722          * power wells while we are system suspended.
2723          */
2724         if (!i915.disable_power_well)
2725                 intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
2726
2727         if (IS_GEN9_BC(dev_priv))
2728                 skl_display_core_uninit(dev_priv);
2729         else if (IS_GEN9_LP(dev_priv))
2730                 bxt_display_core_uninit(dev_priv);
2731 }
2732
2733 /**
2734  * intel_runtime_pm_get - grab a runtime pm reference
2735  * @dev_priv: i915 device instance
2736  *
2737  * This function grabs a device-level runtime pm reference (mostly used for GEM
2738  * code to ensure the GTT or GT is on) and ensures that it is powered up.
2739  *
2740  * Any runtime pm reference obtained by this function must have a symmetric
2741  * call to intel_runtime_pm_put() to release the reference again.
2742  */
2743 void intel_runtime_pm_get(struct drm_i915_private *dev_priv)
2744 {
2745         struct pci_dev *pdev = dev_priv->drm.pdev;
2746         struct device *kdev = &pdev->dev;
2747
2748         pm_runtime_get_sync(kdev);
2749
2750         atomic_inc(&dev_priv->pm.wakeref_count);
2751         assert_rpm_wakelock_held(dev_priv);
2752 }
2753
2754 /**
2755  * intel_runtime_pm_get_if_in_use - grab a runtime pm reference if device in use
2756  * @dev_priv: i915 device instance
2757  *
2758  * This function grabs a device-level runtime pm reference if the device is
2759  * already in use and ensures that it is powered up.
2760  *
2761  * Any runtime pm reference obtained by this function must have a symmetric
2762  * call to intel_runtime_pm_put() to release the reference again.
2763  */
2764 bool intel_runtime_pm_get_if_in_use(struct drm_i915_private *dev_priv)
2765 {
2766         struct pci_dev *pdev = dev_priv->drm.pdev;
2767         struct device *kdev = &pdev->dev;
2768
2769         if (IS_ENABLED(CONFIG_PM)) {
2770                 int ret = pm_runtime_get_if_in_use(kdev);
2771
2772                 /*
2773                  * In cases runtime PM is disabled by the RPM core and we get
2774                  * an -EINVAL return value we are not supposed to call this
2775                  * function, since the power state is undefined. This applies
2776                  * atm to the late/early system suspend/resume handlers.
2777                  */
2778                 WARN_ON_ONCE(ret < 0);
2779                 if (ret <= 0)
2780                         return false;
2781         }
2782
2783         atomic_inc(&dev_priv->pm.wakeref_count);
2784         assert_rpm_wakelock_held(dev_priv);
2785
2786         return true;
2787 }
2788
2789 /**
2790  * intel_runtime_pm_get_noresume - grab a runtime pm reference
2791  * @dev_priv: i915 device instance
2792  *
2793  * This function grabs a device-level runtime pm reference (mostly used for GEM
2794  * code to ensure the GTT or GT is on).
2795  *
2796  * It will _not_ power up the device but instead only check that it's powered
2797  * on.  Therefore it is only valid to call this functions from contexts where
2798  * the device is known to be powered up and where trying to power it up would
2799  * result in hilarity and deadlocks. That pretty much means only the system
2800  * suspend/resume code where this is used to grab runtime pm references for
2801  * delayed setup down in work items.
2802  *
2803  * Any runtime pm reference obtained by this function must have a symmetric
2804  * call to intel_runtime_pm_put() to release the reference again.
2805  */
2806 void intel_runtime_pm_get_noresume(struct drm_i915_private *dev_priv)
2807 {
2808         struct pci_dev *pdev = dev_priv->drm.pdev;
2809         struct device *kdev = &pdev->dev;
2810
2811         assert_rpm_wakelock_held(dev_priv);
2812         pm_runtime_get_noresume(kdev);
2813
2814         atomic_inc(&dev_priv->pm.wakeref_count);
2815 }
2816
2817 /**
2818  * intel_runtime_pm_put - release a runtime pm reference
2819  * @dev_priv: i915 device instance
2820  *
2821  * This function drops the device-level runtime pm reference obtained by
2822  * intel_runtime_pm_get() and might power down the corresponding
2823  * hardware block right away if this is the last reference.
2824  */
2825 void intel_runtime_pm_put(struct drm_i915_private *dev_priv)
2826 {
2827         struct pci_dev *pdev = dev_priv->drm.pdev;
2828         struct device *kdev = &pdev->dev;
2829
2830         assert_rpm_wakelock_held(dev_priv);
2831         atomic_dec(&dev_priv->pm.wakeref_count);
2832
2833         pm_runtime_mark_last_busy(kdev);
2834         pm_runtime_put_autosuspend(kdev);
2835 }
2836
2837 /**
2838  * intel_runtime_pm_enable - enable runtime pm
2839  * @dev_priv: i915 device instance
2840  *
2841  * This function enables runtime pm at the end of the driver load sequence.
2842  *
2843  * Note that this function does currently not enable runtime pm for the
2844  * subordinate display power domains. That is only done on the first modeset
2845  * using intel_display_set_init_power().
2846  */
2847 void intel_runtime_pm_enable(struct drm_i915_private *dev_priv)
2848 {
2849         struct pci_dev *pdev = dev_priv->drm.pdev;
2850         struct device *kdev = &pdev->dev;
2851
2852         pm_runtime_set_autosuspend_delay(kdev, 10000); /* 10s */
2853         pm_runtime_mark_last_busy(kdev);
2854
2855         /*
2856          * Take a permanent reference to disable the RPM functionality and drop
2857          * it only when unloading the driver. Use the low level get/put helpers,
2858          * so the driver's own RPM reference tracking asserts also work on
2859          * platforms without RPM support.
2860          */
2861         if (!HAS_RUNTIME_PM(dev_priv)) {
2862                 pm_runtime_dont_use_autosuspend(kdev);
2863                 pm_runtime_get_sync(kdev);
2864         } else {
2865                 pm_runtime_use_autosuspend(kdev);
2866         }
2867
2868         /*
2869          * The core calls the driver load handler with an RPM reference held.
2870          * We drop that here and will reacquire it during unloading in
2871          * intel_power_domains_fini().
2872          */
2873         pm_runtime_put_autosuspend(kdev);
2874 }