drivers/gpu/drm/i915/intel_ringbuffer.c

   1 /*
   2  * Copyright © 2008-2010 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *    Zou Nan hai <nanhai.zou@intel.com>
  26  *    Xiang Hai hao<haihao.xiang@intel.com>
  27  *
  28  */
  29
  30 #include <linux/log2.h>
  31 #include <drm/drmP.h>
  32 #include "i915_drv.h"
  33 #include <drm/i915_drm.h>
  34 #include "i915_trace.h"
  35 #include "intel_drv.h"
  36
  37 int __intel_ring_space(int head, int tail, int size)
  38 {
  39         int space = head - tail;
  40         if (space <= 0)
  41                 space += size;
  42         return space - I915_RING_FREE_SPACE;
  43 }
  44
  45 void intel_ring_update_space(struct intel_ringbuffer *ringbuf)
  46 {
  47         if (ringbuf->last_retired_head != -1) {
  48                 ringbuf->head = ringbuf->last_retired_head;
  49                 ringbuf->last_retired_head = -1;
  50         }
  51
  52         ringbuf->space = __intel_ring_space(ringbuf->head & HEAD_ADDR,
  53                                             ringbuf->tail, ringbuf->size);
  54 }
  55
  56 int intel_ring_space(struct intel_ringbuffer *ringbuf)
  57 {
  58         intel_ring_update_space(ringbuf);
  59         return ringbuf->space;
  60 }
  61
  62 bool intel_ring_stopped(struct intel_engine_cs *ring)
  63 {
  64         struct drm_i915_private *dev_priv = ring->dev->dev_private;
  65         return dev_priv->gpu_error.stop_rings & intel_ring_flag(ring);
  66 }
  67
  68 static void __intel_ring_advance(struct intel_engine_cs *ring)
  69 {
  70         struct intel_ringbuffer *ringbuf = ring->buffer;
  71         ringbuf->tail &= ringbuf->size - 1;
  72         if (intel_ring_stopped(ring))
  73                 return;
  74         ring->write_tail(ring, ringbuf->tail);
  75 }
  76
  77 static int
  78 gen2_render_ring_flush(struct drm_i915_gem_request *req,
  79                        u32      invalidate_domains,
  80                        u32      flush_domains)
  81 {
  82         struct intel_engine_cs *ring = req->ring;
  83         u32 cmd;
  84         int ret;
  85
  86         cmd = MI_FLUSH;
  87         if (((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER) == 0)
  88                 cmd |= MI_NO_WRITE_FLUSH;
  89
  90         if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
  91                 cmd |= MI_READ_FLUSH;
  92
  93         ret = intel_ring_begin(req, 2);
  94         if (ret)
  95                 return ret;
  96
  97         intel_ring_emit(ring, cmd);
  98         intel_ring_emit(ring, MI_NOOP);
  99         intel_ring_advance(ring);
 100
 101         return 0;
 102 }
 103
 104 static int
 105 gen4_render_ring_flush(struct drm_i915_gem_request *req,
 106                        u32      invalidate_domains,
 107                        u32      flush_domains)
 108 {
 109         struct intel_engine_cs *ring = req->ring;
 110         struct drm_device *dev = ring->dev;
 111         u32 cmd;
 112         int ret;
 113
 114         /*
 115          * read/write caches:
 116          *
 117          * I915_GEM_DOMAIN_RENDER is always invalidated, but is
 118          * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
 119          * also flushed at 2d versus 3d pipeline switches.
 120          *
 121          * read-only caches:
 122          *
 123          * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
 124          * MI_READ_FLUSH is set, and is always flushed on 965.
 125          *
 126          * I915_GEM_DOMAIN_COMMAND may not exist?
 127          *
 128          * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
 129          * invalidated when MI_EXE_FLUSH is set.
 130          *
 131          * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
 132          * invalidated with every MI_FLUSH.
 133          *
 134          * TLBs:
 135          *
 136          * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
 137          * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
 138          * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
 139          * are flushed at any MI_FLUSH.
 140          */
 141
 142         cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
 143         if ((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER)
 144                 cmd &= ~MI_NO_WRITE_FLUSH;
 145         if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
 146                 cmd |= MI_EXE_FLUSH;
 147
 148         if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
 149             (IS_G4X(dev) || IS_GEN5(dev)))
 150                 cmd |= MI_INVALIDATE_ISP;
 151
 152         ret = intel_ring_begin(req, 2);
 153         if (ret)
 154                 return ret;
 155
 156         intel_ring_emit(ring, cmd);
 157         intel_ring_emit(ring, MI_NOOP);
 158         intel_ring_advance(ring);
 159
 160         return 0;
 161 }
 162
 163 /**
 164  * Emits a PIPE_CONTROL with a non-zero post-sync operation, for
 165  * implementing two workarounds on gen6.  From section 1.4.7.1
 166  * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1:
 167  *
 168  * [DevSNB-C+{W/A}] Before any depth stall flush (including those
 169  * produced by non-pipelined state commands), software needs to first
 170  * send a PIPE_CONTROL with no bits set except Post-Sync Operation !=
 171  * 0.
 172  *
 173  * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable
 174  * =1, a PIPE_CONTROL with any non-zero post-sync-op is required.
 175  *
 176  * And the workaround for these two requires this workaround first:
 177  *
 178  * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent
 179  * BEFORE the pipe-control with a post-sync op and no write-cache
 180  * flushes.
 181  *
 182  * And this last workaround is tricky because of the requirements on
 183  * that bit.  From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM
 184  * volume 2 part 1:
 185  *
 186  *     "1 of the following must also be set:
 187  *      - Render Target Cache Flush Enable ([12] of DW1)
 188  *      - Depth Cache Flush Enable ([0] of DW1)
 189  *      - Stall at Pixel Scoreboard ([1] of DW1)
 190  *      - Depth Stall ([13] of DW1)
 191  *      - Post-Sync Operation ([13] of DW1)
 192  *      - Notify Enable ([8] of DW1)"
 193  *
 194  * The cache flushes require the workaround flush that triggered this
 195  * one, so we can't use it.  Depth stall would trigger the same.
 196  * Post-sync nonzero is what triggered this second workaround, so we
 197  * can't use that one either.  Notify enable is IRQs, which aren't
 198  * really our business.  That leaves only stall at scoreboard.
 199  */
 200 static int
 201 intel_emit_post_sync_nonzero_flush(struct drm_i915_gem_request *req)
 202 {
 203         struct intel_engine_cs *ring = req->ring;
 204         u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
 205         int ret;
 206
 207         ret = intel_ring_begin(req, 6);
 208         if (ret)
 209                 return ret;
 210
 211         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
 212         intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |
 213                         PIPE_CONTROL_STALL_AT_SCOREBOARD);
 214         intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
 215         intel_ring_emit(ring, 0); /* low dword */
 216         intel_ring_emit(ring, 0); /* high dword */
 217         intel_ring_emit(ring, MI_NOOP);
 218         intel_ring_advance(ring);
 219
 220         ret = intel_ring_begin(req, 6);
 221         if (ret)
 222                 return ret;
 223
 224         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
 225         intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE);
 226         intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
 227         intel_ring_emit(ring, 0);
 228         intel_ring_emit(ring, 0);
 229         intel_ring_emit(ring, MI_NOOP);
 230         intel_ring_advance(ring);
 231
 232         return 0;
 233 }
 234
 235 static int
 236 gen6_render_ring_flush(struct drm_i915_gem_request *req,
 237                        u32 invalidate_domains, u32 flush_domains)
 238 {
 239         struct intel_engine_cs *ring = req->ring;
 240         u32 flags = 0;
 241         u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
 242         int ret;
 243
 244         /* Force SNB workarounds for PIPE_CONTROL flushes */
 245         ret = intel_emit_post_sync_nonzero_flush(req);
 246         if (ret)
 247                 return ret;
 248
 249         /* Just flush everything.  Experiments have shown that reducing the
 250          * number of bits based on the write domains has little performance
 251          * impact.
 252          */
 253         if (flush_domains) {
 254                 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
 255                 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
 256                 /*
 257                  * Ensure that any following seqno writes only happen
 258                  * when the render cache is indeed flushed.
 259                  */
 260                 flags |= PIPE_CONTROL_CS_STALL;
 261         }
 262         if (invalidate_domains) {
 263                 flags |= PIPE_CONTROL_TLB_INVALIDATE;
 264                 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
 265                 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
 266                 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
 267                 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
 268                 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
 269                 /*
 270                  * TLB invalidate requires a post-sync write.
 271                  */
 272                 flags |= PIPE_CONTROL_QW_WRITE | PIPE_CONTROL_CS_STALL;
 273         }
 274
 275         ret = intel_ring_begin(req, 4);
 276         if (ret)
 277                 return ret;
 278
 279         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
 280         intel_ring_emit(ring, flags);
 281         intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);
 282         intel_ring_emit(ring, 0);
 283         intel_ring_advance(ring);
 284
 285         return 0;
 286 }
 287
 288 static int
 289 gen7_render_ring_cs_stall_wa(struct drm_i915_gem_request *req)
 290 {
 291         struct intel_engine_cs *ring = req->ring;
 292         int ret;
 293
 294         ret = intel_ring_begin(req, 4);
 295         if (ret)
 296                 return ret;
 297
 298         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
 299         intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |
 300                               PIPE_CONTROL_STALL_AT_SCOREBOARD);
 301         intel_ring_emit(ring, 0);
 302         intel_ring_emit(ring, 0);
 303         intel_ring_advance(ring);
 304
 305         return 0;
 306 }
 307
 308 static int
 309 gen7_render_ring_flush(struct drm_i915_gem_request *req,
 310                        u32 invalidate_domains, u32 flush_domains)
 311 {
 312         struct intel_engine_cs *ring = req->ring;
 313         u32 flags = 0;
 314         u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
 315         int ret;
 316
 317         /*
 318          * Ensure that any following seqno writes only happen when the render
 319          * cache is indeed flushed.
 320          *
 321          * Workaround: 4th PIPE_CONTROL command (except the ones with only
 322          * read-cache invalidate bits set) must have the CS_STALL bit set. We
 323          * don't try to be clever and just set it unconditionally.
 324          */
 325         flags |= PIPE_CONTROL_CS_STALL;
 326
 327         /* Just flush everything.  Experiments have shown that reducing the
 328          * number of bits based on the write domains has little performance
 329          * impact.
 330          */
 331         if (flush_domains) {
 332                 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
 333                 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
 334                 flags |= PIPE_CONTROL_FLUSH_ENABLE;
 335         }
 336         if (invalidate_domains) {
 337                 flags |= PIPE_CONTROL_TLB_INVALIDATE;
 338                 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
 339                 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
 340                 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
 341                 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
 342                 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
 343                 flags |= PIPE_CONTROL_MEDIA_STATE_CLEAR;
 344                 /*
 345                  * TLB invalidate requires a post-sync write.
 346                  */
 347                 flags |= PIPE_CONTROL_QW_WRITE;
 348                 flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
 349
 350                 flags |= PIPE_CONTROL_STALL_AT_SCOREBOARD;
 351
 352                 /* Workaround: we must issue a pipe_control with CS-stall bit
 353                  * set before a pipe_control command that has the state cache
 354                  * invalidate bit set. */
 355                 gen7_render_ring_cs_stall_wa(req);
 356         }
 357
 358         ret = intel_ring_begin(req, 4);
 359         if (ret)
 360                 return ret;
 361
 362         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
 363         intel_ring_emit(ring, flags);
 364         intel_ring_emit(ring, scratch_addr);
 365         intel_ring_emit(ring, 0);
 366         intel_ring_advance(ring);
 367
 368         return 0;
 369 }
 370
 371 static int
 372 gen8_emit_pipe_control(struct drm_i915_gem_request *req,
 373                        u32 flags, u32 scratch_addr)
 374 {
 375         struct intel_engine_cs *ring = req->ring;
 376         int ret;
 377
 378         ret = intel_ring_begin(req, 6);
 379         if (ret)
 380                 return ret;
 381
 382         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(6));
 383         intel_ring_emit(ring, flags);
 384         intel_ring_emit(ring, scratch_addr);
 385         intel_ring_emit(ring, 0);
 386         intel_ring_emit(ring, 0);
 387         intel_ring_emit(ring, 0);
 388         intel_ring_advance(ring);
 389
 390         return 0;
 391 }
 392
 393 static int
 394 gen8_render_ring_flush(struct drm_i915_gem_request *req,
 395                        u32 invalidate_domains, u32 flush_domains)
 396 {
 397         u32 flags = 0;
 398         u32 scratch_addr = req->ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
 399         int ret;
 400
 401         flags |= PIPE_CONTROL_CS_STALL;
 402
 403         if (flush_domains) {
 404                 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
 405                 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
 406                 flags |= PIPE_CONTROL_FLUSH_ENABLE;
 407         }
 408         if (invalidate_domains) {
 409                 flags |= PIPE_CONTROL_TLB_INVALIDATE;
 410                 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
 411                 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
 412                 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
 413                 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
 414                 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
 415                 flags |= PIPE_CONTROL_QW_WRITE;
 416                 flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
 417
 418                 /* WaCsStallBeforeStateCacheInvalidate:bdw,chv */
 419                 ret = gen8_emit_pipe_control(req,
 420                                              PIPE_CONTROL_CS_STALL |
 421                                              PIPE_CONTROL_STALL_AT_SCOREBOARD,
 422                                              0);
 423                 if (ret)
 424                         return ret;
 425         }
 426
 427         return gen8_emit_pipe_control(req, flags, scratch_addr);
 428 }
 429
 430 static void ring_write_tail(struct intel_engine_cs *ring,
 431                             u32 value)
 432 {
 433         struct drm_i915_private *dev_priv = ring->dev->dev_private;
 434         I915_WRITE_TAIL(ring, value);
 435 }
 436
 437 u64 intel_ring_get_active_head(struct intel_engine_cs *ring)
 438 {
 439         struct drm_i915_private *dev_priv = ring->dev->dev_private;
 440         u64 acthd;
 441
 442         if (INTEL_INFO(ring->dev)->gen >= 8)
 443                 acthd = I915_READ64_2x32(RING_ACTHD(ring->mmio_base),
 444                                          RING_ACTHD_UDW(ring->mmio_base));
 445         else if (INTEL_INFO(ring->dev)->gen >= 4)
 446                 acthd = I915_READ(RING_ACTHD(ring->mmio_base));
 447         else
 448                 acthd = I915_READ(ACTHD);
 449
 450         return acthd;
 451 }
 452
 453 static void ring_setup_phys_status_page(struct intel_engine_cs *ring)
 454 {
 455         struct drm_i915_private *dev_priv = ring->dev->dev_private;
 456         u32 addr;
 457
 458         addr = dev_priv->status_page_dmah->busaddr;
 459         if (INTEL_INFO(ring->dev)->gen >= 4)
 460                 addr |= (dev_priv->status_page_dmah->busaddr >> 28) & 0xf0;
 461         I915_WRITE(HWS_PGA, addr);
 462 }
 463
 464 static void intel_ring_setup_status_page(struct intel_engine_cs *ring)
 465 {
 466         struct drm_device *dev = ring->dev;
 467         struct drm_i915_private *dev_priv = ring->dev->dev_private;
 468         i915_reg_t mmio;
 469
 470         /* The ring status page addresses are no longer next to the rest of
 471          * the ring registers as of gen7.
 472          */
 473         if (IS_GEN7(dev)) {
 474                 switch (ring->id) {
 475                 case RCS:
 476                         mmio = RENDER_HWS_PGA_GEN7;
 477                         break;
 478                 case BCS:
 479                         mmio = BLT_HWS_PGA_GEN7;
 480                         break;
 481                 /*
 482                  * VCS2 actually doesn't exist on Gen7. Only shut up
 483                  * gcc switch check warning
 484                  */
 485                 case VCS2:
 486                 case VCS:
 487                         mmio = BSD_HWS_PGA_GEN7;
 488                         break;
 489                 case VECS:
 490                         mmio = VEBOX_HWS_PGA_GEN7;
 491                         break;
 492                 }
 493         } else if (IS_GEN6(ring->dev)) {
 494                 mmio = RING_HWS_PGA_GEN6(ring->mmio_base);
 495         } else {
 496                 /* XXX: gen8 returns to sanity */
 497                 mmio = RING_HWS_PGA(ring->mmio_base);
 498         }
 499
 500         I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
 501         POSTING_READ(mmio);
 502
 503         /*
 504          * Flush the TLB for this page
 505          *
 506          * FIXME: These two bits have disappeared on gen8, so a question
 507          * arises: do we still need this and if so how should we go about
 508          * invalidating the TLB?
 509          */
 510         if (INTEL_INFO(dev)->gen >= 6 && INTEL_INFO(dev)->gen < 8) {
 511                 i915_reg_t reg = RING_INSTPM(ring->mmio_base);
 512
 513                 /* ring should be idle before issuing a sync flush*/
 514                 WARN_ON((I915_READ_MODE(ring) & MODE_IDLE) == 0);
 515
 516                 I915_WRITE(reg,
 517                            _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE |
 518                                               INSTPM_SYNC_FLUSH));
 519                 if (wait_for((I915_READ(reg) & INSTPM_SYNC_FLUSH) == 0,
 520                              1000))
 521                         DRM_ERROR("%s: wait for SyncFlush to complete for TLB invalidation timed out\n",
 522                                   ring->name);
 523         }
 524 }
 525
 526 static bool stop_ring(struct intel_engine_cs *ring)
 527 {
 528         struct drm_i915_private *dev_priv = to_i915(ring->dev);
 529
 530         if (!IS_GEN2(ring->dev)) {
 531                 I915_WRITE_MODE(ring, _MASKED_BIT_ENABLE(STOP_RING));
 532                 if (wait_for((I915_READ_MODE(ring) & MODE_IDLE) != 0, 1000)) {
 533                         DRM_ERROR("%s : timed out trying to stop ring\n", ring->name);
 534                         /* Sometimes we observe that the idle flag is not
 535                          * set even though the ring is empty. So double
 536                          * check before giving up.
 537                          */
 538                         if (I915_READ_HEAD(ring) != I915_READ_TAIL(ring))
 539                                 return false;
 540                 }
 541         }
 542
 543         I915_WRITE_CTL(ring, 0);
 544         I915_WRITE_HEAD(ring, 0);
 545         ring->write_tail(ring, 0);
 546
 547         if (!IS_GEN2(ring->dev)) {
 548                 (void)I915_READ_CTL(ring);
 549                 I915_WRITE_MODE(ring, _MASKED_BIT_DISABLE(STOP_RING));
 550         }
 551
 552         return (I915_READ_HEAD(ring) & HEAD_ADDR) == 0;
 553 }
 554
 555 static int init_ring_common(struct intel_engine_cs *ring)
 556 {
 557         struct drm_device *dev = ring->dev;
 558         struct drm_i915_private *dev_priv = dev->dev_private;
 559         struct intel_ringbuffer *ringbuf = ring->buffer;
 560         struct drm_i915_gem_object *obj = ringbuf->obj;
 561         int ret = 0;
 562
 563         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
 564
 565         if (!stop_ring(ring)) {
 566                 /* G45 ring initialization often fails to reset head to zero */
 567                 DRM_DEBUG_KMS("%s head not reset to zero "
 568                               "ctl %08x head %08x tail %08x start %08x\n",
 569                               ring->name,
 570                               I915_READ_CTL(ring),
 571                               I915_READ_HEAD(ring),
 572                               I915_READ_TAIL(ring),
 573                               I915_READ_START(ring));
 574
 575                 if (!stop_ring(ring)) {
 576                         DRM_ERROR("failed to set %s head to zero "
 577                                   "ctl %08x head %08x tail %08x start %08x\n",
 578                                   ring->name,
 579                                   I915_READ_CTL(ring),
 580                                   I915_READ_HEAD(ring),
 581                                   I915_READ_TAIL(ring),
 582                                   I915_READ_START(ring));
 583                         ret = -EIO;
 584                         goto out;
 585                 }
 586         }
 587
 588         if (I915_NEED_GFX_HWS(dev))
 589                 intel_ring_setup_status_page(ring);
 590         else
 591                 ring_setup_phys_status_page(ring);
 592
 593         /* Enforce ordering by reading HEAD register back */
 594         I915_READ_HEAD(ring);
 595
 596         /* Initialize the ring. This must happen _after_ we've cleared the ring
 597          * registers with the above sequence (the readback of the HEAD registers
 598          * also enforces ordering), otherwise the hw might lose the new ring
 599          * register values. */
 600         I915_WRITE_START(ring, i915_gem_obj_ggtt_offset(obj));
 601
 602         /* WaClearRingBufHeadRegAtInit:ctg,elk */
 603         if (I915_READ_HEAD(ring))
 604                 DRM_DEBUG("%s initialization failed [head=%08x], fudging\n",
 605                           ring->name, I915_READ_HEAD(ring));
 606         I915_WRITE_HEAD(ring, 0);
 607         (void)I915_READ_HEAD(ring);
 608
 609         I915_WRITE_CTL(ring,
 610                         ((ringbuf->size - PAGE_SIZE) & RING_NR_PAGES)
 611                         | RING_VALID);
 612
 613         /* If the head is still not zero, the ring is dead */
 614         if (wait_for((I915_READ_CTL(ring) & RING_VALID) != 0 &&
 615                      I915_READ_START(ring) == i915_gem_obj_ggtt_offset(obj) &&
 616                      (I915_READ_HEAD(ring) & HEAD_ADDR) == 0, 50)) {
 617                 DRM_ERROR("%s initialization failed "
 618                           "ctl %08x (valid? %d) head %08x tail %08x start %08x [expected %08lx]\n",
 619                           ring->name,
 620                           I915_READ_CTL(ring), I915_READ_CTL(ring) & RING_VALID,
 621                           I915_READ_HEAD(ring), I915_READ_TAIL(ring),
 622                           I915_READ_START(ring), (unsigned long)i915_gem_obj_ggtt_offset(obj));
 623                 ret = -EIO;
 624                 goto out;
 625         }
 626
 627         ringbuf->last_retired_head = -1;
 628         ringbuf->head = I915_READ_HEAD(ring);
 629         ringbuf->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
 630         intel_ring_update_space(ringbuf);
 631
 632         memset(&ring->hangcheck, 0, sizeof(ring->hangcheck));
 633
 634 out:
 635         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
 636
 637         return ret;
 638 }
 639
 640 void
 641 intel_fini_pipe_control(struct intel_engine_cs *ring)
 642 {
 643         struct drm_device *dev = ring->dev;
 644
 645         if (ring->scratch.obj == NULL)
 646                 return;
 647
 648         if (INTEL_INFO(dev)->gen >= 5) {
 649                 kunmap(sg_page(ring->scratch.obj->pages->sgl));
 650                 i915_gem_object_ggtt_unpin(ring->scratch.obj);
 651         }
 652
 653         drm_gem_object_unreference(&ring->scratch.obj->base);
 654         ring->scratch.obj = NULL;
 655 }
 656
 657 int
 658 intel_init_pipe_control(struct intel_engine_cs *ring)
 659 {
 660         int ret;
 661
 662         WARN_ON(ring->scratch.obj);
 663
 664         ring->scratch.obj = i915_gem_alloc_object(ring->dev, 4096);
 665         if (ring->scratch.obj == NULL) {
 666                 DRM_ERROR("Failed to allocate seqno page\n");
 667                 ret = -ENOMEM;
 668                 goto err;
 669         }
 670
 671         ret = i915_gem_object_set_cache_level(ring->scratch.obj, I915_CACHE_LLC);
 672         if (ret)
 673                 goto err_unref;
 674
 675         ret = i915_gem_obj_ggtt_pin(ring->scratch.obj, 4096, 0);
 676         if (ret)
 677                 goto err_unref;
 678
 679         ring->scratch.gtt_offset = i915_gem_obj_ggtt_offset(ring->scratch.obj);
 680         ring->scratch.cpu_page = kmap(sg_page(ring->scratch.obj->pages->sgl));
 681         if (ring->scratch.cpu_page == NULL) {
 682                 ret = -ENOMEM;
 683                 goto err_unpin;
 684         }
 685
 686         DRM_DEBUG_DRIVER("%s pipe control offset: 0x%08x\n",
 687                          ring->name, ring->scratch.gtt_offset);
 688         return 0;
 689
 690 err_unpin:
 691         i915_gem_object_ggtt_unpin(ring->scratch.obj);
 692 err_unref:
 693         drm_gem_object_unreference(&ring->scratch.obj->base);
 694 err:
 695         return ret;
 696 }
 697
 698 static int intel_ring_workarounds_emit(struct drm_i915_gem_request *req)
 699 {
 700         int ret, i;
 701         struct intel_engine_cs *ring = req->ring;
 702         struct drm_device *dev = ring->dev;
 703         struct drm_i915_private *dev_priv = dev->dev_private;
 704         struct i915_workarounds *w = &dev_priv->workarounds;
 705
 706         if (w->count == 0)
 707                 return 0;
 708
 709         ring->gpu_caches_dirty = true;
 710         ret = intel_ring_flush_all_caches(req);
 711         if (ret)
 712                 return ret;
 713
 714         ret = intel_ring_begin(req, (w->count * 2 + 2));
 715         if (ret)
 716                 return ret;
 717
 718         intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(w->count));
 719         for (i = 0; i < w->count; i++) {
 720                 intel_ring_emit_reg(ring, w->reg[i].addr);
 721                 intel_ring_emit(ring, w->reg[i].value);
 722         }
 723         intel_ring_emit(ring, MI_NOOP);
 724
 725         intel_ring_advance(ring);
 726
 727         ring->gpu_caches_dirty = true;
 728         ret = intel_ring_flush_all_caches(req);
 729         if (ret)
 730                 return ret;
 731
 732         DRM_DEBUG_DRIVER("Number of Workarounds emitted: %d\n", w->count);
 733
 734         return 0;
 735 }
 736
 737 static int intel_rcs_ctx_init(struct drm_i915_gem_request *req)
 738 {
 739         int ret;
 740
 741         ret = intel_ring_workarounds_emit(req);
 742         if (ret != 0)
 743                 return ret;
 744
 745         ret = i915_gem_render_state_init(req);
 746         if (ret)
 747                 DRM_ERROR("init render state: %d\n", ret);
 748
 749         return ret;
 750 }
 751
 752 static int wa_add(struct drm_i915_private *dev_priv,
 753                   i915_reg_t addr,
 754                   const u32 mask, const u32 val)
 755 {
 756         const u32 idx = dev_priv->workarounds.count;
 757
 758         if (WARN_ON(idx >= I915_MAX_WA_REGS))
 759                 return -ENOSPC;
 760
 761         dev_priv->workarounds.reg[idx].addr = addr;
 762         dev_priv->workarounds.reg[idx].value = val;
 763         dev_priv->workarounds.reg[idx].mask = mask;
 764
 765         dev_priv->workarounds.count++;
 766
 767         return 0;
 768 }
 769
 770 #define WA_REG(addr, mask, val) do { \
 771                 const int r = wa_add(dev_priv, (addr), (mask), (val)); \
 772                 if (r) \
 773                         return r; \
 774         } while (0)
 775
 776 #define WA_SET_BIT_MASKED(addr, mask) \
 777         WA_REG(addr, (mask), _MASKED_BIT_ENABLE(mask))
 778
 779 #define WA_CLR_BIT_MASKED(addr, mask) \
 780         WA_REG(addr, (mask), _MASKED_BIT_DISABLE(mask))
 781
 782 #define WA_SET_FIELD_MASKED(addr, mask, value) \
 783         WA_REG(addr, mask, _MASKED_FIELD(mask, value))
 784
 785 #define WA_SET_BIT(addr, mask) WA_REG(addr, mask, I915_READ(addr) | (mask))
 786 #define WA_CLR_BIT(addr, mask) WA_REG(addr, mask, I915_READ(addr) & ~(mask))
 787
 788 #define WA_WRITE(addr, val) WA_REG(addr, 0xffffffff, val)
 789
 790 static int gen8_init_workarounds(struct intel_engine_cs *ring)
 791 {
 792         struct drm_device *dev = ring->dev;
 793         struct drm_i915_private *dev_priv = dev->dev_private;
 794
 795         WA_SET_BIT_MASKED(INSTPM, INSTPM_FORCE_ORDERING);
 796
 797         /* WaDisableAsyncFlipPerfMode:bdw,chv */
 798         WA_SET_BIT_MASKED(MI_MODE, ASYNC_FLIP_PERF_DISABLE);
 799
 800         /* WaDisablePartialInstShootdown:bdw,chv */
 801         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
 802                           PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
 803
 804         /* Use Force Non-Coherent whenever executing a 3D context. This is a
 805          * workaround for for a possible hang in the unlikely event a TLB
 806          * invalidation occurs during a PSD flush.
 807          */
 808         /* WaForceEnableNonCoherent:bdw,chv */
 809         /* WaHdcDisableFetchWhenMasked:bdw,chv */
 810         WA_SET_BIT_MASKED(HDC_CHICKEN0,
 811                           HDC_DONOT_FETCH_MEM_WHEN_MASKED |
 812                           HDC_FORCE_NON_COHERENT);
 813
 814         /* From the Haswell PRM, Command Reference: Registers, CACHE_MODE_0:
 815          * "The Hierarchical Z RAW Stall Optimization allows non-overlapping
 816          *  polygons in the same 8x4 pixel/sample area to be processed without
 817          *  stalling waiting for the earlier ones to write to Hierarchical Z
 818          *  buffer."
 819          *
 820          * This optimization is off by default for BDW and CHV; turn it on.
 821          */
 822         WA_CLR_BIT_MASKED(CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE);
 823
 824         /* Wa4x4STCOptimizationDisable:bdw,chv */
 825         WA_SET_BIT_MASKED(CACHE_MODE_1, GEN8_4x4_STC_OPTIMIZATION_DISABLE);
 826
 827         /*
 828          * BSpec recommends 8x4 when MSAA is used,
 829          * however in practice 16x4 seems fastest.
 830          *
 831          * Note that PS/WM thread counts depend on the WIZ hashing
 832          * disable bit, which we don't touch here, but it's good
 833          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
 834          */
 835         WA_SET_FIELD_MASKED(GEN7_GT_MODE,
 836                             GEN6_WIZ_HASHING_MASK,
 837                             GEN6_WIZ_HASHING_16x4);
 838
 839         return 0;
 840 }
 841
 842 static int bdw_init_workarounds(struct intel_engine_cs *ring)
 843 {
 844         int ret;
 845         struct drm_device *dev = ring->dev;
 846         struct drm_i915_private *dev_priv = dev->dev_private;
 847
 848         ret = gen8_init_workarounds(ring);
 849         if (ret)
 850                 return ret;
 851
 852         /* WaDisableThreadStallDopClockGating:bdw (pre-production) */
 853         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
 854
 855         /* WaDisableDopClockGating:bdw */
 856         WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
 857                           DOP_CLOCK_GATING_DISABLE);
 858
 859         WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
 860                           GEN8_SAMPLER_POWER_BYPASS_DIS);
 861
 862         WA_SET_BIT_MASKED(HDC_CHICKEN0,
 863                           /* WaForceContextSaveRestoreNonCoherent:bdw */
 864                           HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
 865                           /* WaDisableFenceDestinationToSLM:bdw (pre-prod) */
 866                           (IS_BDW_GT3(dev) ? HDC_FENCE_DEST_SLM_DISABLE : 0));
 867
 868         return 0;
 869 }
 870
 871 static int chv_init_workarounds(struct intel_engine_cs *ring)
 872 {
 873         int ret;
 874         struct drm_device *dev = ring->dev;
 875         struct drm_i915_private *dev_priv = dev->dev_private;
 876
 877         ret = gen8_init_workarounds(ring);
 878         if (ret)
 879                 return ret;
 880
 881         /* WaDisableThreadStallDopClockGating:chv */
 882         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
 883
 884         /* Improve HiZ throughput on CHV. */
 885         WA_SET_BIT_MASKED(HIZ_CHICKEN, CHV_HZ_8X8_MODE_IN_1X);
 886
 887         return 0;
 888 }
 889
 890 static int gen9_init_workarounds(struct intel_engine_cs *ring)
 891 {
 892         struct drm_device *dev = ring->dev;
 893         struct drm_i915_private *dev_priv = dev->dev_private;
 894         uint32_t tmp;
 895
 896         /* WaEnableLbsSlaRetryTimerDecrement:skl */
 897         I915_WRITE(BDW_SCRATCH1, I915_READ(BDW_SCRATCH1) |
 898                    GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE);
 899
 900         /* WaDisableKillLogic:bxt,skl */
 901         I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) |
 902                    ECOCHK_DIS_TLB);
 903
 904         /* WaDisablePartialInstShootdown:skl,bxt */
 905         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
 906                           PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
 907
 908         /* Syncing dependencies between camera and graphics:skl,bxt */
 909         WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
 910                           GEN9_DISABLE_OCL_OOB_SUPPRESS_LOGIC);
 911
 912         /* WaDisableDgMirrorFixInHalfSliceChicken5:skl,bxt */
 913         if (IS_SKL_REVID(dev, 0, SKL_REVID_B0) ||
 914             IS_BXT_REVID(dev, 0, BXT_REVID_A1))
 915                 WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
 916                                   GEN9_DG_MIRROR_FIX_ENABLE);
 917
 918         /* WaSetDisablePixMaskCammingAndRhwoInCommonSliceChicken:skl,bxt */
 919         if (IS_SKL_REVID(dev, 0, SKL_REVID_B0) ||
 920             IS_BXT_REVID(dev, 0, BXT_REVID_A1)) {
 921                 WA_SET_BIT_MASKED(GEN7_COMMON_SLICE_CHICKEN1,
 922                                   GEN9_RHWO_OPTIMIZATION_DISABLE);
 923                 /*
 924                  * WA also requires GEN9_SLICE_COMMON_ECO_CHICKEN0[14:14] to be set
 925                  * but we do that in per ctx batchbuffer as there is an issue
 926                  * with this register not getting restored on ctx restore
 927                  */
 928         }
 929
 930         /* WaEnableYV12BugFixInHalfSliceChicken7:skl,bxt */
 931         if (IS_SKL_REVID(dev, SKL_REVID_C0, REVID_FOREVER) || IS_BROXTON(dev))
 932                 WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
 933                                   GEN9_ENABLE_YV12_BUGFIX);
 934
 935         /* Wa4x4STCOptimizationDisable:skl,bxt */
 936         /* WaDisablePartialResolveInVc:skl,bxt */
 937         WA_SET_BIT_MASKED(CACHE_MODE_1, (GEN8_4x4_STC_OPTIMIZATION_DISABLE |
 938                                          GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE));
 939
 940         /* WaCcsTlbPrefetchDisable:skl,bxt */
 941         WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
 942                           GEN9_CCS_TLB_PREFETCH_ENABLE);
 943
 944         /* WaDisableMaskBasedCammingInRCC:skl,bxt */
 945         if (IS_SKL_REVID(dev, SKL_REVID_C0, SKL_REVID_C0) ||
 946             IS_BXT_REVID(dev, 0, BXT_REVID_A1))
 947                 WA_SET_BIT_MASKED(SLICE_ECO_CHICKEN0,
 948                                   PIXEL_MASK_CAMMING_DISABLE);
 949
 950         /* WaForceContextSaveRestoreNonCoherent:skl,bxt */
 951         tmp = HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT;
 952         if (IS_SKL_REVID(dev, SKL_REVID_F0, SKL_REVID_F0) ||
 953             IS_BXT_REVID(dev, BXT_REVID_B0, REVID_FOREVER))
 954                 tmp |= HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE;
 955         WA_SET_BIT_MASKED(HDC_CHICKEN0, tmp);
 956
 957         /* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt */
 958         if (IS_SKYLAKE(dev) || IS_BXT_REVID(dev, 0, BXT_REVID_B0))
 959                 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
 960                                   GEN8_SAMPLER_POWER_BYPASS_DIS);
 961
 962         /* WaDisableSTUnitPowerOptimization:skl,bxt */
 963         WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN2, GEN8_ST_PO_DISABLE);
 964
 965         return 0;
 966 }
 967
 968 static int skl_tune_iz_hashing(struct intel_engine_cs *ring)
 969 {
 970         struct drm_device *dev = ring->dev;
 971         struct drm_i915_private *dev_priv = dev->dev_private;
 972         u8 vals[3] = { 0, 0, 0 };
 973         unsigned int i;
 974
 975         for (i = 0; i < 3; i++) {
 976                 u8 ss;
 977
 978                 /*
 979                  * Only consider slices where one, and only one, subslice has 7
 980                  * EUs
 981                  */
 982                 if (!is_power_of_2(dev_priv->info.subslice_7eu[i]))
 983                         continue;
 984
 985                 /*
 986                  * subslice_7eu[i] != 0 (because of the check above) and
 987                  * ss_max == 4 (maximum number of subslices possible per slice)
 988                  *
 989                  * ->    0 <= ss <= 3;
 990                  */
 991                 ss = ffs(dev_priv->info.subslice_7eu[i]) - 1;
 992                 vals[i] = 3 - ss;
 993         }
 994
 995         if (vals[0] == 0 && vals[1] == 0 && vals[2] == 0)
 996                 return 0;
 997
 998         /* Tune IZ hashing. See intel_device_info_runtime_init() */
 999         WA_SET_FIELD_MASKED(GEN7_GT_MODE,
1000                             GEN9_IZ_HASHING_MASK(2) |
1001                             GEN9_IZ_HASHING_MASK(1) |
1002                             GEN9_IZ_HASHING_MASK(0),
1003                             GEN9_IZ_HASHING(2, vals[2]) |
1004                             GEN9_IZ_HASHING(1, vals[1]) |
1005                             GEN9_IZ_HASHING(0, vals[0]));
1006
1007         return 0;
1008 }
1009
1010 static int skl_init_workarounds(struct intel_engine_cs *ring)
1011 {
1012         int ret;
1013         struct drm_device *dev = ring->dev;
1014         struct drm_i915_private *dev_priv = dev->dev_private;
1015
1016         ret = gen9_init_workarounds(ring);
1017         if (ret)
1018                 return ret;
1019
1020         if (IS_SKL_REVID(dev, 0, SKL_REVID_D0)) {
1021                 /* WaDisableChickenBitTSGBarrierAckForFFSliceCS:skl */
1022                 I915_WRITE(FF_SLICE_CS_CHICKEN2,
1023                            _MASKED_BIT_ENABLE(GEN9_TSG_BARRIER_ACK_DISABLE));
1024         }
1025
1026         /* GEN8_L3SQCREG4 has a dependency with WA batch so any new changes
1027          * involving this register should also be added to WA batch as required.
1028          */
1029         if (IS_SKL_REVID(dev, 0, SKL_REVID_E0))
1030                 /* WaDisableLSQCROPERFforOCL:skl */
1031                 I915_WRITE(GEN8_L3SQCREG4, I915_READ(GEN8_L3SQCREG4) |
1032                            GEN8_LQSC_RO_PERF_DIS);
1033
1034         /* WaEnableGapsTsvCreditFix:skl */
1035         if (IS_SKL_REVID(dev, SKL_REVID_C0, REVID_FOREVER)) {
1036                 I915_WRITE(GEN8_GARBCNTL, (I915_READ(GEN8_GARBCNTL) |
1037                                            GEN9_GAPS_TSV_CREDIT_DISABLE));
1038         }
1039
1040         /* WaDisablePowerCompilerClockGating:skl */
1041         if (IS_SKL_REVID(dev, SKL_REVID_B0, SKL_REVID_B0))
1042                 WA_SET_BIT_MASKED(HIZ_CHICKEN,
1043                                   BDW_HIZ_POWER_COMPILER_CLOCK_GATING_DISABLE);
1044
1045         if (IS_SKL_REVID(dev, 0, SKL_REVID_F0)) {
1046                 /*
1047                  *Use Force Non-Coherent whenever executing a 3D context. This
1048                  * is a workaround for a possible hang in the unlikely event
1049                  * a TLB invalidation occurs during a PSD flush.
1050                  */
1051                 /* WaForceEnableNonCoherent:skl */
1052                 WA_SET_BIT_MASKED(HDC_CHICKEN0,
1053                                   HDC_FORCE_NON_COHERENT);
1054
1055                 /* WaDisableHDCInvalidation:skl */
1056                 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) |
1057                            BDW_DISABLE_HDC_INVALIDATION);
1058         }
1059
1060         /* WaBarrierPerformanceFixDisable:skl */
1061         if (IS_SKL_REVID(dev, SKL_REVID_C0, SKL_REVID_D0))
1062                 WA_SET_BIT_MASKED(HDC_CHICKEN0,
1063                                   HDC_FENCE_DEST_SLM_DISABLE |
1064                                   HDC_BARRIER_PERFORMANCE_DISABLE);
1065
1066         /* WaDisableSbeCacheDispatchPortSharing:skl */
1067         if (IS_SKL_REVID(dev, 0, SKL_REVID_F0))
1068                 WA_SET_BIT_MASKED(
1069                         GEN7_HALF_SLICE_CHICKEN1,
1070                         GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
1071
1072         return skl_tune_iz_hashing(ring);
1073 }
1074
1075 static int bxt_init_workarounds(struct intel_engine_cs *ring)
1076 {
1077         int ret;
1078         struct drm_device *dev = ring->dev;
1079         struct drm_i915_private *dev_priv = dev->dev_private;
1080
1081         ret = gen9_init_workarounds(ring);
1082         if (ret)
1083                 return ret;
1084
1085         /* WaStoreMultiplePTEenable:bxt */
1086         /* This is a requirement according to Hardware specification */
1087         if (IS_BXT_REVID(dev, 0, BXT_REVID_A1))
1088                 I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_TLBPF);
1089
1090         /* WaSetClckGatingDisableMedia:bxt */
1091         if (IS_BXT_REVID(dev, 0, BXT_REVID_A1)) {
1092                 I915_WRITE(GEN7_MISCCPCTL, (I915_READ(GEN7_MISCCPCTL) &
1093                                             ~GEN8_DOP_CLOCK_GATE_MEDIA_ENABLE));
1094         }
1095
1096         /* WaDisableThreadStallDopClockGating:bxt */
1097         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
1098                           STALL_DOP_GATING_DISABLE);
1099
1100         /* WaDisableSbeCacheDispatchPortSharing:bxt */
1101         if (IS_BXT_REVID(dev, 0, BXT_REVID_B0)) {
1102                 WA_SET_BIT_MASKED(
1103                         GEN7_HALF_SLICE_CHICKEN1,
1104                         GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
1105         }
1106
1107         return 0;
1108 }
1109
1110 int init_workarounds_ring(struct intel_engine_cs *ring)
1111 {
1112         struct drm_device *dev = ring->dev;
1113         struct drm_i915_private *dev_priv = dev->dev_private;
1114
1115         WARN_ON(ring->id != RCS);
1116
1117         dev_priv->workarounds.count = 0;
1118
1119         if (IS_BROADWELL(dev))
1120                 return bdw_init_workarounds(ring);
1121
1122         if (IS_CHERRYVIEW(dev))
1123                 return chv_init_workarounds(ring);
1124
1125         if (IS_SKYLAKE(dev))
1126                 return skl_init_workarounds(ring);
1127
1128         if (IS_BROXTON(dev))
1129                 return bxt_init_workarounds(ring);
1130
1131         return 0;
1132 }
1133
1134 static int init_render_ring(struct intel_engine_cs *ring)
1135 {
1136         struct drm_device *dev = ring->dev;
1137         struct drm_i915_private *dev_priv = dev->dev_private;
1138         int ret = init_ring_common(ring);
1139         if (ret)
1140                 return ret;
1141
1142         /* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */
1143         if (INTEL_INFO(dev)->gen >= 4 && INTEL_INFO(dev)->gen < 7)
1144                 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH));
1145
1146         /* We need to disable the AsyncFlip performance optimisations in order
1147          * to use MI_WAIT_FOR_EVENT within the CS. It should already be
1148          * programmed to '1' on all products.
1149          *
1150          * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv
1151          */
1152         if (INTEL_INFO(dev)->gen >= 6 && INTEL_INFO(dev)->gen < 8)
1153                 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE));
1154
1155         /* Required for the hardware to program scanline values for waiting */
1156         /* WaEnableFlushTlbInvalidationMode:snb */
1157         if (INTEL_INFO(dev)->gen == 6)
1158                 I915_WRITE(GFX_MODE,
1159                            _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT));
1160
1161         /* WaBCSVCSTlbInvalidationMode:ivb,vlv,hsw */
1162         if (IS_GEN7(dev))
1163                 I915_WRITE(GFX_MODE_GEN7,
1164                            _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT) |
1165                            _MASKED_BIT_ENABLE(GFX_REPLAY_MODE));
1166
1167         if (IS_GEN6(dev)) {
1168                 /* From the Sandybridge PRM, volume 1 part 3, page 24:
1169                  * "If this bit is set, STCunit will have LRA as replacement
1170                  *  policy. [...] This bit must be reset.  LRA replacement
1171                  *  policy is not supported."
1172                  */
1173                 I915_WRITE(CACHE_MODE_0,
1174                            _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
1175         }
1176
1177         if (INTEL_INFO(dev)->gen >= 6 && INTEL_INFO(dev)->gen < 8)
1178                 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
1179
1180         if (HAS_L3_DPF(dev))
1181                 I915_WRITE_IMR(ring, ~GT_PARITY_ERROR(dev));
1182
1183         return init_workarounds_ring(ring);
1184 }
1185
1186 static void render_ring_cleanup(struct intel_engine_cs *ring)
1187 {
1188         struct drm_device *dev = ring->dev;
1189         struct drm_i915_private *dev_priv = dev->dev_private;
1190
1191         if (dev_priv->semaphore_obj) {
1192                 i915_gem_object_ggtt_unpin(dev_priv->semaphore_obj);
1193                 drm_gem_object_unreference(&dev_priv->semaphore_obj->base);
1194                 dev_priv->semaphore_obj = NULL;
1195         }
1196
1197         intel_fini_pipe_control(ring);
1198 }
1199
1200 static int gen8_rcs_signal(struct drm_i915_gem_request *signaller_req,
1201                            unsigned int num_dwords)
1202 {
1203 #define MBOX_UPDATE_DWORDS 8
1204         struct intel_engine_cs *signaller = signaller_req->ring;
1205         struct drm_device *dev = signaller->dev;
1206         struct drm_i915_private *dev_priv = dev->dev_private;
1207         struct intel_engine_cs *waiter;
1208         int i, ret, num_rings;
1209
1210         num_rings = hweight32(INTEL_INFO(dev)->ring_mask);
1211         num_dwords += (num_rings-1) * MBOX_UPDATE_DWORDS;
1212 #undef MBOX_UPDATE_DWORDS
1213
1214         ret = intel_ring_begin(signaller_req, num_dwords);
1215         if (ret)
1216                 return ret;
1217
1218         for_each_ring(waiter, dev_priv, i) {
1219                 u32 seqno;
1220                 u64 gtt_offset = signaller->semaphore.signal_ggtt[i];
1221                 if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID)
1222                         continue;
1223
1224                 seqno = i915_gem_request_get_seqno(signaller_req);
1225                 intel_ring_emit(signaller, GFX_OP_PIPE_CONTROL(6));
1226                 intel_ring_emit(signaller, PIPE_CONTROL_GLOBAL_GTT_IVB |
1227                                            PIPE_CONTROL_QW_WRITE |
1228                                            PIPE_CONTROL_FLUSH_ENABLE);
1229                 intel_ring_emit(signaller, lower_32_bits(gtt_offset));
1230                 intel_ring_emit(signaller, upper_32_bits(gtt_offset));
1231                 intel_ring_emit(signaller, seqno);
1232                 intel_ring_emit(signaller, 0);
1233                 intel_ring_emit(signaller, MI_SEMAPHORE_SIGNAL |
1234                                            MI_SEMAPHORE_TARGET(waiter->id));
1235                 intel_ring_emit(signaller, 0);
1236         }
1237
1238         return 0;
1239 }
1240
1241 static int gen8_xcs_signal(struct drm_i915_gem_request *signaller_req,
1242                            unsigned int num_dwords)
1243 {
1244 #define MBOX_UPDATE_DWORDS 6
1245         struct intel_engine_cs *signaller = signaller_req->ring;
1246         struct drm_device *dev = signaller->dev;
1247         struct drm_i915_private *dev_priv = dev->dev_private;
1248         struct intel_engine_cs *waiter;
1249         int i, ret, num_rings;
1250
1251         num_rings = hweight32(INTEL_INFO(dev)->ring_mask);
1252         num_dwords += (num_rings-1) * MBOX_UPDATE_DWORDS;
1253 #undef MBOX_UPDATE_DWORDS
1254
1255         ret = intel_ring_begin(signaller_req, num_dwords);
1256         if (ret)
1257                 return ret;
1258
1259         for_each_ring(waiter, dev_priv, i) {
1260                 u32 seqno;
1261                 u64 gtt_offset = signaller->semaphore.signal_ggtt[i];
1262                 if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID)
1263                         continue;
1264
1265                 seqno = i915_gem_request_get_seqno(signaller_req);
1266                 intel_ring_emit(signaller, (MI_FLUSH_DW + 1) |
1267                                            MI_FLUSH_DW_OP_STOREDW);
1268                 intel_ring_emit(signaller, lower_32_bits(gtt_offset) |
1269                                            MI_FLUSH_DW_USE_GTT);
1270                 intel_ring_emit(signaller, upper_32_bits(gtt_offset));
1271                 intel_ring_emit(signaller, seqno);
1272                 intel_ring_emit(signaller, MI_SEMAPHORE_SIGNAL |
1273                                            MI_SEMAPHORE_TARGET(waiter->id));
1274                 intel_ring_emit(signaller, 0);
1275         }
1276
1277         return 0;
1278 }
1279
1280 static int gen6_signal(struct drm_i915_gem_request *signaller_req,
1281                        unsigned int num_dwords)
1282 {
1283         struct intel_engine_cs *signaller = signaller_req->ring;
1284         struct drm_device *dev = signaller->dev;
1285         struct drm_i915_private *dev_priv = dev->dev_private;
1286         struct intel_engine_cs *useless;
1287         int i, ret, num_rings;
1288
1289 #define MBOX_UPDATE_DWORDS 3
1290         num_rings = hweight32(INTEL_INFO(dev)->ring_mask);
1291         num_dwords += round_up((num_rings-1) * MBOX_UPDATE_DWORDS, 2);
1292 #undef MBOX_UPDATE_DWORDS
1293
1294         ret = intel_ring_begin(signaller_req, num_dwords);
1295         if (ret)
1296                 return ret;
1297
1298         for_each_ring(useless, dev_priv, i) {
1299                 i915_reg_t mbox_reg = signaller->semaphore.mbox.signal[i];
1300
1301                 if (i915_mmio_reg_valid(mbox_reg)) {
1302                         u32 seqno = i915_gem_request_get_seqno(signaller_req);
1303
1304                         intel_ring_emit(signaller, MI_LOAD_REGISTER_IMM(1));
1305                         intel_ring_emit_reg(signaller, mbox_reg);
1306                         intel_ring_emit(signaller, seqno);
1307                 }
1308         }
1309
1310         /* If num_dwords was rounded, make sure the tail pointer is correct */
1311         if (num_rings % 2 == 0)
1312                 intel_ring_emit(signaller, MI_NOOP);
1313
1314         return 0;
1315 }
1316
1317 /**
1318  * gen6_add_request - Update the semaphore mailbox registers
1319  *
1320  * @request - request to write to the ring
1321  *
1322  * Update the mailbox registers in the *other* rings with the current seqno.
1323  * This acts like a signal in the canonical semaphore.
1324  */
1325 static int
1326 gen6_add_request(struct drm_i915_gem_request *req)
1327 {
1328         struct intel_engine_cs *ring = req->ring;
1329         int ret;
1330
1331         if (ring->semaphore.signal)
1332                 ret = ring->semaphore.signal(req, 4);
1333         else
1334                 ret = intel_ring_begin(req, 4);
1335
1336         if (ret)
1337                 return ret;
1338
1339         intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
1340         intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
1341         intel_ring_emit(ring, i915_gem_request_get_seqno(req));
1342         intel_ring_emit(ring, MI_USER_INTERRUPT);
1343         __intel_ring_advance(ring);
1344
1345         return 0;
1346 }
1347
1348 static inline bool i915_gem_has_seqno_wrapped(struct drm_device *dev,
1349                                               u32 seqno)
1350 {
1351         struct drm_i915_private *dev_priv = dev->dev_private;
1352         return dev_priv->last_seqno < seqno;
1353 }
1354
1355 /**
1356  * intel_ring_sync - sync the waiter to the signaller on seqno
1357  *
1358  * @waiter - ring that is waiting
1359  * @signaller - ring which has, or will signal
1360  * @seqno - seqno which the waiter will block on
1361  */
1362
1363 static int
1364 gen8_ring_sync(struct drm_i915_gem_request *waiter_req,
1365                struct intel_engine_cs *signaller,
1366                u32 seqno)
1367 {
1368         struct intel_engine_cs *waiter = waiter_req->ring;
1369         struct drm_i915_private *dev_priv = waiter->dev->dev_private;
1370         int ret;
1371
1372         ret = intel_ring_begin(waiter_req, 4);
1373         if (ret)
1374                 return ret;
1375
1376         intel_ring_emit(waiter, MI_SEMAPHORE_WAIT |
1377                                 MI_SEMAPHORE_GLOBAL_GTT |
1378                                 MI_SEMAPHORE_POLL |
1379                                 MI_SEMAPHORE_SAD_GTE_SDD);
1380         intel_ring_emit(waiter, seqno);
1381         intel_ring_emit(waiter,
1382                         lower_32_bits(GEN8_WAIT_OFFSET(waiter, signaller->id)));
1383         intel_ring_emit(waiter,
1384                         upper_32_bits(GEN8_WAIT_OFFSET(waiter, signaller->id)));
1385         intel_ring_advance(waiter);
1386         return 0;
1387 }
1388
1389 static int
1390 gen6_ring_sync(struct drm_i915_gem_request *waiter_req,
1391                struct intel_engine_cs *signaller,
1392                u32 seqno)
1393 {
1394         struct intel_engine_cs *waiter = waiter_req->ring;
1395         u32 dw1 = MI_SEMAPHORE_MBOX |
1396                   MI_SEMAPHORE_COMPARE |
1397                   MI_SEMAPHORE_REGISTER;
1398         u32 wait_mbox = signaller->semaphore.mbox.wait[waiter->id];
1399         int ret;
1400
1401         /* Throughout all of the GEM code, seqno passed implies our current
1402          * seqno is >= the last seqno executed. However for hardware the
1403          * comparison is strictly greater than.
1404          */
1405         seqno -= 1;
1406
1407         WARN_ON(wait_mbox == MI_SEMAPHORE_SYNC_INVALID);
1408
1409         ret = intel_ring_begin(waiter_req, 4);
1410         if (ret)
1411                 return ret;
1412
1413         /* If seqno wrap happened, omit the wait with no-ops */
1414         if (likely(!i915_gem_has_seqno_wrapped(waiter->dev, seqno))) {
1415                 intel_ring_emit(waiter, dw1 | wait_mbox);
1416                 intel_ring_emit(waiter, seqno);
1417                 intel_ring_emit(waiter, 0);
1418                 intel_ring_emit(waiter, MI_NOOP);
1419         } else {
1420                 intel_ring_emit(waiter, MI_NOOP);
1421                 intel_ring_emit(waiter, MI_NOOP);
1422                 intel_ring_emit(waiter, MI_NOOP);
1423                 intel_ring_emit(waiter, MI_NOOP);
1424         }
1425         intel_ring_advance(waiter);
1426
1427         return 0;
1428 }
1429
1430 #define PIPE_CONTROL_FLUSH(ring__, addr__)                                      \
1431 do {                                                                    \
1432         intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |                \
1433                  PIPE_CONTROL_DEPTH_STALL);                             \
1434         intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT);                    \
1435         intel_ring_emit(ring__, 0);                                                     \
1436         intel_ring_emit(ring__, 0);                                                     \
1437 } while (0)
1438
1439 static int
1440 pc_render_add_request(struct drm_i915_gem_request *req)
1441 {
1442         struct intel_engine_cs *ring = req->ring;
1443         u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
1444         int ret;
1445
1446         /* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently
1447          * incoherent with writes to memory, i.e. completely fubar,
1448          * so we need to use PIPE_NOTIFY instead.
1449          *
1450          * However, we also need to workaround the qword write
1451          * incoherence by flushing the 6 PIPE_NOTIFY buffers out to
1452          * memory before requesting an interrupt.
1453          */
1454         ret = intel_ring_begin(req, 32);
1455         if (ret)
1456                 return ret;
1457
1458         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
1459                         PIPE_CONTROL_WRITE_FLUSH |
1460                         PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
1461         intel_ring_emit(ring, ring->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
1462         intel_ring_emit(ring, i915_gem_request_get_seqno(req));
1463         intel_ring_emit(ring, 0);
1464         PIPE_CONTROL_FLUSH(ring, scratch_addr);
1465         scratch_addr += 2 * CACHELINE_BYTES; /* write to separate cachelines */
1466         PIPE_CONTROL_FLUSH(ring, scratch_addr);
1467         scratch_addr += 2 * CACHELINE_BYTES;
1468         PIPE_CONTROL_FLUSH(ring, scratch_addr);
1469         scratch_addr += 2 * CACHELINE_BYTES;
1470         PIPE_CONTROL_FLUSH(ring, scratch_addr);
1471         scratch_addr += 2 * CACHELINE_BYTES;
1472         PIPE_CONTROL_FLUSH(ring, scratch_addr);
1473         scratch_addr += 2 * CACHELINE_BYTES;
1474         PIPE_CONTROL_FLUSH(ring, scratch_addr);
1475
1476         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
1477                         PIPE_CONTROL_WRITE_FLUSH |
1478                         PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
1479                         PIPE_CONTROL_NOTIFY);
1480         intel_ring_emit(ring, ring->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
1481         intel_ring_emit(ring, i915_gem_request_get_seqno(req));
1482         intel_ring_emit(ring, 0);
1483         __intel_ring_advance(ring);
1484
1485         return 0;
1486 }
1487
1488 static u32
1489 gen6_ring_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency)
1490 {
1491         /* Workaround to force correct ordering between irq and seqno writes on
1492          * ivb (and maybe also on snb) by reading from a CS register (like
1493          * ACTHD) before reading the status page. */
1494         if (!lazy_coherency) {
1495                 struct drm_i915_private *dev_priv = ring->dev->dev_private;
1496                 POSTING_READ(RING_ACTHD(ring->mmio_base));
1497         }
1498
1499         return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
1500 }
1501
1502 static u32
1503 ring_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency)
1504 {
1505         return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
1506 }
1507
1508 static void
1509 ring_set_seqno(struct intel_engine_cs *ring, u32 seqno)
1510 {
1511         intel_write_status_page(ring, I915_GEM_HWS_INDEX, seqno);
1512 }
1513
1514 static u32
1515 pc_render_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency)
1516 {
1517         return ring->scratch.cpu_page[0];
1518 }
1519
1520 static void
1521 pc_render_set_seqno(struct intel_engine_cs *ring, u32 seqno)
1522 {
1523         ring->scratch.cpu_page[0] = seqno;
1524 }
1525
1526 static bool
1527 gen5_ring_get_irq(struct intel_engine_cs *ring)
1528 {
1529         struct drm_device *dev = ring->dev;
1530         struct drm_i915_private *dev_priv = dev->dev_private;
1531         unsigned long flags;
1532
1533         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
1534                 return false;
1535
1536         spin_lock_irqsave(&dev_priv->irq_lock, flags);
1537         if (ring->irq_refcount++ == 0)
1538                 gen5_enable_gt_irq(dev_priv, ring->irq_enable_mask);
1539         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1540
1541         return true;
1542 }
1543
1544 static void
1545 gen5_ring_put_irq(struct intel_engine_cs *ring)
1546 {
1547         struct drm_device *dev = ring->dev;
1548         struct drm_i915_private *dev_priv = dev->dev_private;
1549         unsigned long flags;
1550
1551         spin_lock_irqsave(&dev_priv->irq_lock, flags);
1552         if (--ring->irq_refcount == 0)
1553                 gen5_disable_gt_irq(dev_priv, ring->irq_enable_mask);
1554         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1555 }
1556
1557 static bool
1558 i9xx_ring_get_irq(struct intel_engine_cs *ring)
1559 {
1560         struct drm_device *dev = ring->dev;
1561         struct drm_i915_private *dev_priv = dev->dev_private;
1562         unsigned long flags;
1563
1564         if (!intel_irqs_enabled(dev_priv))
1565                 return false;
1566
1567         spin_lock_irqsave(&dev_priv->irq_lock, flags);
1568         if (ring->irq_refcount++ == 0) {
1569                 dev_priv->irq_mask &= ~ring->irq_enable_mask;
1570                 I915_WRITE(IMR, dev_priv->irq_mask);
1571                 POSTING_READ(IMR);
1572         }
1573         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1574
1575         return true;
1576 }
1577
1578 static void
1579 i9xx_ring_put_irq(struct intel_engine_cs *ring)
1580 {
1581         struct drm_device *dev = ring->dev;
1582         struct drm_i915_private *dev_priv = dev->dev_private;
1583         unsigned long flags;
1584
1585         spin_lock_irqsave(&dev_priv->irq_lock, flags);
1586         if (--ring->irq_refcount == 0) {
1587                 dev_priv->irq_mask |= ring->irq_enable_mask;
1588                 I915_WRITE(IMR, dev_priv->irq_mask);
1589                 POSTING_READ(IMR);
1590         }
1591         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1592 }
1593
1594 static bool
1595 i8xx_ring_get_irq(struct intel_engine_cs *ring)
1596 {
1597         struct drm_device *dev = ring->dev;
1598         struct drm_i915_private *dev_priv = dev->dev_private;
1599         unsigned long flags;
1600
1601         if (!intel_irqs_enabled(dev_priv))
1602                 return false;
1603
1604         spin_lock_irqsave(&dev_priv->irq_lock, flags);
1605         if (ring->irq_refcount++ == 0) {
1606                 dev_priv->irq_mask &= ~ring->irq_enable_mask;
1607                 I915_WRITE16(IMR, dev_priv->irq_mask);
1608                 POSTING_READ16(IMR);
1609         }
1610         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1611
1612         return true;
1613 }
1614
1615 static void
1616 i8xx_ring_put_irq(struct intel_engine_cs *ring)
1617 {
1618         struct drm_device *dev = ring->dev;
1619         struct drm_i915_private *dev_priv = dev->dev_private;
1620         unsigned long flags;
1621
1622         spin_lock_irqsave(&dev_priv->irq_lock, flags);
1623         if (--ring->irq_refcount == 0) {
1624                 dev_priv->irq_mask |= ring->irq_enable_mask;
1625                 I915_WRITE16(IMR, dev_priv->irq_mask);
1626                 POSTING_READ16(IMR);
1627         }
1628         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1629 }
1630
1631 static int
1632 bsd_ring_flush(struct drm_i915_gem_request *req,
1633                u32     invalidate_domains,
1634                u32     flush_domains)
1635 {
1636         struct intel_engine_cs *ring = req->ring;
1637         int ret;
1638
1639         ret = intel_ring_begin(req, 2);
1640         if (ret)
1641                 return ret;
1642
1643         intel_ring_emit(ring, MI_FLUSH);
1644         intel_ring_emit(ring, MI_NOOP);
1645         intel_ring_advance(ring);
1646         return 0;
1647 }
1648
1649 static int
1650 i9xx_add_request(struct drm_i915_gem_request *req)
1651 {
1652         struct intel_engine_cs *ring = req->ring;
1653         int ret;
1654
1655         ret = intel_ring_begin(req, 4);
1656         if (ret)
1657                 return ret;
1658
1659         intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
1660         intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
1661         intel_ring_emit(ring, i915_gem_request_get_seqno(req));
1662         intel_ring_emit(ring, MI_USER_INTERRUPT);
1663         __intel_ring_advance(ring);
1664
1665         return 0;
1666 }
1667
1668 static bool
1669 gen6_ring_get_irq(struct intel_engine_cs *ring)
1670 {
1671         struct drm_device *dev = ring->dev;
1672         struct drm_i915_private *dev_priv = dev->dev_private;
1673         unsigned long flags;
1674
1675         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
1676                 return false;
1677
1678         spin_lock_irqsave(&dev_priv->irq_lock, flags);
1679         if (ring->irq_refcount++ == 0) {
1680                 if (HAS_L3_DPF(dev) && ring->id == RCS)
1681                         I915_WRITE_IMR(ring,
1682                                        ~(ring->irq_enable_mask |
1683                                          GT_PARITY_ERROR(dev)));
1684                 else
1685                         I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
1686                 gen5_enable_gt_irq(dev_priv, ring->irq_enable_mask);
1687         }
1688         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1689
1690         return true;
1691 }
1692
1693 static void
1694 gen6_ring_put_irq(struct intel_engine_cs *ring)
1695 {
1696         struct drm_device *dev = ring->dev;
1697         struct drm_i915_private *dev_priv = dev->dev_private;
1698         unsigned long flags;
1699
1700         spin_lock_irqsave(&dev_priv->irq_lock, flags);
1701         if (--ring->irq_refcount == 0) {
1702                 if (HAS_L3_DPF(dev) && ring->id == RCS)
1703                         I915_WRITE_IMR(ring, ~GT_PARITY_ERROR(dev));
1704                 else
1705                         I915_WRITE_IMR(ring, ~0);
1706                 gen5_disable_gt_irq(dev_priv, ring->irq_enable_mask);
1707         }
1708         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1709 }
1710
1711 static bool
1712 hsw_vebox_get_irq(struct intel_engine_cs *ring)
1713 {
1714         struct drm_device *dev = ring->dev;
1715         struct drm_i915_private *dev_priv = dev->dev_private;
1716         unsigned long flags;
1717
1718         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
1719                 return false;
1720
1721         spin_lock_irqsave(&dev_priv->irq_lock, flags);
1722         if (ring->irq_refcount++ == 0) {
1723                 I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
1724                 gen6_enable_pm_irq(dev_priv, ring->irq_enable_mask);
1725         }
1726         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1727
1728         return true;
1729 }
1730
1731 static void
1732 hsw_vebox_put_irq(struct intel_engine_cs *ring)
1733 {
1734         struct drm_device *dev = ring->dev;
1735         struct drm_i915_private *dev_priv = dev->dev_private;
1736         unsigned long flags;
1737
1738         spin_lock_irqsave(&dev_priv->irq_lock, flags);
1739         if (--ring->irq_refcount == 0) {
1740                 I915_WRITE_IMR(ring, ~0);
1741                 gen6_disable_pm_irq(dev_priv, ring->irq_enable_mask);
1742         }
1743         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1744 }
1745
1746 static bool
1747 gen8_ring_get_irq(struct intel_engine_cs *ring)
1748 {
1749         struct drm_device *dev = ring->dev;
1750         struct drm_i915_private *dev_priv = dev->dev_private;
1751         unsigned long flags;
1752
1753         if (WARN_ON(!intel_irqs_enabled(dev_priv)))
1754                 return false;
1755
1756         spin_lock_irqsave(&dev_priv->irq_lock, flags);
1757         if (ring->irq_refcount++ == 0) {
1758                 if (HAS_L3_DPF(dev) && ring->id == RCS) {
1759                         I915_WRITE_IMR(ring,
1760                                        ~(ring->irq_enable_mask |
1761                                          GT_RENDER_L3_PARITY_ERROR_INTERRUPT));
1762                 } else {
1763                         I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
1764                 }
1765                 POSTING_READ(RING_IMR(ring->mmio_base));
1766         }
1767         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1768
1769         return true;
1770 }
1771
1772 static void
1773 gen8_ring_put_irq(struct intel_engine_cs *ring)
1774 {
1775         struct drm_device *dev = ring->dev;
1776         struct drm_i915_private *dev_priv = dev->dev_private;
1777         unsigned long flags;
1778
1779         spin_lock_irqsave(&dev_priv->irq_lock, flags);
1780         if (--ring->irq_refcount == 0) {
1781                 if (HAS_L3_DPF(dev) && ring->id == RCS) {
1782                         I915_WRITE_IMR(ring,
1783                                        ~GT_RENDER_L3_PARITY_ERROR_INTERRUPT);
1784                 } else {
1785                         I915_WRITE_IMR(ring, ~0);
1786                 }
1787                 POSTING_READ(RING_IMR(ring->mmio_base));
1788         }
1789         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
1790 }
1791
1792 static int
1793 i965_dispatch_execbuffer(struct drm_i915_gem_request *req,
1794                          u64 offset, u32 length,
1795                          unsigned dispatch_flags)
1796 {
1797         struct intel_engine_cs *ring = req->ring;
1798         int ret;
1799
1800         ret = intel_ring_begin(req, 2);
1801         if (ret)
1802                 return ret;
1803
1804         intel_ring_emit(ring,
1805                         MI_BATCH_BUFFER_START |
1806                         MI_BATCH_GTT |
1807                         (dispatch_flags & I915_DISPATCH_SECURE ?
1808                          0 : MI_BATCH_NON_SECURE_I965));
1809         intel_ring_emit(ring, offset);
1810         intel_ring_advance(ring);
1811
1812         return 0;
1813 }
1814
1815 /* Just userspace ABI convention to limit the wa batch bo to a resonable size */
1816 #define I830_BATCH_LIMIT (256*1024)
1817 #define I830_TLB_ENTRIES (2)
1818 #define I830_WA_SIZE max(I830_TLB_ENTRIES*4096, I830_BATCH_LIMIT)
1819 static int
1820 i830_dispatch_execbuffer(struct drm_i915_gem_request *req,
1821                          u64 offset, u32 len,
1822                          unsigned dispatch_flags)
1823 {
1824         struct intel_engine_cs *ring = req->ring;
1825         u32 cs_offset = ring->scratch.gtt_offset;
1826         int ret;
1827
1828         ret = intel_ring_begin(req, 6);
1829         if (ret)
1830                 return ret;
1831
1832         /* Evict the invalid PTE TLBs */
1833         intel_ring_emit(ring, COLOR_BLT_CMD | BLT_WRITE_RGBA);
1834         intel_ring_emit(ring, BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | 4096);
1835         intel_ring_emit(ring, I830_TLB_ENTRIES << 16 | 4); /* load each page */
1836         intel_ring_emit(ring, cs_offset);
1837         intel_ring_emit(ring, 0xdeadbeef);
1838         intel_ring_emit(ring, MI_NOOP);
1839         intel_ring_advance(ring);
1840
1841         if ((dispatch_flags & I915_DISPATCH_PINNED) == 0) {
1842                 if (len > I830_BATCH_LIMIT)
1843                         return -ENOSPC;
1844
1845                 ret = intel_ring_begin(req, 6 + 2);
1846                 if (ret)
1847                         return ret;
1848
1849                 /* Blit the batch (which has now all relocs applied) to the
1850                  * stable batch scratch bo area (so that the CS never
1851                  * stumbles over its tlb invalidation bug) ...
1852                  */
1853                 intel_ring_emit(ring, SRC_COPY_BLT_CMD | BLT_WRITE_RGBA);
1854                 intel_ring_emit(ring, BLT_DEPTH_32 | BLT_ROP_SRC_COPY | 4096);
1855                 intel_ring_emit(ring, DIV_ROUND_UP(len, 4096) << 16 | 4096);
1856                 intel_ring_emit(ring, cs_offset);
1857                 intel_ring_emit(ring, 4096);
1858                 intel_ring_emit(ring, offset);
1859
1860                 intel_ring_emit(ring, MI_FLUSH);
1861                 intel_ring_emit(ring, MI_NOOP);
1862                 intel_ring_advance(ring);
1863
1864                 /* ... and execute it. */
1865                 offset = cs_offset;
1866         }
1867
1868         ret = intel_ring_begin(req, 4);
1869         if (ret)
1870                 return ret;
1871
1872         intel_ring_emit(ring, MI_BATCH_BUFFER);
1873         intel_ring_emit(ring, offset | (dispatch_flags & I915_DISPATCH_SECURE ?
1874                                         0 : MI_BATCH_NON_SECURE));
1875         intel_ring_emit(ring, offset + len - 8);
1876         intel_ring_emit(ring, MI_NOOP);
1877         intel_ring_advance(ring);
1878
1879         return 0;
1880 }
1881
1882 static int
1883 i915_dispatch_execbuffer(struct drm_i915_gem_request *req,
1884                          u64 offset, u32 len,
1885                          unsigned dispatch_flags)
1886 {
1887         struct intel_engine_cs *ring = req->ring;
1888         int ret;
1889
1890         ret = intel_ring_begin(req, 2);
1891         if (ret)
1892                 return ret;
1893
1894         intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_GTT);
1895         intel_ring_emit(ring, offset | (dispatch_flags & I915_DISPATCH_SECURE ?
1896                                         0 : MI_BATCH_NON_SECURE));
1897         intel_ring_advance(ring);
1898
1899         return 0;
1900 }
1901
1902 static void cleanup_status_page(struct intel_engine_cs *ring)
1903 {
1904         struct drm_i915_gem_object *obj;
1905
1906         obj = ring->status_page.obj;
1907         if (obj == NULL)
1908                 return;
1909
1910         kunmap(sg_page(obj->pages->sgl));
1911         i915_gem_object_ggtt_unpin(obj);
1912         drm_gem_object_unreference(&obj->base);
1913         ring->status_page.obj = NULL;
1914 }
1915
1916 static int init_status_page(struct intel_engine_cs *ring)
1917 {
1918         struct drm_i915_gem_object *obj;
1919
1920         if ((obj = ring->status_page.obj) == NULL) {
1921                 unsigned flags;
1922                 int ret;
1923
1924                 obj = i915_gem_alloc_object(ring->dev, 4096);
1925                 if (obj == NULL) {
1926                         DRM_ERROR("Failed to allocate status page\n");
1927                         return -ENOMEM;
1928                 }
1929
1930                 ret = i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
1931                 if (ret)
1932                         goto err_unref;
1933
1934                 flags = 0;
1935                 if (!HAS_LLC(ring->dev))
1936                         /* On g33, we cannot place HWS above 256MiB, so
1937                          * restrict its pinning to the low mappable arena.
1938                          * Though this restriction is not documented for
1939                          * gen4, gen5, or byt, they also behave similarly
1940                          * and hang if the HWS is placed at the top of the
1941                          * GTT. To generalise, it appears that all !llc
1942                          * platforms have issues with us placing the HWS
1943                          * above the mappable region (even though we never
1944                          * actualy map it).
1945                          */
1946                         flags |= PIN_MAPPABLE;
1947                 ret = i915_gem_obj_ggtt_pin(obj, 4096, flags);
1948                 if (ret) {
1949 err_unref:
1950                         drm_gem_object_unreference(&obj->base);
1951                         return ret;
1952                 }
1953
1954                 ring->status_page.obj = obj;
1955         }
1956
1957         ring->status_page.gfx_addr = i915_gem_obj_ggtt_offset(obj);
1958         ring->status_page.page_addr = kmap(sg_page(obj->pages->sgl));
1959         memset(ring->status_page.page_addr, 0, PAGE_SIZE);
1960
1961         DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
1962                         ring->name, ring->status_page.gfx_addr);
1963
1964         return 0;
1965 }
1966
1967 static int init_phys_status_page(struct intel_engine_cs *ring)
1968 {
1969         struct drm_i915_private *dev_priv = ring->dev->dev_private;
1970
1971         if (!dev_priv->status_page_dmah) {
1972                 dev_priv->status_page_dmah =
1973                         drm_pci_alloc(ring->dev, PAGE_SIZE, PAGE_SIZE);
1974                 if (!dev_priv->status_page_dmah)
1975                         return -ENOMEM;
1976         }
1977
1978         ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
1979         memset(ring->status_page.page_addr, 0, PAGE_SIZE);
1980
1981         return 0;
1982 }
1983
1984 void intel_unpin_ringbuffer_obj(struct intel_ringbuffer *ringbuf)
1985 {
1986         if (HAS_LLC(ringbuf->obj->base.dev) && !ringbuf->obj->stolen)
1987                 vunmap(ringbuf->virtual_start);
1988         else
1989                 iounmap(ringbuf->virtual_start);
1990         ringbuf->virtual_start = NULL;
1991         i915_gem_object_ggtt_unpin(ringbuf->obj);
1992 }
1993
1994 static u32 *vmap_obj(struct drm_i915_gem_object *obj)
1995 {
1996         struct sg_page_iter sg_iter;
1997         struct page **pages;
1998         void *addr;
1999         int i;
2000
2001         pages = drm_malloc_ab(obj->base.size >> PAGE_SHIFT, sizeof(*pages));
2002         if (pages == NULL)
2003                 return NULL;
2004
2005         i = 0;
2006         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0)
2007                 pages[i++] = sg_page_iter_page(&sg_iter);
2008
2009         addr = vmap(pages, i, 0, PAGE_KERNEL);
2010         drm_free_large(pages);
2011
2012         return addr;
2013 }
2014
2015 int intel_pin_and_map_ringbuffer_obj(struct drm_device *dev,
2016                                      struct intel_ringbuffer *ringbuf)
2017 {
2018         struct drm_i915_private *dev_priv = to_i915(dev);
2019         struct drm_i915_gem_object *obj = ringbuf->obj;
2020         int ret;
2021
2022         if (HAS_LLC(dev_priv) && !obj->stolen) {
2023                 ret = i915_gem_obj_ggtt_pin(obj, PAGE_SIZE, 0);
2024                 if (ret)
2025                         return ret;
2026
2027                 ret = i915_gem_object_set_to_cpu_domain(obj, true);
2028                 if (ret) {
2029                         i915_gem_object_ggtt_unpin(obj);
2030                         return ret;
2031                 }
2032
2033                 ringbuf->virtual_start = vmap_obj(obj);
2034                 if (ringbuf->virtual_start == NULL) {
2035                         i915_gem_object_ggtt_unpin(obj);
2036                         return -ENOMEM;
2037                 }
2038         } else {
2039                 ret = i915_gem_obj_ggtt_pin(obj, PAGE_SIZE, PIN_MAPPABLE);
2040                 if (ret)
2041                         return ret;
2042
2043                 ret = i915_gem_object_set_to_gtt_domain(obj, true);
2044                 if (ret) {
2045                         i915_gem_object_ggtt_unpin(obj);
2046                         return ret;
2047                 }
2048
2049                 ringbuf->virtual_start = ioremap_wc(dev_priv->gtt.mappable_base +
2050                                                     i915_gem_obj_ggtt_offset(obj), ringbuf->size);
2051                 if (ringbuf->virtual_start == NULL) {
2052                         i915_gem_object_ggtt_unpin(obj);
2053                         return -EINVAL;
2054                 }
2055         }
2056
2057         return 0;
2058 }
2059
2060 static void intel_destroy_ringbuffer_obj(struct intel_ringbuffer *ringbuf)
2061 {
2062         drm_gem_object_unreference(&ringbuf->obj->base);
2063         ringbuf->obj = NULL;
2064 }
2065
2066 static int intel_alloc_ringbuffer_obj(struct drm_device *dev,
2067                                       struct intel_ringbuffer *ringbuf)
2068 {
2069         struct drm_i915_gem_object *obj;
2070
2071         obj = NULL;
2072         if (!HAS_LLC(dev))
2073                 obj = i915_gem_object_create_stolen(dev, ringbuf->size);
2074         if (obj == NULL)
2075                 obj = i915_gem_alloc_object(dev, ringbuf->size);
2076         if (obj == NULL)
2077                 return -ENOMEM;
2078
2079         /* mark ring buffers as read-only from GPU side by default */
2080         obj->gt_ro = 1;
2081
2082         ringbuf->obj = obj;
2083
2084         return 0;
2085 }
2086
2087 struct intel_ringbuffer *
2088 intel_engine_create_ringbuffer(struct intel_engine_cs *engine, int size)
2089 {
2090         struct intel_ringbuffer *ring;
2091         int ret;
2092
2093         ring = kzalloc(sizeof(*ring), GFP_KERNEL);
2094         if (ring == NULL) {
2095                 DRM_DEBUG_DRIVER("Failed to allocate ringbuffer %s\n",
2096                                  engine->name);
2097                 return ERR_PTR(-ENOMEM);
2098         }
2099
2100         ring->ring = engine;
2101         list_add(&ring->link, &engine->buffers);
2102
2103         ring->size = size;
2104         /* Workaround an erratum on the i830 which causes a hang if
2105          * the TAIL pointer points to within the last 2 cachelines
2106          * of the buffer.
2107          */
2108         ring->effective_size = size;
2109         if (IS_I830(engine->dev) || IS_845G(engine->dev))
2110                 ring->effective_size -= 2 * CACHELINE_BYTES;
2111
2112         ring->last_retired_head = -1;
2113         intel_ring_update_space(ring);
2114
2115         ret = intel_alloc_ringbuffer_obj(engine->dev, ring);
2116         if (ret) {
2117                 DRM_DEBUG_DRIVER("Failed to allocate ringbuffer %s: %d\n",
2118                                  engine->name, ret);
2119                 list_del(&ring->link);
2120                 kfree(ring);
2121                 return ERR_PTR(ret);
2122         }
2123
2124         return ring;
2125 }
2126
2127 void
2128 intel_ringbuffer_free(struct intel_ringbuffer *ring)
2129 {
2130         intel_destroy_ringbuffer_obj(ring);
2131         list_del(&ring->link);
2132         kfree(ring);
2133 }
2134
2135 static int intel_init_ring_buffer(struct drm_device *dev,
2136                                   struct intel_engine_cs *ring)
2137 {
2138         struct intel_ringbuffer *ringbuf;
2139         int ret;
2140
2141         WARN_ON(ring->buffer);
2142
2143         ring->dev = dev;
2144         INIT_LIST_HEAD(&ring->active_list);
2145         INIT_LIST_HEAD(&ring->request_list);
2146         INIT_LIST_HEAD(&ring->execlist_queue);
2147         INIT_LIST_HEAD(&ring->buffers);
2148         i915_gem_batch_pool_init(dev, &ring->batch_pool);
2149         memset(ring->semaphore.sync_seqno, 0, sizeof(ring->semaphore.sync_seqno));
2150
2151         init_waitqueue_head(&ring->irq_queue);
2152
2153         ringbuf = intel_engine_create_ringbuffer(ring, 32 * PAGE_SIZE);
2154         if (IS_ERR(ringbuf)) {
2155                 ret = PTR_ERR(ringbuf);
2156                 goto error;
2157         }
2158         ring->buffer = ringbuf;
2159
2160         if (I915_NEED_GFX_HWS(dev)) {
2161                 ret = init_status_page(ring);
2162                 if (ret)
2163                         goto error;
2164         } else {
2165                 BUG_ON(ring->id != RCS);
2166                 ret = init_phys_status_page(ring);
2167                 if (ret)
2168                         goto error;
2169         }
2170
2171         ret = intel_pin_and_map_ringbuffer_obj(dev, ringbuf);
2172         if (ret) {
2173                 DRM_ERROR("Failed to pin and map ringbuffer %s: %d\n",
2174                                 ring->name, ret);
2175                 intel_destroy_ringbuffer_obj(ringbuf);
2176                 goto error;
2177         }
2178
2179         ret = i915_cmd_parser_init_ring(ring);
2180         if (ret)
2181                 goto error;
2182
2183         return 0;
2184
2185 error:
2186         intel_cleanup_ring_buffer(ring);
2187         return ret;
2188 }
2189
2190 void intel_cleanup_ring_buffer(struct intel_engine_cs *ring)
2191 {
2192         struct drm_i915_private *dev_priv;
2193
2194         if (!intel_ring_initialized(ring))
2195                 return;
2196
2197         dev_priv = to_i915(ring->dev);
2198
2199         if (ring->buffer) {
2200                 intel_stop_ring_buffer(ring);
2201                 WARN_ON(!IS_GEN2(ring->dev) && (I915_READ_MODE(ring) & MODE_IDLE) == 0);
2202
2203                 intel_unpin_ringbuffer_obj(ring->buffer);
2204                 intel_ringbuffer_free(ring->buffer);
2205                 ring->buffer = NULL;
2206         }
2207
2208         if (ring->cleanup)
2209                 ring->cleanup(ring);
2210
2211         cleanup_status_page(ring);
2212
2213         i915_cmd_parser_fini_ring(ring);
2214         i915_gem_batch_pool_fini(&ring->batch_pool);
2215         ring->dev = NULL;
2216 }
2217
2218 static int ring_wait_for_space(struct intel_engine_cs *ring, int n)
2219 {
2220         struct intel_ringbuffer *ringbuf = ring->buffer;
2221         struct drm_i915_gem_request *request;
2222         unsigned space;
2223         int ret;
2224
2225         if (intel_ring_space(ringbuf) >= n)
2226                 return 0;
2227
2228         /* The whole point of reserving space is to not wait! */
2229         WARN_ON(ringbuf->reserved_in_use);
2230
2231         list_for_each_entry(request, &ring->request_list, list) {
2232                 space = __intel_ring_space(request->postfix, ringbuf->tail,
2233                                            ringbuf->size);
2234                 if (space >= n)
2235                         break;
2236         }
2237
2238         if (WARN_ON(&request->list == &ring->request_list))
2239                 return -ENOSPC;
2240
2241         ret = i915_wait_request(request);
2242         if (ret)
2243                 return ret;
2244
2245         ringbuf->space = space;
2246         return 0;
2247 }
2248
2249 static void __wrap_ring_buffer(struct intel_ringbuffer *ringbuf)
2250 {
2251         uint32_t __iomem *virt;
2252         int rem = ringbuf->size - ringbuf->tail;
2253
2254         virt = ringbuf->virtual_start + ringbuf->tail;
2255         rem /= 4;
2256         while (rem--)
2257                 iowrite32(MI_NOOP, virt++);
2258
2259         ringbuf->tail = 0;
2260         intel_ring_update_space(ringbuf);
2261 }
2262
2263 int intel_ring_idle(struct intel_engine_cs *ring)
2264 {
2265         struct drm_i915_gem_request *req;
2266
2267         /* Wait upon the last request to be completed */
2268         if (list_empty(&ring->request_list))
2269                 return 0;
2270
2271         req = list_entry(ring->request_list.prev,
2272                         struct drm_i915_gem_request,
2273                         list);
2274
2275         /* Make sure we do not trigger any retires */
2276         return __i915_wait_request(req,
2277                                    atomic_read(&to_i915(ring->dev)->gpu_error.reset_counter),
2278                                    to_i915(ring->dev)->mm.interruptible,
2279                                    NULL, NULL);
2280 }
2281
2282 int intel_ring_alloc_request_extras(struct drm_i915_gem_request *request)
2283 {
2284         request->ringbuf = request->ring->buffer;
2285         return 0;
2286 }
2287
2288 int intel_ring_reserve_space(struct drm_i915_gem_request *request)
2289 {
2290         /*
2291          * The first call merely notes the reserve request and is common for
2292          * all back ends. The subsequent localised _begin() call actually
2293          * ensures that the reservation is available. Without the begin, if
2294          * the request creator immediately submitted the request without
2295          * adding any commands to it then there might not actually be
2296          * sufficient room for the submission commands.
2297          */
2298         intel_ring_reserved_space_reserve(request->ringbuf, MIN_SPACE_FOR_ADD_REQUEST);
2299
2300         return intel_ring_begin(request, 0);
2301 }
2302
2303 void intel_ring_reserved_space_reserve(struct intel_ringbuffer *ringbuf, int size)
2304 {
2305         WARN_ON(ringbuf->reserved_size);
2306         WARN_ON(ringbuf->reserved_in_use);
2307
2308         ringbuf->reserved_size = size;
2309 }
2310
2311 void intel_ring_reserved_space_cancel(struct intel_ringbuffer *ringbuf)
2312 {
2313         WARN_ON(ringbuf->reserved_in_use);
2314
2315         ringbuf->reserved_size   = 0;
2316         ringbuf->reserved_in_use = false;
2317 }
2318
2319 void intel_ring_reserved_space_use(struct intel_ringbuffer *ringbuf)
2320 {
2321         WARN_ON(ringbuf->reserved_in_use);
2322
2323         ringbuf->reserved_in_use = true;
2324         ringbuf->reserved_tail   = ringbuf->tail;
2325 }
2326
2327 void intel_ring_reserved_space_end(struct intel_ringbuffer *ringbuf)
2328 {
2329         WARN_ON(!ringbuf->reserved_in_use);
2330         if (ringbuf->tail > ringbuf->reserved_tail) {
2331                 WARN(ringbuf->tail > ringbuf->reserved_tail + ringbuf->reserved_size,
2332                      "request reserved size too small: %d vs %d!\n",
2333                      ringbuf->tail - ringbuf->reserved_tail, ringbuf->reserved_size);
2334         } else {
2335                 /*
2336                  * The ring was wrapped while the reserved space was in use.
2337                  * That means that some unknown amount of the ring tail was
2338                  * no-op filled and skipped. Thus simply adding the ring size
2339                  * to the tail and doing the above space check will not work.
2340                  * Rather than attempt to track how much tail was skipped,
2341                  * it is much simpler to say that also skipping the sanity
2342                  * check every once in a while is not a big issue.
2343                  */
2344         }
2345
2346         ringbuf->reserved_size   = 0;
2347         ringbuf->reserved_in_use = false;
2348 }
2349
2350 static int __intel_ring_prepare(struct intel_engine_cs *ring, int bytes)
2351 {
2352         struct intel_ringbuffer *ringbuf = ring->buffer;
2353         int remain_usable = ringbuf->effective_size - ringbuf->tail;
2354         int remain_actual = ringbuf->size - ringbuf->tail;
2355         int ret, total_bytes, wait_bytes = 0;
2356         bool need_wrap = false;
2357
2358         if (ringbuf->reserved_in_use)
2359                 total_bytes = bytes;
2360         else
2361                 total_bytes = bytes + ringbuf->reserved_size;
2362
2363         if (unlikely(bytes > remain_usable)) {
2364                 /*
2365                  * Not enough space for the basic request. So need to flush
2366                  * out the remainder and then wait for base + reserved.
2367                  */
2368                 wait_bytes = remain_actual + total_bytes;
2369                 need_wrap = true;
2370         } else {
2371                 if (unlikely(total_bytes > remain_usable)) {
2372                         /*
2373                          * The base request will fit but the reserved space
2374                          * falls off the end. So only need to to wait for the
2375                          * reserved size after flushing out the remainder.
2376                          */
2377                         wait_bytes = remain_actual + ringbuf->reserved_size;
2378                         need_wrap = true;
2379                 } else if (total_bytes > ringbuf->space) {
2380                         /* No wrapping required, just waiting. */
2381                         wait_bytes = total_bytes;
2382                 }
2383         }
2384
2385         if (wait_bytes) {
2386                 ret = ring_wait_for_space(ring, wait_bytes);
2387                 if (unlikely(ret))
2388                         return ret;
2389
2390                 if (need_wrap)
2391                         __wrap_ring_buffer(ringbuf);
2392         }
2393
2394         return 0;
2395 }
2396
2397 int intel_ring_begin(struct drm_i915_gem_request *req,
2398                      int num_dwords)
2399 {
2400         struct intel_engine_cs *ring;
2401         struct drm_i915_private *dev_priv;
2402         int ret;
2403
2404         WARN_ON(req == NULL);
2405         ring = req->ring;
2406         dev_priv = ring->dev->dev_private;
2407
2408         ret = i915_gem_check_wedge(&dev_priv->gpu_error,
2409                                    dev_priv->mm.interruptible);
2410         if (ret)
2411                 return ret;
2412
2413         ret = __intel_ring_prepare(ring, num_dwords * sizeof(uint32_t));
2414         if (ret)
2415                 return ret;
2416
2417         ring->buffer->space -= num_dwords * sizeof(uint32_t);
2418         return 0;
2419 }
2420
2421 /* Align the ring tail to a cacheline boundary */
2422 int intel_ring_cacheline_align(struct drm_i915_gem_request *req)
2423 {
2424         struct intel_engine_cs *ring = req->ring;
2425         int num_dwords = (ring->buffer->tail & (CACHELINE_BYTES - 1)) / sizeof(uint32_t);
2426         int ret;
2427
2428         if (num_dwords == 0)
2429                 return 0;
2430
2431         num_dwords = CACHELINE_BYTES / sizeof(uint32_t) - num_dwords;
2432         ret = intel_ring_begin(req, num_dwords);
2433         if (ret)
2434                 return ret;
2435
2436         while (num_dwords--)
2437                 intel_ring_emit(ring, MI_NOOP);
2438
2439         intel_ring_advance(ring);
2440
2441         return 0;
2442 }
2443
2444 void intel_ring_init_seqno(struct intel_engine_cs *ring, u32 seqno)
2445 {
2446         struct drm_device *dev = ring->dev;
2447         struct drm_i915_private *dev_priv = dev->dev_private;
2448
2449         if (INTEL_INFO(dev)->gen == 6 || INTEL_INFO(dev)->gen == 7) {
2450                 I915_WRITE(RING_SYNC_0(ring->mmio_base), 0);
2451                 I915_WRITE(RING_SYNC_1(ring->mmio_base), 0);
2452                 if (HAS_VEBOX(dev))
2453                         I915_WRITE(RING_SYNC_2(ring->mmio_base), 0);
2454         }
2455
2456         ring->set_seqno(ring, seqno);
2457         ring->hangcheck.seqno = seqno;
2458 }
2459
2460 static void gen6_bsd_ring_write_tail(struct intel_engine_cs *ring,
2461                                      u32 value)
2462 {
2463         struct drm_i915_private *dev_priv = ring->dev->dev_private;
2464
2465        /* Every tail move must follow the sequence below */
2466
2467         /* Disable notification that the ring is IDLE. The GT
2468          * will then assume that it is busy and bring it out of rc6.
2469          */
2470         I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
2471                    _MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
2472
2473         /* Clear the context id. Here be magic! */
2474         I915_WRITE64(GEN6_BSD_RNCID, 0x0);
2475
2476         /* Wait for the ring not to be idle, i.e. for it to wake up. */
2477         if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
2478                       GEN6_BSD_SLEEP_INDICATOR) == 0,
2479                      50))
2480                 DRM_ERROR("timed out waiting for the BSD ring to wake up\n");
2481
2482         /* Now that the ring is fully powered up, update the tail */
2483         I915_WRITE_TAIL(ring, value);
2484         POSTING_READ(RING_TAIL(ring->mmio_base));
2485
2486         /* Let the ring send IDLE messages to the GT again,
2487          * and so let it sleep to conserve power when idle.
2488          */
2489         I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
2490                    _MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
2491 }
2492
2493 static int gen6_bsd_ring_flush(struct drm_i915_gem_request *req,
2494                                u32 invalidate, u32 flush)
2495 {
2496         struct intel_engine_cs *ring = req->ring;
2497         uint32_t cmd;
2498         int ret;
2499
2500         ret = intel_ring_begin(req, 4);
2501         if (ret)
2502                 return ret;
2503
2504         cmd = MI_FLUSH_DW;
2505         if (INTEL_INFO(ring->dev)->gen >= 8)
2506                 cmd += 1;
2507
2508         /* We always require a command barrier so that subsequent
2509          * commands, such as breadcrumb interrupts, are strictly ordered
2510          * wrt the contents of the write cache being flushed to memory
2511          * (and thus being coherent from the CPU).
2512          */
2513         cmd |= MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW;
2514
2515         /*
2516          * Bspec vol 1c.5 - video engine command streamer:
2517          * "If ENABLED, all TLBs will be invalidated once the flush
2518          * operation is complete. This bit is only valid when the
2519          * Post-Sync Operation field is a value of 1h or 3h."
2520          */
2521         if (invalidate & I915_GEM_GPU_DOMAINS)
2522                 cmd |= MI_INVALIDATE_TLB | MI_INVALIDATE_BSD;
2523
2524         intel_ring_emit(ring, cmd);
2525         intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT);
2526         if (INTEL_INFO(ring->dev)->gen >= 8) {
2527                 intel_ring_emit(ring, 0); /* upper addr */
2528                 intel_ring_emit(ring, 0); /* value */
2529         } else  {
2530                 intel_ring_emit(ring, 0);
2531                 intel_ring_emit(ring, MI_NOOP);
2532         }
2533         intel_ring_advance(ring);
2534         return 0;
2535 }
2536
2537 static int
2538 gen8_ring_dispatch_execbuffer(struct drm_i915_gem_request *req,
2539                               u64 offset, u32 len,
2540                               unsigned dispatch_flags)
2541 {
2542         struct intel_engine_cs *ring = req->ring;
2543         bool ppgtt = USES_PPGTT(ring->dev) &&
2544                         !(dispatch_flags & I915_DISPATCH_SECURE);
2545         int ret;
2546
2547         ret = intel_ring_begin(req, 4);
2548         if (ret)
2549                 return ret;
2550
2551         /* FIXME(BDW): Address space and security selectors. */
2552         intel_ring_emit(ring, MI_BATCH_BUFFER_START_GEN8 | (ppgtt<<8) |
2553                         (dispatch_flags & I915_DISPATCH_RS ?
2554                          MI_BATCH_RESOURCE_STREAMER : 0));
2555         intel_ring_emit(ring, lower_32_bits(offset));
2556         intel_ring_emit(ring, upper_32_bits(offset));
2557         intel_ring_emit(ring, MI_NOOP);
2558         intel_ring_advance(ring);
2559
2560         return 0;
2561 }
2562
2563 static int
2564 hsw_ring_dispatch_execbuffer(struct drm_i915_gem_request *req,
2565                              u64 offset, u32 len,
2566                              unsigned dispatch_flags)
2567 {
2568         struct intel_engine_cs *ring = req->ring;
2569         int ret;
2570
2571         ret = intel_ring_begin(req, 2);
2572         if (ret)
2573                 return ret;
2574
2575         intel_ring_emit(ring,
2576                         MI_BATCH_BUFFER_START |
2577                         (dispatch_flags & I915_DISPATCH_SECURE ?
2578                          0 : MI_BATCH_PPGTT_HSW | MI_BATCH_NON_SECURE_HSW) |
2579                         (dispatch_flags & I915_DISPATCH_RS ?
2580                          MI_BATCH_RESOURCE_STREAMER : 0));
2581         /* bit0-7 is the length on GEN6+ */
2582         intel_ring_emit(ring, offset);
2583         intel_ring_advance(ring);
2584
2585         return 0;
2586 }
2587
2588 static int
2589 gen6_ring_dispatch_execbuffer(struct drm_i915_gem_request *req,
2590                               u64 offset, u32 len,
2591                               unsigned dispatch_flags)
2592 {
2593         struct intel_engine_cs *ring = req->ring;
2594         int ret;
2595
2596         ret = intel_ring_begin(req, 2);
2597         if (ret)
2598                 return ret;
2599
2600         intel_ring_emit(ring,
2601                         MI_BATCH_BUFFER_START |
2602                         (dispatch_flags & I915_DISPATCH_SECURE ?
2603                          0 : MI_BATCH_NON_SECURE_I965));
2604         /* bit0-7 is the length on GEN6+ */
2605         intel_ring_emit(ring, offset);
2606         intel_ring_advance(ring);
2607
2608         return 0;
2609 }
2610
2611 /* Blitter support (SandyBridge+) */
2612
2613 static int gen6_ring_flush(struct drm_i915_gem_request *req,
2614                            u32 invalidate, u32 flush)
2615 {
2616         struct intel_engine_cs *ring = req->ring;
2617         struct drm_device *dev = ring->dev;
2618         uint32_t cmd;
2619         int ret;
2620
2621         ret = intel_ring_begin(req, 4);
2622         if (ret)
2623                 return ret;
2624
2625         cmd = MI_FLUSH_DW;
2626         if (INTEL_INFO(dev)->gen >= 8)
2627                 cmd += 1;
2628
2629         /* We always require a command barrier so that subsequent
2630          * commands, such as breadcrumb interrupts, are strictly ordered
2631          * wrt the contents of the write cache being flushed to memory
2632          * (and thus being coherent from the CPU).
2633          */
2634         cmd |= MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW;
2635
2636         /*
2637          * Bspec vol 1c.3 - blitter engine command streamer:
2638          * "If ENABLED, all TLBs will be invalidated once the flush
2639          * operation is complete. This bit is only valid when the
2640          * Post-Sync Operation field is a value of 1h or 3h."
2641          */
2642         if (invalidate & I915_GEM_DOMAIN_RENDER)
2643                 cmd |= MI_INVALIDATE_TLB;
2644         intel_ring_emit(ring, cmd);
2645         intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT);
2646         if (INTEL_INFO(dev)->gen >= 8) {
2647                 intel_ring_emit(ring, 0); /* upper addr */
2648                 intel_ring_emit(ring, 0); /* value */
2649         } else  {
2650                 intel_ring_emit(ring, 0);
2651                 intel_ring_emit(ring, MI_NOOP);
2652         }
2653         intel_ring_advance(ring);
2654
2655         return 0;
2656 }
2657
2658 int intel_init_render_ring_buffer(struct drm_device *dev)
2659 {
2660         struct drm_i915_private *dev_priv = dev->dev_private;
2661         struct intel_engine_cs *ring = &dev_priv->ring[RCS];
2662         struct drm_i915_gem_object *obj;
2663         int ret;
2664
2665         ring->name = "render ring";
2666         ring->id = RCS;
2667         ring->mmio_base = RENDER_RING_BASE;
2668
2669         if (INTEL_INFO(dev)->gen >= 8) {
2670                 if (i915_semaphore_is_enabled(dev)) {
2671                         obj = i915_gem_alloc_object(dev, 4096);
2672                         if (obj == NULL) {
2673                                 DRM_ERROR("Failed to allocate semaphore bo. Disabling semaphores\n");
2674                                 i915.semaphores = 0;
2675                         } else {
2676                                 i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
2677                                 ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_NONBLOCK);
2678                                 if (ret != 0) {
2679                                         drm_gem_object_unreference(&obj->base);
2680                                         DRM_ERROR("Failed to pin semaphore bo. Disabling semaphores\n");
2681                                         i915.semaphores = 0;
2682                                 } else
2683                                         dev_priv->semaphore_obj = obj;
2684                         }
2685                 }
2686
2687                 ring->init_context = intel_rcs_ctx_init;
2688                 ring->add_request = gen6_add_request;
2689                 ring->flush = gen8_render_ring_flush;
2690                 ring->irq_get = gen8_ring_get_irq;
2691                 ring->irq_put = gen8_ring_put_irq;
2692                 ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
2693                 ring->get_seqno = gen6_ring_get_seqno;
2694                 ring->set_seqno = ring_set_seqno;
2695                 if (i915_semaphore_is_enabled(dev)) {
2696                         WARN_ON(!dev_priv->semaphore_obj);
2697                         ring->semaphore.sync_to = gen8_ring_sync;
2698                         ring->semaphore.signal = gen8_rcs_signal;
2699                         GEN8_RING_SEMAPHORE_INIT;
2700                 }
2701         } else if (INTEL_INFO(dev)->gen >= 6) {
2702                 ring->init_context = intel_rcs_ctx_init;
2703                 ring->add_request = gen6_add_request;
2704                 ring->flush = gen7_render_ring_flush;
2705                 if (INTEL_INFO(dev)->gen == 6)
2706                         ring->flush = gen6_render_ring_flush;
2707                 ring->irq_get = gen6_ring_get_irq;
2708                 ring->irq_put = gen6_ring_put_irq;
2709                 ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
2710                 ring->get_seqno = gen6_ring_get_seqno;
2711                 ring->set_seqno = ring_set_seqno;
2712                 if (i915_semaphore_is_enabled(dev)) {
2713                         ring->semaphore.sync_to = gen6_ring_sync;
2714                         ring->semaphore.signal = gen6_signal;
2715                         /*
2716                          * The current semaphore is only applied on pre-gen8
2717                          * platform.  And there is no VCS2 ring on the pre-gen8
2718                          * platform. So the semaphore between RCS and VCS2 is
2719                          * initialized as INVALID.  Gen8 will initialize the
2720                          * sema between VCS2 and RCS later.
2721                          */
2722                         ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_INVALID;
2723                         ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_RV;
2724                         ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_RB;
2725                         ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_RVE;
2726                         ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
2727                         ring->semaphore.mbox.signal[RCS] = GEN6_NOSYNC;
2728                         ring->semaphore.mbox.signal[VCS] = GEN6_VRSYNC;
2729                         ring->semaphore.mbox.signal[BCS] = GEN6_BRSYNC;
2730                         ring->semaphore.mbox.signal[VECS] = GEN6_VERSYNC;
2731                         ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
2732                 }
2733         } else if (IS_GEN5(dev)) {
2734                 ring->add_request = pc_render_add_request;
2735                 ring->flush = gen4_render_ring_flush;
2736                 ring->get_seqno = pc_render_get_seqno;
2737                 ring->set_seqno = pc_render_set_seqno;
2738                 ring->irq_get = gen5_ring_get_irq;
2739                 ring->irq_put = gen5_ring_put_irq;
2740                 ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT |
2741                                         GT_RENDER_PIPECTL_NOTIFY_INTERRUPT;
2742         } else {
2743                 ring->add_request = i9xx_add_request;
2744                 if (INTEL_INFO(dev)->gen < 4)
2745                         ring->flush = gen2_render_ring_flush;
2746                 else
2747                         ring->flush = gen4_render_ring_flush;
2748                 ring->get_seqno = ring_get_seqno;
2749                 ring->set_seqno = ring_set_seqno;
2750                 if (IS_GEN2(dev)) {
2751                         ring->irq_get = i8xx_ring_get_irq;
2752                         ring->irq_put = i8xx_ring_put_irq;
2753                 } else {
2754                         ring->irq_get = i9xx_ring_get_irq;
2755                         ring->irq_put = i9xx_ring_put_irq;
2756                 }
2757                 ring->irq_enable_mask = I915_USER_INTERRUPT;
2758         }
2759         ring->write_tail = ring_write_tail;
2760
2761         if (IS_HASWELL(dev))
2762                 ring->dispatch_execbuffer = hsw_ring_dispatch_execbuffer;
2763         else if (IS_GEN8(dev))
2764                 ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
2765         else if (INTEL_INFO(dev)->gen >= 6)
2766                 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
2767         else if (INTEL_INFO(dev)->gen >= 4)
2768                 ring->dispatch_execbuffer = i965_dispatch_execbuffer;
2769         else if (IS_I830(dev) || IS_845G(dev))
2770                 ring->dispatch_execbuffer = i830_dispatch_execbuffer;
2771         else
2772                 ring->dispatch_execbuffer = i915_dispatch_execbuffer;
2773         ring->init_hw = init_render_ring;
2774         ring->cleanup = render_ring_cleanup;
2775
2776         /* Workaround batchbuffer to combat CS tlb bug. */
2777         if (HAS_BROKEN_CS_TLB(dev)) {
2778                 obj = i915_gem_alloc_object(dev, I830_WA_SIZE);
2779                 if (obj == NULL) {
2780                         DRM_ERROR("Failed to allocate batch bo\n");
2781                         return -ENOMEM;
2782                 }
2783
2784                 ret = i915_gem_obj_ggtt_pin(obj, 0, 0);
2785                 if (ret != 0) {
2786                         drm_gem_object_unreference(&obj->base);
2787                         DRM_ERROR("Failed to ping batch bo\n");
2788                         return ret;
2789                 }
2790
2791                 ring->scratch.obj = obj;
2792                 ring->scratch.gtt_offset = i915_gem_obj_ggtt_offset(obj);
2793         }
2794
2795         ret = intel_init_ring_buffer(dev, ring);
2796         if (ret)
2797                 return ret;
2798
2799         if (INTEL_INFO(dev)->gen >= 5) {
2800                 ret = intel_init_pipe_control(ring);
2801                 if (ret)
2802                         return ret;
2803         }
2804
2805         return 0;
2806 }
2807
2808 int intel_init_bsd_ring_buffer(struct drm_device *dev)
2809 {
2810         struct drm_i915_private *dev_priv = dev->dev_private;
2811         struct intel_engine_cs *ring = &dev_priv->ring[VCS];
2812
2813         ring->name = "bsd ring";
2814         ring->id = VCS;
2815
2816         ring->write_tail = ring_write_tail;
2817         if (INTEL_INFO(dev)->gen >= 6) {
2818                 ring->mmio_base = GEN6_BSD_RING_BASE;
2819                 /* gen6 bsd needs a special wa for tail updates */
2820                 if (IS_GEN6(dev))
2821                         ring->write_tail = gen6_bsd_ring_write_tail;
2822                 ring->flush = gen6_bsd_ring_flush;
2823                 ring->add_request = gen6_add_request;
2824                 ring->get_seqno = gen6_ring_get_seqno;
2825                 ring->set_seqno = ring_set_seqno;
2826                 if (INTEL_INFO(dev)->gen >= 8) {
2827                         ring->irq_enable_mask =
2828                                 GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT;
2829                         ring->irq_get = gen8_ring_get_irq;
2830                         ring->irq_put = gen8_ring_put_irq;
2831                         ring->dispatch_execbuffer =
2832                                 gen8_ring_dispatch_execbuffer;
2833                         if (i915_semaphore_is_enabled(dev)) {
2834                                 ring->semaphore.sync_to = gen8_ring_sync;
2835                                 ring->semaphore.signal = gen8_xcs_signal;
2836                                 GEN8_RING_SEMAPHORE_INIT;
2837                         }
2838                 } else {
2839                         ring->irq_enable_mask = GT_BSD_USER_INTERRUPT;
2840                         ring->irq_get = gen6_ring_get_irq;
2841                         ring->irq_put = gen6_ring_put_irq;
2842                         ring->dispatch_execbuffer =
2843                                 gen6_ring_dispatch_execbuffer;
2844                         if (i915_semaphore_is_enabled(dev)) {
2845                                 ring->semaphore.sync_to = gen6_ring_sync;
2846                                 ring->semaphore.signal = gen6_signal;
2847                                 ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VR;
2848                                 ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_INVALID;
2849                                 ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VB;
2850                                 ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_VVE;
2851                                 ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
2852                                 ring->semaphore.mbox.signal[RCS] = GEN6_RVSYNC;
2853                                 ring->semaphore.mbox.signal[VCS] = GEN6_NOSYNC;
2854                                 ring->semaphore.mbox.signal[BCS] = GEN6_BVSYNC;
2855                                 ring->semaphore.mbox.signal[VECS] = GEN6_VEVSYNC;
2856                                 ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
2857                         }
2858                 }
2859         } else {
2860                 ring->mmio_base = BSD_RING_BASE;
2861                 ring->flush = bsd_ring_flush;
2862                 ring->add_request = i9xx_add_request;
2863                 ring->get_seqno = ring_get_seqno;
2864                 ring->set_seqno = ring_set_seqno;
2865                 if (IS_GEN5(dev)) {
2866                         ring->irq_enable_mask = ILK_BSD_USER_INTERRUPT;
2867                         ring->irq_get = gen5_ring_get_irq;
2868                         ring->irq_put = gen5_ring_put_irq;
2869                 } else {
2870                         ring->irq_enable_mask = I915_BSD_USER_INTERRUPT;
2871                         ring->irq_get = i9xx_ring_get_irq;
2872                         ring->irq_put = i9xx_ring_put_irq;
2873                 }
2874                 ring->dispatch_execbuffer = i965_dispatch_execbuffer;
2875         }
2876         ring->init_hw = init_ring_common;
2877
2878         return intel_init_ring_buffer(dev, ring);
2879 }
2880
2881 /**
2882  * Initialize the second BSD ring (eg. Broadwell GT3, Skylake GT3)
2883  */
2884 int intel_init_bsd2_ring_buffer(struct drm_device *dev)
2885 {
2886         struct drm_i915_private *dev_priv = dev->dev_private;
2887         struct intel_engine_cs *ring = &dev_priv->ring[VCS2];
2888
2889         ring->name = "bsd2 ring";
2890         ring->id = VCS2;
2891
2892         ring->write_tail = ring_write_tail;
2893         ring->mmio_base = GEN8_BSD2_RING_BASE;
2894         ring->flush = gen6_bsd_ring_flush;
2895         ring->add_request = gen6_add_request;
2896         ring->get_seqno = gen6_ring_get_seqno;
2897         ring->set_seqno = ring_set_seqno;
2898         ring->irq_enable_mask =
2899                         GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT;
2900         ring->irq_get = gen8_ring_get_irq;
2901         ring->irq_put = gen8_ring_put_irq;
2902         ring->dispatch_execbuffer =
2903                         gen8_ring_dispatch_execbuffer;
2904         if (i915_semaphore_is_enabled(dev)) {
2905                 ring->semaphore.sync_to = gen8_ring_sync;
2906                 ring->semaphore.signal = gen8_xcs_signal;
2907                 GEN8_RING_SEMAPHORE_INIT;
2908         }
2909         ring->init_hw = init_ring_common;
2910
2911         return intel_init_ring_buffer(dev, ring);
2912 }
2913
2914 int intel_init_blt_ring_buffer(struct drm_device *dev)
2915 {
2916         struct drm_i915_private *dev_priv = dev->dev_private;
2917         struct intel_engine_cs *ring = &dev_priv->ring[BCS];
2918
2919         ring->name = "blitter ring";
2920         ring->id = BCS;
2921
2922         ring->mmio_base = BLT_RING_BASE;
2923         ring->write_tail = ring_write_tail;
2924         ring->flush = gen6_ring_flush;
2925         ring->add_request = gen6_add_request;
2926         ring->get_seqno = gen6_ring_get_seqno;
2927         ring->set_seqno = ring_set_seqno;
2928         if (INTEL_INFO(dev)->gen >= 8) {
2929                 ring->irq_enable_mask =
2930                         GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT;
2931                 ring->irq_get = gen8_ring_get_irq;
2932                 ring->irq_put = gen8_ring_put_irq;
2933                 ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
2934                 if (i915_semaphore_is_enabled(dev)) {
2935                         ring->semaphore.sync_to = gen8_ring_sync;
2936                         ring->semaphore.signal = gen8_xcs_signal;
2937                         GEN8_RING_SEMAPHORE_INIT;
2938                 }
2939         } else {
2940                 ring->irq_enable_mask = GT_BLT_USER_INTERRUPT;
2941                 ring->irq_get = gen6_ring_get_irq;
2942                 ring->irq_put = gen6_ring_put_irq;
2943                 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
2944                 if (i915_semaphore_is_enabled(dev)) {
2945                         ring->semaphore.signal = gen6_signal;
2946                         ring->semaphore.sync_to = gen6_ring_sync;
2947                         /*
2948                          * The current semaphore is only applied on pre-gen8
2949                          * platform.  And there is no VCS2 ring on the pre-gen8
2950                          * platform. So the semaphore between BCS and VCS2 is
2951                          * initialized as INVALID.  Gen8 will initialize the
2952                          * sema between BCS and VCS2 later.
2953                          */
2954                         ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_BR;
2955                         ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_BV;
2956                         ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_INVALID;
2957                         ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_BVE;
2958                         ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
2959                         ring->semaphore.mbox.signal[RCS] = GEN6_RBSYNC;
2960                         ring->semaphore.mbox.signal[VCS] = GEN6_VBSYNC;
2961                         ring->semaphore.mbox.signal[BCS] = GEN6_NOSYNC;
2962                         ring->semaphore.mbox.signal[VECS] = GEN6_VEBSYNC;
2963                         ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
2964                 }
2965         }
2966         ring->init_hw = init_ring_common;
2967
2968         return intel_init_ring_buffer(dev, ring);
2969 }
2970
2971 int intel_init_vebox_ring_buffer(struct drm_device *dev)
2972 {
2973         struct drm_i915_private *dev_priv = dev->dev_private;
2974         struct intel_engine_cs *ring = &dev_priv->ring[VECS];
2975
2976         ring->name = "video enhancement ring";
2977         ring->id = VECS;
2978
2979         ring->mmio_base = VEBOX_RING_BASE;
2980         ring->write_tail = ring_write_tail;
2981         ring->flush = gen6_ring_flush;
2982         ring->add_request = gen6_add_request;
2983         ring->get_seqno = gen6_ring_get_seqno;
2984         ring->set_seqno = ring_set_seqno;
2985
2986         if (INTEL_INFO(dev)->gen >= 8) {
2987                 ring->irq_enable_mask =
2988                         GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT;
2989                 ring->irq_get = gen8_ring_get_irq;
2990                 ring->irq_put = gen8_ring_put_irq;
2991                 ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
2992                 if (i915_semaphore_is_enabled(dev)) {
2993                         ring->semaphore.sync_to = gen8_ring_sync;
2994                         ring->semaphore.signal = gen8_xcs_signal;
2995                         GEN8_RING_SEMAPHORE_INIT;
2996                 }
2997         } else {
2998                 ring->irq_enable_mask = PM_VEBOX_USER_INTERRUPT;
2999                 ring->irq_get = hsw_vebox_get_irq;
3000                 ring->irq_put = hsw_vebox_put_irq;
3001                 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
3002                 if (i915_semaphore_is_enabled(dev)) {
3003                         ring->semaphore.sync_to = gen6_ring_sync;
3004                         ring->semaphore.signal = gen6_signal;
3005                         ring->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VER;
3006                         ring->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_VEV;
3007                         ring->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VEB;
3008                         ring->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_INVALID;
3009                         ring->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
3010                         ring->semaphore.mbox.signal[RCS] = GEN6_RVESYNC;
3011                         ring->semaphore.mbox.signal[VCS] = GEN6_VVESYNC;
3012                         ring->semaphore.mbox.signal[BCS] = GEN6_BVESYNC;
3013                         ring->semaphore.mbox.signal[VECS] = GEN6_NOSYNC;
3014                         ring->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
3015                 }
3016         }
3017         ring->init_hw = init_ring_common;
3018
3019         return intel_init_ring_buffer(dev, ring);
3020 }
3021
3022 int
3023 intel_ring_flush_all_caches(struct drm_i915_gem_request *req)
3024 {
3025         struct intel_engine_cs *ring = req->ring;
3026         int ret;
3027
3028         if (!ring->gpu_caches_dirty)
3029                 return 0;
3030
3031         ret = ring->flush(req, 0, I915_GEM_GPU_DOMAINS);
3032         if (ret)
3033                 return ret;
3034
3035         trace_i915_gem_ring_flush(req, 0, I915_GEM_GPU_DOMAINS);
3036
3037         ring->gpu_caches_dirty = false;
3038         return 0;
3039 }
3040
3041 int
3042 intel_ring_invalidate_all_caches(struct drm_i915_gem_request *req)
3043 {
3044         struct intel_engine_cs *ring = req->ring;
3045         uint32_t flush_domains;
3046         int ret;
3047
3048         flush_domains = 0;
3049         if (ring->gpu_caches_dirty)
3050                 flush_domains = I915_GEM_GPU_DOMAINS;
3051
3052         ret = ring->flush(req, I915_GEM_GPU_DOMAINS, flush_domains);
3053         if (ret)
3054                 return ret;
3055
3056         trace_i915_gem_ring_flush(req, I915_GEM_GPU_DOMAINS, flush_domains);
3057
3058         ring->gpu_caches_dirty = false;
3059         return 0;
3060 }
3061
3062 void
3063 intel_stop_ring_buffer(struct intel_engine_cs *ring)
3064 {
3065         int ret;
3066
3067         if (!intel_ring_initialized(ring))
3068                 return;
3069
3070         ret = intel_ring_idle(ring);
3071         if (ret && !i915_reset_in_progress(&to_i915(ring->dev)->gpu_error))
3072                 DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n",
3073                           ring->name, ret);
3074
3075         stop_ring(ring);
3076 }