drivers/gpu/drm/i915/intel_ringbuffer.c

   1 /*
   2  * Copyright © 2008-2010 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *    Zou Nan hai <nanhai.zou@intel.com>
  26  *    Xiang Hai hao<haihao.xiang@intel.com>
  27  *
  28  */
  29
  30 #include <linux/log2.h>
  31 #include <drm/drmP.h>
  32 #include "i915_drv.h"
  33 #include <drm/i915_drm.h>
  34 #include "i915_trace.h"
  35 #include "intel_drv.h"
  36
  37 /* Rough estimate of the typical request size, performing a flush,
  38  * set-context and then emitting the batch.
  39  */
  40 #define LEGACY_REQUEST_SIZE 200
  41
  42 int __intel_ring_space(int head, int tail, int size)
  43 {
  44         int space = head - tail;
  45         if (space <= 0)
  46                 space += size;
  47         return space - I915_RING_FREE_SPACE;
  48 }
  49
  50 void intel_ring_update_space(struct intel_ring *ring)
  51 {
  52         if (ring->last_retired_head != -1) {
  53                 ring->head = ring->last_retired_head;
  54                 ring->last_retired_head = -1;
  55         }
  56
  57         ring->space = __intel_ring_space(ring->head & HEAD_ADDR,
  58                                          ring->tail, ring->size);
  59 }
  60
  61 static int
  62 gen2_render_ring_flush(struct drm_i915_gem_request *req, u32 mode)
  63 {
  64         struct intel_ring *ring = req->ring;
  65         u32 cmd;
  66         int ret;
  67
  68         cmd = MI_FLUSH;
  69
  70         if (mode & EMIT_INVALIDATE)
  71                 cmd |= MI_READ_FLUSH;
  72
  73         ret = intel_ring_begin(req, 2);
  74         if (ret)
  75                 return ret;
  76
  77         intel_ring_emit(ring, cmd);
  78         intel_ring_emit(ring, MI_NOOP);
  79         intel_ring_advance(ring);
  80
  81         return 0;
  82 }
  83
  84 static int
  85 gen4_render_ring_flush(struct drm_i915_gem_request *req, u32 mode)
  86 {
  87         struct intel_ring *ring = req->ring;
  88         u32 cmd;
  89         int ret;
  90
  91         /*
  92          * read/write caches:
  93          *
  94          * I915_GEM_DOMAIN_RENDER is always invalidated, but is
  95          * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
  96          * also flushed at 2d versus 3d pipeline switches.
  97          *
  98          * read-only caches:
  99          *
 100          * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
 101          * MI_READ_FLUSH is set, and is always flushed on 965.
 102          *
 103          * I915_GEM_DOMAIN_COMMAND may not exist?
 104          *
 105          * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
 106          * invalidated when MI_EXE_FLUSH is set.
 107          *
 108          * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
 109          * invalidated with every MI_FLUSH.
 110          *
 111          * TLBs:
 112          *
 113          * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
 114          * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
 115          * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
 116          * are flushed at any MI_FLUSH.
 117          */
 118
 119         cmd = MI_FLUSH;
 120         if (mode & EMIT_INVALIDATE) {
 121                 cmd |= MI_EXE_FLUSH;
 122                 if (IS_G4X(req->i915) || IS_GEN5(req->i915))
 123                         cmd |= MI_INVALIDATE_ISP;
 124         }
 125
 126         ret = intel_ring_begin(req, 2);
 127         if (ret)
 128                 return ret;
 129
 130         intel_ring_emit(ring, cmd);
 131         intel_ring_emit(ring, MI_NOOP);
 132         intel_ring_advance(ring);
 133
 134         return 0;
 135 }
 136
 137 /**
 138  * Emits a PIPE_CONTROL with a non-zero post-sync operation, for
 139  * implementing two workarounds on gen6.  From section 1.4.7.1
 140  * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1:
 141  *
 142  * [DevSNB-C+{W/A}] Before any depth stall flush (including those
 143  * produced by non-pipelined state commands), software needs to first
 144  * send a PIPE_CONTROL with no bits set except Post-Sync Operation !=
 145  * 0.
 146  *
 147  * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable
 148  * =1, a PIPE_CONTROL with any non-zero post-sync-op is required.
 149  *
 150  * And the workaround for these two requires this workaround first:
 151  *
 152  * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent
 153  * BEFORE the pipe-control with a post-sync op and no write-cache
 154  * flushes.
 155  *
 156  * And this last workaround is tricky because of the requirements on
 157  * that bit.  From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM
 158  * volume 2 part 1:
 159  *
 160  *     "1 of the following must also be set:
 161  *      - Render Target Cache Flush Enable ([12] of DW1)
 162  *      - Depth Cache Flush Enable ([0] of DW1)
 163  *      - Stall at Pixel Scoreboard ([1] of DW1)
 164  *      - Depth Stall ([13] of DW1)
 165  *      - Post-Sync Operation ([13] of DW1)
 166  *      - Notify Enable ([8] of DW1)"
 167  *
 168  * The cache flushes require the workaround flush that triggered this
 169  * one, so we can't use it.  Depth stall would trigger the same.
 170  * Post-sync nonzero is what triggered this second workaround, so we
 171  * can't use that one either.  Notify enable is IRQs, which aren't
 172  * really our business.  That leaves only stall at scoreboard.
 173  */
 174 static int
 175 intel_emit_post_sync_nonzero_flush(struct drm_i915_gem_request *req)
 176 {
 177         struct intel_ring *ring = req->ring;
 178         u32 scratch_addr =
 179                 i915_ggtt_offset(req->engine->scratch) + 2 * CACHELINE_BYTES;
 180         int ret;
 181
 182         ret = intel_ring_begin(req, 6);
 183         if (ret)
 184                 return ret;
 185
 186         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
 187         intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |
 188                         PIPE_CONTROL_STALL_AT_SCOREBOARD);
 189         intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);
 190         intel_ring_emit(ring, 0); /* low dword */
 191         intel_ring_emit(ring, 0); /* high dword */
 192         intel_ring_emit(ring, MI_NOOP);
 193         intel_ring_advance(ring);
 194
 195         ret = intel_ring_begin(req, 6);
 196         if (ret)
 197                 return ret;
 198
 199         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
 200         intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE);
 201         intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);
 202         intel_ring_emit(ring, 0);
 203         intel_ring_emit(ring, 0);
 204         intel_ring_emit(ring, MI_NOOP);
 205         intel_ring_advance(ring);
 206
 207         return 0;
 208 }
 209
 210 static int
 211 gen6_render_ring_flush(struct drm_i915_gem_request *req, u32 mode)
 212 {
 213         struct intel_ring *ring = req->ring;
 214         u32 scratch_addr =
 215                 i915_ggtt_offset(req->engine->scratch) + 2 * CACHELINE_BYTES;
 216         u32 flags = 0;
 217         int ret;
 218
 219         /* Force SNB workarounds for PIPE_CONTROL flushes */
 220         ret = intel_emit_post_sync_nonzero_flush(req);
 221         if (ret)
 222                 return ret;
 223
 224         /* Just flush everything.  Experiments have shown that reducing the
 225          * number of bits based on the write domains has little performance
 226          * impact.
 227          */
 228         if (mode & EMIT_FLUSH) {
 229                 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
 230                 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
 231                 /*
 232                  * Ensure that any following seqno writes only happen
 233                  * when the render cache is indeed flushed.
 234                  */
 235                 flags |= PIPE_CONTROL_CS_STALL;
 236         }
 237         if (mode & EMIT_INVALIDATE) {
 238                 flags |= PIPE_CONTROL_TLB_INVALIDATE;
 239                 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
 240                 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
 241                 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
 242                 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
 243                 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
 244                 /*
 245                  * TLB invalidate requires a post-sync write.
 246                  */
 247                 flags |= PIPE_CONTROL_QW_WRITE | PIPE_CONTROL_CS_STALL;
 248         }
 249
 250         ret = intel_ring_begin(req, 4);
 251         if (ret)
 252                 return ret;
 253
 254         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
 255         intel_ring_emit(ring, flags);
 256         intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);
 257         intel_ring_emit(ring, 0);
 258         intel_ring_advance(ring);
 259
 260         return 0;
 261 }
 262
 263 static int
 264 gen7_render_ring_cs_stall_wa(struct drm_i915_gem_request *req)
 265 {
 266         struct intel_ring *ring = req->ring;
 267         int ret;
 268
 269         ret = intel_ring_begin(req, 4);
 270         if (ret)
 271                 return ret;
 272
 273         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
 274         intel_ring_emit(ring,
 275                         PIPE_CONTROL_CS_STALL |
 276                         PIPE_CONTROL_STALL_AT_SCOREBOARD);
 277         intel_ring_emit(ring, 0);
 278         intel_ring_emit(ring, 0);
 279         intel_ring_advance(ring);
 280
 281         return 0;
 282 }
 283
 284 static int
 285 gen7_render_ring_flush(struct drm_i915_gem_request *req, u32 mode)
 286 {
 287         struct intel_ring *ring = req->ring;
 288         u32 scratch_addr =
 289                 i915_ggtt_offset(req->engine->scratch) + 2 * CACHELINE_BYTES;
 290         u32 flags = 0;
 291         int ret;
 292
 293         /*
 294          * Ensure that any following seqno writes only happen when the render
 295          * cache is indeed flushed.
 296          *
 297          * Workaround: 4th PIPE_CONTROL command (except the ones with only
 298          * read-cache invalidate bits set) must have the CS_STALL bit set. We
 299          * don't try to be clever and just set it unconditionally.
 300          */
 301         flags |= PIPE_CONTROL_CS_STALL;
 302
 303         /* Just flush everything.  Experiments have shown that reducing the
 304          * number of bits based on the write domains has little performance
 305          * impact.
 306          */
 307         if (mode & EMIT_FLUSH) {
 308                 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
 309                 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
 310                 flags |= PIPE_CONTROL_DC_FLUSH_ENABLE;
 311                 flags |= PIPE_CONTROL_FLUSH_ENABLE;
 312         }
 313         if (mode & EMIT_INVALIDATE) {
 314                 flags |= PIPE_CONTROL_TLB_INVALIDATE;
 315                 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
 316                 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
 317                 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
 318                 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
 319                 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
 320                 flags |= PIPE_CONTROL_MEDIA_STATE_CLEAR;
 321                 /*
 322                  * TLB invalidate requires a post-sync write.
 323                  */
 324                 flags |= PIPE_CONTROL_QW_WRITE;
 325                 flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
 326
 327                 flags |= PIPE_CONTROL_STALL_AT_SCOREBOARD;
 328
 329                 /* Workaround: we must issue a pipe_control with CS-stall bit
 330                  * set before a pipe_control command that has the state cache
 331                  * invalidate bit set. */
 332                 gen7_render_ring_cs_stall_wa(req);
 333         }
 334
 335         ret = intel_ring_begin(req, 4);
 336         if (ret)
 337                 return ret;
 338
 339         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
 340         intel_ring_emit(ring, flags);
 341         intel_ring_emit(ring, scratch_addr);
 342         intel_ring_emit(ring, 0);
 343         intel_ring_advance(ring);
 344
 345         return 0;
 346 }
 347
 348 static int
 349 gen8_emit_pipe_control(struct drm_i915_gem_request *req,
 350                        u32 flags, u32 scratch_addr)
 351 {
 352         struct intel_ring *ring = req->ring;
 353         int ret;
 354
 355         ret = intel_ring_begin(req, 6);
 356         if (ret)
 357                 return ret;
 358
 359         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(6));
 360         intel_ring_emit(ring, flags);
 361         intel_ring_emit(ring, scratch_addr);
 362         intel_ring_emit(ring, 0);
 363         intel_ring_emit(ring, 0);
 364         intel_ring_emit(ring, 0);
 365         intel_ring_advance(ring);
 366
 367         return 0;
 368 }
 369
 370 static int
 371 gen8_render_ring_flush(struct drm_i915_gem_request *req, u32 mode)
 372 {
 373         u32 scratch_addr =
 374                 i915_ggtt_offset(req->engine->scratch) + 2 * CACHELINE_BYTES;
 375         u32 flags = 0;
 376         int ret;
 377
 378         flags |= PIPE_CONTROL_CS_STALL;
 379
 380         if (mode & EMIT_FLUSH) {
 381                 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
 382                 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
 383                 flags |= PIPE_CONTROL_DC_FLUSH_ENABLE;
 384                 flags |= PIPE_CONTROL_FLUSH_ENABLE;
 385         }
 386         if (mode & EMIT_INVALIDATE) {
 387                 flags |= PIPE_CONTROL_TLB_INVALIDATE;
 388                 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
 389                 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
 390                 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
 391                 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
 392                 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
 393                 flags |= PIPE_CONTROL_QW_WRITE;
 394                 flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
 395
 396                 /* WaCsStallBeforeStateCacheInvalidate:bdw,chv */
 397                 ret = gen8_emit_pipe_control(req,
 398                                              PIPE_CONTROL_CS_STALL |
 399                                              PIPE_CONTROL_STALL_AT_SCOREBOARD,
 400                                              0);
 401                 if (ret)
 402                         return ret;
 403         }
 404
 405         return gen8_emit_pipe_control(req, flags, scratch_addr);
 406 }
 407
 408 static void ring_setup_phys_status_page(struct intel_engine_cs *engine)
 409 {
 410         struct drm_i915_private *dev_priv = engine->i915;
 411         u32 addr;
 412
 413         addr = dev_priv->status_page_dmah->busaddr;
 414         if (INTEL_GEN(dev_priv) >= 4)
 415                 addr |= (dev_priv->status_page_dmah->busaddr >> 28) & 0xf0;
 416         I915_WRITE(HWS_PGA, addr);
 417 }
 418
 419 static void intel_ring_setup_status_page(struct intel_engine_cs *engine)
 420 {
 421         struct drm_i915_private *dev_priv = engine->i915;
 422         i915_reg_t mmio;
 423
 424         /* The ring status page addresses are no longer next to the rest of
 425          * the ring registers as of gen7.
 426          */
 427         if (IS_GEN7(dev_priv)) {
 428                 switch (engine->id) {
 429                 case RCS:
 430                         mmio = RENDER_HWS_PGA_GEN7;
 431                         break;
 432                 case BCS:
 433                         mmio = BLT_HWS_PGA_GEN7;
 434                         break;
 435                 /*
 436                  * VCS2 actually doesn't exist on Gen7. Only shut up
 437                  * gcc switch check warning
 438                  */
 439                 case VCS2:
 440                 case VCS:
 441                         mmio = BSD_HWS_PGA_GEN7;
 442                         break;
 443                 case VECS:
 444                         mmio = VEBOX_HWS_PGA_GEN7;
 445                         break;
 446                 }
 447         } else if (IS_GEN6(dev_priv)) {
 448                 mmio = RING_HWS_PGA_GEN6(engine->mmio_base);
 449         } else {
 450                 /* XXX: gen8 returns to sanity */
 451                 mmio = RING_HWS_PGA(engine->mmio_base);
 452         }
 453
 454         I915_WRITE(mmio, engine->status_page.ggtt_offset);
 455         POSTING_READ(mmio);
 456
 457         /*
 458          * Flush the TLB for this page
 459          *
 460          * FIXME: These two bits have disappeared on gen8, so a question
 461          * arises: do we still need this and if so how should we go about
 462          * invalidating the TLB?
 463          */
 464         if (IS_GEN(dev_priv, 6, 7)) {
 465                 i915_reg_t reg = RING_INSTPM(engine->mmio_base);
 466
 467                 /* ring should be idle before issuing a sync flush*/
 468                 WARN_ON((I915_READ_MODE(engine) & MODE_IDLE) == 0);
 469
 470                 I915_WRITE(reg,
 471                            _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE |
 472                                               INSTPM_SYNC_FLUSH));
 473                 if (intel_wait_for_register(dev_priv,
 474                                             reg, INSTPM_SYNC_FLUSH, 0,
 475                                             1000))
 476                         DRM_ERROR("%s: wait for SyncFlush to complete for TLB invalidation timed out\n",
 477                                   engine->name);
 478         }
 479 }
 480
 481 static bool stop_ring(struct intel_engine_cs *engine)
 482 {
 483         struct drm_i915_private *dev_priv = engine->i915;
 484
 485         if (INTEL_GEN(dev_priv) > 2) {
 486                 I915_WRITE_MODE(engine, _MASKED_BIT_ENABLE(STOP_RING));
 487                 if (intel_wait_for_register(dev_priv,
 488                                             RING_MI_MODE(engine->mmio_base),
 489                                             MODE_IDLE,
 490                                             MODE_IDLE,
 491                                             1000)) {
 492                         DRM_ERROR("%s : timed out trying to stop ring\n",
 493                                   engine->name);
 494                         /* Sometimes we observe that the idle flag is not
 495                          * set even though the ring is empty. So double
 496                          * check before giving up.
 497                          */
 498                         if (I915_READ_HEAD(engine) != I915_READ_TAIL(engine))
 499                                 return false;
 500                 }
 501         }
 502
 503         I915_WRITE_CTL(engine, 0);
 504         I915_WRITE_HEAD(engine, 0);
 505         I915_WRITE_TAIL(engine, 0);
 506
 507         if (INTEL_GEN(dev_priv) > 2) {
 508                 (void)I915_READ_CTL(engine);
 509                 I915_WRITE_MODE(engine, _MASKED_BIT_DISABLE(STOP_RING));
 510         }
 511
 512         return (I915_READ_HEAD(engine) & HEAD_ADDR) == 0;
 513 }
 514
 515 static int init_ring_common(struct intel_engine_cs *engine)
 516 {
 517         struct drm_i915_private *dev_priv = engine->i915;
 518         struct intel_ring *ring = engine->buffer;
 519         int ret = 0;
 520
 521         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
 522
 523         if (!stop_ring(engine)) {
 524                 /* G45 ring initialization often fails to reset head to zero */
 525                 DRM_DEBUG_KMS("%s head not reset to zero "
 526                               "ctl %08x head %08x tail %08x start %08x\n",
 527                               engine->name,
 528                               I915_READ_CTL(engine),
 529                               I915_READ_HEAD(engine),
 530                               I915_READ_TAIL(engine),
 531                               I915_READ_START(engine));
 532
 533                 if (!stop_ring(engine)) {
 534                         DRM_ERROR("failed to set %s head to zero "
 535                                   "ctl %08x head %08x tail %08x start %08x\n",
 536                                   engine->name,
 537                                   I915_READ_CTL(engine),
 538                                   I915_READ_HEAD(engine),
 539                                   I915_READ_TAIL(engine),
 540                                   I915_READ_START(engine));
 541                         ret = -EIO;
 542                         goto out;
 543                 }
 544         }
 545
 546         if (HWS_NEEDS_PHYSICAL(dev_priv))
 547                 ring_setup_phys_status_page(engine);
 548         else
 549                 intel_ring_setup_status_page(engine);
 550
 551         intel_engine_reset_breadcrumbs(engine);
 552
 553         /* Enforce ordering by reading HEAD register back */
 554         I915_READ_HEAD(engine);
 555
 556         /* Initialize the ring. This must happen _after_ we've cleared the ring
 557          * registers with the above sequence (the readback of the HEAD registers
 558          * also enforces ordering), otherwise the hw might lose the new ring
 559          * register values. */
 560         I915_WRITE_START(engine, i915_ggtt_offset(ring->vma));
 561
 562         /* WaClearRingBufHeadRegAtInit:ctg,elk */
 563         if (I915_READ_HEAD(engine))
 564                 DRM_DEBUG("%s initialization failed [head=%08x], fudging\n",
 565                           engine->name, I915_READ_HEAD(engine));
 566
 567         intel_ring_update_space(ring);
 568         I915_WRITE_HEAD(engine, ring->head);
 569         I915_WRITE_TAIL(engine, ring->tail);
 570         (void)I915_READ_TAIL(engine);
 571
 572         I915_WRITE_CTL(engine, RING_CTL_SIZE(ring->size) | RING_VALID);
 573
 574         /* If the head is still not zero, the ring is dead */
 575         if (intel_wait_for_register_fw(dev_priv, RING_CTL(engine->mmio_base),
 576                                        RING_VALID, RING_VALID,
 577                                        50)) {
 578                 DRM_ERROR("%s initialization failed "
 579                           "ctl %08x (valid? %d) head %08x [%08x] tail %08x [%08x] start %08x [expected %08x]\n",
 580                           engine->name,
 581                           I915_READ_CTL(engine),
 582                           I915_READ_CTL(engine) & RING_VALID,
 583                           I915_READ_HEAD(engine), ring->head,
 584                           I915_READ_TAIL(engine), ring->tail,
 585                           I915_READ_START(engine),
 586                           i915_ggtt_offset(ring->vma));
 587                 ret = -EIO;
 588                 goto out;
 589         }
 590
 591         intel_engine_init_hangcheck(engine);
 592
 593 out:
 594         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
 595
 596         return ret;
 597 }
 598
 599 static void reset_ring_common(struct intel_engine_cs *engine,
 600                               struct drm_i915_gem_request *request)
 601 {
 602         struct intel_ring *ring = request->ring;
 603
 604         ring->head = request->postfix;
 605         ring->last_retired_head = -1;
 606 }
 607
 608 static int intel_ring_workarounds_emit(struct drm_i915_gem_request *req)
 609 {
 610         struct intel_ring *ring = req->ring;
 611         struct i915_workarounds *w = &req->i915->workarounds;
 612         int ret, i;
 613
 614         if (w->count == 0)
 615                 return 0;
 616
 617         ret = req->engine->emit_flush(req, EMIT_BARRIER);
 618         if (ret)
 619                 return ret;
 620
 621         ret = intel_ring_begin(req, (w->count * 2 + 2));
 622         if (ret)
 623                 return ret;
 624
 625         intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(w->count));
 626         for (i = 0; i < w->count; i++) {
 627                 intel_ring_emit_reg(ring, w->reg[i].addr);
 628                 intel_ring_emit(ring, w->reg[i].value);
 629         }
 630         intel_ring_emit(ring, MI_NOOP);
 631
 632         intel_ring_advance(ring);
 633
 634         ret = req->engine->emit_flush(req, EMIT_BARRIER);
 635         if (ret)
 636                 return ret;
 637
 638         DRM_DEBUG_DRIVER("Number of Workarounds emitted: %d\n", w->count);
 639
 640         return 0;
 641 }
 642
 643 static int intel_rcs_ctx_init(struct drm_i915_gem_request *req)
 644 {
 645         int ret;
 646
 647         ret = intel_ring_workarounds_emit(req);
 648         if (ret != 0)
 649                 return ret;
 650
 651         ret = i915_gem_render_state_emit(req);
 652         if (ret)
 653                 return ret;
 654
 655         return 0;
 656 }
 657
 658 static int wa_add(struct drm_i915_private *dev_priv,
 659                   i915_reg_t addr,
 660                   const u32 mask, const u32 val)
 661 {
 662         const u32 idx = dev_priv->workarounds.count;
 663
 664         if (WARN_ON(idx >= I915_MAX_WA_REGS))
 665                 return -ENOSPC;
 666
 667         dev_priv->workarounds.reg[idx].addr = addr;
 668         dev_priv->workarounds.reg[idx].value = val;
 669         dev_priv->workarounds.reg[idx].mask = mask;
 670
 671         dev_priv->workarounds.count++;
 672
 673         return 0;
 674 }
 675
 676 #define WA_REG(addr, mask, val) do { \
 677                 const int r = wa_add(dev_priv, (addr), (mask), (val)); \
 678                 if (r) \
 679                         return r; \
 680         } while (0)
 681
 682 #define WA_SET_BIT_MASKED(addr, mask) \
 683         WA_REG(addr, (mask), _MASKED_BIT_ENABLE(mask))
 684
 685 #define WA_CLR_BIT_MASKED(addr, mask) \
 686         WA_REG(addr, (mask), _MASKED_BIT_DISABLE(mask))
 687
 688 #define WA_SET_FIELD_MASKED(addr, mask, value) \
 689         WA_REG(addr, mask, _MASKED_FIELD(mask, value))
 690
 691 #define WA_SET_BIT(addr, mask) WA_REG(addr, mask, I915_READ(addr) | (mask))
 692 #define WA_CLR_BIT(addr, mask) WA_REG(addr, mask, I915_READ(addr) & ~(mask))
 693
 694 #define WA_WRITE(addr, val) WA_REG(addr, 0xffffffff, val)
 695
 696 static int wa_ring_whitelist_reg(struct intel_engine_cs *engine,
 697                                  i915_reg_t reg)
 698 {
 699         struct drm_i915_private *dev_priv = engine->i915;
 700         struct i915_workarounds *wa = &dev_priv->workarounds;
 701         const uint32_t index = wa->hw_whitelist_count[engine->id];
 702
 703         if (WARN_ON(index >= RING_MAX_NONPRIV_SLOTS))
 704                 return -EINVAL;
 705
 706         WA_WRITE(RING_FORCE_TO_NONPRIV(engine->mmio_base, index),
 707                  i915_mmio_reg_offset(reg));
 708         wa->hw_whitelist_count[engine->id]++;
 709
 710         return 0;
 711 }
 712
 713 static int gen8_init_workarounds(struct intel_engine_cs *engine)
 714 {
 715         struct drm_i915_private *dev_priv = engine->i915;
 716
 717         WA_SET_BIT_MASKED(INSTPM, INSTPM_FORCE_ORDERING);
 718
 719         /* WaDisableAsyncFlipPerfMode:bdw,chv */
 720         WA_SET_BIT_MASKED(MI_MODE, ASYNC_FLIP_PERF_DISABLE);
 721
 722         /* WaDisablePartialInstShootdown:bdw,chv */
 723         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
 724                           PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
 725
 726         /* Use Force Non-Coherent whenever executing a 3D context. This is a
 727          * workaround for for a possible hang in the unlikely event a TLB
 728          * invalidation occurs during a PSD flush.
 729          */
 730         /* WaForceEnableNonCoherent:bdw,chv */
 731         /* WaHdcDisableFetchWhenMasked:bdw,chv */
 732         WA_SET_BIT_MASKED(HDC_CHICKEN0,
 733                           HDC_DONOT_FETCH_MEM_WHEN_MASKED |
 734                           HDC_FORCE_NON_COHERENT);
 735
 736         /* From the Haswell PRM, Command Reference: Registers, CACHE_MODE_0:
 737          * "The Hierarchical Z RAW Stall Optimization allows non-overlapping
 738          *  polygons in the same 8x4 pixel/sample area to be processed without
 739          *  stalling waiting for the earlier ones to write to Hierarchical Z
 740          *  buffer."
 741          *
 742          * This optimization is off by default for BDW and CHV; turn it on.
 743          */
 744         WA_CLR_BIT_MASKED(CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE);
 745
 746         /* Wa4x4STCOptimizationDisable:bdw,chv */
 747         WA_SET_BIT_MASKED(CACHE_MODE_1, GEN8_4x4_STC_OPTIMIZATION_DISABLE);
 748
 749         /*
 750          * BSpec recommends 8x4 when MSAA is used,
 751          * however in practice 16x4 seems fastest.
 752          *
 753          * Note that PS/WM thread counts depend on the WIZ hashing
 754          * disable bit, which we don't touch here, but it's good
 755          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
 756          */
 757         WA_SET_FIELD_MASKED(GEN7_GT_MODE,
 758                             GEN6_WIZ_HASHING_MASK,
 759                             GEN6_WIZ_HASHING_16x4);
 760
 761         return 0;
 762 }
 763
 764 static int bdw_init_workarounds(struct intel_engine_cs *engine)
 765 {
 766         struct drm_i915_private *dev_priv = engine->i915;
 767         int ret;
 768
 769         ret = gen8_init_workarounds(engine);
 770         if (ret)
 771                 return ret;
 772
 773         /* WaDisableThreadStallDopClockGating:bdw (pre-production) */
 774         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
 775
 776         /* WaDisableDopClockGating:bdw */
 777         WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
 778                           DOP_CLOCK_GATING_DISABLE);
 779
 780         WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
 781                           GEN8_SAMPLER_POWER_BYPASS_DIS);
 782
 783         WA_SET_BIT_MASKED(HDC_CHICKEN0,
 784                           /* WaForceContextSaveRestoreNonCoherent:bdw */
 785                           HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
 786                           /* WaDisableFenceDestinationToSLM:bdw (pre-prod) */
 787                           (IS_BDW_GT3(dev_priv) ? HDC_FENCE_DEST_SLM_DISABLE : 0));
 788
 789         return 0;
 790 }
 791
 792 static int chv_init_workarounds(struct intel_engine_cs *engine)
 793 {
 794         struct drm_i915_private *dev_priv = engine->i915;
 795         int ret;
 796
 797         ret = gen8_init_workarounds(engine);
 798         if (ret)
 799                 return ret;
 800
 801         /* WaDisableThreadStallDopClockGating:chv */
 802         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
 803
 804         /* Improve HiZ throughput on CHV. */
 805         WA_SET_BIT_MASKED(HIZ_CHICKEN, CHV_HZ_8X8_MODE_IN_1X);
 806
 807         return 0;
 808 }
 809
 810 static int gen9_init_workarounds(struct intel_engine_cs *engine)
 811 {
 812         struct drm_i915_private *dev_priv = engine->i915;
 813         int ret;
 814
 815         /* WaConextSwitchWithConcurrentTLBInvalidate:skl,bxt,kbl */
 816         I915_WRITE(GEN9_CSFE_CHICKEN1_RCS, _MASKED_BIT_ENABLE(GEN9_PREEMPT_GPGPU_SYNC_SWITCH_DISABLE));
 817
 818         /* WaEnableLbsSlaRetryTimerDecrement:skl,bxt,kbl */
 819         I915_WRITE(BDW_SCRATCH1, I915_READ(BDW_SCRATCH1) |
 820                    GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE);
 821
 822         /* WaDisableKillLogic:bxt,skl,kbl */
 823         I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) |
 824                    ECOCHK_DIS_TLB);
 825
 826         /* WaClearFlowControlGpgpuContextSave:skl,bxt,kbl */
 827         /* WaDisablePartialInstShootdown:skl,bxt,kbl */
 828         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
 829                           FLOW_CONTROL_ENABLE |
 830                           PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
 831
 832         /* Syncing dependencies between camera and graphics:skl,bxt,kbl */
 833         WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
 834                           GEN9_DISABLE_OCL_OOB_SUPPRESS_LOGIC);
 835
 836         /* WaDisableDgMirrorFixInHalfSliceChicken5:bxt */
 837         if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
 838                 WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
 839                                   GEN9_DG_MIRROR_FIX_ENABLE);
 840
 841         /* WaSetDisablePixMaskCammingAndRhwoInCommonSliceChicken:bxt */
 842         if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
 843                 WA_SET_BIT_MASKED(GEN7_COMMON_SLICE_CHICKEN1,
 844                                   GEN9_RHWO_OPTIMIZATION_DISABLE);
 845                 /*
 846                  * WA also requires GEN9_SLICE_COMMON_ECO_CHICKEN0[14:14] to be set
 847                  * but we do that in per ctx batchbuffer as there is an issue
 848                  * with this register not getting restored on ctx restore
 849                  */
 850         }
 851
 852         /* WaEnableSamplerGPGPUPreemptionSupport:skl,bxt,kbl */
 853         WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
 854                           GEN9_ENABLE_GPGPU_PREEMPTION);
 855
 856         /* Wa4x4STCOptimizationDisable:skl,bxt,kbl */
 857         /* WaDisablePartialResolveInVc:skl,bxt,kbl */
 858         WA_SET_BIT_MASKED(CACHE_MODE_1, (GEN8_4x4_STC_OPTIMIZATION_DISABLE |
 859                                          GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE));
 860
 861         /* WaCcsTlbPrefetchDisable:skl,bxt,kbl */
 862         WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
 863                           GEN9_CCS_TLB_PREFETCH_ENABLE);
 864
 865         /* WaDisableMaskBasedCammingInRCC:bxt */
 866         if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
 867                 WA_SET_BIT_MASKED(SLICE_ECO_CHICKEN0,
 868                                   PIXEL_MASK_CAMMING_DISABLE);
 869
 870         /* WaForceContextSaveRestoreNonCoherent:skl,bxt,kbl */
 871         WA_SET_BIT_MASKED(HDC_CHICKEN0,
 872                           HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
 873                           HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE);
 874
 875         /* WaForceEnableNonCoherent and WaDisableHDCInvalidation are
 876          * both tied to WaForceContextSaveRestoreNonCoherent
 877          * in some hsds for skl. We keep the tie for all gen9. The
 878          * documentation is a bit hazy and so we want to get common behaviour,
 879          * even though there is no clear evidence we would need both on kbl/bxt.
 880          * This area has been source of system hangs so we play it safe
 881          * and mimic the skl regardless of what bspec says.
 882          *
 883          * Use Force Non-Coherent whenever executing a 3D context. This
 884          * is a workaround for a possible hang in the unlikely event
 885          * a TLB invalidation occurs during a PSD flush.
 886          */
 887
 888         /* WaForceEnableNonCoherent:skl,bxt,kbl */
 889         WA_SET_BIT_MASKED(HDC_CHICKEN0,
 890                           HDC_FORCE_NON_COHERENT);
 891
 892         /* WaDisableHDCInvalidation:skl,bxt,kbl */
 893         I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) |
 894                    BDW_DISABLE_HDC_INVALIDATION);
 895
 896         /* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt,kbl */
 897         if (IS_SKYLAKE(dev_priv) ||
 898             IS_KABYLAKE(dev_priv) ||
 899             IS_BXT_REVID(dev_priv, 0, BXT_REVID_B0))
 900                 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
 901                                   GEN8_SAMPLER_POWER_BYPASS_DIS);
 902
 903         /* WaDisableSTUnitPowerOptimization:skl,bxt,kbl */
 904         WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN2, GEN8_ST_PO_DISABLE);
 905
 906         /* WaOCLCoherentLineFlush:skl,bxt,kbl */
 907         I915_WRITE(GEN8_L3SQCREG4, (I915_READ(GEN8_L3SQCREG4) |
 908                                     GEN8_LQSC_FLUSH_COHERENT_LINES));
 909
 910         /* WaVFEStateAfterPipeControlwithMediaStateClear:skl,bxt */
 911         ret = wa_ring_whitelist_reg(engine, GEN9_CTX_PREEMPT_REG);
 912         if (ret)
 913                 return ret;
 914
 915         /* WaEnablePreemptionGranularityControlByUMD:skl,bxt,kbl */
 916         ret= wa_ring_whitelist_reg(engine, GEN8_CS_CHICKEN1);
 917         if (ret)
 918                 return ret;
 919
 920         /* WaAllowUMDToModifyHDCChicken1:skl,bxt,kbl */
 921         ret = wa_ring_whitelist_reg(engine, GEN8_HDC_CHICKEN1);
 922         if (ret)
 923                 return ret;
 924
 925         return 0;
 926 }
 927
 928 static int skl_tune_iz_hashing(struct intel_engine_cs *engine)
 929 {
 930         struct drm_i915_private *dev_priv = engine->i915;
 931         u8 vals[3] = { 0, 0, 0 };
 932         unsigned int i;
 933
 934         for (i = 0; i < 3; i++) {
 935                 u8 ss;
 936
 937                 /*
 938                  * Only consider slices where one, and only one, subslice has 7
 939                  * EUs
 940                  */
 941                 if (!is_power_of_2(INTEL_INFO(dev_priv)->sseu.subslice_7eu[i]))
 942                         continue;
 943
 944                 /*
 945                  * subslice_7eu[i] != 0 (because of the check above) and
 946                  * ss_max == 4 (maximum number of subslices possible per slice)
 947                  *
 948                  * ->    0 <= ss <= 3;
 949                  */
 950                 ss = ffs(INTEL_INFO(dev_priv)->sseu.subslice_7eu[i]) - 1;
 951                 vals[i] = 3 - ss;
 952         }
 953
 954         if (vals[0] == 0 && vals[1] == 0 && vals[2] == 0)
 955                 return 0;
 956
 957         /* Tune IZ hashing. See intel_device_info_runtime_init() */
 958         WA_SET_FIELD_MASKED(GEN7_GT_MODE,
 959                             GEN9_IZ_HASHING_MASK(2) |
 960                             GEN9_IZ_HASHING_MASK(1) |
 961                             GEN9_IZ_HASHING_MASK(0),
 962                             GEN9_IZ_HASHING(2, vals[2]) |
 963                             GEN9_IZ_HASHING(1, vals[1]) |
 964                             GEN9_IZ_HASHING(0, vals[0]));
 965
 966         return 0;
 967 }
 968
 969 static int skl_init_workarounds(struct intel_engine_cs *engine)
 970 {
 971         struct drm_i915_private *dev_priv = engine->i915;
 972         int ret;
 973
 974         ret = gen9_init_workarounds(engine);
 975         if (ret)
 976                 return ret;
 977
 978         /*
 979          * Actual WA is to disable percontext preemption granularity control
 980          * until D0 which is the default case so this is equivalent to
 981          * !WaDisablePerCtxtPreemptionGranularityControl:skl
 982          */
 983         I915_WRITE(GEN7_FF_SLICE_CS_CHICKEN1,
 984                    _MASKED_BIT_ENABLE(GEN9_FFSC_PERCTX_PREEMPT_CTRL));
 985
 986         /* WaEnableGapsTsvCreditFix:skl */
 987         I915_WRITE(GEN8_GARBCNTL, (I915_READ(GEN8_GARBCNTL) |
 988                                    GEN9_GAPS_TSV_CREDIT_DISABLE));
 989
 990         /* WaDisableGafsUnitClkGating:skl */
 991         WA_SET_BIT(GEN7_UCGCTL4, GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
 992
 993         /* WaInPlaceDecompressionHang:skl */
 994         if (IS_SKL_REVID(dev_priv, SKL_REVID_H0, REVID_FOREVER))
 995                 WA_SET_BIT(GEN9_GAMT_ECO_REG_RW_IA,
 996                            GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
 997
 998         /* WaDisableLSQCROPERFforOCL:skl */
 999         ret = wa_ring_whitelist_reg(engine, GEN8_L3SQCREG4);
1000         if (ret)
1001                 return ret;
1002
1003         return skl_tune_iz_hashing(engine);
1004 }
1005
1006 static int bxt_init_workarounds(struct intel_engine_cs *engine)
1007 {
1008         struct drm_i915_private *dev_priv = engine->i915;
1009         int ret;
1010
1011         ret = gen9_init_workarounds(engine);
1012         if (ret)
1013                 return ret;
1014
1015         /* WaStoreMultiplePTEenable:bxt */
1016         /* This is a requirement according to Hardware specification */
1017         if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
1018                 I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_TLBPF);
1019
1020         /* WaSetClckGatingDisableMedia:bxt */
1021         if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
1022                 I915_WRITE(GEN7_MISCCPCTL, (I915_READ(GEN7_MISCCPCTL) &
1023                                             ~GEN8_DOP_CLOCK_GATE_MEDIA_ENABLE));
1024         }
1025
1026         /* WaDisableThreadStallDopClockGating:bxt */
1027         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
1028                           STALL_DOP_GATING_DISABLE);
1029
1030         /* WaDisablePooledEuLoadBalancingFix:bxt */
1031         if (IS_BXT_REVID(dev_priv, BXT_REVID_B0, REVID_FOREVER)) {
1032                 WA_SET_BIT_MASKED(FF_SLICE_CS_CHICKEN2,
1033                                   GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE);
1034         }
1035
1036         /* WaDisableSbeCacheDispatchPortSharing:bxt */
1037         if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_B0)) {
1038                 WA_SET_BIT_MASKED(
1039                         GEN7_HALF_SLICE_CHICKEN1,
1040                         GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
1041         }
1042
1043         /* WaDisableObjectLevelPreemptionForTrifanOrPolygon:bxt */
1044         /* WaDisableObjectLevelPreemptionForInstancedDraw:bxt */
1045         /* WaDisableObjectLevelPreemtionForInstanceId:bxt */
1046         /* WaDisableLSQCROPERFforOCL:bxt */
1047         if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
1048                 ret = wa_ring_whitelist_reg(engine, GEN9_CS_DEBUG_MODE1);
1049                 if (ret)
1050                         return ret;
1051
1052                 ret = wa_ring_whitelist_reg(engine, GEN8_L3SQCREG4);
1053                 if (ret)
1054                         return ret;
1055         }
1056
1057         /* WaProgramL3SqcReg1DefaultForPerf:bxt */
1058         if (IS_BXT_REVID(dev_priv, BXT_REVID_B0, REVID_FOREVER))
1059                 I915_WRITE(GEN8_L3SQCREG1, L3_GENERAL_PRIO_CREDITS(62) |
1060                                            L3_HIGH_PRIO_CREDITS(2));
1061
1062         /* WaToEnableHwFixForPushConstHWBug:bxt */
1063         if (IS_BXT_REVID(dev_priv, BXT_REVID_C0, REVID_FOREVER))
1064                 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
1065                                   GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
1066
1067         /* WaInPlaceDecompressionHang:bxt */
1068         if (IS_BXT_REVID(dev_priv, BXT_REVID_C0, REVID_FOREVER))
1069                 WA_SET_BIT(GEN9_GAMT_ECO_REG_RW_IA,
1070                            GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
1071
1072         return 0;
1073 }
1074
1075 static int kbl_init_workarounds(struct intel_engine_cs *engine)
1076 {
1077         struct drm_i915_private *dev_priv = engine->i915;
1078         int ret;
1079
1080         ret = gen9_init_workarounds(engine);
1081         if (ret)
1082                 return ret;
1083
1084         /* WaEnableGapsTsvCreditFix:kbl */
1085         I915_WRITE(GEN8_GARBCNTL, (I915_READ(GEN8_GARBCNTL) |
1086                                    GEN9_GAPS_TSV_CREDIT_DISABLE));
1087
1088         /* WaDisableDynamicCreditSharing:kbl */
1089         if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
1090                 WA_SET_BIT(GAMT_CHKN_BIT_REG,
1091                            GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING);
1092
1093         /* WaDisableFenceDestinationToSLM:kbl (pre-prod) */
1094         if (IS_KBL_REVID(dev_priv, KBL_REVID_A0, KBL_REVID_A0))
1095                 WA_SET_BIT_MASKED(HDC_CHICKEN0,
1096                                   HDC_FENCE_DEST_SLM_DISABLE);
1097
1098         /* GEN8_L3SQCREG4 has a dependency with WA batch so any new changes
1099          * involving this register should also be added to WA batch as required.
1100          */
1101         if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_E0))
1102                 /* WaDisableLSQCROPERFforOCL:kbl */
1103                 I915_WRITE(GEN8_L3SQCREG4, I915_READ(GEN8_L3SQCREG4) |
1104                            GEN8_LQSC_RO_PERF_DIS);
1105
1106         /* WaToEnableHwFixForPushConstHWBug:kbl */
1107         if (IS_KBL_REVID(dev_priv, KBL_REVID_C0, REVID_FOREVER))
1108                 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
1109                                   GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
1110
1111         /* WaDisableGafsUnitClkGating:kbl */
1112         WA_SET_BIT(GEN7_UCGCTL4, GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
1113
1114         /* WaDisableSbeCacheDispatchPortSharing:kbl */
1115         WA_SET_BIT_MASKED(
1116                 GEN7_HALF_SLICE_CHICKEN1,
1117                 GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
1118
1119         /* WaInPlaceDecompressionHang:kbl */
1120         WA_SET_BIT(GEN9_GAMT_ECO_REG_RW_IA,
1121                    GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
1122
1123         /* WaDisableLSQCROPERFforOCL:kbl */
1124         ret = wa_ring_whitelist_reg(engine, GEN8_L3SQCREG4);
1125         if (ret)
1126                 return ret;
1127
1128         return 0;
1129 }
1130
1131 int init_workarounds_ring(struct intel_engine_cs *engine)
1132 {
1133         struct drm_i915_private *dev_priv = engine->i915;
1134
1135         WARN_ON(engine->id != RCS);
1136
1137         dev_priv->workarounds.count = 0;
1138         dev_priv->workarounds.hw_whitelist_count[RCS] = 0;
1139
1140         if (IS_BROADWELL(dev_priv))
1141                 return bdw_init_workarounds(engine);
1142
1143         if (IS_CHERRYVIEW(dev_priv))
1144                 return chv_init_workarounds(engine);
1145
1146         if (IS_SKYLAKE(dev_priv))
1147                 return skl_init_workarounds(engine);
1148
1149         if (IS_BROXTON(dev_priv))
1150                 return bxt_init_workarounds(engine);
1151
1152         if (IS_KABYLAKE(dev_priv))
1153                 return kbl_init_workarounds(engine);
1154
1155         return 0;
1156 }
1157
1158 static int init_render_ring(struct intel_engine_cs *engine)
1159 {
1160         struct drm_i915_private *dev_priv = engine->i915;
1161         int ret = init_ring_common(engine);
1162         if (ret)
1163                 return ret;
1164
1165         /* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */
1166         if (IS_GEN(dev_priv, 4, 6))
1167                 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH));
1168
1169         /* We need to disable the AsyncFlip performance optimisations in order
1170          * to use MI_WAIT_FOR_EVENT within the CS. It should already be
1171          * programmed to '1' on all products.
1172          *
1173          * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv
1174          */
1175         if (IS_GEN(dev_priv, 6, 7))
1176                 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE));
1177
1178         /* Required for the hardware to program scanline values for waiting */
1179         /* WaEnableFlushTlbInvalidationMode:snb */
1180         if (IS_GEN6(dev_priv))
1181                 I915_WRITE(GFX_MODE,
1182                            _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT));
1183
1184         /* WaBCSVCSTlbInvalidationMode:ivb,vlv,hsw */
1185         if (IS_GEN7(dev_priv))
1186                 I915_WRITE(GFX_MODE_GEN7,
1187                            _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT) |
1188                            _MASKED_BIT_ENABLE(GFX_REPLAY_MODE));
1189
1190         if (IS_GEN6(dev_priv)) {
1191                 /* From the Sandybridge PRM, volume 1 part 3, page 24:
1192                  * "If this bit is set, STCunit will have LRA as replacement
1193                  *  policy. [...] This bit must be reset.  LRA replacement
1194                  *  policy is not supported."
1195                  */
1196                 I915_WRITE(CACHE_MODE_0,
1197                            _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
1198         }
1199
1200         if (IS_GEN(dev_priv, 6, 7))
1201                 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
1202
1203         if (INTEL_INFO(dev_priv)->gen >= 6)
1204                 I915_WRITE_IMR(engine, ~engine->irq_keep_mask);
1205
1206         return init_workarounds_ring(engine);
1207 }
1208
1209 static void render_ring_cleanup(struct intel_engine_cs *engine)
1210 {
1211         struct drm_i915_private *dev_priv = engine->i915;
1212
1213         i915_vma_unpin_and_release(&dev_priv->semaphore);
1214 }
1215
1216 static u32 *gen8_rcs_signal(struct drm_i915_gem_request *req, u32 *out)
1217 {
1218         struct drm_i915_private *dev_priv = req->i915;
1219         struct intel_engine_cs *waiter;
1220         enum intel_engine_id id;
1221
1222         for_each_engine(waiter, dev_priv, id) {
1223                 u64 gtt_offset = req->engine->semaphore.signal_ggtt[id];
1224                 if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID)
1225                         continue;
1226
1227                 *out++ = GFX_OP_PIPE_CONTROL(6);
1228                 *out++ = (PIPE_CONTROL_GLOBAL_GTT_IVB |
1229                           PIPE_CONTROL_QW_WRITE |
1230                           PIPE_CONTROL_CS_STALL);
1231                 *out++ = lower_32_bits(gtt_offset);
1232                 *out++ = upper_32_bits(gtt_offset);
1233                 *out++ = req->global_seqno;
1234                 *out++ = 0;
1235                 *out++ = (MI_SEMAPHORE_SIGNAL |
1236                           MI_SEMAPHORE_TARGET(waiter->hw_id));
1237                 *out++ = 0;
1238         }
1239
1240         return out;
1241 }
1242
1243 static u32 *gen8_xcs_signal(struct drm_i915_gem_request *req, u32 *out)
1244 {
1245         struct drm_i915_private *dev_priv = req->i915;
1246         struct intel_engine_cs *waiter;
1247         enum intel_engine_id id;
1248
1249         for_each_engine(waiter, dev_priv, id) {
1250                 u64 gtt_offset = req->engine->semaphore.signal_ggtt[id];
1251                 if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID)
1252                         continue;
1253
1254                 *out++ = (MI_FLUSH_DW + 1) | MI_FLUSH_DW_OP_STOREDW;
1255                 *out++ = lower_32_bits(gtt_offset) | MI_FLUSH_DW_USE_GTT;
1256                 *out++ = upper_32_bits(gtt_offset);
1257                 *out++ = req->global_seqno;
1258                 *out++ = (MI_SEMAPHORE_SIGNAL |
1259                           MI_SEMAPHORE_TARGET(waiter->hw_id));
1260                 *out++ = 0;
1261         }
1262
1263         return out;
1264 }
1265
1266 static u32 *gen6_signal(struct drm_i915_gem_request *req, u32 *out)
1267 {
1268         struct drm_i915_private *dev_priv = req->i915;
1269         struct intel_engine_cs *engine;
1270         enum intel_engine_id id;
1271         int num_rings = 0;
1272
1273         for_each_engine(engine, dev_priv, id) {
1274                 i915_reg_t mbox_reg;
1275
1276                 if (!(BIT(engine->hw_id) & GEN6_SEMAPHORES_MASK))
1277                         continue;
1278
1279                 mbox_reg = req->engine->semaphore.mbox.signal[engine->hw_id];
1280                 if (i915_mmio_reg_valid(mbox_reg)) {
1281                         *out++ = MI_LOAD_REGISTER_IMM(1);
1282                         *out++ = i915_mmio_reg_offset(mbox_reg);
1283                         *out++ = req->global_seqno;
1284                         num_rings++;
1285                 }
1286         }
1287         if (num_rings & 1)
1288                 *out++ = MI_NOOP;
1289
1290         return out;
1291 }
1292
1293 static void i9xx_submit_request(struct drm_i915_gem_request *request)
1294 {
1295         struct drm_i915_private *dev_priv = request->i915;
1296
1297         i915_gem_request_submit(request);
1298
1299         I915_WRITE_TAIL(request->engine, request->tail);
1300 }
1301
1302 static void i9xx_emit_breadcrumb(struct drm_i915_gem_request *req,
1303                                  u32 *out)
1304 {
1305         *out++ = MI_STORE_DWORD_INDEX;
1306         *out++ = I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT;
1307         *out++ = req->global_seqno;
1308         *out++ = MI_USER_INTERRUPT;
1309
1310         req->tail = intel_ring_offset(req->ring, out);
1311 }
1312
1313 static const int i9xx_emit_breadcrumb_sz = 4;
1314
1315 /**
1316  * gen6_sema_emit_breadcrumb - Update the semaphore mailbox registers
1317  *
1318  * @request - request to write to the ring
1319  *
1320  * Update the mailbox registers in the *other* rings with the current seqno.
1321  * This acts like a signal in the canonical semaphore.
1322  */
1323 static void gen6_sema_emit_breadcrumb(struct drm_i915_gem_request *req,
1324                                       u32 *out)
1325 {
1326         return i9xx_emit_breadcrumb(req,
1327                                     req->engine->semaphore.signal(req, out));
1328 }
1329
1330 static void gen8_render_emit_breadcrumb(struct drm_i915_gem_request *req,
1331                                         u32 *out)
1332 {
1333         struct intel_engine_cs *engine = req->engine;
1334
1335         if (engine->semaphore.signal)
1336                 out = engine->semaphore.signal(req, out);
1337
1338         *out++ = GFX_OP_PIPE_CONTROL(6);
1339         *out++ = (PIPE_CONTROL_GLOBAL_GTT_IVB |
1340                                PIPE_CONTROL_CS_STALL |
1341                                PIPE_CONTROL_QW_WRITE);
1342         *out++ = intel_hws_seqno_address(engine);
1343         *out++ = 0;
1344         *out++ = req->global_seqno;
1345         /* We're thrashing one dword of HWS. */
1346         *out++ = 0;
1347         *out++ = MI_USER_INTERRUPT;
1348         *out++ = MI_NOOP;
1349
1350         req->tail = intel_ring_offset(req->ring, out);
1351 }
1352
1353 static const int gen8_render_emit_breadcrumb_sz = 8;
1354
1355 /**
1356  * intel_ring_sync - sync the waiter to the signaller on seqno
1357  *
1358  * @waiter - ring that is waiting
1359  * @signaller - ring which has, or will signal
1360  * @seqno - seqno which the waiter will block on
1361  */
1362
1363 static int
1364 gen8_ring_sync_to(struct drm_i915_gem_request *req,
1365                   struct drm_i915_gem_request *signal)
1366 {
1367         struct intel_ring *ring = req->ring;
1368         struct drm_i915_private *dev_priv = req->i915;
1369         u64 offset = GEN8_WAIT_OFFSET(req->engine, signal->engine->id);
1370         struct i915_hw_ppgtt *ppgtt;
1371         int ret;
1372
1373         ret = intel_ring_begin(req, 4);
1374         if (ret)
1375                 return ret;
1376
1377         intel_ring_emit(ring,
1378                         MI_SEMAPHORE_WAIT |
1379                         MI_SEMAPHORE_GLOBAL_GTT |
1380                         MI_SEMAPHORE_SAD_GTE_SDD);
1381         intel_ring_emit(ring, signal->global_seqno);
1382         intel_ring_emit(ring, lower_32_bits(offset));
1383         intel_ring_emit(ring, upper_32_bits(offset));
1384         intel_ring_advance(ring);
1385
1386         /* When the !RCS engines idle waiting upon a semaphore, they lose their
1387          * pagetables and we must reload them before executing the batch.
1388          * We do this on the i915_switch_context() following the wait and
1389          * before the dispatch.
1390          */
1391         ppgtt = req->ctx->ppgtt;
1392         if (ppgtt && req->engine->id != RCS)
1393                 ppgtt->pd_dirty_rings |= intel_engine_flag(req->engine);
1394         return 0;
1395 }
1396
1397 static int
1398 gen6_ring_sync_to(struct drm_i915_gem_request *req,
1399                   struct drm_i915_gem_request *signal)
1400 {
1401         struct intel_ring *ring = req->ring;
1402         u32 dw1 = MI_SEMAPHORE_MBOX |
1403                   MI_SEMAPHORE_COMPARE |
1404                   MI_SEMAPHORE_REGISTER;
1405         u32 wait_mbox = signal->engine->semaphore.mbox.wait[req->engine->hw_id];
1406         int ret;
1407
1408         WARN_ON(wait_mbox == MI_SEMAPHORE_SYNC_INVALID);
1409
1410         ret = intel_ring_begin(req, 4);
1411         if (ret)
1412                 return ret;
1413
1414         intel_ring_emit(ring, dw1 | wait_mbox);
1415         /* Throughout all of the GEM code, seqno passed implies our current
1416          * seqno is >= the last seqno executed. However for hardware the
1417          * comparison is strictly greater than.
1418          */
1419         intel_ring_emit(ring, signal->global_seqno - 1);
1420         intel_ring_emit(ring, 0);
1421         intel_ring_emit(ring, MI_NOOP);
1422         intel_ring_advance(ring);
1423
1424         return 0;
1425 }
1426
1427 static void
1428 gen5_seqno_barrier(struct intel_engine_cs *engine)
1429 {
1430         /* MI_STORE are internally buffered by the GPU and not flushed
1431          * either by MI_FLUSH or SyncFlush or any other combination of
1432          * MI commands.
1433          *
1434          * "Only the submission of the store operation is guaranteed.
1435          * The write result will be complete (coherent) some time later
1436          * (this is practically a finite period but there is no guaranteed
1437          * latency)."
1438          *
1439          * Empirically, we observe that we need a delay of at least 75us to
1440          * be sure that the seqno write is visible by the CPU.
1441          */
1442         usleep_range(125, 250);
1443 }
1444
1445 static void
1446 gen6_seqno_barrier(struct intel_engine_cs *engine)
1447 {
1448         struct drm_i915_private *dev_priv = engine->i915;
1449
1450         /* Workaround to force correct ordering between irq and seqno writes on
1451          * ivb (and maybe also on snb) by reading from a CS register (like
1452          * ACTHD) before reading the status page.
1453          *
1454          * Note that this effectively stalls the read by the time it takes to
1455          * do a memory transaction, which more or less ensures that the write
1456          * from the GPU has sufficient time to invalidate the CPU cacheline.
1457          * Alternatively we could delay the interrupt from the CS ring to give
1458          * the write time to land, but that would incur a delay after every
1459          * batch i.e. much more frequent than a delay when waiting for the
1460          * interrupt (with the same net latency).
1461          *
1462          * Also note that to prevent whole machine hangs on gen7, we have to
1463          * take the spinlock to guard against concurrent cacheline access.
1464          */
1465         spin_lock_irq(&dev_priv->uncore.lock);
1466         POSTING_READ_FW(RING_ACTHD(engine->mmio_base));
1467         spin_unlock_irq(&dev_priv->uncore.lock);
1468 }
1469
1470 static void
1471 gen5_irq_enable(struct intel_engine_cs *engine)
1472 {
1473         gen5_enable_gt_irq(engine->i915, engine->irq_enable_mask);
1474 }
1475
1476 static void
1477 gen5_irq_disable(struct intel_engine_cs *engine)
1478 {
1479         gen5_disable_gt_irq(engine->i915, engine->irq_enable_mask);
1480 }
1481
1482 static void
1483 i9xx_irq_enable(struct intel_engine_cs *engine)
1484 {
1485         struct drm_i915_private *dev_priv = engine->i915;
1486
1487         dev_priv->irq_mask &= ~engine->irq_enable_mask;
1488         I915_WRITE(IMR, dev_priv->irq_mask);
1489         POSTING_READ_FW(RING_IMR(engine->mmio_base));
1490 }
1491
1492 static void
1493 i9xx_irq_disable(struct intel_engine_cs *engine)
1494 {
1495         struct drm_i915_private *dev_priv = engine->i915;
1496
1497         dev_priv->irq_mask |= engine->irq_enable_mask;
1498         I915_WRITE(IMR, dev_priv->irq_mask);
1499 }
1500
1501 static void
1502 i8xx_irq_enable(struct intel_engine_cs *engine)
1503 {
1504         struct drm_i915_private *dev_priv = engine->i915;
1505
1506         dev_priv->irq_mask &= ~engine->irq_enable_mask;
1507         I915_WRITE16(IMR, dev_priv->irq_mask);
1508         POSTING_READ16(RING_IMR(engine->mmio_base));
1509 }
1510
1511 static void
1512 i8xx_irq_disable(struct intel_engine_cs *engine)
1513 {
1514         struct drm_i915_private *dev_priv = engine->i915;
1515
1516         dev_priv->irq_mask |= engine->irq_enable_mask;
1517         I915_WRITE16(IMR, dev_priv->irq_mask);
1518 }
1519
1520 static int
1521 bsd_ring_flush(struct drm_i915_gem_request *req, u32 mode)
1522 {
1523         struct intel_ring *ring = req->ring;
1524         int ret;
1525
1526         ret = intel_ring_begin(req, 2);
1527         if (ret)
1528                 return ret;
1529
1530         intel_ring_emit(ring, MI_FLUSH);
1531         intel_ring_emit(ring, MI_NOOP);
1532         intel_ring_advance(ring);
1533         return 0;
1534 }
1535
1536 static void
1537 gen6_irq_enable(struct intel_engine_cs *engine)
1538 {
1539         struct drm_i915_private *dev_priv = engine->i915;
1540
1541         I915_WRITE_IMR(engine,
1542                        ~(engine->irq_enable_mask |
1543                          engine->irq_keep_mask));
1544         gen5_enable_gt_irq(dev_priv, engine->irq_enable_mask);
1545 }
1546
1547 static void
1548 gen6_irq_disable(struct intel_engine_cs *engine)
1549 {
1550         struct drm_i915_private *dev_priv = engine->i915;
1551
1552         I915_WRITE_IMR(engine, ~engine->irq_keep_mask);
1553         gen5_disable_gt_irq(dev_priv, engine->irq_enable_mask);
1554 }
1555
1556 static void
1557 hsw_vebox_irq_enable(struct intel_engine_cs *engine)
1558 {
1559         struct drm_i915_private *dev_priv = engine->i915;
1560
1561         I915_WRITE_IMR(engine, ~engine->irq_enable_mask);
1562         gen6_unmask_pm_irq(dev_priv, engine->irq_enable_mask);
1563 }
1564
1565 static void
1566 hsw_vebox_irq_disable(struct intel_engine_cs *engine)
1567 {
1568         struct drm_i915_private *dev_priv = engine->i915;
1569
1570         I915_WRITE_IMR(engine, ~0);
1571         gen6_mask_pm_irq(dev_priv, engine->irq_enable_mask);
1572 }
1573
1574 static void
1575 gen8_irq_enable(struct intel_engine_cs *engine)
1576 {
1577         struct drm_i915_private *dev_priv = engine->i915;
1578
1579         I915_WRITE_IMR(engine,
1580                        ~(engine->irq_enable_mask |
1581                          engine->irq_keep_mask));
1582         POSTING_READ_FW(RING_IMR(engine->mmio_base));
1583 }
1584
1585 static void
1586 gen8_irq_disable(struct intel_engine_cs *engine)
1587 {
1588         struct drm_i915_private *dev_priv = engine->i915;
1589
1590         I915_WRITE_IMR(engine, ~engine->irq_keep_mask);
1591 }
1592
1593 static int
1594 i965_emit_bb_start(struct drm_i915_gem_request *req,
1595                    u64 offset, u32 length,
1596                    unsigned int dispatch_flags)
1597 {
1598         struct intel_ring *ring = req->ring;
1599         int ret;
1600
1601         ret = intel_ring_begin(req, 2);
1602         if (ret)
1603                 return ret;
1604
1605         intel_ring_emit(ring,
1606                         MI_BATCH_BUFFER_START |
1607                         MI_BATCH_GTT |
1608                         (dispatch_flags & I915_DISPATCH_SECURE ?
1609                          0 : MI_BATCH_NON_SECURE_I965));
1610         intel_ring_emit(ring, offset);
1611         intel_ring_advance(ring);
1612
1613         return 0;
1614 }
1615
1616 /* Just userspace ABI convention to limit the wa batch bo to a resonable size */
1617 #define I830_BATCH_LIMIT (256*1024)
1618 #define I830_TLB_ENTRIES (2)
1619 #define I830_WA_SIZE max(I830_TLB_ENTRIES*4096, I830_BATCH_LIMIT)
1620 static int
1621 i830_emit_bb_start(struct drm_i915_gem_request *req,
1622                    u64 offset, u32 len,
1623                    unsigned int dispatch_flags)
1624 {
1625         struct intel_ring *ring = req->ring;
1626         u32 cs_offset = i915_ggtt_offset(req->engine->scratch);
1627         int ret;
1628
1629         ret = intel_ring_begin(req, 6);
1630         if (ret)
1631                 return ret;
1632
1633         /* Evict the invalid PTE TLBs */
1634         intel_ring_emit(ring, COLOR_BLT_CMD | BLT_WRITE_RGBA);
1635         intel_ring_emit(ring, BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | 4096);
1636         intel_ring_emit(ring, I830_TLB_ENTRIES << 16 | 4); /* load each page */
1637         intel_ring_emit(ring, cs_offset);
1638         intel_ring_emit(ring, 0xdeadbeef);
1639         intel_ring_emit(ring, MI_NOOP);
1640         intel_ring_advance(ring);
1641
1642         if ((dispatch_flags & I915_DISPATCH_PINNED) == 0) {
1643                 if (len > I830_BATCH_LIMIT)
1644                         return -ENOSPC;
1645
1646                 ret = intel_ring_begin(req, 6 + 2);
1647                 if (ret)
1648                         return ret;
1649
1650                 /* Blit the batch (which has now all relocs applied) to the
1651                  * stable batch scratch bo area (so that the CS never
1652                  * stumbles over its tlb invalidation bug) ...
1653                  */
1654                 intel_ring_emit(ring, SRC_COPY_BLT_CMD | BLT_WRITE_RGBA);
1655                 intel_ring_emit(ring,
1656                                 BLT_DEPTH_32 | BLT_ROP_SRC_COPY | 4096);
1657                 intel_ring_emit(ring, DIV_ROUND_UP(len, 4096) << 16 | 4096);
1658                 intel_ring_emit(ring, cs_offset);
1659                 intel_ring_emit(ring, 4096);
1660                 intel_ring_emit(ring, offset);
1661
1662                 intel_ring_emit(ring, MI_FLUSH);
1663                 intel_ring_emit(ring, MI_NOOP);
1664                 intel_ring_advance(ring);
1665
1666                 /* ... and execute it. */
1667                 offset = cs_offset;
1668         }
1669
1670         ret = intel_ring_begin(req, 2);
1671         if (ret)
1672                 return ret;
1673
1674         intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_GTT);
1675         intel_ring_emit(ring, offset | (dispatch_flags & I915_DISPATCH_SECURE ?
1676                                         0 : MI_BATCH_NON_SECURE));
1677         intel_ring_advance(ring);
1678
1679         return 0;
1680 }
1681
1682 static int
1683 i915_emit_bb_start(struct drm_i915_gem_request *req,
1684                    u64 offset, u32 len,
1685                    unsigned int dispatch_flags)
1686 {
1687         struct intel_ring *ring = req->ring;
1688         int ret;
1689
1690         ret = intel_ring_begin(req, 2);
1691         if (ret)
1692                 return ret;
1693
1694         intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_GTT);
1695         intel_ring_emit(ring, offset | (dispatch_flags & I915_DISPATCH_SECURE ?
1696                                         0 : MI_BATCH_NON_SECURE));
1697         intel_ring_advance(ring);
1698
1699         return 0;
1700 }
1701
1702 static void cleanup_phys_status_page(struct intel_engine_cs *engine)
1703 {
1704         struct drm_i915_private *dev_priv = engine->i915;
1705
1706         if (!dev_priv->status_page_dmah)
1707                 return;
1708
1709         drm_pci_free(&dev_priv->drm, dev_priv->status_page_dmah);
1710         engine->status_page.page_addr = NULL;
1711 }
1712
1713 static void cleanup_status_page(struct intel_engine_cs *engine)
1714 {
1715         struct i915_vma *vma;
1716         struct drm_i915_gem_object *obj;
1717
1718         vma = fetch_and_zero(&engine->status_page.vma);
1719         if (!vma)
1720                 return;
1721
1722         obj = vma->obj;
1723
1724         i915_vma_unpin(vma);
1725         i915_vma_close(vma);
1726
1727         i915_gem_object_unpin_map(obj);
1728         __i915_gem_object_release_unless_active(obj);
1729 }
1730
1731 static int init_status_page(struct intel_engine_cs *engine)
1732 {
1733         struct drm_i915_gem_object *obj;
1734         struct i915_vma *vma;
1735         unsigned int flags;
1736         void *vaddr;
1737         int ret;
1738
1739         obj = i915_gem_object_create_internal(engine->i915, 4096);
1740         if (IS_ERR(obj)) {
1741                 DRM_ERROR("Failed to allocate status page\n");
1742                 return PTR_ERR(obj);
1743         }
1744
1745         ret = i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
1746         if (ret)
1747                 goto err;
1748
1749         vma = i915_vma_create(obj, &engine->i915->ggtt.base, NULL);
1750         if (IS_ERR(vma)) {
1751                 ret = PTR_ERR(vma);
1752                 goto err;
1753         }
1754
1755         flags = PIN_GLOBAL;
1756         if (!HAS_LLC(engine->i915))
1757                 /* On g33, we cannot place HWS above 256MiB, so
1758                  * restrict its pinning to the low mappable arena.
1759                  * Though this restriction is not documented for
1760                  * gen4, gen5, or byt, they also behave similarly
1761                  * and hang if the HWS is placed at the top of the
1762                  * GTT. To generalise, it appears that all !llc
1763                  * platforms have issues with us placing the HWS
1764                  * above the mappable region (even though we never
1765                  * actualy map it).
1766                  */
1767                 flags |= PIN_MAPPABLE;
1768         ret = i915_vma_pin(vma, 0, 4096, flags);
1769         if (ret)
1770                 goto err;
1771
1772         vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
1773         if (IS_ERR(vaddr)) {
1774                 ret = PTR_ERR(vaddr);
1775                 goto err_unpin;
1776         }
1777
1778         engine->status_page.vma = vma;
1779         engine->status_page.ggtt_offset = i915_ggtt_offset(vma);
1780         engine->status_page.page_addr = memset(vaddr, 0, 4096);
1781
1782         DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
1783                          engine->name, i915_ggtt_offset(vma));
1784         return 0;
1785
1786 err_unpin:
1787         i915_vma_unpin(vma);
1788 err:
1789         i915_gem_object_put(obj);
1790         return ret;
1791 }
1792
1793 static int init_phys_status_page(struct intel_engine_cs *engine)
1794 {
1795         struct drm_i915_private *dev_priv = engine->i915;
1796
1797         dev_priv->status_page_dmah =
1798                 drm_pci_alloc(&dev_priv->drm, PAGE_SIZE, PAGE_SIZE);
1799         if (!dev_priv->status_page_dmah)
1800                 return -ENOMEM;
1801
1802         engine->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
1803         memset(engine->status_page.page_addr, 0, PAGE_SIZE);
1804
1805         return 0;
1806 }
1807
1808 int intel_ring_pin(struct intel_ring *ring)
1809 {
1810         /* Ring wraparound at offset 0 sometimes hangs. No idea why. */
1811         unsigned int flags = PIN_GLOBAL | PIN_OFFSET_BIAS | 4096;
1812         enum i915_map_type map;
1813         struct i915_vma *vma = ring->vma;
1814         void *addr;
1815         int ret;
1816
1817         GEM_BUG_ON(ring->vaddr);
1818
1819         map = HAS_LLC(ring->engine->i915) ? I915_MAP_WB : I915_MAP_WC;
1820
1821         if (vma->obj->stolen)
1822                 flags |= PIN_MAPPABLE;
1823
1824         if (!(vma->flags & I915_VMA_GLOBAL_BIND)) {
1825                 if (flags & PIN_MAPPABLE || map == I915_MAP_WC)
1826                         ret = i915_gem_object_set_to_gtt_domain(vma->obj, true);
1827                 else
1828                         ret = i915_gem_object_set_to_cpu_domain(vma->obj, true);
1829                 if (unlikely(ret))
1830                         return ret;
1831         }
1832
1833         ret = i915_vma_pin(vma, 0, PAGE_SIZE, flags);
1834         if (unlikely(ret))
1835                 return ret;
1836
1837         if (i915_vma_is_map_and_fenceable(vma))
1838                 addr = (void __force *)i915_vma_pin_iomap(vma);
1839         else
1840                 addr = i915_gem_object_pin_map(vma->obj, map);
1841         if (IS_ERR(addr))
1842                 goto err;
1843
1844         ring->vaddr = addr;
1845         return 0;
1846
1847 err:
1848         i915_vma_unpin(vma);
1849         return PTR_ERR(addr);
1850 }
1851
1852 void intel_ring_unpin(struct intel_ring *ring)
1853 {
1854         GEM_BUG_ON(!ring->vma);
1855         GEM_BUG_ON(!ring->vaddr);
1856
1857         if (i915_vma_is_map_and_fenceable(ring->vma))
1858                 i915_vma_unpin_iomap(ring->vma);
1859         else
1860                 i915_gem_object_unpin_map(ring->vma->obj);
1861         ring->vaddr = NULL;
1862
1863         i915_vma_unpin(ring->vma);
1864 }
1865
1866 static struct i915_vma *
1867 intel_ring_create_vma(struct drm_i915_private *dev_priv, int size)
1868 {
1869         struct drm_i915_gem_object *obj;
1870         struct i915_vma *vma;
1871
1872         obj = i915_gem_object_create_stolen(&dev_priv->drm, size);
1873         if (!obj)
1874                 obj = i915_gem_object_create(&dev_priv->drm, size);
1875         if (IS_ERR(obj))
1876                 return ERR_CAST(obj);
1877
1878         /* mark ring buffers as read-only from GPU side by default */
1879         obj->gt_ro = 1;
1880
1881         vma = i915_vma_create(obj, &dev_priv->ggtt.base, NULL);
1882         if (IS_ERR(vma))
1883                 goto err;
1884
1885         return vma;
1886
1887 err:
1888         i915_gem_object_put(obj);
1889         return vma;
1890 }
1891
1892 struct intel_ring *
1893 intel_engine_create_ring(struct intel_engine_cs *engine, int size)
1894 {
1895         struct intel_ring *ring;
1896         struct i915_vma *vma;
1897
1898         GEM_BUG_ON(!is_power_of_2(size));
1899         GEM_BUG_ON(RING_CTL_SIZE(size) & ~RING_NR_PAGES);
1900
1901         ring = kzalloc(sizeof(*ring), GFP_KERNEL);
1902         if (!ring)
1903                 return ERR_PTR(-ENOMEM);
1904
1905         ring->engine = engine;
1906
1907         INIT_LIST_HEAD(&ring->request_list);
1908
1909         ring->size = size;
1910         /* Workaround an erratum on the i830 which causes a hang if
1911          * the TAIL pointer points to within the last 2 cachelines
1912          * of the buffer.
1913          */
1914         ring->effective_size = size;
1915         if (IS_I830(engine->i915) || IS_845G(engine->i915))
1916                 ring->effective_size -= 2 * CACHELINE_BYTES;
1917
1918         ring->last_retired_head = -1;
1919         intel_ring_update_space(ring);
1920
1921         vma = intel_ring_create_vma(engine->i915, size);
1922         if (IS_ERR(vma)) {
1923                 kfree(ring);
1924                 return ERR_CAST(vma);
1925         }
1926         ring->vma = vma;
1927
1928         return ring;
1929 }
1930
1931 void
1932 intel_ring_free(struct intel_ring *ring)
1933 {
1934         struct drm_i915_gem_object *obj = ring->vma->obj;
1935
1936         i915_vma_close(ring->vma);
1937         __i915_gem_object_release_unless_active(obj);
1938
1939         kfree(ring);
1940 }
1941
1942 static int intel_ring_context_pin(struct i915_gem_context *ctx,
1943                                   struct intel_engine_cs *engine)
1944 {
1945         struct intel_context *ce = &ctx->engine[engine->id];
1946         int ret;
1947
1948         lockdep_assert_held(&ctx->i915->drm.struct_mutex);
1949
1950         if (ce->pin_count++)
1951                 return 0;
1952
1953         if (ce->state) {
1954                 struct i915_vma *vma;
1955
1956                 vma = i915_gem_context_pin_legacy(ctx, PIN_HIGH);
1957                 if (IS_ERR(vma)) {
1958                         ret = PTR_ERR(vma);
1959                         goto error;
1960                 }
1961         }
1962
1963         /* The kernel context is only used as a placeholder for flushing the
1964          * active context. It is never used for submitting user rendering and
1965          * as such never requires the golden render context, and so we can skip
1966          * emitting it when we switch to the kernel context. This is required
1967          * as during eviction we cannot allocate and pin the renderstate in
1968          * order to initialise the context.
1969          */
1970         if (ctx == ctx->i915->kernel_context)
1971                 ce->initialised = true;
1972
1973         i915_gem_context_get(ctx);
1974         return 0;
1975
1976 error:
1977         ce->pin_count = 0;
1978         return ret;
1979 }
1980
1981 static void intel_ring_context_unpin(struct i915_gem_context *ctx,
1982                                      struct intel_engine_cs *engine)
1983 {
1984         struct intel_context *ce = &ctx->engine[engine->id];
1985
1986         lockdep_assert_held(&ctx->i915->drm.struct_mutex);
1987
1988         if (--ce->pin_count)
1989                 return;
1990
1991         if (ce->state)
1992                 i915_vma_unpin(ce->state);
1993
1994         i915_gem_context_put(ctx);
1995 }
1996
1997 static int intel_init_ring_buffer(struct intel_engine_cs *engine)
1998 {
1999         struct drm_i915_private *dev_priv = engine->i915;
2000         struct intel_ring *ring;
2001         int ret;
2002
2003         WARN_ON(engine->buffer);
2004
2005         intel_engine_setup_common(engine);
2006
2007         ret = intel_engine_init_common(engine);
2008         if (ret)
2009                 goto error;
2010
2011         /* We may need to do things with the shrinker which
2012          * require us to immediately switch back to the default
2013          * context. This can cause a problem as pinning the
2014          * default context also requires GTT space which may not
2015          * be available. To avoid this we always pin the default
2016          * context.
2017          */
2018         ret = intel_ring_context_pin(dev_priv->kernel_context, engine);
2019         if (ret)
2020                 goto error;
2021
2022         ring = intel_engine_create_ring(engine, 32 * PAGE_SIZE);
2023         if (IS_ERR(ring)) {
2024                 ret = PTR_ERR(ring);
2025                 goto error;
2026         }
2027
2028         if (HWS_NEEDS_PHYSICAL(dev_priv)) {
2029                 WARN_ON(engine->id != RCS);
2030                 ret = init_phys_status_page(engine);
2031                 if (ret)
2032                         goto error;
2033         } else {
2034                 ret = init_status_page(engine);
2035                 if (ret)
2036                         goto error;
2037         }
2038
2039         ret = intel_ring_pin(ring);
2040         if (ret) {
2041                 intel_ring_free(ring);
2042                 goto error;
2043         }
2044         engine->buffer = ring;
2045
2046         return 0;
2047
2048 error:
2049         intel_engine_cleanup(engine);
2050         return ret;
2051 }
2052
2053 void intel_engine_cleanup(struct intel_engine_cs *engine)
2054 {
2055         struct drm_i915_private *dev_priv;
2056
2057         dev_priv = engine->i915;
2058
2059         if (engine->buffer) {
2060                 WARN_ON(INTEL_GEN(dev_priv) > 2 &&
2061                         (I915_READ_MODE(engine) & MODE_IDLE) == 0);
2062
2063                 intel_ring_unpin(engine->buffer);
2064                 intel_ring_free(engine->buffer);
2065                 engine->buffer = NULL;
2066         }
2067
2068         if (engine->cleanup)
2069                 engine->cleanup(engine);
2070
2071         if (HWS_NEEDS_PHYSICAL(dev_priv)) {
2072                 WARN_ON(engine->id != RCS);
2073                 cleanup_phys_status_page(engine);
2074         } else {
2075                 cleanup_status_page(engine);
2076         }
2077
2078         intel_engine_cleanup_common(engine);
2079
2080         intel_ring_context_unpin(dev_priv->kernel_context, engine);
2081
2082         engine->i915 = NULL;
2083         dev_priv->engine[engine->id] = NULL;
2084         kfree(engine);
2085 }
2086
2087 void intel_legacy_submission_resume(struct drm_i915_private *dev_priv)
2088 {
2089         struct intel_engine_cs *engine;
2090         enum intel_engine_id id;
2091
2092         for_each_engine(engine, dev_priv, id) {
2093                 engine->buffer->head = engine->buffer->tail;
2094                 engine->buffer->last_retired_head = -1;
2095         }
2096 }
2097
2098 int intel_ring_alloc_request_extras(struct drm_i915_gem_request *request)
2099 {
2100         int ret;
2101
2102         /* Flush enough space to reduce the likelihood of waiting after
2103          * we start building the request - in which case we will just
2104          * have to repeat work.
2105          */
2106         request->reserved_space += LEGACY_REQUEST_SIZE;
2107
2108         request->ring = request->engine->buffer;
2109
2110         ret = intel_ring_begin(request, 0);
2111         if (ret)
2112                 return ret;
2113
2114         request->reserved_space -= LEGACY_REQUEST_SIZE;
2115         return 0;
2116 }
2117
2118 static int wait_for_space(struct drm_i915_gem_request *req, int bytes)
2119 {
2120         struct intel_ring *ring = req->ring;
2121         struct drm_i915_gem_request *target;
2122         long timeout;
2123
2124         lockdep_assert_held(&req->i915->drm.struct_mutex);
2125
2126         intel_ring_update_space(ring);
2127         if (ring->space >= bytes)
2128                 return 0;
2129
2130         /*
2131          * Space is reserved in the ringbuffer for finalising the request,
2132          * as that cannot be allowed to fail. During request finalisation,
2133          * reserved_space is set to 0 to stop the overallocation and the
2134          * assumption is that then we never need to wait (which has the
2135          * risk of failing with EINTR).
2136          *
2137          * See also i915_gem_request_alloc() and i915_add_request().
2138          */
2139         GEM_BUG_ON(!req->reserved_space);
2140
2141         list_for_each_entry(target, &ring->request_list, ring_link) {
2142                 unsigned space;
2143
2144                 /* Would completion of this request free enough space? */
2145                 space = __intel_ring_space(target->postfix, ring->tail,
2146                                            ring->size);
2147                 if (space >= bytes)
2148                         break;
2149         }
2150
2151         if (WARN_ON(&target->ring_link == &ring->request_list))
2152                 return -ENOSPC;
2153
2154         timeout = i915_wait_request(target,
2155                                     I915_WAIT_INTERRUPTIBLE | I915_WAIT_LOCKED,
2156                                     MAX_SCHEDULE_TIMEOUT);
2157         if (timeout < 0)
2158                 return timeout;
2159
2160         i915_gem_request_retire_upto(target);
2161
2162         intel_ring_update_space(ring);
2163         GEM_BUG_ON(ring->space < bytes);
2164         return 0;
2165 }
2166
2167 int intel_ring_begin(struct drm_i915_gem_request *req, int num_dwords)
2168 {
2169         struct intel_ring *ring = req->ring;
2170         int remain_actual = ring->size - ring->tail;
2171         int remain_usable = ring->effective_size - ring->tail;
2172         int bytes = num_dwords * sizeof(u32);
2173         int total_bytes, wait_bytes;
2174         bool need_wrap = false;
2175
2176         total_bytes = bytes + req->reserved_space;
2177
2178         if (unlikely(bytes > remain_usable)) {
2179                 /*
2180                  * Not enough space for the basic request. So need to flush
2181                  * out the remainder and then wait for base + reserved.
2182                  */
2183                 wait_bytes = remain_actual + total_bytes;
2184                 need_wrap = true;
2185         } else if (unlikely(total_bytes > remain_usable)) {
2186                 /*
2187                  * The base request will fit but the reserved space
2188                  * falls off the end. So we don't need an immediate wrap
2189                  * and only need to effectively wait for the reserved
2190                  * size space from the start of ringbuffer.
2191                  */
2192                 wait_bytes = remain_actual + req->reserved_space;
2193         } else {
2194                 /* No wrapping required, just waiting. */
2195                 wait_bytes = total_bytes;
2196         }
2197
2198         if (wait_bytes > ring->space) {
2199                 int ret = wait_for_space(req, wait_bytes);
2200                 if (unlikely(ret))
2201                         return ret;
2202         }
2203
2204         if (unlikely(need_wrap)) {
2205                 GEM_BUG_ON(remain_actual > ring->space);
2206                 GEM_BUG_ON(ring->tail + remain_actual > ring->size);
2207
2208                 /* Fill the tail with MI_NOOP */
2209                 memset(ring->vaddr + ring->tail, 0, remain_actual);
2210                 ring->tail = 0;
2211                 ring->space -= remain_actual;
2212         }
2213
2214         ring->space -= bytes;
2215         GEM_BUG_ON(ring->space < 0);
2216         return 0;
2217 }
2218
2219 /* Align the ring tail to a cacheline boundary */
2220 int intel_ring_cacheline_align(struct drm_i915_gem_request *req)
2221 {
2222         struct intel_ring *ring = req->ring;
2223         int num_dwords =
2224                 (ring->tail & (CACHELINE_BYTES - 1)) / sizeof(uint32_t);
2225         int ret;
2226
2227         if (num_dwords == 0)
2228                 return 0;
2229
2230         num_dwords = CACHELINE_BYTES / sizeof(uint32_t) - num_dwords;
2231         ret = intel_ring_begin(req, num_dwords);
2232         if (ret)
2233                 return ret;
2234
2235         while (num_dwords--)
2236                 intel_ring_emit(ring, MI_NOOP);
2237
2238         intel_ring_advance(ring);
2239
2240         return 0;
2241 }
2242
2243 static void gen6_bsd_submit_request(struct drm_i915_gem_request *request)
2244 {
2245         struct drm_i915_private *dev_priv = request->i915;
2246
2247         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
2248
2249        /* Every tail move must follow the sequence below */
2250
2251         /* Disable notification that the ring is IDLE. The GT
2252          * will then assume that it is busy and bring it out of rc6.
2253          */
2254         I915_WRITE_FW(GEN6_BSD_SLEEP_PSMI_CONTROL,
2255                       _MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
2256
2257         /* Clear the context id. Here be magic! */
2258         I915_WRITE64_FW(GEN6_BSD_RNCID, 0x0);
2259
2260         /* Wait for the ring not to be idle, i.e. for it to wake up. */
2261         if (intel_wait_for_register_fw(dev_priv,
2262                                        GEN6_BSD_SLEEP_PSMI_CONTROL,
2263                                        GEN6_BSD_SLEEP_INDICATOR,
2264                                        0,
2265                                        50))
2266                 DRM_ERROR("timed out waiting for the BSD ring to wake up\n");
2267
2268         /* Now that the ring is fully powered up, update the tail */
2269         i9xx_submit_request(request);
2270
2271         /* Let the ring send IDLE messages to the GT again,
2272          * and so let it sleep to conserve power when idle.
2273          */
2274         I915_WRITE_FW(GEN6_BSD_SLEEP_PSMI_CONTROL,
2275                       _MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
2276
2277         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
2278 }
2279
2280 static int gen6_bsd_ring_flush(struct drm_i915_gem_request *req, u32 mode)
2281 {
2282         struct intel_ring *ring = req->ring;
2283         uint32_t cmd;
2284         int ret;
2285
2286         ret = intel_ring_begin(req, 4);
2287         if (ret)
2288                 return ret;
2289
2290         cmd = MI_FLUSH_DW;
2291         if (INTEL_GEN(req->i915) >= 8)
2292                 cmd += 1;
2293
2294         /* We always require a command barrier so that subsequent
2295          * commands, such as breadcrumb interrupts, are strictly ordered
2296          * wrt the contents of the write cache being flushed to memory
2297          * (and thus being coherent from the CPU).
2298          */
2299         cmd |= MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW;
2300
2301         /*
2302          * Bspec vol 1c.5 - video engine command streamer:
2303          * "If ENABLED, all TLBs will be invalidated once the flush
2304          * operation is complete. This bit is only valid when the
2305          * Post-Sync Operation field is a value of 1h or 3h."
2306          */
2307         if (mode & EMIT_INVALIDATE)
2308                 cmd |= MI_INVALIDATE_TLB | MI_INVALIDATE_BSD;
2309
2310         intel_ring_emit(ring, cmd);
2311         intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT);
2312         if (INTEL_GEN(req->i915) >= 8) {
2313                 intel_ring_emit(ring, 0); /* upper addr */
2314                 intel_ring_emit(ring, 0); /* value */
2315         } else  {
2316                 intel_ring_emit(ring, 0);
2317                 intel_ring_emit(ring, MI_NOOP);
2318         }
2319         intel_ring_advance(ring);
2320         return 0;
2321 }
2322
2323 static int
2324 gen8_emit_bb_start(struct drm_i915_gem_request *req,
2325                    u64 offset, u32 len,
2326                    unsigned int dispatch_flags)
2327 {
2328         struct intel_ring *ring = req->ring;
2329         bool ppgtt = USES_PPGTT(req->i915) &&
2330                         !(dispatch_flags & I915_DISPATCH_SECURE);
2331         int ret;
2332
2333         ret = intel_ring_begin(req, 4);
2334         if (ret)
2335                 return ret;
2336
2337         /* FIXME(BDW): Address space and security selectors. */
2338         intel_ring_emit(ring, MI_BATCH_BUFFER_START_GEN8 | (ppgtt<<8) |
2339                         (dispatch_flags & I915_DISPATCH_RS ?
2340                          MI_BATCH_RESOURCE_STREAMER : 0));
2341         intel_ring_emit(ring, lower_32_bits(offset));
2342         intel_ring_emit(ring, upper_32_bits(offset));
2343         intel_ring_emit(ring, MI_NOOP);
2344         intel_ring_advance(ring);
2345
2346         return 0;
2347 }
2348
2349 static int
2350 hsw_emit_bb_start(struct drm_i915_gem_request *req,
2351                   u64 offset, u32 len,
2352                   unsigned int dispatch_flags)
2353 {
2354         struct intel_ring *ring = req->ring;
2355         int ret;
2356
2357         ret = intel_ring_begin(req, 2);
2358         if (ret)
2359                 return ret;
2360
2361         intel_ring_emit(ring,
2362                         MI_BATCH_BUFFER_START |
2363                         (dispatch_flags & I915_DISPATCH_SECURE ?
2364                          0 : MI_BATCH_PPGTT_HSW | MI_BATCH_NON_SECURE_HSW) |
2365                         (dispatch_flags & I915_DISPATCH_RS ?
2366                          MI_BATCH_RESOURCE_STREAMER : 0));
2367         /* bit0-7 is the length on GEN6+ */
2368         intel_ring_emit(ring, offset);
2369         intel_ring_advance(ring);
2370
2371         return 0;
2372 }
2373
2374 static int
2375 gen6_emit_bb_start(struct drm_i915_gem_request *req,
2376                    u64 offset, u32 len,
2377                    unsigned int dispatch_flags)
2378 {
2379         struct intel_ring *ring = req->ring;
2380         int ret;
2381
2382         ret = intel_ring_begin(req, 2);
2383         if (ret)
2384                 return ret;
2385
2386         intel_ring_emit(ring,
2387                         MI_BATCH_BUFFER_START |
2388                         (dispatch_flags & I915_DISPATCH_SECURE ?
2389                          0 : MI_BATCH_NON_SECURE_I965));
2390         /* bit0-7 is the length on GEN6+ */
2391         intel_ring_emit(ring, offset);
2392         intel_ring_advance(ring);
2393
2394         return 0;
2395 }
2396
2397 /* Blitter support (SandyBridge+) */
2398
2399 static int gen6_ring_flush(struct drm_i915_gem_request *req, u32 mode)
2400 {
2401         struct intel_ring *ring = req->ring;
2402         uint32_t cmd;
2403         int ret;
2404
2405         ret = intel_ring_begin(req, 4);
2406         if (ret)
2407                 return ret;
2408
2409         cmd = MI_FLUSH_DW;
2410         if (INTEL_GEN(req->i915) >= 8)
2411                 cmd += 1;
2412
2413         /* We always require a command barrier so that subsequent
2414          * commands, such as breadcrumb interrupts, are strictly ordered
2415          * wrt the contents of the write cache being flushed to memory
2416          * (and thus being coherent from the CPU).
2417          */
2418         cmd |= MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW;
2419
2420         /*
2421          * Bspec vol 1c.3 - blitter engine command streamer:
2422          * "If ENABLED, all TLBs will be invalidated once the flush
2423          * operation is complete. This bit is only valid when the
2424          * Post-Sync Operation field is a value of 1h or 3h."
2425          */
2426         if (mode & EMIT_INVALIDATE)
2427                 cmd |= MI_INVALIDATE_TLB;
2428         intel_ring_emit(ring, cmd);
2429         intel_ring_emit(ring,
2430                         I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT);
2431         if (INTEL_GEN(req->i915) >= 8) {
2432                 intel_ring_emit(ring, 0); /* upper addr */
2433                 intel_ring_emit(ring, 0); /* value */
2434         } else  {
2435                 intel_ring_emit(ring, 0);
2436                 intel_ring_emit(ring, MI_NOOP);
2437         }
2438         intel_ring_advance(ring);
2439
2440         return 0;
2441 }
2442
2443 static void intel_ring_init_semaphores(struct drm_i915_private *dev_priv,
2444                                        struct intel_engine_cs *engine)
2445 {
2446         struct drm_i915_gem_object *obj;
2447         int ret, i;
2448
2449         if (!i915.semaphores)
2450                 return;
2451
2452         if (INTEL_GEN(dev_priv) >= 8 && !dev_priv->semaphore) {
2453                 struct i915_vma *vma;
2454
2455                 obj = i915_gem_object_create(&dev_priv->drm, 4096);
2456                 if (IS_ERR(obj))
2457                         goto err;
2458
2459                 vma = i915_vma_create(obj, &dev_priv->ggtt.base, NULL);
2460                 if (IS_ERR(vma))
2461                         goto err_obj;
2462
2463                 ret = i915_gem_object_set_to_gtt_domain(obj, false);
2464                 if (ret)
2465                         goto err_obj;
2466
2467                 ret = i915_vma_pin(vma, 0, 0, PIN_GLOBAL | PIN_HIGH);
2468                 if (ret)
2469                         goto err_obj;
2470
2471                 dev_priv->semaphore = vma;
2472         }
2473
2474         if (INTEL_GEN(dev_priv) >= 8) {
2475                 u32 offset = i915_ggtt_offset(dev_priv->semaphore);
2476
2477                 engine->semaphore.sync_to = gen8_ring_sync_to;
2478                 engine->semaphore.signal = gen8_xcs_signal;
2479
2480                 for (i = 0; i < I915_NUM_ENGINES; i++) {
2481                         u32 ring_offset;
2482
2483                         if (i != engine->id)
2484                                 ring_offset = offset + GEN8_SEMAPHORE_OFFSET(engine->id, i);
2485                         else
2486                                 ring_offset = MI_SEMAPHORE_SYNC_INVALID;
2487
2488                         engine->semaphore.signal_ggtt[i] = ring_offset;
2489                 }
2490         } else if (INTEL_GEN(dev_priv) >= 6) {
2491                 engine->semaphore.sync_to = gen6_ring_sync_to;
2492                 engine->semaphore.signal = gen6_signal;
2493
2494                 /*
2495                  * The current semaphore is only applied on pre-gen8
2496                  * platform.  And there is no VCS2 ring on the pre-gen8
2497                  * platform. So the semaphore between RCS and VCS2 is
2498                  * initialized as INVALID.  Gen8 will initialize the
2499                  * sema between VCS2 and RCS later.
2500                  */
2501                 for (i = 0; i < GEN6_NUM_SEMAPHORES; i++) {
2502                         static const struct {
2503                                 u32 wait_mbox;
2504                                 i915_reg_t mbox_reg;
2505                         } sem_data[GEN6_NUM_SEMAPHORES][GEN6_NUM_SEMAPHORES] = {
2506                                 [RCS_HW] = {
2507                                         [VCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_RV,  .mbox_reg = GEN6_VRSYNC },
2508                                         [BCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_RB,  .mbox_reg = GEN6_BRSYNC },
2509                                         [VECS_HW] = { .wait_mbox = MI_SEMAPHORE_SYNC_RVE, .mbox_reg = GEN6_VERSYNC },
2510                                 },
2511                                 [VCS_HW] = {
2512                                         [RCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_VR,  .mbox_reg = GEN6_RVSYNC },
2513                                         [BCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_VB,  .mbox_reg = GEN6_BVSYNC },
2514                                         [VECS_HW] = { .wait_mbox = MI_SEMAPHORE_SYNC_VVE, .mbox_reg = GEN6_VEVSYNC },
2515                                 },
2516                                 [BCS_HW] = {
2517                                         [RCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_BR,  .mbox_reg = GEN6_RBSYNC },
2518                                         [VCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_BV,  .mbox_reg = GEN6_VBSYNC },
2519                                         [VECS_HW] = { .wait_mbox = MI_SEMAPHORE_SYNC_BVE, .mbox_reg = GEN6_VEBSYNC },
2520                                 },
2521                                 [VECS_HW] = {
2522                                         [RCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_VER, .mbox_reg = GEN6_RVESYNC },
2523                                         [VCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_VEV, .mbox_reg = GEN6_VVESYNC },
2524                                         [BCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_VEB, .mbox_reg = GEN6_BVESYNC },
2525                                 },
2526                         };
2527                         u32 wait_mbox;
2528                         i915_reg_t mbox_reg;
2529
2530                         if (i == engine->hw_id) {
2531                                 wait_mbox = MI_SEMAPHORE_SYNC_INVALID;
2532                                 mbox_reg = GEN6_NOSYNC;
2533                         } else {
2534                                 wait_mbox = sem_data[engine->hw_id][i].wait_mbox;
2535                                 mbox_reg = sem_data[engine->hw_id][i].mbox_reg;
2536                         }
2537
2538                         engine->semaphore.mbox.wait[i] = wait_mbox;
2539                         engine->semaphore.mbox.signal[i] = mbox_reg;
2540                 }
2541         }
2542
2543         return;
2544
2545 err_obj:
2546         i915_gem_object_put(obj);
2547 err:
2548         DRM_DEBUG_DRIVER("Failed to allocate space for semaphores, disabling\n");
2549         i915.semaphores = 0;
2550 }
2551
2552 static void intel_ring_init_irq(struct drm_i915_private *dev_priv,
2553                                 struct intel_engine_cs *engine)
2554 {
2555         engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT << engine->irq_shift;
2556
2557         if (INTEL_GEN(dev_priv) >= 8) {
2558                 engine->irq_enable = gen8_irq_enable;
2559                 engine->irq_disable = gen8_irq_disable;
2560                 engine->irq_seqno_barrier = gen6_seqno_barrier;
2561         } else if (INTEL_GEN(dev_priv) >= 6) {
2562                 engine->irq_enable = gen6_irq_enable;
2563                 engine->irq_disable = gen6_irq_disable;
2564                 engine->irq_seqno_barrier = gen6_seqno_barrier;
2565         } else if (INTEL_GEN(dev_priv) >= 5) {
2566                 engine->irq_enable = gen5_irq_enable;
2567                 engine->irq_disable = gen5_irq_disable;
2568                 engine->irq_seqno_barrier = gen5_seqno_barrier;
2569         } else if (INTEL_GEN(dev_priv) >= 3) {
2570                 engine->irq_enable = i9xx_irq_enable;
2571                 engine->irq_disable = i9xx_irq_disable;
2572         } else {
2573                 engine->irq_enable = i8xx_irq_enable;
2574                 engine->irq_disable = i8xx_irq_disable;
2575         }
2576 }
2577
2578 static void intel_ring_default_vfuncs(struct drm_i915_private *dev_priv,
2579                                       struct intel_engine_cs *engine)
2580 {
2581         intel_ring_init_irq(dev_priv, engine);
2582         intel_ring_init_semaphores(dev_priv, engine);
2583
2584         engine->init_hw = init_ring_common;
2585         engine->reset_hw = reset_ring_common;
2586
2587         engine->emit_breadcrumb = i9xx_emit_breadcrumb;
2588         engine->emit_breadcrumb_sz = i9xx_emit_breadcrumb_sz;
2589         if (i915.semaphores) {
2590                 int num_rings;
2591
2592                 engine->emit_breadcrumb = gen6_sema_emit_breadcrumb;
2593
2594                 num_rings = hweight32(INTEL_INFO(dev_priv)->ring_mask) - 1;
2595                 if (INTEL_GEN(dev_priv) >= 8) {
2596                         engine->emit_breadcrumb_sz += num_rings * 6;
2597                 } else {
2598                         engine->emit_breadcrumb_sz += num_rings * 3;
2599                         if (num_rings & 1)
2600                                 engine->emit_breadcrumb_sz++;
2601                 }
2602         }
2603         engine->submit_request = i9xx_submit_request;
2604
2605         if (INTEL_GEN(dev_priv) >= 8)
2606                 engine->emit_bb_start = gen8_emit_bb_start;
2607         else if (INTEL_GEN(dev_priv) >= 6)
2608                 engine->emit_bb_start = gen6_emit_bb_start;
2609         else if (INTEL_GEN(dev_priv) >= 4)
2610                 engine->emit_bb_start = i965_emit_bb_start;
2611         else if (IS_I830(dev_priv) || IS_845G(dev_priv))
2612                 engine->emit_bb_start = i830_emit_bb_start;
2613         else
2614                 engine->emit_bb_start = i915_emit_bb_start;
2615 }
2616
2617 int intel_init_render_ring_buffer(struct intel_engine_cs *engine)
2618 {
2619         struct drm_i915_private *dev_priv = engine->i915;
2620         int ret;
2621
2622         intel_ring_default_vfuncs(dev_priv, engine);
2623
2624         if (HAS_L3_DPF(dev_priv))
2625                 engine->irq_keep_mask = GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
2626
2627         if (INTEL_GEN(dev_priv) >= 8) {
2628                 engine->init_context = intel_rcs_ctx_init;
2629                 engine->emit_breadcrumb = gen8_render_emit_breadcrumb;
2630                 engine->emit_breadcrumb_sz = gen8_render_emit_breadcrumb_sz;
2631                 engine->emit_flush = gen8_render_ring_flush;
2632                 if (i915.semaphores) {
2633                         int num_rings;
2634
2635                         engine->semaphore.signal = gen8_rcs_signal;
2636
2637                         num_rings =
2638                                 hweight32(INTEL_INFO(dev_priv)->ring_mask) - 1;
2639                         engine->emit_breadcrumb_sz += num_rings * 6;
2640                 }
2641         } else if (INTEL_GEN(dev_priv) >= 6) {
2642                 engine->init_context = intel_rcs_ctx_init;
2643                 engine->emit_flush = gen7_render_ring_flush;
2644                 if (IS_GEN6(dev_priv))
2645                         engine->emit_flush = gen6_render_ring_flush;
2646         } else if (IS_GEN5(dev_priv)) {
2647                 engine->emit_flush = gen4_render_ring_flush;
2648         } else {
2649                 if (INTEL_GEN(dev_priv) < 4)
2650                         engine->emit_flush = gen2_render_ring_flush;
2651                 else
2652                         engine->emit_flush = gen4_render_ring_flush;
2653                 engine->irq_enable_mask = I915_USER_INTERRUPT;
2654         }
2655
2656         if (IS_HASWELL(dev_priv))
2657                 engine->emit_bb_start = hsw_emit_bb_start;
2658
2659         engine->init_hw = init_render_ring;
2660         engine->cleanup = render_ring_cleanup;
2661
2662         ret = intel_init_ring_buffer(engine);
2663         if (ret)
2664                 return ret;
2665
2666         if (INTEL_GEN(dev_priv) >= 6) {
2667                 ret = intel_engine_create_scratch(engine, 4096);
2668                 if (ret)
2669                         return ret;
2670         } else if (HAS_BROKEN_CS_TLB(dev_priv)) {
2671                 ret = intel_engine_create_scratch(engine, I830_WA_SIZE);
2672                 if (ret)
2673                         return ret;
2674         }
2675
2676         return 0;
2677 }
2678
2679 int intel_init_bsd_ring_buffer(struct intel_engine_cs *engine)
2680 {
2681         struct drm_i915_private *dev_priv = engine->i915;
2682
2683         intel_ring_default_vfuncs(dev_priv, engine);
2684
2685         if (INTEL_GEN(dev_priv) >= 6) {
2686                 /* gen6 bsd needs a special wa for tail updates */
2687                 if (IS_GEN6(dev_priv))
2688                         engine->submit_request = gen6_bsd_submit_request;
2689                 engine->emit_flush = gen6_bsd_ring_flush;
2690                 if (INTEL_GEN(dev_priv) < 8)
2691                         engine->irq_enable_mask = GT_BSD_USER_INTERRUPT;
2692         } else {
2693                 engine->mmio_base = BSD_RING_BASE;
2694                 engine->emit_flush = bsd_ring_flush;
2695                 if (IS_GEN5(dev_priv))
2696                         engine->irq_enable_mask = ILK_BSD_USER_INTERRUPT;
2697                 else
2698                         engine->irq_enable_mask = I915_BSD_USER_INTERRUPT;
2699         }
2700
2701         return intel_init_ring_buffer(engine);
2702 }
2703
2704 /**
2705  * Initialize the second BSD ring (eg. Broadwell GT3, Skylake GT3)
2706  */
2707 int intel_init_bsd2_ring_buffer(struct intel_engine_cs *engine)
2708 {
2709         struct drm_i915_private *dev_priv = engine->i915;
2710
2711         intel_ring_default_vfuncs(dev_priv, engine);
2712
2713         engine->emit_flush = gen6_bsd_ring_flush;
2714
2715         return intel_init_ring_buffer(engine);
2716 }
2717
2718 int intel_init_blt_ring_buffer(struct intel_engine_cs *engine)
2719 {
2720         struct drm_i915_private *dev_priv = engine->i915;
2721
2722         intel_ring_default_vfuncs(dev_priv, engine);
2723
2724         engine->emit_flush = gen6_ring_flush;
2725         if (INTEL_GEN(dev_priv) < 8)
2726                 engine->irq_enable_mask = GT_BLT_USER_INTERRUPT;
2727
2728         return intel_init_ring_buffer(engine);
2729 }
2730
2731 int intel_init_vebox_ring_buffer(struct intel_engine_cs *engine)
2732 {
2733         struct drm_i915_private *dev_priv = engine->i915;
2734
2735         intel_ring_default_vfuncs(dev_priv, engine);
2736
2737         engine->emit_flush = gen6_ring_flush;
2738
2739         if (INTEL_GEN(dev_priv) < 8) {
2740                 engine->irq_enable_mask = PM_VEBOX_USER_INTERRUPT;
2741                 engine->irq_enable = hsw_vebox_irq_enable;
2742                 engine->irq_disable = hsw_vebox_irq_disable;
2743         }
2744
2745         return intel_init_ring_buffer(engine);
2746 }