drivers/gpu/drm/i915/intel_ringbuffer.c

   1 /*
   2  * Copyright © 2008-2010 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *    Zou Nan hai <nanhai.zou@intel.com>
  26  *    Xiang Hai hao<haihao.xiang@intel.com>
  27  *
  28  */
  29
  30 #include "drmP.h"
  31 #include "i915_drv.h"
  32 #include "i915_drm.h"
  33 #include "i915_trace.h"
  34 #include "intel_drv.h"
  35
  36 /*
  37  * 965+ support PIPE_CONTROL commands, which provide finer grained control
  38  * over cache flushing.
  39  */
  40 struct pipe_control {
  41         struct drm_i915_gem_object *obj;
  42         volatile u32 *cpu_page;
  43         u32 gtt_offset;
  44 };
  45
  46 static inline int ring_space(struct intel_ring_buffer *ring)
  47 {
  48         int space = (ring->head & HEAD_ADDR) - (ring->tail + 8);
  49         if (space < 0)
  50                 space += ring->size;
  51         return space;
  52 }
  53
  54 static int
  55 gen2_render_ring_flush(struct intel_ring_buffer *ring,
  56                        u32      invalidate_domains,
  57                        u32      flush_domains)
  58 {
  59         u32 cmd;
  60         int ret;
  61
  62         cmd = MI_FLUSH;
  63         if (((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER) == 0)
  64                 cmd |= MI_NO_WRITE_FLUSH;
  65
  66         if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
  67                 cmd |= MI_READ_FLUSH;
  68
  69         ret = intel_ring_begin(ring, 2);
  70         if (ret)
  71                 return ret;
  72
  73         intel_ring_emit(ring, cmd);
  74         intel_ring_emit(ring, MI_NOOP);
  75         intel_ring_advance(ring);
  76
  77         return 0;
  78 }
  79
  80 static int
  81 gen4_render_ring_flush(struct intel_ring_buffer *ring,
  82                        u32      invalidate_domains,
  83                        u32      flush_domains)
  84 {
  85         struct drm_device *dev = ring->dev;
  86         u32 cmd;
  87         int ret;
  88
  89         /*
  90          * read/write caches:
  91          *
  92          * I915_GEM_DOMAIN_RENDER is always invalidated, but is
  93          * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
  94          * also flushed at 2d versus 3d pipeline switches.
  95          *
  96          * read-only caches:
  97          *
  98          * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
  99          * MI_READ_FLUSH is set, and is always flushed on 965.
 100          *
 101          * I915_GEM_DOMAIN_COMMAND may not exist?
 102          *
 103          * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
 104          * invalidated when MI_EXE_FLUSH is set.
 105          *
 106          * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
 107          * invalidated with every MI_FLUSH.
 108          *
 109          * TLBs:
 110          *
 111          * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
 112          * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
 113          * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
 114          * are flushed at any MI_FLUSH.
 115          */
 116
 117         cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
 118         if ((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER)
 119                 cmd &= ~MI_NO_WRITE_FLUSH;
 120         if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
 121                 cmd |= MI_EXE_FLUSH;
 122
 123         if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
 124             (IS_G4X(dev) || IS_GEN5(dev)))
 125                 cmd |= MI_INVALIDATE_ISP;
 126
 127         ret = intel_ring_begin(ring, 2);
 128         if (ret)
 129                 return ret;
 130
 131         intel_ring_emit(ring, cmd);
 132         intel_ring_emit(ring, MI_NOOP);
 133         intel_ring_advance(ring);
 134
 135         return 0;
 136 }
 137
 138 /**
 139  * Emits a PIPE_CONTROL with a non-zero post-sync operation, for
 140  * implementing two workarounds on gen6.  From section 1.4.7.1
 141  * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1:
 142  *
 143  * [DevSNB-C+{W/A}] Before any depth stall flush (including those
 144  * produced by non-pipelined state commands), software needs to first
 145  * send a PIPE_CONTROL with no bits set except Post-Sync Operation !=
 146  * 0.
 147  *
 148  * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable
 149  * =1, a PIPE_CONTROL with any non-zero post-sync-op is required.
 150  *
 151  * And the workaround for these two requires this workaround first:
 152  *
 153  * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent
 154  * BEFORE the pipe-control with a post-sync op and no write-cache
 155  * flushes.
 156  *
 157  * And this last workaround is tricky because of the requirements on
 158  * that bit.  From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM
 159  * volume 2 part 1:
 160  *
 161  *     "1 of the following must also be set:
 162  *      - Render Target Cache Flush Enable ([12] of DW1)
 163  *      - Depth Cache Flush Enable ([0] of DW1)
 164  *      - Stall at Pixel Scoreboard ([1] of DW1)
 165  *      - Depth Stall ([13] of DW1)
 166  *      - Post-Sync Operation ([13] of DW1)
 167  *      - Notify Enable ([8] of DW1)"
 168  *
 169  * The cache flushes require the workaround flush that triggered this
 170  * one, so we can't use it.  Depth stall would trigger the same.
 171  * Post-sync nonzero is what triggered this second workaround, so we
 172  * can't use that one either.  Notify enable is IRQs, which aren't
 173  * really our business.  That leaves only stall at scoreboard.
 174  */
 175 static int
 176 intel_emit_post_sync_nonzero_flush(struct intel_ring_buffer *ring)
 177 {
 178         struct pipe_control *pc = ring->private;
 179         u32 scratch_addr = pc->gtt_offset + 128;
 180         int ret;
 181
 182
 183         ret = intel_ring_begin(ring, 6);
 184         if (ret)
 185                 return ret;
 186
 187         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
 188         intel_ring_emit(ring, PIPE_CONTROL_CS_STALL |
 189                         PIPE_CONTROL_STALL_AT_SCOREBOARD);
 190         intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
 191         intel_ring_emit(ring, 0); /* low dword */
 192         intel_ring_emit(ring, 0); /* high dword */
 193         intel_ring_emit(ring, MI_NOOP);
 194         intel_ring_advance(ring);
 195
 196         ret = intel_ring_begin(ring, 6);
 197         if (ret)
 198                 return ret;
 199
 200         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(5));
 201         intel_ring_emit(ring, PIPE_CONTROL_QW_WRITE);
 202         intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT); /* address */
 203         intel_ring_emit(ring, 0);
 204         intel_ring_emit(ring, 0);
 205         intel_ring_emit(ring, MI_NOOP);
 206         intel_ring_advance(ring);
 207
 208         return 0;
 209 }
 210
 211 static int
 212 gen6_render_ring_flush(struct intel_ring_buffer *ring,
 213                          u32 invalidate_domains, u32 flush_domains)
 214 {
 215         u32 flags = 0;
 216         struct pipe_control *pc = ring->private;
 217         u32 scratch_addr = pc->gtt_offset + 128;
 218         int ret;
 219
 220         /* Force SNB workarounds for PIPE_CONTROL flushes */
 221         ret = intel_emit_post_sync_nonzero_flush(ring);
 222         if (ret)
 223                 return ret;
 224
 225         /* Just flush everything.  Experiments have shown that reducing the
 226          * number of bits based on the write domains has little performance
 227          * impact.
 228          */
 229         if (flush_domains) {
 230                 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
 231                 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
 232                 /*
 233                  * Ensure that any following seqno writes only happen
 234                  * when the render cache is indeed flushed.
 235                  */
 236                 flags |= PIPE_CONTROL_CS_STALL;
 237         }
 238         if (invalidate_domains) {
 239                 flags |= PIPE_CONTROL_TLB_INVALIDATE;
 240                 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
 241                 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
 242                 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
 243                 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
 244                 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
 245                 /*
 246                  * TLB invalidate requires a post-sync write.
 247                  */
 248                 flags |= PIPE_CONTROL_QW_WRITE;
 249         }
 250
 251         ret = intel_ring_begin(ring, 4);
 252         if (ret)
 253                 return ret;
 254
 255         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4));
 256         intel_ring_emit(ring, flags);
 257         intel_ring_emit(ring, scratch_addr | PIPE_CONTROL_GLOBAL_GTT);
 258         intel_ring_emit(ring, 0);
 259         intel_ring_advance(ring);
 260
 261         return 0;
 262 }
 263
 264 static void ring_write_tail(struct intel_ring_buffer *ring,
 265                             u32 value)
 266 {
 267         drm_i915_private_t *dev_priv = ring->dev->dev_private;
 268         I915_WRITE_TAIL(ring, value);
 269 }
 270
 271 u32 intel_ring_get_active_head(struct intel_ring_buffer *ring)
 272 {
 273         drm_i915_private_t *dev_priv = ring->dev->dev_private;
 274         u32 acthd_reg = INTEL_INFO(ring->dev)->gen >= 4 ?
 275                         RING_ACTHD(ring->mmio_base) : ACTHD;
 276
 277         return I915_READ(acthd_reg);
 278 }
 279
 280 static int init_ring_common(struct intel_ring_buffer *ring)
 281 {
 282         struct drm_device *dev = ring->dev;
 283         drm_i915_private_t *dev_priv = dev->dev_private;
 284         struct drm_i915_gem_object *obj = ring->obj;
 285         int ret = 0;
 286         u32 head;
 287
 288         if (HAS_FORCE_WAKE(dev))
 289                 gen6_gt_force_wake_get(dev_priv);
 290
 291         /* Stop the ring if it's running. */
 292         I915_WRITE_CTL(ring, 0);
 293         I915_WRITE_HEAD(ring, 0);
 294         ring->write_tail(ring, 0);
 295
 296         head = I915_READ_HEAD(ring) & HEAD_ADDR;
 297
 298         /* G45 ring initialization fails to reset head to zero */
 299         if (head != 0) {
 300                 DRM_DEBUG_KMS("%s head not reset to zero "
 301                               "ctl %08x head %08x tail %08x start %08x\n",
 302                               ring->name,
 303                               I915_READ_CTL(ring),
 304                               I915_READ_HEAD(ring),
 305                               I915_READ_TAIL(ring),
 306                               I915_READ_START(ring));
 307
 308                 I915_WRITE_HEAD(ring, 0);
 309
 310                 if (I915_READ_HEAD(ring) & HEAD_ADDR) {
 311                         DRM_ERROR("failed to set %s head to zero "
 312                                   "ctl %08x head %08x tail %08x start %08x\n",
 313                                   ring->name,
 314                                   I915_READ_CTL(ring),
 315                                   I915_READ_HEAD(ring),
 316                                   I915_READ_TAIL(ring),
 317                                   I915_READ_START(ring));
 318                 }
 319         }
 320
 321         /* Initialize the ring. This must happen _after_ we've cleared the ring
 322          * registers with the above sequence (the readback of the HEAD registers
 323          * also enforces ordering), otherwise the hw might lose the new ring
 324          * register values. */
 325         I915_WRITE_START(ring, obj->gtt_offset);
 326         I915_WRITE_CTL(ring,
 327                         ((ring->size - PAGE_SIZE) & RING_NR_PAGES)
 328                         | RING_VALID);
 329
 330         /* If the head is still not zero, the ring is dead */
 331         if (wait_for((I915_READ_CTL(ring) & RING_VALID) != 0 &&
 332                      I915_READ_START(ring) == obj->gtt_offset &&
 333                      (I915_READ_HEAD(ring) & HEAD_ADDR) == 0, 50)) {
 334                 DRM_ERROR("%s initialization failed "
 335                                 "ctl %08x head %08x tail %08x start %08x\n",
 336                                 ring->name,
 337                                 I915_READ_CTL(ring),
 338                                 I915_READ_HEAD(ring),
 339                                 I915_READ_TAIL(ring),
 340                                 I915_READ_START(ring));
 341                 ret = -EIO;
 342                 goto out;
 343         }
 344
 345         if (!drm_core_check_feature(ring->dev, DRIVER_MODESET))
 346                 i915_kernel_lost_context(ring->dev);
 347         else {
 348                 ring->head = I915_READ_HEAD(ring);
 349                 ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
 350                 ring->space = ring_space(ring);
 351                 ring->last_retired_head = -1;
 352         }
 353
 354 out:
 355         if (HAS_FORCE_WAKE(dev))
 356                 gen6_gt_force_wake_put(dev_priv);
 357
 358         return ret;
 359 }
 360
 361 static int
 362 init_pipe_control(struct intel_ring_buffer *ring)
 363 {
 364         struct pipe_control *pc;
 365         struct drm_i915_gem_object *obj;
 366         int ret;
 367
 368         if (ring->private)
 369                 return 0;
 370
 371         pc = kmalloc(sizeof(*pc), GFP_KERNEL);
 372         if (!pc)
 373                 return -ENOMEM;
 374
 375         obj = i915_gem_alloc_object(ring->dev, 4096);
 376         if (obj == NULL) {
 377                 DRM_ERROR("Failed to allocate seqno page\n");
 378                 ret = -ENOMEM;
 379                 goto err;
 380         }
 381
 382         i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
 383
 384         ret = i915_gem_object_pin(obj, 4096, true);
 385         if (ret)
 386                 goto err_unref;
 387
 388         pc->gtt_offset = obj->gtt_offset;
 389         pc->cpu_page =  kmap(obj->pages[0]);
 390         if (pc->cpu_page == NULL)
 391                 goto err_unpin;
 392
 393         pc->obj = obj;
 394         ring->private = pc;
 395         return 0;
 396
 397 err_unpin:
 398         i915_gem_object_unpin(obj);
 399 err_unref:
 400         drm_gem_object_unreference(&obj->base);
 401 err:
 402         kfree(pc);
 403         return ret;
 404 }
 405
 406 static void
 407 cleanup_pipe_control(struct intel_ring_buffer *ring)
 408 {
 409         struct pipe_control *pc = ring->private;
 410         struct drm_i915_gem_object *obj;
 411
 412         if (!ring->private)
 413                 return;
 414
 415         obj = pc->obj;
 416         kunmap(obj->pages[0]);
 417         i915_gem_object_unpin(obj);
 418         drm_gem_object_unreference(&obj->base);
 419
 420         kfree(pc);
 421         ring->private = NULL;
 422 }
 423
 424 static int init_render_ring(struct intel_ring_buffer *ring)
 425 {
 426         struct drm_device *dev = ring->dev;
 427         struct drm_i915_private *dev_priv = dev->dev_private;
 428         int ret = init_ring_common(ring);
 429
 430         if (INTEL_INFO(dev)->gen > 3) {
 431                 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH));
 432                 if (IS_GEN7(dev))
 433                         I915_WRITE(GFX_MODE_GEN7,
 434                                    _MASKED_BIT_DISABLE(GFX_TLB_INVALIDATE_ALWAYS) |
 435                                    _MASKED_BIT_ENABLE(GFX_REPLAY_MODE));
 436         }
 437
 438         if (INTEL_INFO(dev)->gen >= 5) {
 439                 ret = init_pipe_control(ring);
 440                 if (ret)
 441                         return ret;
 442         }
 443
 444         if (IS_GEN6(dev)) {
 445                 /* From the Sandybridge PRM, volume 1 part 3, page 24:
 446                  * "If this bit is set, STCunit will have LRA as replacement
 447                  *  policy. [...] This bit must be reset.  LRA replacement
 448                  *  policy is not supported."
 449                  */
 450                 I915_WRITE(CACHE_MODE_0,
 451                            _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
 452
 453                 /* This is not explicitly set for GEN6, so read the register.
 454                  * see intel_ring_mi_set_context() for why we care.
 455                  * TODO: consider explicitly setting the bit for GEN5
 456                  */
 457                 ring->itlb_before_ctx_switch =
 458                         !!(I915_READ(GFX_MODE) & GFX_TLB_INVALIDATE_ALWAYS);
 459         }
 460
 461         if (INTEL_INFO(dev)->gen >= 6)
 462                 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
 463
 464         if (IS_IVYBRIDGE(dev))
 465                 I915_WRITE_IMR(ring, ~GEN6_RENDER_L3_PARITY_ERROR);
 466
 467         return ret;
 468 }
 469
 470 static void render_ring_cleanup(struct intel_ring_buffer *ring)
 471 {
 472         if (!ring->private)
 473                 return;
 474
 475         cleanup_pipe_control(ring);
 476 }
 477
 478 static void
 479 update_mboxes(struct intel_ring_buffer *ring,
 480             u32 seqno,
 481             u32 mmio_offset)
 482 {
 483         intel_ring_emit(ring, MI_SEMAPHORE_MBOX |
 484                               MI_SEMAPHORE_GLOBAL_GTT |
 485                               MI_SEMAPHORE_REGISTER |
 486                               MI_SEMAPHORE_UPDATE);
 487         intel_ring_emit(ring, seqno);
 488         intel_ring_emit(ring, mmio_offset);
 489 }
 490
 491 /**
 492  * gen6_add_request - Update the semaphore mailbox registers
 493  *
 494  * @ring - ring that is adding a request
 495  * @seqno - return seqno stuck into the ring
 496  *
 497  * Update the mailbox registers in the *other* rings with the current seqno.
 498  * This acts like a signal in the canonical semaphore.
 499  */
 500 static int
 501 gen6_add_request(struct intel_ring_buffer *ring,
 502                  u32 *seqno)
 503 {
 504         u32 mbox1_reg;
 505         u32 mbox2_reg;
 506         int ret;
 507
 508         ret = intel_ring_begin(ring, 10);
 509         if (ret)
 510                 return ret;
 511
 512         mbox1_reg = ring->signal_mbox[0];
 513         mbox2_reg = ring->signal_mbox[1];
 514
 515         *seqno = i915_gem_next_request_seqno(ring);
 516
 517         update_mboxes(ring, *seqno, mbox1_reg);
 518         update_mboxes(ring, *seqno, mbox2_reg);
 519         intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
 520         intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
 521         intel_ring_emit(ring, *seqno);
 522         intel_ring_emit(ring, MI_USER_INTERRUPT);
 523         intel_ring_advance(ring);
 524
 525         return 0;
 526 }
 527
 528 /**
 529  * intel_ring_sync - sync the waiter to the signaller on seqno
 530  *
 531  * @waiter - ring that is waiting
 532  * @signaller - ring which has, or will signal
 533  * @seqno - seqno which the waiter will block on
 534  */
 535 static int
 536 gen6_ring_sync(struct intel_ring_buffer *waiter,
 537                struct intel_ring_buffer *signaller,
 538                u32 seqno)
 539 {
 540         int ret;
 541         u32 dw1 = MI_SEMAPHORE_MBOX |
 542                   MI_SEMAPHORE_COMPARE |
 543                   MI_SEMAPHORE_REGISTER;
 544
 545         /* Throughout all of the GEM code, seqno passed implies our current
 546          * seqno is >= the last seqno executed. However for hardware the
 547          * comparison is strictly greater than.
 548          */
 549         seqno -= 1;
 550
 551         WARN_ON(signaller->semaphore_register[waiter->id] ==
 552                 MI_SEMAPHORE_SYNC_INVALID);
 553
 554         ret = intel_ring_begin(waiter, 4);
 555         if (ret)
 556                 return ret;
 557
 558         intel_ring_emit(waiter,
 559                         dw1 | signaller->semaphore_register[waiter->id]);
 560         intel_ring_emit(waiter, seqno);
 561         intel_ring_emit(waiter, 0);
 562         intel_ring_emit(waiter, MI_NOOP);
 563         intel_ring_advance(waiter);
 564
 565         return 0;
 566 }
 567
 568 #define PIPE_CONTROL_FLUSH(ring__, addr__)                                      \
 569 do {                                                                    \
 570         intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |                \
 571                  PIPE_CONTROL_DEPTH_STALL);                             \
 572         intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT);                    \
 573         intel_ring_emit(ring__, 0);                                                     \
 574         intel_ring_emit(ring__, 0);                                                     \
 575 } while (0)
 576
 577 static int
 578 pc_render_add_request(struct intel_ring_buffer *ring,
 579                       u32 *result)
 580 {
 581         u32 seqno = i915_gem_next_request_seqno(ring);
 582         struct pipe_control *pc = ring->private;
 583         u32 scratch_addr = pc->gtt_offset + 128;
 584         int ret;
 585
 586         /* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently
 587          * incoherent with writes to memory, i.e. completely fubar,
 588          * so we need to use PIPE_NOTIFY instead.
 589          *
 590          * However, we also need to workaround the qword write
 591          * incoherence by flushing the 6 PIPE_NOTIFY buffers out to
 592          * memory before requesting an interrupt.
 593          */
 594         ret = intel_ring_begin(ring, 32);
 595         if (ret)
 596                 return ret;
 597
 598         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
 599                         PIPE_CONTROL_WRITE_FLUSH |
 600                         PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
 601         intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
 602         intel_ring_emit(ring, seqno);
 603         intel_ring_emit(ring, 0);
 604         PIPE_CONTROL_FLUSH(ring, scratch_addr);
 605         scratch_addr += 128; /* write to separate cachelines */
 606         PIPE_CONTROL_FLUSH(ring, scratch_addr);
 607         scratch_addr += 128;
 608         PIPE_CONTROL_FLUSH(ring, scratch_addr);
 609         scratch_addr += 128;
 610         PIPE_CONTROL_FLUSH(ring, scratch_addr);
 611         scratch_addr += 128;
 612         PIPE_CONTROL_FLUSH(ring, scratch_addr);
 613         scratch_addr += 128;
 614         PIPE_CONTROL_FLUSH(ring, scratch_addr);
 615
 616         intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
 617                         PIPE_CONTROL_WRITE_FLUSH |
 618                         PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
 619                         PIPE_CONTROL_NOTIFY);
 620         intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
 621         intel_ring_emit(ring, seqno);
 622         intel_ring_emit(ring, 0);
 623         intel_ring_advance(ring);
 624
 625         *result = seqno;
 626         return 0;
 627 }
 628
 629 static u32
 630 gen6_ring_get_seqno(struct intel_ring_buffer *ring)
 631 {
 632         struct drm_device *dev = ring->dev;
 633
 634         /* Workaround to force correct ordering between irq and seqno writes on
 635          * ivb (and maybe also on snb) by reading from a CS register (like
 636          * ACTHD) before reading the status page. */
 637         if (IS_GEN6(dev) || IS_GEN7(dev))
 638                 intel_ring_get_active_head(ring);
 639         return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
 640 }
 641
 642 static u32
 643 ring_get_seqno(struct intel_ring_buffer *ring)
 644 {
 645         return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
 646 }
 647
 648 static u32
 649 pc_render_get_seqno(struct intel_ring_buffer *ring)
 650 {
 651         struct pipe_control *pc = ring->private;
 652         return pc->cpu_page[0];
 653 }
 654
 655 static bool
 656 gen5_ring_get_irq(struct intel_ring_buffer *ring)
 657 {
 658         struct drm_device *dev = ring->dev;
 659         drm_i915_private_t *dev_priv = dev->dev_private;
 660         unsigned long flags;
 661
 662         if (!dev->irq_enabled)
 663                 return false;
 664
 665         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 666         if (ring->irq_refcount++ == 0) {
 667                 dev_priv->gt_irq_mask &= ~ring->irq_enable_mask;
 668                 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
 669                 POSTING_READ(GTIMR);
 670         }
 671         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 672
 673         return true;
 674 }
 675
 676 static void
 677 gen5_ring_put_irq(struct intel_ring_buffer *ring)
 678 {
 679         struct drm_device *dev = ring->dev;
 680         drm_i915_private_t *dev_priv = dev->dev_private;
 681         unsigned long flags;
 682
 683         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 684         if (--ring->irq_refcount == 0) {
 685                 dev_priv->gt_irq_mask |= ring->irq_enable_mask;
 686                 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
 687                 POSTING_READ(GTIMR);
 688         }
 689         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 690 }
 691
 692 static bool
 693 i9xx_ring_get_irq(struct intel_ring_buffer *ring)
 694 {
 695         struct drm_device *dev = ring->dev;
 696         drm_i915_private_t *dev_priv = dev->dev_private;
 697         unsigned long flags;
 698
 699         if (!dev->irq_enabled)
 700                 return false;
 701
 702         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 703         if (ring->irq_refcount++ == 0) {
 704                 dev_priv->irq_mask &= ~ring->irq_enable_mask;
 705                 I915_WRITE(IMR, dev_priv->irq_mask);
 706                 POSTING_READ(IMR);
 707         }
 708         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 709
 710         return true;
 711 }
 712
 713 static void
 714 i9xx_ring_put_irq(struct intel_ring_buffer *ring)
 715 {
 716         struct drm_device *dev = ring->dev;
 717         drm_i915_private_t *dev_priv = dev->dev_private;
 718         unsigned long flags;
 719
 720         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 721         if (--ring->irq_refcount == 0) {
 722                 dev_priv->irq_mask |= ring->irq_enable_mask;
 723                 I915_WRITE(IMR, dev_priv->irq_mask);
 724                 POSTING_READ(IMR);
 725         }
 726         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 727 }
 728
 729 static bool
 730 i8xx_ring_get_irq(struct intel_ring_buffer *ring)
 731 {
 732         struct drm_device *dev = ring->dev;
 733         drm_i915_private_t *dev_priv = dev->dev_private;
 734         unsigned long flags;
 735
 736         if (!dev->irq_enabled)
 737                 return false;
 738
 739         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 740         if (ring->irq_refcount++ == 0) {
 741                 dev_priv->irq_mask &= ~ring->irq_enable_mask;
 742                 I915_WRITE16(IMR, dev_priv->irq_mask);
 743                 POSTING_READ16(IMR);
 744         }
 745         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 746
 747         return true;
 748 }
 749
 750 static void
 751 i8xx_ring_put_irq(struct intel_ring_buffer *ring)
 752 {
 753         struct drm_device *dev = ring->dev;
 754         drm_i915_private_t *dev_priv = dev->dev_private;
 755         unsigned long flags;
 756
 757         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 758         if (--ring->irq_refcount == 0) {
 759                 dev_priv->irq_mask |= ring->irq_enable_mask;
 760                 I915_WRITE16(IMR, dev_priv->irq_mask);
 761                 POSTING_READ16(IMR);
 762         }
 763         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 764 }
 765
 766 void intel_ring_setup_status_page(struct intel_ring_buffer *ring)
 767 {
 768         struct drm_device *dev = ring->dev;
 769         drm_i915_private_t *dev_priv = ring->dev->dev_private;
 770         u32 mmio = 0;
 771
 772         /* The ring status page addresses are no longer next to the rest of
 773          * the ring registers as of gen7.
 774          */
 775         if (IS_GEN7(dev)) {
 776                 switch (ring->id) {
 777                 case RCS:
 778                         mmio = RENDER_HWS_PGA_GEN7;
 779                         break;
 780                 case BCS:
 781                         mmio = BLT_HWS_PGA_GEN7;
 782                         break;
 783                 case VCS:
 784                         mmio = BSD_HWS_PGA_GEN7;
 785                         break;
 786                 }
 787         } else if (IS_GEN6(ring->dev)) {
 788                 mmio = RING_HWS_PGA_GEN6(ring->mmio_base);
 789         } else {
 790                 mmio = RING_HWS_PGA(ring->mmio_base);
 791         }
 792
 793         I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
 794         POSTING_READ(mmio);
 795 }
 796
 797 static int
 798 bsd_ring_flush(struct intel_ring_buffer *ring,
 799                u32     invalidate_domains,
 800                u32     flush_domains)
 801 {
 802         int ret;
 803
 804         ret = intel_ring_begin(ring, 2);
 805         if (ret)
 806                 return ret;
 807
 808         intel_ring_emit(ring, MI_FLUSH);
 809         intel_ring_emit(ring, MI_NOOP);
 810         intel_ring_advance(ring);
 811         return 0;
 812 }
 813
 814 static int
 815 i9xx_add_request(struct intel_ring_buffer *ring,
 816                  u32 *result)
 817 {
 818         u32 seqno;
 819         int ret;
 820
 821         ret = intel_ring_begin(ring, 4);
 822         if (ret)
 823                 return ret;
 824
 825         seqno = i915_gem_next_request_seqno(ring);
 826
 827         intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
 828         intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
 829         intel_ring_emit(ring, seqno);
 830         intel_ring_emit(ring, MI_USER_INTERRUPT);
 831         intel_ring_advance(ring);
 832
 833         *result = seqno;
 834         return 0;
 835 }
 836
 837 static bool
 838 gen6_ring_get_irq(struct intel_ring_buffer *ring)
 839 {
 840         struct drm_device *dev = ring->dev;
 841         drm_i915_private_t *dev_priv = dev->dev_private;
 842         unsigned long flags;
 843
 844         if (!dev->irq_enabled)
 845                return false;
 846
 847         /* It looks like we need to prevent the gt from suspending while waiting
 848          * for an notifiy irq, otherwise irqs seem to get lost on at least the
 849          * blt/bsd rings on ivb. */
 850         gen6_gt_force_wake_get(dev_priv);
 851
 852         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 853         if (ring->irq_refcount++ == 0) {
 854                 if (IS_IVYBRIDGE(dev) && ring->id == RCS)
 855                         I915_WRITE_IMR(ring, ~(ring->irq_enable_mask |
 856                                                 GEN6_RENDER_L3_PARITY_ERROR));
 857                 else
 858                         I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
 859                 dev_priv->gt_irq_mask &= ~ring->irq_enable_mask;
 860                 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
 861                 POSTING_READ(GTIMR);
 862         }
 863         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 864
 865         return true;
 866 }
 867
 868 static void
 869 gen6_ring_put_irq(struct intel_ring_buffer *ring)
 870 {
 871         struct drm_device *dev = ring->dev;
 872         drm_i915_private_t *dev_priv = dev->dev_private;
 873         unsigned long flags;
 874
 875         spin_lock_irqsave(&dev_priv->irq_lock, flags);
 876         if (--ring->irq_refcount == 0) {
 877                 if (IS_IVYBRIDGE(dev) && ring->id == RCS)
 878                         I915_WRITE_IMR(ring, ~GEN6_RENDER_L3_PARITY_ERROR);
 879                 else
 880                         I915_WRITE_IMR(ring, ~0);
 881                 dev_priv->gt_irq_mask |= ring->irq_enable_mask;
 882                 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
 883                 POSTING_READ(GTIMR);
 884         }
 885         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 886
 887         gen6_gt_force_wake_put(dev_priv);
 888 }
 889
 890 static int
 891 i965_dispatch_execbuffer(struct intel_ring_buffer *ring, u32 offset, u32 length)
 892 {
 893         int ret;
 894
 895         ret = intel_ring_begin(ring, 2);
 896         if (ret)
 897                 return ret;
 898
 899         intel_ring_emit(ring,
 900                         MI_BATCH_BUFFER_START |
 901                         MI_BATCH_GTT |
 902                         MI_BATCH_NON_SECURE_I965);
 903         intel_ring_emit(ring, offset);
 904         intel_ring_advance(ring);
 905
 906         return 0;
 907 }
 908
 909 static int
 910 i830_dispatch_execbuffer(struct intel_ring_buffer *ring,
 911                                 u32 offset, u32 len)
 912 {
 913         int ret;
 914
 915         ret = intel_ring_begin(ring, 4);
 916         if (ret)
 917                 return ret;
 918
 919         intel_ring_emit(ring, MI_BATCH_BUFFER);
 920         intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
 921         intel_ring_emit(ring, offset + len - 8);
 922         intel_ring_emit(ring, 0);
 923         intel_ring_advance(ring);
 924
 925         return 0;
 926 }
 927
 928 static int
 929 i915_dispatch_execbuffer(struct intel_ring_buffer *ring,
 930                                 u32 offset, u32 len)
 931 {
 932         int ret;
 933
 934         ret = intel_ring_begin(ring, 2);
 935         if (ret)
 936                 return ret;
 937
 938         intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_GTT);
 939         intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
 940         intel_ring_advance(ring);
 941
 942         return 0;
 943 }
 944
 945 static void cleanup_status_page(struct intel_ring_buffer *ring)
 946 {
 947         struct drm_i915_gem_object *obj;
 948
 949         obj = ring->status_page.obj;
 950         if (obj == NULL)
 951                 return;
 952
 953         kunmap(obj->pages[0]);
 954         i915_gem_object_unpin(obj);
 955         drm_gem_object_unreference(&obj->base);
 956         ring->status_page.obj = NULL;
 957 }
 958
 959 static int init_status_page(struct intel_ring_buffer *ring)
 960 {
 961         struct drm_device *dev = ring->dev;
 962         struct drm_i915_gem_object *obj;
 963         int ret;
 964
 965         obj = i915_gem_alloc_object(dev, 4096);
 966         if (obj == NULL) {
 967                 DRM_ERROR("Failed to allocate status page\n");
 968                 ret = -ENOMEM;
 969                 goto err;
 970         }
 971
 972         i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
 973
 974         ret = i915_gem_object_pin(obj, 4096, true);
 975         if (ret != 0) {
 976                 goto err_unref;
 977         }
 978
 979         ring->status_page.gfx_addr = obj->gtt_offset;
 980         ring->status_page.page_addr = kmap(obj->pages[0]);
 981         if (ring->status_page.page_addr == NULL) {
 982                 ret = -ENOMEM;
 983                 goto err_unpin;
 984         }
 985         ring->status_page.obj = obj;
 986         memset(ring->status_page.page_addr, 0, PAGE_SIZE);
 987
 988         intel_ring_setup_status_page(ring);
 989         DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
 990                         ring->name, ring->status_page.gfx_addr);
 991
 992         return 0;
 993
 994 err_unpin:
 995         i915_gem_object_unpin(obj);
 996 err_unref:
 997         drm_gem_object_unreference(&obj->base);
 998 err:
 999         return ret;
1000 }
1001
1002 static int intel_init_ring_buffer(struct drm_device *dev,
1003                                   struct intel_ring_buffer *ring)
1004 {
1005         struct drm_i915_gem_object *obj;
1006         struct drm_i915_private *dev_priv = dev->dev_private;
1007         int ret;
1008
1009         ring->dev = dev;
1010         INIT_LIST_HEAD(&ring->active_list);
1011         INIT_LIST_HEAD(&ring->request_list);
1012         INIT_LIST_HEAD(&ring->gpu_write_list);
1013         ring->size = 32 * PAGE_SIZE;
1014
1015         init_waitqueue_head(&ring->irq_queue);
1016
1017         if (I915_NEED_GFX_HWS(dev)) {
1018                 ret = init_status_page(ring);
1019                 if (ret)
1020                         return ret;
1021         }
1022
1023         obj = i915_gem_alloc_object(dev, ring->size);
1024         if (obj == NULL) {
1025                 DRM_ERROR("Failed to allocate ringbuffer\n");
1026                 ret = -ENOMEM;
1027                 goto err_hws;
1028         }
1029
1030         ring->obj = obj;
1031
1032         ret = i915_gem_object_pin(obj, PAGE_SIZE, true);
1033         if (ret)
1034                 goto err_unref;
1035
1036         ret = i915_gem_object_set_to_gtt_domain(obj, true);
1037         if (ret)
1038                 goto err_unpin;
1039
1040         ring->virtual_start =
1041                 ioremap_wc(dev_priv->mm.gtt->gma_bus_addr + obj->gtt_offset,
1042                            ring->size);
1043         if (ring->virtual_start == NULL) {
1044                 DRM_ERROR("Failed to map ringbuffer.\n");
1045                 ret = -EINVAL;
1046                 goto err_unpin;
1047         }
1048
1049         ret = ring->init(ring);
1050         if (ret)
1051                 goto err_unmap;
1052
1053         /* Workaround an erratum on the i830 which causes a hang if
1054          * the TAIL pointer points to within the last 2 cachelines
1055          * of the buffer.
1056          */
1057         ring->effective_size = ring->size;
1058         if (IS_I830(ring->dev) || IS_845G(ring->dev))
1059                 ring->effective_size -= 128;
1060
1061         return 0;
1062
1063 err_unmap:
1064         iounmap(ring->virtual_start);
1065 err_unpin:
1066         i915_gem_object_unpin(obj);
1067 err_unref:
1068         drm_gem_object_unreference(&obj->base);
1069         ring->obj = NULL;
1070 err_hws:
1071         cleanup_status_page(ring);
1072         return ret;
1073 }
1074
1075 void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring)
1076 {
1077         struct drm_i915_private *dev_priv;
1078         int ret;
1079
1080         if (ring->obj == NULL)
1081                 return;
1082
1083         /* Disable the ring buffer. The ring must be idle at this point */
1084         dev_priv = ring->dev->dev_private;
1085         ret = intel_wait_ring_idle(ring);
1086         if (ret)
1087                 DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n",
1088                           ring->name, ret);
1089
1090         I915_WRITE_CTL(ring, 0);
1091
1092         iounmap(ring->virtual_start);
1093
1094         i915_gem_object_unpin(ring->obj);
1095         drm_gem_object_unreference(&ring->obj->base);
1096         ring->obj = NULL;
1097
1098         if (ring->cleanup)
1099                 ring->cleanup(ring);
1100
1101         cleanup_status_page(ring);
1102 }
1103
1104 static int intel_wrap_ring_buffer(struct intel_ring_buffer *ring)
1105 {
1106         uint32_t __iomem *virt;
1107         int rem = ring->size - ring->tail;
1108
1109         if (ring->space < rem) {
1110                 int ret = intel_wait_ring_buffer(ring, rem);
1111                 if (ret)
1112                         return ret;
1113         }
1114
1115         virt = ring->virtual_start + ring->tail;
1116         rem /= 4;
1117         while (rem--)
1118                 iowrite32(MI_NOOP, virt++);
1119
1120         ring->tail = 0;
1121         ring->space = ring_space(ring);
1122
1123         return 0;
1124 }
1125
1126 static int intel_ring_wait_seqno(struct intel_ring_buffer *ring, u32 seqno)
1127 {
1128         int ret;
1129
1130         ret = i915_wait_seqno(ring, seqno);
1131         if (!ret)
1132                 i915_gem_retire_requests_ring(ring);
1133
1134         return ret;
1135 }
1136
1137 static int intel_ring_wait_request(struct intel_ring_buffer *ring, int n)
1138 {
1139         struct drm_i915_gem_request *request;
1140         u32 seqno = 0;
1141         int ret;
1142
1143         i915_gem_retire_requests_ring(ring);
1144
1145         if (ring->last_retired_head != -1) {
1146                 ring->head = ring->last_retired_head;
1147                 ring->last_retired_head = -1;
1148                 ring->space = ring_space(ring);
1149                 if (ring->space >= n)
1150                         return 0;
1151         }
1152
1153         list_for_each_entry(request, &ring->request_list, list) {
1154                 int space;
1155
1156                 if (request->tail == -1)
1157                         continue;
1158
1159                 space = request->tail - (ring->tail + 8);
1160                 if (space < 0)
1161                         space += ring->size;
1162                 if (space >= n) {
1163                         seqno = request->seqno;
1164                         break;
1165                 }
1166
1167                 /* Consume this request in case we need more space than
1168                  * is available and so need to prevent a race between
1169                  * updating last_retired_head and direct reads of
1170                  * I915_RING_HEAD. It also provides a nice sanity check.
1171                  */
1172                 request->tail = -1;
1173         }
1174
1175         if (seqno == 0)
1176                 return -ENOSPC;
1177
1178         ret = intel_ring_wait_seqno(ring, seqno);
1179         if (ret)
1180                 return ret;
1181
1182         if (WARN_ON(ring->last_retired_head == -1))
1183                 return -ENOSPC;
1184
1185         ring->head = ring->last_retired_head;
1186         ring->last_retired_head = -1;
1187         ring->space = ring_space(ring);
1188         if (WARN_ON(ring->space < n))
1189                 return -ENOSPC;
1190
1191         return 0;
1192 }
1193
1194 int intel_wait_ring_buffer(struct intel_ring_buffer *ring, int n)
1195 {
1196         struct drm_device *dev = ring->dev;
1197         struct drm_i915_private *dev_priv = dev->dev_private;
1198         unsigned long end;
1199         int ret;
1200
1201         ret = intel_ring_wait_request(ring, n);
1202         if (ret != -ENOSPC)
1203                 return ret;
1204
1205         trace_i915_ring_wait_begin(ring);
1206         /* With GEM the hangcheck timer should kick us out of the loop,
1207          * leaving it early runs the risk of corrupting GEM state (due
1208          * to running on almost untested codepaths). But on resume
1209          * timers don't work yet, so prevent a complete hang in that
1210          * case by choosing an insanely large timeout. */
1211         end = jiffies + 60 * HZ;
1212
1213         do {
1214                 ring->head = I915_READ_HEAD(ring);
1215                 ring->space = ring_space(ring);
1216                 if (ring->space >= n) {
1217                         trace_i915_ring_wait_end(ring);
1218                         return 0;
1219                 }
1220
1221                 if (dev->primary->master) {
1222                         struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
1223                         if (master_priv->sarea_priv)
1224                                 master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
1225                 }
1226
1227                 msleep(1);
1228
1229                 ret = i915_gem_check_wedge(dev_priv, dev_priv->mm.interruptible);
1230                 if (ret)
1231                         return ret;
1232         } while (!time_after(jiffies, end));
1233         trace_i915_ring_wait_end(ring);
1234         return -EBUSY;
1235 }
1236
1237 int intel_ring_begin(struct intel_ring_buffer *ring,
1238                      int num_dwords)
1239 {
1240         drm_i915_private_t *dev_priv = ring->dev->dev_private;
1241         int n = 4*num_dwords;
1242         int ret;
1243
1244         ret = i915_gem_check_wedge(dev_priv, dev_priv->mm.interruptible);
1245         if (ret)
1246                 return ret;
1247
1248         if (unlikely(ring->tail + n > ring->effective_size)) {
1249                 ret = intel_wrap_ring_buffer(ring);
1250                 if (unlikely(ret))
1251                         return ret;
1252         }
1253
1254         if (unlikely(ring->space < n)) {
1255                 ret = intel_wait_ring_buffer(ring, n);
1256                 if (unlikely(ret))
1257                         return ret;
1258         }
1259
1260         ring->space -= n;
1261         return 0;
1262 }
1263
1264 void intel_ring_advance(struct intel_ring_buffer *ring)
1265 {
1266         struct drm_i915_private *dev_priv = ring->dev->dev_private;
1267
1268         ring->tail &= ring->size - 1;
1269         if (dev_priv->stop_rings & intel_ring_flag(ring))
1270                 return;
1271         ring->write_tail(ring, ring->tail);
1272 }
1273
1274
1275 static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring,
1276                                      u32 value)
1277 {
1278         drm_i915_private_t *dev_priv = ring->dev->dev_private;
1279
1280        /* Every tail move must follow the sequence below */
1281
1282         /* Disable notification that the ring is IDLE. The GT
1283          * will then assume that it is busy and bring it out of rc6.
1284          */
1285         I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
1286                    _MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
1287
1288         /* Clear the context id. Here be magic! */
1289         I915_WRITE64(GEN6_BSD_RNCID, 0x0);
1290
1291         /* Wait for the ring not to be idle, i.e. for it to wake up. */
1292         if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
1293                       GEN6_BSD_SLEEP_INDICATOR) == 0,
1294                      50))
1295                 DRM_ERROR("timed out waiting for the BSD ring to wake up\n");
1296
1297         /* Now that the ring is fully powered up, update the tail */
1298         I915_WRITE_TAIL(ring, value);
1299         POSTING_READ(RING_TAIL(ring->mmio_base));
1300
1301         /* Let the ring send IDLE messages to the GT again,
1302          * and so let it sleep to conserve power when idle.
1303          */
1304         I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
1305                    _MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
1306 }
1307
1308 static int gen6_ring_flush(struct intel_ring_buffer *ring,
1309                            u32 invalidate, u32 flush)
1310 {
1311         uint32_t cmd;
1312         int ret;
1313
1314         ret = intel_ring_begin(ring, 4);
1315         if (ret)
1316                 return ret;
1317
1318         cmd = MI_FLUSH_DW;
1319         if (invalidate & I915_GEM_GPU_DOMAINS)
1320                 cmd |= MI_INVALIDATE_TLB | MI_INVALIDATE_BSD;
1321         intel_ring_emit(ring, cmd);
1322         intel_ring_emit(ring, 0);
1323         intel_ring_emit(ring, 0);
1324         intel_ring_emit(ring, MI_NOOP);
1325         intel_ring_advance(ring);
1326         return 0;
1327 }
1328
1329 static int
1330 gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
1331                               u32 offset, u32 len)
1332 {
1333         int ret;
1334
1335         ret = intel_ring_begin(ring, 2);
1336         if (ret)
1337                 return ret;
1338
1339         intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_NON_SECURE_I965);
1340         /* bit0-7 is the length on GEN6+ */
1341         intel_ring_emit(ring, offset);
1342         intel_ring_advance(ring);
1343
1344         return 0;
1345 }
1346
1347 /* Blitter support (SandyBridge+) */
1348
1349 static int blt_ring_flush(struct intel_ring_buffer *ring,
1350                           u32 invalidate, u32 flush)
1351 {
1352         uint32_t cmd;
1353         int ret;
1354
1355         ret = intel_ring_begin(ring, 4);
1356         if (ret)
1357                 return ret;
1358
1359         cmd = MI_FLUSH_DW;
1360         if (invalidate & I915_GEM_DOMAIN_RENDER)
1361                 cmd |= MI_INVALIDATE_TLB;
1362         intel_ring_emit(ring, cmd);
1363         intel_ring_emit(ring, 0);
1364         intel_ring_emit(ring, 0);
1365         intel_ring_emit(ring, MI_NOOP);
1366         intel_ring_advance(ring);
1367         return 0;
1368 }
1369
1370 int intel_init_render_ring_buffer(struct drm_device *dev)
1371 {
1372         drm_i915_private_t *dev_priv = dev->dev_private;
1373         struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
1374
1375         ring->name = "render ring";
1376         ring->id = RCS;
1377         ring->mmio_base = RENDER_RING_BASE;
1378
1379         if (INTEL_INFO(dev)->gen >= 6) {
1380                 ring->add_request = gen6_add_request;
1381                 ring->flush = gen6_render_ring_flush;
1382                 ring->irq_get = gen6_ring_get_irq;
1383                 ring->irq_put = gen6_ring_put_irq;
1384                 ring->irq_enable_mask = GT_USER_INTERRUPT;
1385                 ring->get_seqno = gen6_ring_get_seqno;
1386                 ring->sync_to = gen6_ring_sync;
1387                 ring->semaphore_register[0] = MI_SEMAPHORE_SYNC_INVALID;
1388                 ring->semaphore_register[1] = MI_SEMAPHORE_SYNC_RV;
1389                 ring->semaphore_register[2] = MI_SEMAPHORE_SYNC_RB;
1390                 ring->signal_mbox[0] = GEN6_VRSYNC;
1391                 ring->signal_mbox[1] = GEN6_BRSYNC;
1392         } else if (IS_GEN5(dev)) {
1393                 ring->add_request = pc_render_add_request;
1394                 ring->flush = gen4_render_ring_flush;
1395                 ring->get_seqno = pc_render_get_seqno;
1396                 ring->irq_get = gen5_ring_get_irq;
1397                 ring->irq_put = gen5_ring_put_irq;
1398                 ring->irq_enable_mask = GT_USER_INTERRUPT | GT_PIPE_NOTIFY;
1399         } else {
1400                 ring->add_request = i9xx_add_request;
1401                 if (INTEL_INFO(dev)->gen < 4)
1402                         ring->flush = gen2_render_ring_flush;
1403                 else
1404                         ring->flush = gen4_render_ring_flush;
1405                 ring->get_seqno = ring_get_seqno;
1406                 if (IS_GEN2(dev)) {
1407                         ring->irq_get = i8xx_ring_get_irq;
1408                         ring->irq_put = i8xx_ring_put_irq;
1409                 } else {
1410                         ring->irq_get = i9xx_ring_get_irq;
1411                         ring->irq_put = i9xx_ring_put_irq;
1412                 }
1413                 ring->irq_enable_mask = I915_USER_INTERRUPT;
1414         }
1415         ring->write_tail = ring_write_tail;
1416         if (INTEL_INFO(dev)->gen >= 6)
1417                 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
1418         else if (INTEL_INFO(dev)->gen >= 4)
1419                 ring->dispatch_execbuffer = i965_dispatch_execbuffer;
1420         else if (IS_I830(dev) || IS_845G(dev))
1421                 ring->dispatch_execbuffer = i830_dispatch_execbuffer;
1422         else
1423                 ring->dispatch_execbuffer = i915_dispatch_execbuffer;
1424         ring->init = init_render_ring;
1425         ring->cleanup = render_ring_cleanup;
1426
1427
1428         if (!I915_NEED_GFX_HWS(dev)) {
1429                 ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
1430                 memset(ring->status_page.page_addr, 0, PAGE_SIZE);
1431         }
1432
1433         return intel_init_ring_buffer(dev, ring);
1434 }
1435
1436 int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size)
1437 {
1438         drm_i915_private_t *dev_priv = dev->dev_private;
1439         struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
1440
1441         ring->name = "render ring";
1442         ring->id = RCS;
1443         ring->mmio_base = RENDER_RING_BASE;
1444
1445         if (INTEL_INFO(dev)->gen >= 6) {
1446                 /* non-kms not supported on gen6+ */
1447                 return -ENODEV;
1448         }
1449
1450         /* Note: gem is not supported on gen5/ilk without kms (the corresponding
1451          * gem_init ioctl returns with -ENODEV). Hence we do not need to set up
1452          * the special gen5 functions. */
1453         ring->add_request = i9xx_add_request;
1454         if (INTEL_INFO(dev)->gen < 4)
1455                 ring->flush = gen2_render_ring_flush;
1456         else
1457                 ring->flush = gen4_render_ring_flush;
1458         ring->get_seqno = ring_get_seqno;
1459         if (IS_GEN2(dev)) {
1460                 ring->irq_get = i8xx_ring_get_irq;
1461                 ring->irq_put = i8xx_ring_put_irq;
1462         } else {
1463                 ring->irq_get = i9xx_ring_get_irq;
1464                 ring->irq_put = i9xx_ring_put_irq;
1465         }
1466         ring->irq_enable_mask = I915_USER_INTERRUPT;
1467         ring->write_tail = ring_write_tail;
1468         if (INTEL_INFO(dev)->gen >= 4)
1469                 ring->dispatch_execbuffer = i965_dispatch_execbuffer;
1470         else if (IS_I830(dev) || IS_845G(dev))
1471                 ring->dispatch_execbuffer = i830_dispatch_execbuffer;
1472         else
1473                 ring->dispatch_execbuffer = i915_dispatch_execbuffer;
1474         ring->init = init_render_ring;
1475         ring->cleanup = render_ring_cleanup;
1476
1477         if (!I915_NEED_GFX_HWS(dev))
1478                 ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
1479
1480         ring->dev = dev;
1481         INIT_LIST_HEAD(&ring->active_list);
1482         INIT_LIST_HEAD(&ring->request_list);
1483         INIT_LIST_HEAD(&ring->gpu_write_list);
1484
1485         ring->size = size;
1486         ring->effective_size = ring->size;
1487         if (IS_I830(ring->dev))
1488                 ring->effective_size -= 128;
1489
1490         ring->virtual_start = ioremap_wc(start, size);
1491         if (ring->virtual_start == NULL) {
1492                 DRM_ERROR("can not ioremap virtual address for"
1493                           " ring buffer\n");
1494                 return -ENOMEM;
1495         }
1496
1497         return 0;
1498 }
1499
1500 int intel_init_bsd_ring_buffer(struct drm_device *dev)
1501 {
1502         drm_i915_private_t *dev_priv = dev->dev_private;
1503         struct intel_ring_buffer *ring = &dev_priv->ring[VCS];
1504
1505         ring->name = "bsd ring";
1506         ring->id = VCS;
1507
1508         ring->write_tail = ring_write_tail;
1509         if (IS_GEN6(dev) || IS_GEN7(dev)) {
1510                 ring->mmio_base = GEN6_BSD_RING_BASE;
1511                 /* gen6 bsd needs a special wa for tail updates */
1512                 if (IS_GEN6(dev))
1513                         ring->write_tail = gen6_bsd_ring_write_tail;
1514                 ring->flush = gen6_ring_flush;
1515                 ring->add_request = gen6_add_request;
1516                 ring->get_seqno = gen6_ring_get_seqno;
1517                 ring->irq_enable_mask = GEN6_BSD_USER_INTERRUPT;
1518                 ring->irq_get = gen6_ring_get_irq;
1519                 ring->irq_put = gen6_ring_put_irq;
1520                 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
1521                 ring->sync_to = gen6_ring_sync;
1522                 ring->semaphore_register[0] = MI_SEMAPHORE_SYNC_VR;
1523                 ring->semaphore_register[1] = MI_SEMAPHORE_SYNC_INVALID;
1524                 ring->semaphore_register[2] = MI_SEMAPHORE_SYNC_VB;
1525                 ring->signal_mbox[0] = GEN6_RVSYNC;
1526                 ring->signal_mbox[1] = GEN6_BVSYNC;
1527         } else {
1528                 ring->mmio_base = BSD_RING_BASE;
1529                 ring->flush = bsd_ring_flush;
1530                 ring->add_request = i9xx_add_request;
1531                 ring->get_seqno = ring_get_seqno;
1532                 if (IS_GEN5(dev)) {
1533                         ring->irq_enable_mask = GT_BSD_USER_INTERRUPT;
1534                         ring->irq_get = gen5_ring_get_irq;
1535                         ring->irq_put = gen5_ring_put_irq;
1536                 } else {
1537                         ring->irq_enable_mask = I915_BSD_USER_INTERRUPT;
1538                         ring->irq_get = i9xx_ring_get_irq;
1539                         ring->irq_put = i9xx_ring_put_irq;
1540                 }
1541                 ring->dispatch_execbuffer = i965_dispatch_execbuffer;
1542         }
1543         ring->init = init_ring_common;
1544
1545
1546         return intel_init_ring_buffer(dev, ring);
1547 }
1548
1549 int intel_init_blt_ring_buffer(struct drm_device *dev)
1550 {
1551         drm_i915_private_t *dev_priv = dev->dev_private;
1552         struct intel_ring_buffer *ring = &dev_priv->ring[BCS];
1553
1554         ring->name = "blitter ring";
1555         ring->id = BCS;
1556
1557         ring->mmio_base = BLT_RING_BASE;
1558         ring->write_tail = ring_write_tail;
1559         ring->flush = blt_ring_flush;
1560         ring->add_request = gen6_add_request;
1561         ring->get_seqno = gen6_ring_get_seqno;
1562         ring->irq_enable_mask = GEN6_BLITTER_USER_INTERRUPT;
1563         ring->irq_get = gen6_ring_get_irq;
1564         ring->irq_put = gen6_ring_put_irq;
1565         ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
1566         ring->sync_to = gen6_ring_sync;
1567         ring->semaphore_register[0] = MI_SEMAPHORE_SYNC_BR;
1568         ring->semaphore_register[1] = MI_SEMAPHORE_SYNC_BV;
1569         ring->semaphore_register[2] = MI_SEMAPHORE_SYNC_INVALID;
1570         ring->signal_mbox[0] = GEN6_RBSYNC;
1571         ring->signal_mbox[1] = GEN6_VBSYNC;
1572         ring->init = init_ring_common;
1573
1574         return intel_init_ring_buffer(dev, ring);
1575 }