drivers/gpu/drm/nouveau/core/engine/fifo/nv04.c

   1 /*
   2  * Copyright 2012 Red Hat Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Ben Skeggs
  23  */
  24
  25 #include <core/client.h>
  26 #include <nvif/unpack.h>
  27 #include <nvif/class.h>
  28 #include <core/engctx.h>
  29 #include <core/namedb.h>
  30 #include <core/handle.h>
  31 #include <core/ramht.h>
  32 #include <core/event.h>
  33
  34 #include <subdev/instmem.h>
  35 #include <subdev/instmem/nv04.h>
  36 #include <subdev/timer.h>
  37 #include <subdev/fb.h>
  38
  39 #include <engine/fifo.h>
  40
  41 #include "nv04.h"
  42
  43 static struct ramfc_desc
  44 nv04_ramfc[] = {
  45         { 32,  0, 0x00,  0, NV04_PFIFO_CACHE1_DMA_PUT },
  46         { 32,  0, 0x04,  0, NV04_PFIFO_CACHE1_DMA_GET },
  47         { 16,  0, 0x08,  0, NV04_PFIFO_CACHE1_DMA_INSTANCE },
  48         { 16, 16, 0x08,  0, NV04_PFIFO_CACHE1_DMA_DCOUNT },
  49         { 32,  0, 0x0c,  0, NV04_PFIFO_CACHE1_DMA_STATE },
  50         { 32,  0, 0x10,  0, NV04_PFIFO_CACHE1_DMA_FETCH },
  51         { 32,  0, 0x14,  0, NV04_PFIFO_CACHE1_ENGINE },
  52         { 32,  0, 0x18,  0, NV04_PFIFO_CACHE1_PULL1 },
  53         {}
  54 };
  55
  56 /*******************************************************************************
  57  * FIFO channel objects
  58  ******************************************************************************/
  59
  60 int
  61 nv04_fifo_object_attach(struct nouveau_object *parent,
  62                         struct nouveau_object *object, u32 handle)
  63 {
  64         struct nv04_fifo_priv *priv = (void *)parent->engine;
  65         struct nv04_fifo_chan *chan = (void *)parent;
  66         u32 context, chid = chan->base.chid;
  67         int ret;
  68
  69         if (nv_iclass(object, NV_GPUOBJ_CLASS))
  70                 context = nv_gpuobj(object)->addr >> 4;
  71         else
  72                 context = 0x00000004; /* just non-zero */
  73
  74         switch (nv_engidx(object->engine)) {
  75         case NVDEV_ENGINE_DMAOBJ:
  76         case NVDEV_ENGINE_SW:
  77                 context |= 0x00000000;
  78                 break;
  79         case NVDEV_ENGINE_GR:
  80                 context |= 0x00010000;
  81                 break;
  82         case NVDEV_ENGINE_MPEG:
  83                 context |= 0x00020000;
  84                 break;
  85         default:
  86                 return -EINVAL;
  87         }
  88
  89         context |= 0x80000000; /* valid */
  90         context |= chid << 24;
  91
  92         mutex_lock(&nv_subdev(priv)->mutex);
  93         ret = nouveau_ramht_insert(priv->ramht, chid, handle, context);
  94         mutex_unlock(&nv_subdev(priv)->mutex);
  95         return ret;
  96 }
  97
  98 void
  99 nv04_fifo_object_detach(struct nouveau_object *parent, int cookie)
 100 {
 101         struct nv04_fifo_priv *priv = (void *)parent->engine;
 102         mutex_lock(&nv_subdev(priv)->mutex);
 103         nouveau_ramht_remove(priv->ramht, cookie);
 104         mutex_unlock(&nv_subdev(priv)->mutex);
 105 }
 106
 107 int
 108 nv04_fifo_context_attach(struct nouveau_object *parent,
 109                          struct nouveau_object *object)
 110 {
 111         nv_engctx(object)->addr = nouveau_fifo_chan(parent)->chid;
 112         return 0;
 113 }
 114
 115 static int
 116 nv04_fifo_chan_ctor(struct nouveau_object *parent,
 117                     struct nouveau_object *engine,
 118                     struct nouveau_oclass *oclass, void *data, u32 size,
 119                     struct nouveau_object **pobject)
 120 {
 121         union {
 122                 struct nv03_channel_dma_v0 v0;
 123         } *args = data;
 124         struct nv04_fifo_priv *priv = (void *)engine;
 125         struct nv04_fifo_chan *chan;
 126         int ret;
 127
 128         nv_ioctl(parent, "create channel dma size %d\n", size);
 129         if (nvif_unpack(args->v0, 0, 0, false)) {
 130                 nv_ioctl(parent, "create channel dma vers %d pushbuf %08x "
 131                                  "offset %016llx\n", args->v0.version,
 132                          args->v0.pushbuf, args->v0.offset);
 133         } else
 134                 return ret;
 135
 136         ret = nouveau_fifo_channel_create(parent, engine, oclass, 0, 0x800000,
 137                                           0x10000, args->v0.pushbuf,
 138                                           (1ULL << NVDEV_ENGINE_DMAOBJ) |
 139                                           (1ULL << NVDEV_ENGINE_SW) |
 140                                           (1ULL << NVDEV_ENGINE_GR), &chan);
 141         *pobject = nv_object(chan);
 142         if (ret)
 143                 return ret;
 144
 145         args->v0.chid = chan->base.chid;
 146
 147         nv_parent(chan)->object_attach = nv04_fifo_object_attach;
 148         nv_parent(chan)->object_detach = nv04_fifo_object_detach;
 149         nv_parent(chan)->context_attach = nv04_fifo_context_attach;
 150         chan->ramfc = chan->base.chid * 32;
 151
 152         nv_wo32(priv->ramfc, chan->ramfc + 0x00, args->v0.offset);
 153         nv_wo32(priv->ramfc, chan->ramfc + 0x04, args->v0.offset);
 154         nv_wo32(priv->ramfc, chan->ramfc + 0x08, chan->base.pushgpu->addr >> 4);
 155         nv_wo32(priv->ramfc, chan->ramfc + 0x10,
 156                              NV_PFIFO_CACHE1_DMA_FETCH_TRIG_128_BYTES |
 157                              NV_PFIFO_CACHE1_DMA_FETCH_SIZE_128_BYTES |
 158 #ifdef __BIG_ENDIAN
 159                              NV_PFIFO_CACHE1_BIG_ENDIAN |
 160 #endif
 161                              NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_8);
 162         return 0;
 163 }
 164
 165 void
 166 nv04_fifo_chan_dtor(struct nouveau_object *object)
 167 {
 168         struct nv04_fifo_priv *priv = (void *)object->engine;
 169         struct nv04_fifo_chan *chan = (void *)object;
 170         struct ramfc_desc *c = priv->ramfc_desc;
 171
 172         do {
 173                 nv_wo32(priv->ramfc, chan->ramfc + c->ctxp, 0x00000000);
 174         } while ((++c)->bits);
 175
 176         nouveau_fifo_channel_destroy(&chan->base);
 177 }
 178
 179 int
 180 nv04_fifo_chan_init(struct nouveau_object *object)
 181 {
 182         struct nv04_fifo_priv *priv = (void *)object->engine;
 183         struct nv04_fifo_chan *chan = (void *)object;
 184         u32 mask = 1 << chan->base.chid;
 185         unsigned long flags;
 186         int ret;
 187
 188         ret = nouveau_fifo_channel_init(&chan->base);
 189         if (ret)
 190                 return ret;
 191
 192         spin_lock_irqsave(&priv->base.lock, flags);
 193         nv_mask(priv, NV04_PFIFO_MODE, mask, mask);
 194         spin_unlock_irqrestore(&priv->base.lock, flags);
 195         return 0;
 196 }
 197
 198 int
 199 nv04_fifo_chan_fini(struct nouveau_object *object, bool suspend)
 200 {
 201         struct nv04_fifo_priv *priv = (void *)object->engine;
 202         struct nv04_fifo_chan *chan = (void *)object;
 203         struct nouveau_gpuobj *fctx = priv->ramfc;
 204         struct ramfc_desc *c;
 205         unsigned long flags;
 206         u32 data = chan->ramfc;
 207         u32 chid;
 208
 209         /* prevent fifo context switches */
 210         spin_lock_irqsave(&priv->base.lock, flags);
 211         nv_wr32(priv, NV03_PFIFO_CACHES, 0);
 212
 213         /* if this channel is active, replace it with a null context */
 214         chid = nv_rd32(priv, NV03_PFIFO_CACHE1_PUSH1) & priv->base.max;
 215         if (chid == chan->base.chid) {
 216                 nv_mask(priv, NV04_PFIFO_CACHE1_DMA_PUSH, 0x00000001, 0);
 217                 nv_wr32(priv, NV03_PFIFO_CACHE1_PUSH0, 0);
 218                 nv_mask(priv, NV04_PFIFO_CACHE1_PULL0, 0x00000001, 0);
 219
 220                 c = priv->ramfc_desc;
 221                 do {
 222                         u32 rm = ((1ULL << c->bits) - 1) << c->regs;
 223                         u32 cm = ((1ULL << c->bits) - 1) << c->ctxs;
 224                         u32 rv = (nv_rd32(priv, c->regp) &  rm) >> c->regs;
 225                         u32 cv = (nv_ro32(fctx, c->ctxp + data) & ~cm);
 226                         nv_wo32(fctx, c->ctxp + data, cv | (rv << c->ctxs));
 227                 } while ((++c)->bits);
 228
 229                 c = priv->ramfc_desc;
 230                 do {
 231                         nv_wr32(priv, c->regp, 0x00000000);
 232                 } while ((++c)->bits);
 233
 234                 nv_wr32(priv, NV03_PFIFO_CACHE1_GET, 0);
 235                 nv_wr32(priv, NV03_PFIFO_CACHE1_PUT, 0);
 236                 nv_wr32(priv, NV03_PFIFO_CACHE1_PUSH1, priv->base.max);
 237                 nv_wr32(priv, NV03_PFIFO_CACHE1_PUSH0, 1);
 238                 nv_wr32(priv, NV04_PFIFO_CACHE1_PULL0, 1);
 239         }
 240
 241         /* restore normal operation, after disabling dma mode */
 242         nv_mask(priv, NV04_PFIFO_MODE, 1 << chan->base.chid, 0);
 243         nv_wr32(priv, NV03_PFIFO_CACHES, 1);
 244         spin_unlock_irqrestore(&priv->base.lock, flags);
 245
 246         return nouveau_fifo_channel_fini(&chan->base, suspend);
 247 }
 248
 249 static struct nouveau_ofuncs
 250 nv04_fifo_ofuncs = {
 251         .ctor = nv04_fifo_chan_ctor,
 252         .dtor = nv04_fifo_chan_dtor,
 253         .init = nv04_fifo_chan_init,
 254         .fini = nv04_fifo_chan_fini,
 255         .map  = _nouveau_fifo_channel_map,
 256         .rd32 = _nouveau_fifo_channel_rd32,
 257         .wr32 = _nouveau_fifo_channel_wr32,
 258         .ntfy = _nouveau_fifo_channel_ntfy
 259 };
 260
 261 static struct nouveau_oclass
 262 nv04_fifo_sclass[] = {
 263         { NV03_CHANNEL_DMA, &nv04_fifo_ofuncs },
 264         {}
 265 };
 266
 267 /*******************************************************************************
 268  * FIFO context - basically just the instmem reserved for the channel
 269  ******************************************************************************/
 270
 271 int
 272 nv04_fifo_context_ctor(struct nouveau_object *parent,
 273                        struct nouveau_object *engine,
 274                        struct nouveau_oclass *oclass, void *data, u32 size,
 275                        struct nouveau_object **pobject)
 276 {
 277         struct nv04_fifo_base *base;
 278         int ret;
 279
 280         ret = nouveau_fifo_context_create(parent, engine, oclass, NULL, 0x1000,
 281                                           0x1000, NVOBJ_FLAG_HEAP, &base);
 282         *pobject = nv_object(base);
 283         if (ret)
 284                 return ret;
 285
 286         return 0;
 287 }
 288
 289 static struct nouveau_oclass
 290 nv04_fifo_cclass = {
 291         .handle = NV_ENGCTX(FIFO, 0x04),
 292         .ofuncs = &(struct nouveau_ofuncs) {
 293                 .ctor = nv04_fifo_context_ctor,
 294                 .dtor = _nouveau_fifo_context_dtor,
 295                 .init = _nouveau_fifo_context_init,
 296                 .fini = _nouveau_fifo_context_fini,
 297                 .rd32 = _nouveau_fifo_context_rd32,
 298                 .wr32 = _nouveau_fifo_context_wr32,
 299         },
 300 };
 301
 302 /*******************************************************************************
 303  * PFIFO engine
 304  ******************************************************************************/
 305
 306 void
 307 nv04_fifo_pause(struct nouveau_fifo *pfifo, unsigned long *pflags)
 308 __acquires(priv->base.lock)
 309 {
 310         struct nv04_fifo_priv *priv = (void *)pfifo;
 311         unsigned long flags;
 312
 313         spin_lock_irqsave(&priv->base.lock, flags);
 314         *pflags = flags;
 315
 316         nv_wr32(priv, NV03_PFIFO_CACHES, 0x00000000);
 317         nv_mask(priv, NV04_PFIFO_CACHE1_PULL0, 0x00000001, 0x00000000);
 318
 319         /* in some cases the puller may be left in an inconsistent state
 320          * if you try to stop it while it's busy translating handles.
 321          * sometimes you get a CACHE_ERROR, sometimes it just fails
 322          * silently; sending incorrect instance offsets to PGRAPH after
 323          * it's started up again.
 324          *
 325          * to avoid this, we invalidate the most recently calculated
 326          * instance.
 327          */
 328         if (!nv_wait(priv, NV04_PFIFO_CACHE1_PULL0,
 329                            NV04_PFIFO_CACHE1_PULL0_HASH_BUSY, 0x00000000))
 330                 nv_warn(priv, "timeout idling puller\n");
 331
 332         if (nv_rd32(priv, NV04_PFIFO_CACHE1_PULL0) &
 333                           NV04_PFIFO_CACHE1_PULL0_HASH_FAILED)
 334                 nv_wr32(priv, NV03_PFIFO_INTR_0, NV_PFIFO_INTR_CACHE_ERROR);
 335
 336         nv_wr32(priv, NV04_PFIFO_CACHE1_HASH, 0x00000000);
 337 }
 338
 339 void
 340 nv04_fifo_start(struct nouveau_fifo *pfifo, unsigned long *pflags)
 341 __releases(priv->base.lock)
 342 {
 343         struct nv04_fifo_priv *priv = (void *)pfifo;
 344         unsigned long flags = *pflags;
 345
 346         nv_mask(priv, NV04_PFIFO_CACHE1_PULL0, 0x00000001, 0x00000001);
 347         nv_wr32(priv, NV03_PFIFO_CACHES, 0x00000001);
 348
 349         spin_unlock_irqrestore(&priv->base.lock, flags);
 350 }
 351
 352 static const char *
 353 nv_dma_state_err(u32 state)
 354 {
 355         static const char * const desc[] = {
 356                 "NONE", "CALL_SUBR_ACTIVE", "INVALID_MTHD", "RET_SUBR_INACTIVE",
 357                 "INVALID_CMD", "IB_EMPTY"/* NV50+ */, "MEM_FAULT", "UNK"
 358         };
 359         return desc[(state >> 29) & 0x7];
 360 }
 361
 362 static bool
 363 nv04_fifo_swmthd(struct nv04_fifo_priv *priv, u32 chid, u32 addr, u32 data)
 364 {
 365         struct nv04_fifo_chan *chan = NULL;
 366         struct nouveau_handle *bind;
 367         const int subc = (addr >> 13) & 0x7;
 368         const int mthd = addr & 0x1ffc;
 369         bool handled = false;
 370         unsigned long flags;
 371         u32 engine;
 372
 373         spin_lock_irqsave(&priv->base.lock, flags);
 374         if (likely(chid >= priv->base.min && chid <= priv->base.max))
 375                 chan = (void *)priv->base.channel[chid];
 376         if (unlikely(!chan))
 377                 goto out;
 378
 379         switch (mthd) {
 380         case 0x0000:
 381                 bind = nouveau_namedb_get(nv_namedb(chan), data);
 382                 if (unlikely(!bind))
 383                         break;
 384
 385                 if (nv_engidx(bind->object->engine) == NVDEV_ENGINE_SW) {
 386                         engine = 0x0000000f << (subc * 4);
 387                         chan->subc[subc] = data;
 388                         handled = true;
 389
 390                         nv_mask(priv, NV04_PFIFO_CACHE1_ENGINE, engine, 0);
 391                 }
 392
 393                 nouveau_namedb_put(bind);
 394                 break;
 395         default:
 396                 engine = nv_rd32(priv, NV04_PFIFO_CACHE1_ENGINE);
 397                 if (unlikely(((engine >> (subc * 4)) & 0xf) != 0))
 398                         break;
 399
 400                 bind = nouveau_namedb_get(nv_namedb(chan), chan->subc[subc]);
 401                 if (likely(bind)) {
 402                         if (!nv_call(bind->object, mthd, data))
 403                                 handled = true;
 404                         nouveau_namedb_put(bind);
 405                 }
 406                 break;
 407         }
 408
 409 out:
 410         spin_unlock_irqrestore(&priv->base.lock, flags);
 411         return handled;
 412 }
 413
 414 static void
 415 nv04_fifo_cache_error(struct nouveau_device *device,
 416                 struct nv04_fifo_priv *priv, u32 chid, u32 get)
 417 {
 418         u32 mthd, data;
 419         int ptr;
 420
 421         /* NV_PFIFO_CACHE1_GET actually goes to 0xffc before wrapping on my
 422          * G80 chips, but CACHE1 isn't big enough for this much data.. Tests
 423          * show that it wraps around to the start at GET=0x800.. No clue as to
 424          * why..
 425          */
 426         ptr = (get & 0x7ff) >> 2;
 427
 428         if (device->card_type < NV_40) {
 429                 mthd = nv_rd32(priv, NV04_PFIFO_CACHE1_METHOD(ptr));
 430                 data = nv_rd32(priv, NV04_PFIFO_CACHE1_DATA(ptr));
 431         } else {
 432                 mthd = nv_rd32(priv, NV40_PFIFO_CACHE1_METHOD(ptr));
 433                 data = nv_rd32(priv, NV40_PFIFO_CACHE1_DATA(ptr));
 434         }
 435
 436         if (!nv04_fifo_swmthd(priv, chid, mthd, data)) {
 437                 const char *client_name =
 438                         nouveau_client_name_for_fifo_chid(&priv->base, chid);
 439                 nv_error(priv,
 440                          "CACHE_ERROR - ch %d [%s] subc %d mthd 0x%04x data 0x%08x\n",
 441                          chid, client_name, (mthd >> 13) & 7, mthd & 0x1ffc,
 442                          data);
 443         }
 444
 445         nv_wr32(priv, NV04_PFIFO_CACHE1_DMA_PUSH, 0);
 446         nv_wr32(priv, NV03_PFIFO_INTR_0, NV_PFIFO_INTR_CACHE_ERROR);
 447
 448         nv_wr32(priv, NV03_PFIFO_CACHE1_PUSH0,
 449                 nv_rd32(priv, NV03_PFIFO_CACHE1_PUSH0) & ~1);
 450         nv_wr32(priv, NV03_PFIFO_CACHE1_GET, get + 4);
 451         nv_wr32(priv, NV03_PFIFO_CACHE1_PUSH0,
 452                 nv_rd32(priv, NV03_PFIFO_CACHE1_PUSH0) | 1);
 453         nv_wr32(priv, NV04_PFIFO_CACHE1_HASH, 0);
 454
 455         nv_wr32(priv, NV04_PFIFO_CACHE1_DMA_PUSH,
 456                 nv_rd32(priv, NV04_PFIFO_CACHE1_DMA_PUSH) | 1);
 457         nv_wr32(priv, NV04_PFIFO_CACHE1_PULL0, 1);
 458 }
 459
 460 static void
 461 nv04_fifo_dma_pusher(struct nouveau_device *device, struct nv04_fifo_priv *priv,
 462                 u32 chid)
 463 {
 464         const char *client_name;
 465         u32 dma_get = nv_rd32(priv, 0x003244);
 466         u32 dma_put = nv_rd32(priv, 0x003240);
 467         u32 push = nv_rd32(priv, 0x003220);
 468         u32 state = nv_rd32(priv, 0x003228);
 469
 470         client_name = nouveau_client_name_for_fifo_chid(&priv->base, chid);
 471
 472         if (device->card_type == NV_50) {
 473                 u32 ho_get = nv_rd32(priv, 0x003328);
 474                 u32 ho_put = nv_rd32(priv, 0x003320);
 475                 u32 ib_get = nv_rd32(priv, 0x003334);
 476                 u32 ib_put = nv_rd32(priv, 0x003330);
 477
 478                 nv_error(priv,
 479                          "DMA_PUSHER - ch %d [%s] get 0x%02x%08x put 0x%02x%08x ib_get 0x%08x ib_put 0x%08x state 0x%08x (err: %s) push 0x%08x\n",
 480                          chid, client_name, ho_get, dma_get, ho_put, dma_put,
 481                          ib_get, ib_put, state, nv_dma_state_err(state), push);
 482
 483                 /* METHOD_COUNT, in DMA_STATE on earlier chipsets */
 484                 nv_wr32(priv, 0x003364, 0x00000000);
 485                 if (dma_get != dma_put || ho_get != ho_put) {
 486                         nv_wr32(priv, 0x003244, dma_put);
 487                         nv_wr32(priv, 0x003328, ho_put);
 488                 } else
 489                 if (ib_get != ib_put)
 490                         nv_wr32(priv, 0x003334, ib_put);
 491         } else {
 492                 nv_error(priv,
 493                          "DMA_PUSHER - ch %d [%s] get 0x%08x put 0x%08x state 0x%08x (err: %s) push 0x%08x\n",
 494                          chid, client_name, dma_get, dma_put, state,
 495                          nv_dma_state_err(state), push);
 496
 497                 if (dma_get != dma_put)
 498                         nv_wr32(priv, 0x003244, dma_put);
 499         }
 500
 501         nv_wr32(priv, 0x003228, 0x00000000);
 502         nv_wr32(priv, 0x003220, 0x00000001);
 503         nv_wr32(priv, 0x002100, NV_PFIFO_INTR_DMA_PUSHER);
 504 }
 505
 506 void
 507 nv04_fifo_intr(struct nouveau_subdev *subdev)
 508 {
 509         struct nouveau_device *device = nv_device(subdev);
 510         struct nv04_fifo_priv *priv = (void *)subdev;
 511         uint32_t status, reassign;
 512         int cnt = 0;
 513
 514         reassign = nv_rd32(priv, NV03_PFIFO_CACHES) & 1;
 515         while ((status = nv_rd32(priv, NV03_PFIFO_INTR_0)) && (cnt++ < 100)) {
 516                 uint32_t chid, get;
 517
 518                 nv_wr32(priv, NV03_PFIFO_CACHES, 0);
 519
 520                 chid = nv_rd32(priv, NV03_PFIFO_CACHE1_PUSH1) & priv->base.max;
 521                 get  = nv_rd32(priv, NV03_PFIFO_CACHE1_GET);
 522
 523                 if (status & NV_PFIFO_INTR_CACHE_ERROR) {
 524                         nv04_fifo_cache_error(device, priv, chid, get);
 525                         status &= ~NV_PFIFO_INTR_CACHE_ERROR;
 526                 }
 527
 528                 if (status & NV_PFIFO_INTR_DMA_PUSHER) {
 529                         nv04_fifo_dma_pusher(device, priv, chid);
 530                         status &= ~NV_PFIFO_INTR_DMA_PUSHER;
 531                 }
 532
 533                 if (status & NV_PFIFO_INTR_SEMAPHORE) {
 534                         uint32_t sem;
 535
 536                         status &= ~NV_PFIFO_INTR_SEMAPHORE;
 537                         nv_wr32(priv, NV03_PFIFO_INTR_0,
 538                                 NV_PFIFO_INTR_SEMAPHORE);
 539
 540                         sem = nv_rd32(priv, NV10_PFIFO_CACHE1_SEMAPHORE);
 541                         nv_wr32(priv, NV10_PFIFO_CACHE1_SEMAPHORE, sem | 0x1);
 542
 543                         nv_wr32(priv, NV03_PFIFO_CACHE1_GET, get + 4);
 544                         nv_wr32(priv, NV04_PFIFO_CACHE1_PULL0, 1);
 545                 }
 546
 547                 if (device->card_type == NV_50) {
 548                         if (status & 0x00000010) {
 549                                 status &= ~0x00000010;
 550                                 nv_wr32(priv, 0x002100, 0x00000010);
 551                         }
 552
 553                         if (status & 0x40000000) {
 554                                 nv_wr32(priv, 0x002100, 0x40000000);
 555                                 nouveau_fifo_uevent(&priv->base);
 556                                 status &= ~0x40000000;
 557                         }
 558                 }
 559
 560                 if (status) {
 561                         nv_warn(priv, "unknown intr 0x%08x, ch %d\n",
 562                                 status, chid);
 563                         nv_wr32(priv, NV03_PFIFO_INTR_0, status);
 564                         status = 0;
 565                 }
 566
 567                 nv_wr32(priv, NV03_PFIFO_CACHES, reassign);
 568         }
 569
 570         if (status) {
 571                 nv_error(priv, "still angry after %d spins, halt\n", cnt);
 572                 nv_wr32(priv, 0x002140, 0);
 573                 nv_wr32(priv, 0x000140, 0);
 574         }
 575
 576         nv_wr32(priv, 0x000100, 0x00000100);
 577 }
 578
 579 static int
 580 nv04_fifo_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
 581                struct nouveau_oclass *oclass, void *data, u32 size,
 582                struct nouveau_object **pobject)
 583 {
 584         struct nv04_instmem_priv *imem = nv04_instmem(parent);
 585         struct nv04_fifo_priv *priv;
 586         int ret;
 587
 588         ret = nouveau_fifo_create(parent, engine, oclass, 0, 15, &priv);
 589         *pobject = nv_object(priv);
 590         if (ret)
 591                 return ret;
 592
 593         nouveau_ramht_ref(imem->ramht, &priv->ramht);
 594         nouveau_gpuobj_ref(imem->ramro, &priv->ramro);
 595         nouveau_gpuobj_ref(imem->ramfc, &priv->ramfc);
 596
 597         nv_subdev(priv)->unit = 0x00000100;
 598         nv_subdev(priv)->intr = nv04_fifo_intr;
 599         nv_engine(priv)->cclass = &nv04_fifo_cclass;
 600         nv_engine(priv)->sclass = nv04_fifo_sclass;
 601         priv->base.pause = nv04_fifo_pause;
 602         priv->base.start = nv04_fifo_start;
 603         priv->ramfc_desc = nv04_ramfc;
 604         return 0;
 605 }
 606
 607 void
 608 nv04_fifo_dtor(struct nouveau_object *object)
 609 {
 610         struct nv04_fifo_priv *priv = (void *)object;
 611         nouveau_gpuobj_ref(NULL, &priv->ramfc);
 612         nouveau_gpuobj_ref(NULL, &priv->ramro);
 613         nouveau_ramht_ref(NULL, &priv->ramht);
 614         nouveau_fifo_destroy(&priv->base);
 615 }
 616
 617 int
 618 nv04_fifo_init(struct nouveau_object *object)
 619 {
 620         struct nv04_fifo_priv *priv = (void *)object;
 621         int ret;
 622
 623         ret = nouveau_fifo_init(&priv->base);
 624         if (ret)
 625                 return ret;
 626
 627         nv_wr32(priv, NV04_PFIFO_DELAY_0, 0x000000ff);
 628         nv_wr32(priv, NV04_PFIFO_DMA_TIMESLICE, 0x0101ffff);
 629
 630         nv_wr32(priv, NV03_PFIFO_RAMHT, (0x03 << 24) /* search 128 */ |
 631                                        ((priv->ramht->bits - 9) << 16) |
 632                                         (priv->ramht->base.addr >> 8));
 633         nv_wr32(priv, NV03_PFIFO_RAMRO, priv->ramro->addr >> 8);
 634         nv_wr32(priv, NV03_PFIFO_RAMFC, priv->ramfc->addr >> 8);
 635
 636         nv_wr32(priv, NV03_PFIFO_CACHE1_PUSH1, priv->base.max);
 637
 638         nv_wr32(priv, NV03_PFIFO_INTR_0, 0xffffffff);
 639         nv_wr32(priv, NV03_PFIFO_INTR_EN_0, 0xffffffff);
 640
 641         nv_wr32(priv, NV03_PFIFO_CACHE1_PUSH0, 1);
 642         nv_wr32(priv, NV04_PFIFO_CACHE1_PULL0, 1);
 643         nv_wr32(priv, NV03_PFIFO_CACHES, 1);
 644         return 0;
 645 }
 646
 647 struct nouveau_oclass *
 648 nv04_fifo_oclass = &(struct nouveau_oclass) {
 649         .handle = NV_ENGINE(FIFO, 0x04),
 650         .ofuncs = &(struct nouveau_ofuncs) {
 651                 .ctor = nv04_fifo_ctor,
 652                 .dtor = nv04_fifo_dtor,
 653                 .init = nv04_fifo_init,
 654                 .fini = _nouveau_fifo_fini,
 655         },
 656 };