drivers/mxc/gpu-viv/hal/kernel/gc_hal_kernel_event.c

   1 /****************************************************************************
   2 *
   3 *    Copyright (C) 2005 - 2014 by Vivante Corp.
   4 *
   5 *    This program is free software; you can redistribute it and/or modify
   6 *    it under the terms of the GNU General Public License as published by
   7 *    the Free Software Foundation; either version 2 of the license, or
   8 *    (at your option) any later version.
   9 *
  10 *    This program is distributed in the hope that it will be useful,
  11 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13 *    GNU General Public License for more details.
  14 *
  15 *    You should have received a copy of the GNU General Public License
  16 *    along with this program; if not write to the Free Software
  17 *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  18 *
  19 *****************************************************************************/
  20
  21
  22 #include "gc_hal_kernel_precomp.h"
  23 #include "gc_hal_kernel_buffer.h"
  24
  25 #ifdef __QNXNTO__
  26 #include <atomic.h>
  27 #include "gc_hal_kernel_qnx.h"
  28 #endif
  29
  30 #define _GC_OBJ_ZONE                    gcvZONE_EVENT
  31
  32 #define gcdEVENT_ALLOCATION_COUNT       (4096 / gcmSIZEOF(gcsHAL_INTERFACE))
  33 #define gcdEVENT_MIN_THRESHOLD          4
  34
  35 /******************************************************************************\
  36 ********************************* Support Code *********************************
  37 \******************************************************************************/
  38
  39 static gceSTATUS
  40 gckEVENT_AllocateQueue(
  41     IN gckEVENT Event,
  42     OUT gcsEVENT_QUEUE_PTR * Queue
  43     )
  44 {
  45     gceSTATUS status;
  46
  47     gcmkHEADER_ARG("Event=0x%x", Event);
  48
  49     /* Verify the arguments. */
  50     gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT);
  51     gcmkVERIFY_ARGUMENT(Queue != gcvNULL);
  52
  53     /* Do we have free queues? */
  54     if (Event->freeList == gcvNULL)
  55     {
  56         gcmkONERROR(gcvSTATUS_OUT_OF_RESOURCES);
  57     }
  58
  59     /* Move one free queue from the free list. */
  60     * Queue = Event->freeList;
  61     Event->freeList = Event->freeList->next;
  62
  63     /* Success. */
  64     gcmkFOOTER_ARG("*Queue=0x%x", gcmOPT_POINTER(Queue));
  65     return gcvSTATUS_OK;
  66
  67 OnError:
  68     /* Return the status. */
  69     gcmkFOOTER();
  70     return status;
  71 }
  72
  73 static gceSTATUS
  74 gckEVENT_FreeQueue(
  75     IN gckEVENT Event,
  76     OUT gcsEVENT_QUEUE_PTR Queue
  77     )
  78 {
  79     gceSTATUS status = gcvSTATUS_OK;
  80
  81     gcmkHEADER_ARG("Event=0x%x", Event);
  82
  83     /* Verify the arguments. */
  84     gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT);
  85     gcmkVERIFY_ARGUMENT(Queue != gcvNULL);
  86
  87     /* Move one free queue from the free list. */
  88     Queue->next = Event->freeList;
  89     Event->freeList = Queue;
  90
  91     /* Success. */
  92     gcmkFOOTER();
  93     return status;
  94 }
  95
  96 static gceSTATUS
  97 gckEVENT_FreeRecord(
  98     IN gckEVENT Event,
  99     IN gcsEVENT_PTR Record
 100     )
 101 {
 102     gceSTATUS status;
 103     gctBOOL acquired = gcvFALSE;
 104
 105     gcmkHEADER_ARG("Event=0x%x Record=0x%x", Event, Record);
 106
 107     /* Verify the arguments. */
 108     gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT);
 109     gcmkVERIFY_ARGUMENT(Record != gcvNULL);
 110
 111     /* Acquire the mutex. */
 112     gcmkONERROR(gckOS_AcquireMutex(Event->os,
 113                                    Event->freeEventMutex,
 114                                    gcvINFINITE));
 115     acquired = gcvTRUE;
 116
 117     /* Push the record on the free list. */
 118     Record->next           = Event->freeEventList;
 119     Event->freeEventList   = Record;
 120     Event->freeEventCount += 1;
 121
 122     /* Release the mutex. */
 123     gcmkONERROR(gckOS_ReleaseMutex(Event->os, Event->freeEventMutex));
 124
 125     /* Success. */
 126     gcmkFOOTER_NO();
 127     return gcvSTATUS_OK;
 128
 129 OnError:
 130     /* Roll back. */
 131     if (acquired)
 132     {
 133         gcmkVERIFY_OK(gckOS_ReleaseMutex(Event->os, Event->freeEventMutex));
 134     }
 135
 136     /* Return the status. */
 137     gcmkFOOTER();
 138     return gcvSTATUS_OK;
 139 }
 140
 141 static gceSTATUS
 142 gckEVENT_IsEmpty(
 143     IN gckEVENT Event,
 144     OUT gctBOOL_PTR IsEmpty
 145     )
 146 {
 147     gceSTATUS status;
 148     gctSIZE_T i;
 149
 150     gcmkHEADER_ARG("Event=0x%x", Event);
 151
 152     /* Verify the arguments. */
 153     gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT);
 154     gcmkVERIFY_ARGUMENT(IsEmpty != gcvNULL);
 155
 156     /* Assume the event queue is empty. */
 157     *IsEmpty = gcvTRUE;
 158
 159     /* Walk the event queue. */
 160     for (i = 0; i < gcmCOUNTOF(Event->queues); ++i)
 161     {
 162         /* Check whether this event is in use. */
 163         if (Event->queues[i].head != gcvNULL)
 164         {
 165             /* The event is in use, hence the queue is not empty. */
 166             *IsEmpty = gcvFALSE;
 167             break;
 168         }
 169     }
 170
 171     /* Try acquiring the mutex. */
 172     status = gckOS_AcquireMutex(Event->os, Event->eventQueueMutex, 0);
 173     if (status == gcvSTATUS_TIMEOUT)
 174     {
 175         /* Timeout - queue is no longer empty. */
 176         *IsEmpty = gcvFALSE;
 177     }
 178     else
 179     {
 180         /* Bail out on error. */
 181         gcmkONERROR(status);
 182
 183         /* Release the mutex. */
 184         gcmkVERIFY_OK(gckOS_ReleaseMutex(Event->os, Event->eventQueueMutex));
 185     }
 186
 187     /* Success. */
 188     gcmkFOOTER_ARG("*IsEmpty=%d", gcmOPT_VALUE(IsEmpty));
 189     return gcvSTATUS_OK;
 190
 191 OnError:
 192     /* Return the status. */
 193     gcmkFOOTER();
 194     return status;
 195 }
 196
 197 static gceSTATUS
 198 _TryToIdleGPU(
 199     IN gckEVENT Event
 200 )
 201 {
 202     gceSTATUS status;
 203     gctBOOL empty = gcvFALSE, idle = gcvFALSE;
 204     gctBOOL powerLocked = gcvFALSE;
 205     gckHARDWARE hardware;
 206
 207     gcmkHEADER_ARG("Event=0x%x", Event);
 208
 209     /* Verify the arguments. */
 210     gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT);
 211
 212     /* Grab gckHARDWARE object. */
 213     hardware = Event->kernel->hardware;
 214     gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE);
 215
 216     /* Check whether the event queue is empty. */
 217     gcmkONERROR(gckEVENT_IsEmpty(Event, &empty));
 218
 219     if (empty)
 220     {
 221         status = gckOS_AcquireMutex(hardware->os, hardware->powerMutex, 0);
 222         if (status == gcvSTATUS_TIMEOUT)
 223         {
 224             gcmkFOOTER_NO();
 225             return gcvSTATUS_OK;
 226         }
 227
 228         powerLocked = gcvTRUE;
 229
 230         /* Query whether the hardware is idle. */
 231         gcmkONERROR(gckHARDWARE_QueryIdle(Event->kernel->hardware, &idle));
 232
 233         gcmkONERROR(gckOS_ReleaseMutex(hardware->os, hardware->powerMutex));
 234         powerLocked = gcvFALSE;
 235
 236         if (idle)
 237         {
 238             /* Inform the system of idle GPU. */
 239             gcmkONERROR(gckOS_Broadcast(Event->os,
 240                                         Event->kernel->hardware,
 241                                         gcvBROADCAST_GPU_IDLE));
 242         }
 243     }
 244
 245     gcmkFOOTER_NO();
 246     return gcvSTATUS_OK;
 247
 248 OnError:
 249     if (powerLocked)
 250     {
 251         gcmkONERROR(gckOS_ReleaseMutex(hardware->os, hardware->powerMutex));
 252     }
 253
 254     gcmkFOOTER();
 255     return status;
 256 }
 257
 258 static gceSTATUS
 259 __RemoveRecordFromProcessDB(
 260     IN gckEVENT Event,
 261     IN gcsEVENT_PTR Record
 262     )
 263 {
 264     gcmkHEADER_ARG("Event=0x%x Record=0x%x", Event, Record);
 265     gcmkVERIFY_ARGUMENT(Record != gcvNULL);
 266
 267     while (Record != gcvNULL)
 268     {
 269         if (Record->info.command == gcvHAL_SIGNAL)
 270         {
 271             /* TODO: Find a better place to bind signal to hardware.*/
 272             gcmkVERIFY_OK(gckOS_SignalSetHardware(Event->os,
 273                         gcmUINT64_TO_PTR(Record->info.u.Signal.signal),
 274                         Event->kernel->hardware));
 275         }
 276
 277         if (Record->fromKernel)
 278         {
 279             /* No need to check db if event is from kernel. */
 280             Record = Record->next;
 281             continue;
 282         }
 283
 284         switch (Record->info.command)
 285         {
 286         case gcvHAL_FREE_NON_PAGED_MEMORY:
 287             gcmkVERIFY_OK(gckKERNEL_RemoveProcessDB(
 288                 Event->kernel,
 289                 Record->processID,
 290                 gcvDB_NON_PAGED,
 291                 gcmUINT64_TO_PTR(Record->info.u.FreeNonPagedMemory.logical)));
 292             break;
 293
 294         case gcvHAL_FREE_CONTIGUOUS_MEMORY:
 295             gcmkVERIFY_OK(gckKERNEL_RemoveProcessDB(
 296                 Event->kernel,
 297                 Record->processID,
 298                 gcvDB_CONTIGUOUS,
 299                 gcmUINT64_TO_PTR(Record->info.u.FreeContiguousMemory.logical)));
 300             break;
 301
 302         case gcvHAL_UNLOCK_VIDEO_MEMORY:
 303             gcmkVERIFY_OK(gckKERNEL_RemoveProcessDB(
 304                 Event->kernel,
 305                 Record->processID,
 306                 gcvDB_VIDEO_MEMORY_LOCKED,
 307                 gcmUINT64_TO_PTR(Record->info.u.UnlockVideoMemory.node)));
 308             break;
 309
 310         case gcvHAL_UNMAP_USER_MEMORY:
 311             gcmkVERIFY_OK(gckKERNEL_RemoveProcessDB(
 312                 Event->kernel,
 313                 Record->processID,
 314                 gcvDB_MAP_USER_MEMORY,
 315                 gcmINT2PTR(Record->info.u.UnmapUserMemory.info)));
 316             break;
 317
 318         case gcvHAL_FREE_VIRTUAL_COMMAND_BUFFER:
 319             gcmkVERIFY_OK(gckKERNEL_RemoveProcessDB(
 320                 Event->kernel,
 321                 Record->processID,
 322                 gcvDB_COMMAND_BUFFER,
 323                 gcmUINT64_TO_PTR(Record->info.u.FreeVirtualCommandBuffer.logical)));
 324             break;
 325
 326         default:
 327             break;
 328         }
 329
 330         Record = Record->next;
 331     }
 332     gcmkFOOTER_NO();
 333     return gcvSTATUS_OK;
 334 }
 335
 336 gceSTATUS
 337 _ReleaseVideoMemoryHandle(
 338     IN gckKERNEL Kernel,
 339     IN OUT gcsEVENT_PTR Record,
 340     IN OUT gcsHAL_INTERFACE * Interface
 341     )
 342 {
 343     gceSTATUS status;
 344     gckVIDMEM_NODE nodeObject;
 345     gctUINT32 handle;
 346
 347     switch(Interface->command)
 348     {
 349     case gcvHAL_UNLOCK_VIDEO_MEMORY:
 350         handle = (gctUINT32)Interface->u.UnlockVideoMemory.node;
 351
 352         gcmkONERROR(gckVIDMEM_HANDLE_Lookup(
 353             Kernel, Record->processID, handle, &nodeObject));
 354
 355         Record->info.u.UnlockVideoMemory.node = gcmPTR_TO_UINT64(nodeObject);
 356
 357         gckVIDMEM_HANDLE_Dereference(Kernel, Record->processID, handle);
 358         break;
 359
 360     default:
 361         break;
 362     }
 363
 364     return gcvSTATUS_OK;
 365 OnError:
 366     return status;
 367 }
 368
 369 /*******************************************************************************
 370 **
 371 **  _QueryFlush
 372 **
 373 **  Check the type of surfaces which will be released by current event and
 374 **  determine the cache needed to flush.
 375 **
 376 */
 377 static gceSTATUS
 378 _QueryFlush(
 379     IN gckEVENT Event,
 380     IN gcsEVENT_PTR Record,
 381     OUT gceKERNEL_FLUSH *Flush
 382     )
 383 {
 384     gceKERNEL_FLUSH flush = 0;
 385     gcmkHEADER_ARG("Event=0x%x Record=0x%x", Event, Record);
 386     gcmkVERIFY_ARGUMENT(Record != gcvNULL);
 387
 388     while (Record != gcvNULL)
 389     {
 390         switch (Record->info.command)
 391         {
 392         case gcvHAL_UNLOCK_VIDEO_MEMORY:
 393             switch(Record->info.u.UnlockVideoMemory.type)
 394             {
 395             case gcvSURF_TILE_STATUS:
 396                 flush |= gcvFLUSH_TILE_STATUS;
 397                 break;
 398             case gcvSURF_RENDER_TARGET:
 399                 flush |= gcvFLUSH_COLOR;
 400                 break;
 401             case gcvSURF_DEPTH:
 402                 flush |= gcvFLUSH_DEPTH;
 403                 break;
 404             case gcvSURF_TEXTURE:
 405                 flush |= gcvFLUSH_TEXTURE;
 406                 break;
 407             case gcvSURF_TYPE_UNKNOWN:
 408                 gcmkASSERT(0);
 409                 break;
 410             default:
 411                 break;
 412             }
 413             break;
 414         case gcvHAL_UNMAP_USER_MEMORY:
 415             *Flush = gcvFLUSH_ALL;
 416             return gcvSTATUS_OK;
 417
 418         default:
 419             break;
 420         }
 421
 422         Record = Record->next;
 423     }
 424
 425     *Flush = flush;
 426
 427     gcmkFOOTER_NO();
 428     return gcvSTATUS_OK;
 429 }
 430
 431 void
 432 _SubmitTimerFunction(
 433     gctPOINTER Data
 434     )
 435 {
 436     gckEVENT event = (gckEVENT)Data;
 437 #if gcdMULTI_GPU
 438     gcmkVERIFY_OK(gckEVENT_Submit(event, gcvTRUE, gcvFALSE, gcvCORE_3D_ALL_MASK));
 439 #else
 440     gcmkVERIFY_OK(gckEVENT_Submit(event, gcvTRUE, gcvFALSE));
 441 #endif
 442 }
 443
 444 /******************************************************************************\
 445 ******************************* gckEVENT API Code *******************************
 446 \******************************************************************************/
 447
 448 /*******************************************************************************
 449 **
 450 **  gckEVENT_Construct
 451 **
 452 **  Construct a new gckEVENT object.
 453 **
 454 **  INPUT:
 455 **
 456 **      gckKERNEL Kernel
 457 **          Pointer to an gckKERNEL object.
 458 **
 459 **  OUTPUT:
 460 **
 461 **      gckEVENT * Event
 462 **          Pointer to a variable that receives the gckEVENT object pointer.
 463 */
 464 gceSTATUS
 465 gckEVENT_Construct(
 466     IN gckKERNEL Kernel,
 467     OUT gckEVENT * Event
 468     )
 469 {
 470     gckOS os;
 471     gceSTATUS status;
 472     gckEVENT eventObj = gcvNULL;
 473     int i;
 474     gcsEVENT_PTR record;
 475     gctPOINTER pointer = gcvNULL;
 476
 477     gcmkHEADER_ARG("Kernel=0x%x", Kernel);
 478
 479     /* Verify the arguments. */
 480     gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL);
 481     gcmkVERIFY_ARGUMENT(Event != gcvNULL);
 482
 483     /* Extract the pointer to the gckOS object. */
 484     os = Kernel->os;
 485     gcmkVERIFY_OBJECT(os, gcvOBJ_OS);
 486
 487     /* Allocate the gckEVENT object. */
 488     gcmkONERROR(gckOS_Allocate(os, gcmSIZEOF(struct _gckEVENT), &pointer));
 489
 490     eventObj = pointer;
 491
 492     /* Reset the object. */
 493     gcmkVERIFY_OK(gckOS_ZeroMemory(eventObj, gcmSIZEOF(struct _gckEVENT)));
 494
 495     /* Initialize the gckEVENT object. */
 496     eventObj->object.type = gcvOBJ_EVENT;
 497     eventObj->kernel      = Kernel;
 498     eventObj->os          = os;
 499
 500     /* Create the mutexes. */
 501     gcmkONERROR(gckOS_CreateMutex(os, &eventObj->eventQueueMutex));
 502     gcmkONERROR(gckOS_CreateMutex(os, &eventObj->freeEventMutex));
 503     gcmkONERROR(gckOS_CreateMutex(os, &eventObj->eventListMutex));
 504
 505     /* Create a bunch of event reccords. */
 506     for (i = 0; i < gcdEVENT_ALLOCATION_COUNT; i += 1)
 507     {
 508         /* Allocate an event record. */
 509         gcmkONERROR(gckOS_Allocate(os, gcmSIZEOF(gcsEVENT), &pointer));
 510
 511         record = pointer;
 512
 513         /* Push it on the free list. */
 514         record->next              = eventObj->freeEventList;
 515         eventObj->freeEventList   = record;
 516         eventObj->freeEventCount += 1;
 517     }
 518
 519     /* Initialize the free list of event queues. */
 520     for (i = 0; i < gcdREPO_LIST_COUNT; i += 1)
 521     {
 522         eventObj->repoList[i].next = eventObj->freeList;
 523         eventObj->freeList = &eventObj->repoList[i];
 524     }
 525
 526     /* Construct the atom. */
 527     gcmkONERROR(gckOS_AtomConstruct(os, &eventObj->freeAtom));
 528     gcmkONERROR(gckOS_AtomSet(os,
 529                               eventObj->freeAtom,
 530                               gcmCOUNTOF(eventObj->queues)));
 531
 532 #if gcdSMP
 533     gcmkONERROR(gckOS_AtomConstruct(os, &eventObj->pending));
 534
 535 #if gcdMULTI_GPU
 536     for (i = 0; i < gcdMULTI_GPU; i++)
 537     {
 538         gcmkONERROR(gckOS_AtomConstruct(os, &eventObj->pending3D[i]));
 539         gcmkONERROR(gckOS_AtomConstruct(os, &eventObj->pending3DMask[i]));
 540     }
 541
 542     gcmkONERROR(gckOS_AtomConstruct(os, &eventObj->pendingMask));
 543 #endif
 544
 545 #endif
 546
 547     gcmkVERIFY_OK(gckOS_CreateTimer(os,
 548                                     _SubmitTimerFunction,
 549                                     (gctPOINTER)eventObj,
 550                                     &eventObj->submitTimer));
 551
 552 #if gcdINTERRUPT_STATISTIC
 553     gcmkONERROR(gckOS_AtomConstruct(os, &eventObj->interruptCount));
 554     gcmkONERROR(gckOS_AtomSet(os,eventObj->interruptCount, 0));
 555 #endif
 556
 557     /* Return pointer to the gckEVENT object. */
 558     *Event = eventObj;
 559
 560     /* Success. */
 561     gcmkFOOTER_ARG("*Event=0x%x", *Event);
 562     return gcvSTATUS_OK;
 563
 564 OnError:
 565     /* Roll back. */
 566     if (eventObj != gcvNULL)
 567     {
 568         if (eventObj->eventQueueMutex != gcvNULL)
 569         {
 570             gcmkVERIFY_OK(gckOS_DeleteMutex(os, eventObj->eventQueueMutex));
 571         }
 572
 573         if (eventObj->freeEventMutex != gcvNULL)
 574         {
 575             gcmkVERIFY_OK(gckOS_DeleteMutex(os, eventObj->freeEventMutex));
 576         }
 577
 578         if (eventObj->eventListMutex != gcvNULL)
 579         {
 580             gcmkVERIFY_OK(gckOS_DeleteMutex(os, eventObj->eventListMutex));
 581         }
 582
 583         while (eventObj->freeEventList != gcvNULL)
 584         {
 585             record = eventObj->freeEventList;
 586             eventObj->freeEventList = record->next;
 587
 588             gcmkVERIFY_OK(gcmkOS_SAFE_FREE(os, record));
 589         }
 590
 591         if (eventObj->freeAtom != gcvNULL)
 592         {
 593             gcmkVERIFY_OK(gckOS_AtomDestroy(os, eventObj->freeAtom));
 594         }
 595
 596 #if gcdSMP
 597         if (eventObj->pending != gcvNULL)
 598         {
 599             gcmkVERIFY_OK(gckOS_AtomDestroy(os, eventObj->pending));
 600         }
 601
 602 #if gcdMULTI_GPU
 603         for (i = 0; i < gcdMULTI_GPU; i++)
 604         {
 605             if (eventObj->pending3D[i] != gcvNULL)
 606             {
 607                 gcmkVERIFY_OK(gckOS_AtomDestroy(os, eventObj->pending3D[i]));
 608             }
 609
 610             if (eventObj->pending3DMask[i] != gcvNULL)
 611             {
 612                 gcmkVERIFY_OK(gckOS_AtomDestroy(os, eventObj->pending3DMask[i]));
 613             }
 614         }
 615 #endif
 616 #endif
 617
 618 #if gcdINTERRUPT_STATISTIC
 619         if (eventObj->interruptCount)
 620         {
 621             gcmkVERIFY_OK(gckOS_AtomDestroy(os, eventObj->interruptCount));
 622         }
 623 #endif
 624         gcmkVERIFY_OK(gcmkOS_SAFE_FREE(os, eventObj));
 625     }
 626
 627     /* Return the status. */
 628     gcmkFOOTER();
 629     return status;
 630 }
 631
 632 /*******************************************************************************
 633 **
 634 **  gckEVENT_Destroy
 635 **
 636 **  Destroy an gckEVENT object.
 637 **
 638 **  INPUT:
 639 **
 640 **      gckEVENT Event
 641 **          Pointer to an gckEVENT object.
 642 **
 643 **  OUTPUT:
 644 **
 645 **      Nothing.
 646 */
 647 gceSTATUS
 648 gckEVENT_Destroy(
 649     IN gckEVENT Event
 650     )
 651 {
 652     gcsEVENT_PTR record;
 653     gcsEVENT_QUEUE_PTR queue;
 654
 655     gcmkHEADER_ARG("Event=0x%x", Event);
 656
 657     /* Verify the arguments. */
 658     gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT);
 659
 660     if (Event->submitTimer != gcvNULL)
 661     {
 662         gcmkVERIFY_OK(gckOS_StopTimer(Event->os, Event->submitTimer));
 663         gcmkVERIFY_OK(gckOS_DestroyTimer(Event->os, Event->submitTimer));
 664     }
 665
 666     /* Delete the queue mutex. */
 667     gcmkVERIFY_OK(gckOS_DeleteMutex(Event->os, Event->eventQueueMutex));
 668
 669     /* Free all free events. */
 670     while (Event->freeEventList != gcvNULL)
 671     {
 672         record = Event->freeEventList;
 673         Event->freeEventList = record->next;
 674
 675         gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Event->os, record));
 676     }
 677
 678     /* Delete the free mutex. */
 679     gcmkVERIFY_OK(gckOS_DeleteMutex(Event->os, Event->freeEventMutex));
 680
 681     /* Free all pending queues. */
 682     while (Event->queueHead != gcvNULL)
 683     {
 684         /* Get the current queue. */
 685         queue = Event->queueHead;
 686
 687         /* Free all pending events. */
 688         while (queue->head != gcvNULL)
 689         {
 690             record      = queue->head;
 691             queue->head = record->next;
 692
 693             gcmkTRACE_ZONE_N(
 694                 gcvLEVEL_WARNING, gcvZONE_EVENT,
 695                 gcmSIZEOF(record) + gcmSIZEOF(queue->source),
 696                 "Event record 0x%x is still pending for %d.",
 697                 record, queue->source
 698                 );
 699
 700             gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Event->os, record));
 701         }
 702
 703         /* Remove the top queue from the list. */
 704         if (Event->queueHead == Event->queueTail)
 705         {
 706             Event->queueHead =
 707             Event->queueTail = gcvNULL;
 708         }
 709         else
 710         {
 711             Event->queueHead = Event->queueHead->next;
 712         }
 713
 714         /* Free the queue. */
 715         gcmkVERIFY_OK(gckEVENT_FreeQueue(Event, queue));
 716     }
 717
 718     /* Delete the list mutex. */
 719     gcmkVERIFY_OK(gckOS_DeleteMutex(Event->os, Event->eventListMutex));
 720
 721     /* Delete the atom. */
 722     gcmkVERIFY_OK(gckOS_AtomDestroy(Event->os, Event->freeAtom));
 723
 724 #if gcdSMP
 725     gcmkVERIFY_OK(gckOS_AtomDestroy(Event->os, Event->pending));
 726
 727 #if gcdMULTI_GPU
 728     {
 729         gctINT i;
 730         for (i = 0; i < gcdMULTI_GPU; i++)
 731         {
 732             gcmkVERIFY_OK(gckOS_AtomDestroy(Event->os, Event->pending3D[i]));
 733             gcmkVERIFY_OK(gckOS_AtomDestroy(Event->os, Event->pending3DMask[i]));
 734         }
 735     }
 736 #endif
 737 #endif
 738
 739 #if gcdINTERRUPT_STATISTIC
 740     gcmkVERIFY_OK(gckOS_AtomDestroy(Event->os, Event->interruptCount));
 741 #endif
 742
 743     /* Mark the gckEVENT object as unknown. */
 744     Event->object.type = gcvOBJ_UNKNOWN;
 745
 746     /* Free the gckEVENT object. */
 747     gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Event->os, Event));
 748
 749     /* Success. */
 750     gcmkFOOTER_NO();
 751     return gcvSTATUS_OK;
 752 }
 753
 754 /*******************************************************************************
 755 **
 756 **  gckEVENT_GetEvent
 757 **
 758 **  Reserve the next available hardware event.
 759 **
 760 **  INPUT:
 761 **
 762 **      gckEVENT Event
 763 **          Pointer to an gckEVENT object.
 764 **
 765 **      gctBOOL Wait
 766 **          Set to gcvTRUE to force the function to wait if no events are
 767 **          immediately available.
 768 **
 769 **      gceKERNEL_WHERE Source
 770 **          Source of the event.
 771 **
 772 **  OUTPUT:
 773 **
 774 **      gctUINT8 * EventID
 775 **          Reserved event ID.
 776 */
 777 #define gcdINVALID_EVENT_PTR    ((gcsEVENT_PTR)gcvMAXUINTPTR_T)
 778
 779 #if gcdMULTI_GPU
 780 gceSTATUS
 781 gckEVENT_GetEvent(
 782     IN gckEVENT Event,
 783     IN gctBOOL Wait,
 784     OUT gctUINT8 * EventID,
 785     IN gceKERNEL_WHERE Source,
 786     IN gceCORE_3D_MASK ChipEnable
 787     )
 788 #else
 789 gceSTATUS
 790 gckEVENT_GetEvent(
 791     IN gckEVENT Event,
 792     IN gctBOOL Wait,
 793     OUT gctUINT8 * EventID,
 794     IN gceKERNEL_WHERE Source
 795     )
 796 #endif
 797 {
 798     gctINT i, id;
 799     gceSTATUS status;
 800     gctBOOL acquired = gcvFALSE;
 801     gctINT32 free;
 802 #if gcdMULTI_GPU
 803     gctINT j;
 804 #endif
 805
 806     gcmkHEADER_ARG("Event=0x%x Source=%d", Event, Source);
 807
 808     while (gcvTRUE)
 809     {
 810         /* Grab the queue mutex. */
 811         gcmkONERROR(gckOS_AcquireMutex(Event->os,
 812                                        Event->eventQueueMutex,
 813                                        gcvINFINITE));
 814         acquired = gcvTRUE;
 815
 816         /* Walk through all events. */
 817         id = Event->lastID;
 818         for (i = 0; i < gcmCOUNTOF(Event->queues); ++i)
 819         {
 820             gctINT nextID = gckMATH_ModuloInt((id + 1),
 821                                               gcmCOUNTOF(Event->queues));
 822
 823             if (Event->queues[id].head == gcvNULL)
 824             {
 825                 *EventID = (gctUINT8) id;
 826
 827                 Event->lastID = (gctUINT8) nextID;
 828
 829                 /* Save time stamp of event. */
 830                 Event->queues[id].head   = gcdINVALID_EVENT_PTR;
 831                 Event->queues[id].stamp  = ++(Event->stamp);
 832                 Event->queues[id].source = Source;
 833
 834 #if gcdMULTI_GPU
 835                 Event->queues[id].chipEnable = ChipEnable;
 836
 837                 if (ChipEnable == gcvCORE_3D_ALL_MASK)
 838                 {
 839                     gckOS_AtomSetMask(Event->pendingMask, (1 << id));
 840
 841                     for (j = 0; j < gcdMULTI_GPU; j++)
 842                     {
 843                         gckOS_AtomSetMask(Event->pending3DMask[j], (1 << id));
 844                     }
 845                 }
 846                 else
 847                 {
 848                     for (j = 0; j < gcdMULTI_GPU; j++)
 849                     {
 850                         if (ChipEnable & (1 << j))
 851                         {
 852                             gckOS_AtomSetMask(Event->pending3DMask[j], (1 << id));
 853                         }
 854                     }
 855                 }
 856 #endif
 857
 858                 gcmkONERROR(gckOS_AtomDecrement(Event->os,
 859                                                 Event->freeAtom,
 860                                                 &free));
 861 #if gcdDYNAMIC_SPEED
 862                 if (free <= gcdDYNAMIC_EVENT_THRESHOLD)
 863                 {
 864                     gcmkONERROR(gckOS_BroadcastHurry(
 865                         Event->os,
 866                         Event->kernel->hardware,
 867                         gcdDYNAMIC_EVENT_THRESHOLD - free));
 868                 }
 869 #endif
 870
 871                 /* Release the queue mutex. */
 872                 gcmkONERROR(gckOS_ReleaseMutex(Event->os,
 873                                                Event->eventQueueMutex));
 874
 875                 /* Success. */
 876                 gcmkTRACE_ZONE_N(
 877                     gcvLEVEL_INFO, gcvZONE_EVENT,
 878                     gcmSIZEOF(id),
 879                     "Using id=%d",
 880                     id
 881                     );
 882
 883                 gcmkFOOTER_ARG("*EventID=%u", *EventID);
 884                 return gcvSTATUS_OK;
 885             }
 886
 887             id = nextID;
 888         }
 889
 890 #if gcdDYNAMIC_SPEED
 891         /* No free events, speed up the GPU right now! */
 892         gcmkONERROR(gckOS_BroadcastHurry(Event->os,
 893                                          Event->kernel->hardware,
 894                                          gcdDYNAMIC_EVENT_THRESHOLD));
 895 #endif
 896
 897         /* Release the queue mutex. */
 898         gcmkONERROR(gckOS_ReleaseMutex(Event->os, Event->eventQueueMutex));
 899         acquired = gcvFALSE;
 900
 901         /* Fail if wait is not requested. */
 902         if (!Wait)
 903         {
 904             /* Out of resources. */
 905             gcmkONERROR(gcvSTATUS_OUT_OF_RESOURCES);
 906         }
 907
 908         /* Delay a while. */
 909         gcmkONERROR(gckOS_Delay(Event->os, 1));
 910     }
 911
 912 OnError:
 913     if (acquired)
 914     {
 915         /* Release the queue mutex. */
 916         gcmkVERIFY_OK(gckOS_ReleaseMutex(Event->os, Event->eventQueueMutex));
 917     }
 918
 919     /* Return the status. */
 920     gcmkFOOTER();
 921     return status;
 922 }
 923
 924 /*******************************************************************************
 925 **
 926 **  gckEVENT_AllocateRecord
 927 **
 928 **  Allocate a record for the new event.
 929 **
 930 **  INPUT:
 931 **
 932 **      gckEVENT Event
 933 **          Pointer to an gckEVENT object.
 934 **
 935 **      gctBOOL AllocateAllowed
 936 **          State for allocation if out of free events.
 937 **
 938 **  OUTPUT:
 939 **
 940 **      gcsEVENT_PTR * Record
 941 **          Allocated event record.
 942 */
 943 gceSTATUS
 944 gckEVENT_AllocateRecord(
 945     IN gckEVENT Event,
 946     IN gctBOOL AllocateAllowed,
 947     OUT gcsEVENT_PTR * Record
 948     )
 949 {
 950     gceSTATUS status;
 951     gctBOOL acquired = gcvFALSE;
 952     gctINT i;
 953     gcsEVENT_PTR record;
 954     gctPOINTER pointer = gcvNULL;
 955
 956     gcmkHEADER_ARG("Event=0x%x AllocateAllowed=%d", Event, AllocateAllowed);
 957
 958     /* Verify the arguments. */
 959     gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT);
 960     gcmkVERIFY_ARGUMENT(Record != gcvNULL);
 961
 962     /* Acquire the mutex. */
 963     gcmkONERROR(gckOS_AcquireMutex(Event->os, Event->freeEventMutex, gcvINFINITE));
 964     acquired = gcvTRUE;
 965
 966     /* Test if we are below the allocation threshold. */
 967     if ( (AllocateAllowed && (Event->freeEventCount < gcdEVENT_MIN_THRESHOLD)) ||
 968          (Event->freeEventCount == 0) )
 969     {
 970         /* Allocate a bunch of records. */
 971         for (i = 0; i < gcdEVENT_ALLOCATION_COUNT; i += 1)
 972         {
 973             /* Allocate an event record. */
 974             gcmkONERROR(gckOS_Allocate(Event->os,
 975                                        gcmSIZEOF(gcsEVENT),
 976                                        &pointer));
 977
 978             record = pointer;
 979
 980             /* Push it on the free list. */
 981             record->next           = Event->freeEventList;
 982             Event->freeEventList   = record;
 983             Event->freeEventCount += 1;
 984         }
 985     }
 986
 987     *Record                = Event->freeEventList;
 988     Event->freeEventList   = Event->freeEventList->next;
 989     Event->freeEventCount -= 1;
 990
 991     /* Release the mutex. */
 992     gcmkONERROR(gckOS_ReleaseMutex(Event->os, Event->freeEventMutex));
 993
 994     /* Success. */
 995     gcmkFOOTER_ARG("*Record=0x%x", gcmOPT_POINTER(Record));
 996     return gcvSTATUS_OK;
 997
 998 OnError:
 999     /* Roll back. */
1000     if (acquired)
1001     {
1002         gcmkVERIFY_OK(gckOS_ReleaseMutex(Event->os, Event->freeEventMutex));
1003     }
1004
1005     /* Return the status. */
1006     gcmkFOOTER();
1007     return status;
1008 }
1009
1010 /*******************************************************************************
1011 **
1012 **  gckEVENT_AddList
1013 **
1014 **  Add a new event to the list of events.
1015 **
1016 **  INPUT:
1017 **
1018 **      gckEVENT Event
1019 **          Pointer to an gckEVENT object.
1020 **
1021 **      gcsHAL_INTERFACE_PTR Interface
1022 **          Pointer to the interface for the event to be added.
1023 **
1024 **      gceKERNEL_WHERE FromWhere
1025 **          Place in the pipe where the event needs to be generated.
1026 **
1027 **      gctBOOL AllocateAllowed
1028 **          State for allocation if out of free events.
1029 **
1030 **  OUTPUT:
1031 **
1032 **      Nothing.
1033 */
1034 gceSTATUS
1035 gckEVENT_AddList(
1036     IN gckEVENT Event,
1037     IN gcsHAL_INTERFACE_PTR Interface,
1038     IN gceKERNEL_WHERE FromWhere,
1039     IN gctBOOL AllocateAllowed,
1040     IN gctBOOL FromKernel
1041     )
1042 {
1043     gceSTATUS status;
1044     gctBOOL acquired = gcvFALSE;
1045     gcsEVENT_PTR record = gcvNULL;
1046     gcsEVENT_QUEUE_PTR queue;
1047     gckVIRTUAL_COMMAND_BUFFER_PTR buffer;
1048     gckKERNEL kernel = Event->kernel;
1049
1050     gcmkHEADER_ARG("Event=0x%x Interface=0x%x",
1051                    Event, Interface);
1052
1053     gcmkTRACE_ZONE(gcvLEVEL_VERBOSE, _GC_OBJ_ZONE,
1054                     "FromWhere=%d AllocateAllowed=%d",
1055                     FromWhere, AllocateAllowed);
1056
1057     /* Verify the arguments. */
1058     gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT);
1059     gcmkVERIFY_ARGUMENT(Interface != gcvNULL);
1060
1061     /* Verify the event command. */
1062     gcmkASSERT
1063         (  (Interface->command == gcvHAL_FREE_NON_PAGED_MEMORY)
1064         || (Interface->command == gcvHAL_FREE_CONTIGUOUS_MEMORY)
1065         || (Interface->command == gcvHAL_WRITE_DATA)
1066         || (Interface->command == gcvHAL_UNLOCK_VIDEO_MEMORY)
1067         || (Interface->command == gcvHAL_SIGNAL)
1068         || (Interface->command == gcvHAL_UNMAP_USER_MEMORY)
1069         || (Interface->command == gcvHAL_TIMESTAMP)
1070         || (Interface->command == gcvHAL_COMMIT_DONE)
1071         || (Interface->command == gcvHAL_FREE_VIRTUAL_COMMAND_BUFFER)
1072         || (Interface->command == gcvHAL_SYNC_POINT)
1073         || (Interface->command == gcvHAL_DESTROY_MMU)
1074         );
1075
1076     /* Validate the source. */
1077     if ((FromWhere != gcvKERNEL_COMMAND) && (FromWhere != gcvKERNEL_PIXEL))
1078     {
1079         /* Invalid argument. */
1080         gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT);
1081     }
1082
1083     /* Allocate a free record. */
1084     gcmkONERROR(gckEVENT_AllocateRecord(Event, AllocateAllowed, &record));
1085
1086     /* Termninate the record. */
1087     record->next = gcvNULL;
1088
1089     /* Record the committer. */
1090     record->fromKernel = FromKernel;
1091
1092     /* Copy the event interface into the record. */
1093     gckOS_MemCopy(&record->info, Interface, gcmSIZEOF(record->info));
1094
1095     /* Get process ID. */
1096     gcmkONERROR(gckOS_GetProcessID(&record->processID));
1097
1098     gcmkONERROR(__RemoveRecordFromProcessDB(Event, record));
1099
1100     /* Handle is belonged to current process, it must be released now. */
1101     if (FromKernel == gcvFALSE)
1102     {
1103         status = _ReleaseVideoMemoryHandle(Event->kernel, record, Interface);
1104
1105         if (gcmIS_ERROR(status))
1106         {
1107             /* Ingore error because there are other events in the queue. */
1108             status = gcvSTATUS_OK;
1109             goto OnError;
1110         }
1111     }
1112
1113 #ifdef __QNXNTO__
1114     record->kernel = Event->kernel;
1115 #endif
1116
1117     /* Acquire the mutex. */
1118     gcmkONERROR(gckOS_AcquireMutex(Event->os, Event->eventListMutex, gcvINFINITE));
1119     acquired = gcvTRUE;
1120
1121     /* Do we need to allocate a new queue? */
1122     if ((Event->queueTail == gcvNULL) || (Event->queueTail->source < FromWhere))
1123     {
1124         /* Allocate a new queue. */
1125         gcmkONERROR(gckEVENT_AllocateQueue(Event, &queue));
1126
1127         /* Initialize the queue. */
1128         queue->source = FromWhere;
1129         queue->head   = gcvNULL;
1130         queue->next   = gcvNULL;
1131
1132         /* Attach it to the list of allocated queues. */
1133         if (Event->queueTail == gcvNULL)
1134         {
1135             Event->queueHead =
1136             Event->queueTail = queue;
1137         }
1138         else
1139         {
1140             Event->queueTail->next = queue;
1141             Event->queueTail       = queue;
1142         }
1143     }
1144     else
1145     {
1146         queue = Event->queueTail;
1147     }
1148
1149     /* Attach the record to the queue. */
1150     if (queue->head == gcvNULL)
1151     {
1152         queue->head = record;
1153         queue->tail = record;
1154     }
1155     else
1156     {
1157         queue->tail->next = record;
1158         queue->tail       = record;
1159     }
1160
1161     /* Unmap user space logical address.
1162      * Linux kernel does not support unmap the memory of other process any more since 3.5.
1163      * Let's unmap memory of self process before submit the event to gpu.
1164      * */
1165     switch(Interface->command)
1166     {
1167     case gcvHAL_FREE_NON_PAGED_MEMORY:
1168         gcmkONERROR(gckOS_UnmapUserLogical(
1169                         Event->os,
1170                         gcmNAME_TO_PTR(Interface->u.FreeNonPagedMemory.physical),
1171                         (gctSIZE_T) Interface->u.FreeNonPagedMemory.bytes,
1172                         gcmUINT64_TO_PTR(Interface->u.FreeNonPagedMemory.logical)));
1173         break;
1174     case gcvHAL_FREE_CONTIGUOUS_MEMORY:
1175         gcmkONERROR(gckOS_UnmapUserLogical(
1176                         Event->os,
1177                         gcmNAME_TO_PTR(Interface->u.FreeContiguousMemory.physical),
1178                         (gctSIZE_T) Interface->u.FreeContiguousMemory.bytes,
1179                         gcmUINT64_TO_PTR(Interface->u.FreeContiguousMemory.logical)));
1180         break;
1181
1182     case gcvHAL_FREE_VIRTUAL_COMMAND_BUFFER:
1183         buffer = (gckVIRTUAL_COMMAND_BUFFER_PTR)gcmNAME_TO_PTR(Interface->u.FreeVirtualCommandBuffer.physical);
1184         if (buffer->userLogical)
1185         {
1186             gcmkONERROR(gckOS_DestroyUserVirtualMapping(
1187                             Event->os,
1188                             buffer->physical,
1189                             (gctSIZE_T) Interface->u.FreeVirtualCommandBuffer.bytes,
1190                             gcmUINT64_TO_PTR(Interface->u.FreeVirtualCommandBuffer.logical)));
1191         }
1192         break;
1193
1194     default:
1195         break;
1196     }
1197
1198     /* Release the mutex. */
1199     gcmkONERROR(gckOS_ReleaseMutex(Event->os, Event->eventListMutex));
1200
1201     /* Success. */
1202     gcmkFOOTER_NO();
1203     return gcvSTATUS_OK;
1204
1205 OnError:
1206     /* Roll back. */
1207     if (acquired)
1208     {
1209         gcmkVERIFY_OK(gckOS_ReleaseMutex(Event->os, Event->eventListMutex));
1210     }
1211
1212     if (record != gcvNULL)
1213     {
1214         gcmkVERIFY_OK(gckEVENT_FreeRecord(Event, record));
1215     }
1216
1217     /* Return the status. */
1218     gcmkFOOTER();
1219     return status;
1220 }
1221
1222 /*******************************************************************************
1223 **
1224 **  gckEVENT_Unlock
1225 **
1226 **  Schedule an event to unlock virtual memory.
1227 **
1228 **  INPUT:
1229 **
1230 **      gckEVENT Event
1231 **          Pointer to an gckEVENT object.
1232 **
1233 **      gceKERNEL_WHERE FromWhere
1234 **          Place in the pipe where the event needs to be generated.
1235 **
1236 **      gcuVIDMEM_NODE_PTR Node
1237 **          Pointer to a gcuVIDMEM_NODE union that specifies the virtual memory
1238 **          to unlock.
1239 **
1240 **      gceSURF_TYPE Type
1241 **          Type of surface to unlock.
1242 **
1243 **  OUTPUT:
1244 **
1245 **      Nothing.
1246 */
1247 gceSTATUS
1248 gckEVENT_Unlock(
1249     IN gckEVENT Event,
1250     IN gceKERNEL_WHERE FromWhere,
1251     IN gctPOINTER Node,
1252     IN gceSURF_TYPE Type
1253     )
1254 {
1255     gceSTATUS status;
1256     gcsHAL_INTERFACE iface;
1257
1258     gcmkHEADER_ARG("Event=0x%x FromWhere=%d Node=0x%x Type=%d",
1259                    Event, FromWhere, Node, Type);
1260
1261     /* Verify the arguments. */
1262     gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT);
1263     gcmkVERIFY_ARGUMENT(Node != gcvNULL);
1264
1265     /* Mark the event as an unlock. */
1266     iface.command                           = gcvHAL_UNLOCK_VIDEO_MEMORY;
1267     iface.u.UnlockVideoMemory.node          = gcmPTR_TO_UINT64(Node);
1268     iface.u.UnlockVideoMemory.type          = Type;
1269     iface.u.UnlockVideoMemory.asynchroneous = 0;
1270
1271     /* Append it to the queue. */
1272     gcmkONERROR(gckEVENT_AddList(Event, &iface, FromWhere, gcvFALSE, gcvTRUE));
1273
1274     /* Success. */
1275     gcmkFOOTER_NO();
1276     return gcvSTATUS_OK;
1277
1278 OnError:
1279     /* Return the status. */
1280     gcmkFOOTER();
1281     return status;
1282 }
1283
1284 /*******************************************************************************
1285 **
1286 **  gckEVENT_FreeNonPagedMemory
1287 **
1288 **  Schedule an event to free non-paged memory.
1289 **
1290 **  INPUT:
1291 **
1292 **      gckEVENT Event
1293 **          Pointer to an gckEVENT object.
1294 **
1295 **      gctSIZE_T Bytes
1296 **          Number of bytes of non-paged memory to free.
1297 **
1298 **      gctPHYS_ADDR Physical
1299 **          Physical address of non-paged memory to free.
1300 **
1301 **      gctPOINTER Logical
1302 **          Logical address of non-paged memory to free.
1303 **
1304 **      gceKERNEL_WHERE FromWhere
1305 **          Place in the pipe where the event needs to be generated.
1306 */
1307 gceSTATUS
1308 gckEVENT_FreeNonPagedMemory(
1309     IN gckEVENT Event,
1310     IN gctSIZE_T Bytes,
1311     IN gctPHYS_ADDR Physical,
1312     IN gctPOINTER Logical,
1313     IN gceKERNEL_WHERE FromWhere
1314     )
1315 {
1316     gceSTATUS status;
1317     gcsHAL_INTERFACE iface;
1318     gckKERNEL kernel = Event->kernel;
1319
1320     gcmkHEADER_ARG("Event=0x%x Bytes=%lu Physical=0x%x Logical=0x%x "
1321                    "FromWhere=%d",
1322                    Event, Bytes, Physical, Logical, FromWhere);
1323
1324     /* Verify the arguments. */
1325     gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT);
1326     gcmkVERIFY_ARGUMENT(Physical != gcvNULL);
1327     gcmkVERIFY_ARGUMENT(Logical != gcvNULL);
1328     gcmkVERIFY_ARGUMENT(Bytes > 0);
1329
1330     /* Create an event. */
1331     iface.command = gcvHAL_FREE_NON_PAGED_MEMORY;
1332     iface.u.FreeNonPagedMemory.bytes    = Bytes;
1333     iface.u.FreeNonPagedMemory.physical = gcmPTR_TO_NAME(Physical);
1334     iface.u.FreeNonPagedMemory.logical  = gcmPTR_TO_UINT64(Logical);
1335
1336     /* Append it to the queue. */
1337     gcmkONERROR(gckEVENT_AddList(Event, &iface, FromWhere, gcvFALSE, gcvTRUE));
1338
1339     /* Success. */
1340     gcmkFOOTER_NO();
1341     return gcvSTATUS_OK;
1342
1343 OnError:
1344     /* Return the status. */
1345     gcmkFOOTER();
1346     return status;
1347 }
1348
1349 gceSTATUS
1350 gckEVENT_DestroyVirtualCommandBuffer(
1351     IN gckEVENT Event,
1352     IN gctSIZE_T Bytes,
1353     IN gctPHYS_ADDR Physical,
1354     IN gctPOINTER Logical,
1355     IN gceKERNEL_WHERE FromWhere
1356     )
1357 {
1358     gceSTATUS status;
1359     gcsHAL_INTERFACE iface;
1360     gckKERNEL kernel = Event->kernel;
1361
1362     gcmkHEADER_ARG("Event=0x%x Bytes=%lu Physical=0x%x Logical=0x%x "
1363                    "FromWhere=%d",
1364                    Event, Bytes, Physical, Logical, FromWhere);
1365
1366     /* Verify the arguments. */
1367     gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT);
1368     gcmkVERIFY_ARGUMENT(Physical != gcvNULL);
1369     gcmkVERIFY_ARGUMENT(Logical != gcvNULL);
1370     gcmkVERIFY_ARGUMENT(Bytes > 0);
1371
1372     /* Create an event. */
1373     iface.command = gcvHAL_FREE_VIRTUAL_COMMAND_BUFFER;
1374     iface.u.FreeVirtualCommandBuffer.bytes    = Bytes;
1375     iface.u.FreeVirtualCommandBuffer.physical = gcmPTR_TO_NAME(Physical);
1376     iface.u.FreeVirtualCommandBuffer.logical  = gcmPTR_TO_UINT64(Logical);
1377
1378     /* Append it to the queue. */
1379     gcmkONERROR(gckEVENT_AddList(Event, &iface, FromWhere, gcvFALSE, gcvTRUE));
1380
1381     /* Success. */
1382     gcmkFOOTER_NO();
1383     return gcvSTATUS_OK;
1384
1385 OnError:
1386     /* Return the status. */
1387     gcmkFOOTER();
1388     return status;
1389 }
1390
1391 /*******************************************************************************
1392 **
1393 **  gckEVENT_FreeContigiuousMemory
1394 **
1395 **  Schedule an event to free contiguous memory.
1396 **
1397 **  INPUT:
1398 **
1399 **      gckEVENT Event
1400 **          Pointer to an gckEVENT object.
1401 **
1402 **      gctSIZE_T Bytes
1403 **          Number of bytes of contiguous memory to free.
1404 **
1405 **      gctPHYS_ADDR Physical
1406 **          Physical address of contiguous memory to free.
1407 **
1408 **      gctPOINTER Logical
1409 **          Logical address of contiguous memory to free.
1410 **
1411 **      gceKERNEL_WHERE FromWhere
1412 **          Place in the pipe where the event needs to be generated.
1413 */
1414 gceSTATUS
1415 gckEVENT_FreeContiguousMemory(
1416     IN gckEVENT Event,
1417     IN gctSIZE_T Bytes,
1418     IN gctPHYS_ADDR Physical,
1419     IN gctPOINTER Logical,
1420     IN gceKERNEL_WHERE FromWhere
1421     )
1422 {
1423     gceSTATUS status;
1424     gcsHAL_INTERFACE iface;
1425     gckKERNEL kernel = Event->kernel;
1426
1427     gcmkHEADER_ARG("Event=0x%x Bytes=%lu Physical=0x%x Logical=0x%x "
1428                    "FromWhere=%d",
1429                    Event, Bytes, Physical, Logical, FromWhere);
1430
1431     /* Verify the arguments. */
1432     gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT);
1433     gcmkVERIFY_ARGUMENT(Physical != gcvNULL);
1434     gcmkVERIFY_ARGUMENT(Logical != gcvNULL);
1435     gcmkVERIFY_ARGUMENT(Bytes > 0);
1436
1437     /* Create an event. */
1438     iface.command = gcvHAL_FREE_CONTIGUOUS_MEMORY;
1439     iface.u.FreeContiguousMemory.bytes    = Bytes;
1440     iface.u.FreeContiguousMemory.physical = gcmPTR_TO_NAME(Physical);
1441     iface.u.FreeContiguousMemory.logical  = gcmPTR_TO_UINT64(Logical);
1442
1443     /* Append it to the queue. */
1444     gcmkONERROR(gckEVENT_AddList(Event, &iface, FromWhere, gcvFALSE, gcvTRUE));
1445
1446     /* Success. */
1447     gcmkFOOTER_NO();
1448     return gcvSTATUS_OK;
1449
1450 OnError:
1451     /* Return the status. */
1452     gcmkFOOTER();
1453     return status;
1454 }
1455
1456 /*******************************************************************************
1457 **
1458 **  gckEVENT_Signal
1459 **
1460 **  Schedule an event to trigger a signal.
1461 **
1462 **  INPUT:
1463 **
1464 **      gckEVENT Event
1465 **          Pointer to an gckEVENT object.
1466 **
1467 **      gctSIGNAL Signal
1468 **          Pointer to the signal to trigger.
1469 **
1470 **      gceKERNEL_WHERE FromWhere
1471 **          Place in the pipe where the event needs to be generated.
1472 **
1473 **  OUTPUT:
1474 **
1475 **      Nothing.
1476 */
1477 gceSTATUS
1478 gckEVENT_Signal(
1479     IN gckEVENT Event,
1480     IN gctSIGNAL Signal,
1481     IN gceKERNEL_WHERE FromWhere
1482     )
1483 {
1484     gceSTATUS status;
1485     gcsHAL_INTERFACE iface;
1486
1487     gcmkHEADER_ARG("Event=0x%x Signal=0x%x FromWhere=%d",
1488                    Event, Signal, FromWhere);
1489
1490     /* Verify the arguments. */
1491     gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT);
1492     gcmkVERIFY_ARGUMENT(Signal != gcvNULL);
1493
1494     /* Mark the event as a signal. */
1495     iface.command            = gcvHAL_SIGNAL;
1496     iface.u.Signal.signal    = gcmPTR_TO_UINT64(Signal);
1497 #ifdef __QNXNTO__
1498     iface.u.Signal.coid      = 0;
1499     iface.u.Signal.rcvid     = 0;
1500 #endif
1501     iface.u.Signal.auxSignal = 0;
1502     iface.u.Signal.process   = 0;
1503
1504     /* Append it to the queue. */
1505     gcmkONERROR(gckEVENT_AddList(Event, &iface, FromWhere, gcvFALSE, gcvTRUE));
1506
1507     /* Success. */
1508     gcmkFOOTER_NO();
1509     return gcvSTATUS_OK;
1510
1511 OnError:
1512     /* Return the status. */
1513     gcmkFOOTER();
1514     return status;
1515 }
1516
1517 /*******************************************************************************
1518 **
1519 **  gckEVENT_CommitDone
1520 **
1521 **  Schedule an event to wake up work thread when commit is done by GPU.
1522 **
1523 **  INPUT:
1524 **
1525 **      gckEVENT Event
1526 **          Pointer to an gckEVENT object.
1527 **
1528 **      gceKERNEL_WHERE FromWhere
1529 **          Place in the pipe where the event needs to be generated.
1530 **
1531 **  OUTPUT:
1532 **
1533 **      Nothing.
1534 */
1535 gceSTATUS
1536 gckEVENT_CommitDone(
1537     IN gckEVENT Event,
1538     IN gceKERNEL_WHERE FromWhere
1539     )
1540 {
1541     gceSTATUS status;
1542     gcsHAL_INTERFACE iface;
1543
1544     gcmkHEADER_ARG("Event=0x%x FromWhere=%d", Event, FromWhere);
1545
1546     /* Verify the arguments. */
1547     gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT);
1548
1549     iface.command = gcvHAL_COMMIT_DONE;
1550
1551     /* Append it to the queue. */
1552     gcmkONERROR(gckEVENT_AddList(Event, &iface, FromWhere, gcvFALSE, gcvTRUE));
1553
1554     /* Success. */
1555     gcmkFOOTER_NO();
1556     return gcvSTATUS_OK;
1557
1558 OnError:
1559     /* Return the status. */
1560     gcmkFOOTER();
1561     return status;
1562 }
1563
1564 #if gcdPROCESS_ADDRESS_SPACE
1565 gceSTATUS
1566 gckEVENT_DestroyMmu(
1567     IN gckEVENT Event,
1568     IN gckMMU Mmu,
1569     IN gceKERNEL_WHERE FromWhere
1570     )
1571 {
1572     gceSTATUS status;
1573     gcsHAL_INTERFACE iface;
1574
1575     gcmkHEADER_ARG("Event=0x%x FromWhere=%d", Event, FromWhere);
1576
1577     /* Verify the arguments. */
1578     gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT);
1579
1580     iface.command = gcvHAL_DESTROY_MMU;
1581     iface.u.DestroyMmu.mmu = gcmPTR_TO_UINT64(Mmu);
1582
1583     /* Append it to the queue. */
1584     gcmkONERROR(gckEVENT_AddList(Event, &iface, FromWhere, gcvFALSE, gcvTRUE));
1585
1586     /* Success. */
1587     gcmkFOOTER_NO();
1588     return gcvSTATUS_OK;
1589
1590 OnError:
1591     /* Return the status. */
1592     gcmkFOOTER();
1593     return status;
1594 }
1595 #endif
1596
1597 /*******************************************************************************
1598 **
1599 **  gckEVENT_Submit
1600 **
1601 **  Submit the current event queue to the GPU.
1602 **
1603 **  INPUT:
1604 **
1605 **      gckEVENT Event
1606 **          Pointer to an gckEVENT object.
1607 **
1608 **      gctBOOL Wait
1609 **          Submit requires one vacant event; if Wait is set to not zero,
1610 **          and there are no vacant events at this time, the function will
1611 **          wait until an event becomes vacant so that submission of the
1612 **          queue is successful.
1613 **
1614 **      gctBOOL FromPower
1615 **          Determines whether the call originates from inside the power
1616 **          management or not.
1617 **
1618 **  OUTPUT:
1619 **
1620 **      Nothing.
1621 */
1622 #if gcdMULTI_GPU
1623 gceSTATUS
1624 gckEVENT_Submit(
1625     IN gckEVENT Event,
1626     IN gctBOOL Wait,
1627     IN gctBOOL FromPower,
1628     IN gceCORE_3D_MASK ChipEnable
1629     )
1630 #else
1631 gceSTATUS
1632 gckEVENT_Submit(
1633     IN gckEVENT Event,
1634     IN gctBOOL Wait,
1635     IN gctBOOL FromPower
1636     )
1637 #endif
1638 {
1639     gceSTATUS status;
1640     gctUINT8 id = 0xFF;
1641     gcsEVENT_QUEUE_PTR queue;
1642     gctBOOL acquired = gcvFALSE;
1643     gckCOMMAND command = gcvNULL;
1644     gctBOOL commitEntered = gcvFALSE;
1645 #if !gcdNULL_DRIVER
1646     gctUINT32 bytes;
1647     gctPOINTER buffer;
1648 #endif
1649
1650 #if gcdMULTI_GPU
1651     gctSIZE_T chipEnableBytes;
1652 #endif
1653
1654 #if gcdINTERRUPT_STATISTIC
1655     gctINT32 oldValue;
1656 #endif
1657
1658 #if gcdSECURITY
1659     gctPOINTER reservedBuffer;
1660 #endif
1661
1662     gctUINT32 flushBytes;
1663     gctUINT32 executeBytes;
1664     gckHARDWARE hardware;
1665
1666     gceKERNEL_FLUSH flush = gcvFALSE;
1667
1668     gcmkHEADER_ARG("Event=0x%x Wait=%d", Event, Wait);
1669
1670     /* Get gckCOMMAND object. */
1671     command = Event->kernel->command;
1672     hardware = Event->kernel->hardware;
1673
1674     gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE);
1675
1676     gckOS_GetTicks(&Event->lastCommitStamp);
1677
1678     /* Are there event queues? */
1679     if (Event->queueHead != gcvNULL)
1680     {
1681         /* Acquire the command queue. */
1682         gcmkONERROR(gckCOMMAND_EnterCommit(command, FromPower));
1683         commitEntered = gcvTRUE;
1684
1685         /* Process all queues. */
1686         while (Event->queueHead != gcvNULL)
1687         {
1688             /* Acquire the list mutex. */
1689             gcmkONERROR(gckOS_AcquireMutex(Event->os,
1690                                            Event->eventListMutex,
1691                                            gcvINFINITE));
1692             acquired = gcvTRUE;
1693
1694             /* Get the current queue. */
1695             queue = Event->queueHead;
1696
1697             /* Allocate an event ID. */
1698 #if gcdMULTI_GPU
1699             gcmkONERROR(gckEVENT_GetEvent(Event, Wait, &id, queue->source, ChipEnable));
1700 #else
1701             gcmkONERROR(gckEVENT_GetEvent(Event, Wait, &id, queue->source));
1702 #endif
1703
1704             /* Copy event list to event ID queue. */
1705             Event->queues[id].head   = queue->head;
1706
1707             /* Remove the top queue from the list. */
1708             if (Event->queueHead == Event->queueTail)
1709             {
1710                 Event->queueHead = gcvNULL;
1711                 Event->queueTail = gcvNULL;
1712             }
1713             else
1714             {
1715                 Event->queueHead = Event->queueHead->next;
1716             }
1717
1718             /* Free the queue. */
1719             gcmkONERROR(gckEVENT_FreeQueue(Event, queue));
1720
1721             /* Release the list mutex. */
1722             gcmkONERROR(gckOS_ReleaseMutex(Event->os, Event->eventListMutex));
1723             acquired = gcvFALSE;
1724
1725             /* Determine cache needed to flush. */
1726             gcmkVERIFY_OK(_QueryFlush(Event, Event->queues[id].head, &flush));
1727
1728 #if gcdINTERRUPT_STATISTIC
1729             gcmkVERIFY_OK(gckOS_AtomIncrement(
1730                 Event->os,
1731                 Event->interruptCount,
1732                 &oldValue
1733                 ));
1734 #endif
1735
1736 #if gcdNULL_DRIVER
1737             /* Notify immediately on infinite hardware. */
1738             gcmkONERROR(gckEVENT_Interrupt(Event, 1 << id));
1739
1740             gcmkONERROR(gckEVENT_Notify(Event, 0));
1741 #else
1742             /* Get the size of the hardware event. */
1743             gcmkONERROR(gckHARDWARE_Event(
1744                 hardware,
1745                 gcvNULL,
1746                 id,
1747                 Event->queues[id].source,
1748                 &bytes
1749                 ));
1750
1751             /* Get the size of flush command. */
1752             gcmkONERROR(gckHARDWARE_Flush(
1753                 hardware,
1754                 flush,
1755                 gcvNULL,
1756                 &flushBytes
1757                 ));
1758
1759             bytes += flushBytes;
1760
1761 #if gcdMULTI_GPU
1762             gcmkONERROR(gckHARDWARE_ChipEnable(
1763                 hardware,
1764                 gcvNULL,
1765                 0,
1766                 &chipEnableBytes
1767                 ));
1768
1769             bytes += chipEnableBytes * 2;
1770 #endif
1771
1772             /* Total bytes need to execute. */
1773             executeBytes = bytes;
1774
1775             /* Reserve space in the command queue. */
1776             gcmkONERROR(gckCOMMAND_Reserve(command, bytes, &buffer, &bytes));
1777 #if gcdSECURITY
1778             reservedBuffer = buffer;
1779 #endif
1780
1781 #if gcdMULTI_GPU
1782             gcmkONERROR(gckHARDWARE_ChipEnable(
1783                 hardware,
1784                 buffer,
1785                 ChipEnable,
1786                 &chipEnableBytes
1787                 ));
1788
1789             buffer = (gctUINT8_PTR)buffer + chipEnableBytes;
1790 #endif
1791
1792             /* Set the flush in the command queue. */
1793             gcmkONERROR(gckHARDWARE_Flush(
1794                 hardware,
1795                 flush,
1796                 buffer,
1797                 &flushBytes
1798                 ));
1799
1800             /* Advance to next command. */
1801             buffer = (gctUINT8_PTR)buffer + flushBytes;
1802
1803             /* Set the hardware event in the command queue. */
1804             gcmkONERROR(gckHARDWARE_Event(
1805                 hardware,
1806                 buffer,
1807                 id,
1808                 Event->queues[id].source,
1809                 &bytes
1810                 ));
1811
1812             /* Advance to next command. */
1813             buffer = (gctUINT8_PTR)buffer + bytes;
1814
1815 #if gcdMULTI_GPU
1816             gcmkONERROR(gckHARDWARE_ChipEnable(
1817                 hardware,
1818                 buffer,
1819                 gcvCORE_3D_ALL_MASK,
1820                 &chipEnableBytes
1821                 ));
1822 #endif
1823
1824 #if gcdSECURITY
1825             gckKERNEL_SecurityExecute(
1826                 Event->kernel,
1827                 reservedBuffer,
1828                 executeBytes
1829                 );
1830 #else
1831             /* Execute the hardware event. */
1832             gcmkONERROR(gckCOMMAND_Execute(command, executeBytes));
1833 #endif
1834 #endif
1835         }
1836
1837         /* Release the command queue. */
1838         gcmkONERROR(gckCOMMAND_ExitCommit(command, FromPower));
1839
1840 #if !gcdNULL_DRIVER
1841         gcmkVERIFY_OK(_TryToIdleGPU(Event));
1842 #endif
1843     }
1844
1845     /* Success. */
1846     gcmkFOOTER_NO();
1847     return gcvSTATUS_OK;
1848
1849 OnError:
1850     if (acquired)
1851     {
1852         /* Need to unroll the mutex acquire. */
1853         gcmkVERIFY_OK(gckOS_ReleaseMutex(Event->os, Event->eventListMutex));
1854     }
1855
1856     if (commitEntered)
1857     {
1858         /* Release the command queue mutex. */
1859         gcmkVERIFY_OK(gckCOMMAND_ExitCommit(command, FromPower));
1860     }
1861
1862     if (id != 0xFF)
1863     {
1864         /* Need to unroll the event allocation. */
1865         Event->queues[id].head = gcvNULL;
1866     }
1867
1868     if (status == gcvSTATUS_GPU_NOT_RESPONDING)
1869     {
1870         /* Broadcast GPU stuck. */
1871         status = gckOS_Broadcast(Event->os,
1872                                  Event->kernel->hardware,
1873                                  gcvBROADCAST_GPU_STUCK);
1874     }
1875
1876     /* Return the status. */
1877     gcmkFOOTER();
1878     return status;
1879 }
1880
1881 /*******************************************************************************
1882 **
1883 **  gckEVENT_Commit
1884 **
1885 **  Commit an event queue from the user.
1886 **
1887 **  INPUT:
1888 **
1889 **      gckEVENT Event
1890 **          Pointer to an gckEVENT object.
1891 **
1892 **      gcsQUEUE_PTR Queue
1893 **          User event queue.
1894 **
1895 **  OUTPUT:
1896 **
1897 **      Nothing.
1898 */
1899 #if gcdMULTI_GPU
1900 gceSTATUS
1901 gckEVENT_Commit(
1902     IN gckEVENT Event,
1903     IN gcsQUEUE_PTR Queue,
1904     IN gceCORE_3D_MASK ChipEnable
1905     )
1906 #else
1907 gceSTATUS
1908 gckEVENT_Commit(
1909     IN gckEVENT Event,
1910     IN gcsQUEUE_PTR Queue
1911     )
1912 #endif
1913 {
1914     gceSTATUS status;
1915     gcsQUEUE_PTR record = gcvNULL, next;
1916     gctUINT32 processID;
1917     gctBOOL needCopy = gcvFALSE;
1918
1919     gcmkHEADER_ARG("Event=0x%x Queue=0x%x", Event, Queue);
1920
1921     /* Verify the arguments. */
1922     gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT);
1923
1924     /* Get the current process ID. */
1925     gcmkONERROR(gckOS_GetProcessID(&processID));
1926
1927     /* Query if we need to copy the client data. */
1928     gcmkONERROR(gckOS_QueryNeedCopy(Event->os, processID, &needCopy));
1929
1930     /* Loop while there are records in the queue. */
1931     while (Queue != gcvNULL)
1932     {
1933         gcsQUEUE queue;
1934
1935         if (needCopy)
1936         {
1937             /* Point to stack record. */
1938             record = &queue;
1939
1940             /* Copy the data from the client. */
1941             gcmkONERROR(gckOS_CopyFromUserData(Event->os,
1942                                                record,
1943                                                Queue,
1944                                                gcmSIZEOF(gcsQUEUE)));
1945         }
1946         else
1947         {
1948             gctPOINTER pointer = gcvNULL;
1949
1950             /* Map record into kernel memory. */
1951             gcmkONERROR(gckOS_MapUserPointer(Event->os,
1952                                              Queue,
1953                                              gcmSIZEOF(gcsQUEUE),
1954                                              &pointer));
1955
1956             record = pointer;
1957         }
1958
1959         /* Append event record to event queue. */
1960         gcmkONERROR(
1961             gckEVENT_AddList(Event, &record->iface, gcvKERNEL_PIXEL, gcvTRUE, gcvFALSE));
1962
1963         /* Next record in the queue. */
1964         next = gcmUINT64_TO_PTR(record->next);
1965
1966         if (!needCopy)
1967         {
1968             /* Unmap record from kernel memory. */
1969             gcmkONERROR(
1970                 gckOS_UnmapUserPointer(Event->os,
1971                                        Queue,
1972                                        gcmSIZEOF(gcsQUEUE),
1973                                        (gctPOINTER *) record));
1974             record = gcvNULL;
1975         }
1976
1977         Queue = next;
1978     }
1979
1980     /* Submit the event list. */
1981 #if gcdMULTI_GPU
1982     gcmkONERROR(gckEVENT_Submit(Event, gcvTRUE, gcvFALSE, ChipEnable));
1983 #else
1984     gcmkONERROR(gckEVENT_Submit(Event, gcvTRUE, gcvFALSE));
1985 #endif
1986
1987     /* Success */
1988     gcmkFOOTER_NO();
1989     return gcvSTATUS_OK;
1990
1991 OnError:
1992     if ((record != gcvNULL) && !needCopy)
1993     {
1994         /* Roll back. */
1995         gcmkVERIFY_OK(gckOS_UnmapUserPointer(Event->os,
1996                                              Queue,
1997                                              gcmSIZEOF(gcsQUEUE),
1998                                              (gctPOINTER *) record));
1999     }
2000
2001     /* Return the status. */
2002     gcmkFOOTER();
2003     return status;
2004 }
2005
2006 /*******************************************************************************
2007 **
2008 **  gckEVENT_Compose
2009 **
2010 **  Schedule a composition event and start a composition.
2011 **
2012 **  INPUT:
2013 **
2014 **      gckEVENT Event
2015 **          Pointer to an gckEVENT object.
2016 **
2017 **      gcsHAL_COMPOSE_PTR Info
2018 **          Pointer to the composition structure.
2019 **
2020 **  OUTPUT:
2021 **
2022 **      Nothing.
2023 */
2024 gceSTATUS
2025 gckEVENT_Compose(
2026     IN gckEVENT Event,
2027     IN gcsHAL_COMPOSE_PTR Info
2028     )
2029 {
2030     gceSTATUS status;
2031     gcsEVENT_PTR headRecord;
2032     gcsEVENT_PTR tailRecord;
2033     gcsEVENT_PTR tempRecord;
2034     gctUINT8 id = 0xFF;
2035     gctUINT32 processID;
2036
2037     gcmkHEADER_ARG("Event=0x%x Info=0x%x", Event, Info);
2038
2039     /* Verify the arguments. */
2040     gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT);
2041     gcmkVERIFY_ARGUMENT(Info != gcvNULL);
2042
2043     /* Allocate an event ID. */
2044 #if gcdMULTI_GPU
2045     gcmkONERROR(gckEVENT_GetEvent(Event, gcvTRUE, &id, gcvKERNEL_PIXEL, gcvCORE_3D_ALL_MASK));
2046 #else
2047     gcmkONERROR(gckEVENT_GetEvent(Event, gcvTRUE, &id, gcvKERNEL_PIXEL));
2048 #endif
2049
2050     /* Get process ID. */
2051     gcmkONERROR(gckOS_GetProcessID(&processID));
2052
2053     /* Allocate a record. */
2054     gcmkONERROR(gckEVENT_AllocateRecord(Event, gcvTRUE, &tempRecord));
2055     headRecord = tailRecord = tempRecord;
2056
2057     /* Initialize the record. */
2058     tempRecord->info.command            = gcvHAL_SIGNAL;
2059     tempRecord->info.u.Signal.process   = Info->process;
2060 #ifdef __QNXNTO__
2061     tempRecord->info.u.Signal.coid      = Info->coid;
2062     tempRecord->info.u.Signal.rcvid     = Info->rcvid;
2063 #endif
2064     tempRecord->info.u.Signal.signal    = Info->signal;
2065     tempRecord->info.u.Signal.auxSignal = 0;
2066     tempRecord->next = gcvNULL;
2067     tempRecord->processID = processID;
2068
2069     /* Allocate another record for user signal #1. */
2070     if (gcmUINT64_TO_PTR(Info->userSignal1) != gcvNULL)
2071     {
2072         /* Allocate a record. */
2073         gcmkONERROR(gckEVENT_AllocateRecord(Event, gcvTRUE, &tempRecord));
2074         tailRecord->next = tempRecord;
2075         tailRecord = tempRecord;
2076
2077         /* Initialize the record. */
2078         tempRecord->info.command            = gcvHAL_SIGNAL;
2079         tempRecord->info.u.Signal.process   = Info->userProcess;
2080 #ifdef __QNXNTO__
2081         tempRecord->info.u.Signal.coid      = Info->coid;
2082         tempRecord->info.u.Signal.rcvid     = Info->rcvid;
2083 #endif
2084         tempRecord->info.u.Signal.signal    = Info->userSignal1;
2085         tempRecord->info.u.Signal.auxSignal = 0;
2086         tempRecord->next = gcvNULL;
2087         tempRecord->processID = processID;
2088     }
2089
2090     /* Allocate another record for user signal #2. */
2091     if (gcmUINT64_TO_PTR(Info->userSignal2) != gcvNULL)
2092     {
2093         /* Allocate a record. */
2094         gcmkONERROR(gckEVENT_AllocateRecord(Event, gcvTRUE, &tempRecord));
2095         tailRecord->next = tempRecord;
2096
2097         /* Initialize the record. */
2098         tempRecord->info.command            = gcvHAL_SIGNAL;
2099         tempRecord->info.u.Signal.process   = Info->userProcess;
2100 #ifdef __QNXNTO__
2101         tempRecord->info.u.Signal.coid      = Info->coid;
2102         tempRecord->info.u.Signal.rcvid     = Info->rcvid;
2103 #endif
2104         tempRecord->info.u.Signal.signal    = Info->userSignal2;
2105         tempRecord->info.u.Signal.auxSignal = 0;
2106         tempRecord->next = gcvNULL;
2107         tempRecord->processID = processID;
2108     }
2109
2110     /* Set the event list. */
2111     Event->queues[id].head = headRecord;
2112
2113     /* Start composition. */
2114     gcmkONERROR(gckHARDWARE_Compose(
2115         Event->kernel->hardware, processID,
2116         gcmUINT64_TO_PTR(Info->physical), gcmUINT64_TO_PTR(Info->logical), Info->offset, Info->size, id
2117         ));
2118
2119     /* Success. */
2120     gcmkFOOTER_NO();
2121     return gcvSTATUS_OK;
2122
2123 OnError:
2124     /* Return the status. */
2125     gcmkFOOTER();
2126     return status;
2127 }
2128
2129 /*******************************************************************************
2130 **
2131 **  gckEVENT_Interrupt
2132 **
2133 **  Called by the interrupt service routine to store the triggered interrupt
2134 **  mask to be later processed by gckEVENT_Notify.
2135 **
2136 **  INPUT:
2137 **
2138 **      gckEVENT Event
2139 **          Pointer to an gckEVENT object.
2140 **
2141 **      gctUINT32 Data
2142 **          Mask for the 32 interrupts.
2143 **
2144 **  OUTPUT:
2145 **
2146 **      Nothing.
2147 */
2148 gceSTATUS
2149 gckEVENT_Interrupt(
2150     IN gckEVENT Event,
2151 #if gcdMULTI_GPU
2152     IN gctUINT CoreId,
2153 #endif
2154     IN gctUINT32 Data
2155     )
2156 {
2157 #if gcdMULTI_GPU
2158 #if defined(WIN32)
2159     gctUINT32 i;
2160 #endif
2161 #endif
2162     gcmkHEADER_ARG("Event=0x%x Data=0x%x", Event, Data);
2163
2164     /* Verify the arguments. */
2165     gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT);
2166
2167     if (Data & 0x20000000)
2168     {
2169         gckENTRYDATA data;
2170         gctUINT32 idle;
2171         Data &= ~0x20000000;
2172
2173 #if gcdMULTI_GPU
2174         if (Event->kernel->core == gcvCORE_MAJOR)
2175 #endif
2176         {
2177             /* Get first entry information. */
2178             gcmkVERIFY_OK(
2179                 gckENTRYQUEUE_Dequeue(&Event->kernel->command->queue, &data));
2180
2181             /* Make sure FE is idle. */
2182             do
2183             {
2184                 gcmkVERIFY_OK(gckOS_ReadRegisterEx(
2185                     Event->os,
2186                     Event->kernel->core,
2187                     0x4,
2188                     &idle));
2189             }
2190             while (idle != 0x7FFFFFFF);
2191
2192             /* Start Command Parser. */
2193             gcmkVERIFY_OK(gckHARDWARE_Execute(
2194                 Event->kernel->hardware,
2195                 data->physical,
2196                 data->bytes
2197                 ));
2198         }
2199     }
2200
2201     /* Combine current interrupt status with pending flags. */
2202 #if gcdSMP
2203 #if gcdMULTI_GPU
2204     if (Event->kernel->core == gcvCORE_MAJOR)
2205     {
2206         gckOS_AtomSetMask(Event->pending3D[CoreId], Data);
2207     }
2208     else
2209 #endif
2210     {
2211         gckOS_AtomSetMask(Event->pending, Data);
2212     }
2213 #elif defined(__QNXNTO__)
2214 #if gcdMULTI_GPU
2215     if (Event->kernel->core == gcvCORE_MAJOR)
2216     {
2217         atomic_set(&Event->pending3D[CoreId], Data);
2218     }
2219     else
2220 #endif
2221     {
2222         atomic_set(&Event->pending, Data);
2223     }
2224 #else
2225 #if gcdMULTI_GPU
2226 #if defined(WIN32)
2227     if (Event->kernel->core == gcvCORE_MAJOR)
2228     {
2229         for (i = 0; i < gcdMULTI_GPU; i++)
2230         {
2231             Event->pending3D[i] |= Data;
2232         }
2233     }
2234     else
2235 #else
2236     if (Event->kernel->core == gcvCORE_MAJOR)
2237     {
2238         Event->pending3D[CoreId] |= Data;
2239     }
2240     else
2241 #endif
2242 #endif
2243     {
2244         Event->pending |= Data;
2245     }
2246 #endif
2247
2248 #if gcdINTERRUPT_STATISTIC
2249     {
2250         gctINT j = 0;
2251         gctINT32 oldValue;
2252
2253         for (j = 0; j < gcmCOUNTOF(Event->queues); j++)
2254         {
2255             if ((Data & (1 << j)))
2256             {
2257                 gcmkVERIFY_OK(gckOS_AtomDecrement(Event->os,
2258                                                   Event->interruptCount,
2259                                                   &oldValue));
2260             }
2261         }
2262     }
2263 #endif
2264
2265     /* Success. */
2266     gcmkFOOTER_NO();
2267     return gcvSTATUS_OK;
2268 }
2269
2270 /*******************************************************************************
2271 **
2272 **  gckEVENT_Notify
2273 **
2274 **  Process all triggered interrupts.
2275 **
2276 **  INPUT:
2277 **
2278 **      gckEVENT Event
2279 **          Pointer to an gckEVENT object.
2280 **
2281 **  OUTPUT:
2282 **
2283 **      Nothing.
2284 */
2285 gceSTATUS
2286 gckEVENT_Notify(
2287     IN gckEVENT Event,
2288     IN gctUINT32 IDs
2289     )
2290 {
2291     gceSTATUS status = gcvSTATUS_OK;
2292     gctINT i;
2293     gcsEVENT_QUEUE * queue;
2294     gctUINT mask = 0;
2295     gctBOOL acquired = gcvFALSE;
2296     gctPOINTER info;
2297     gctSIGNAL signal;
2298     gctUINT pending = 0;
2299     gckKERNEL kernel = Event->kernel;
2300 #if gcdMULTI_GPU
2301     gceCORE core = Event->kernel->core;
2302     gctUINT32 busy;
2303     gctUINT32 oldValue;
2304     gctUINT pendingMask;
2305 #endif
2306 #if !gcdSMP
2307     gctBOOL suspended = gcvFALSE;
2308 #endif
2309 #if gcmIS_DEBUG(gcdDEBUG_TRACE)
2310     gctINT eventNumber = 0;
2311 #endif
2312     gctINT32 free;
2313 #if gcdSECURE_USER
2314     gcskSECURE_CACHE_PTR cache;
2315 #endif
2316     gckVIDMEM_NODE nodeObject;
2317     gcuVIDMEM_NODE_PTR node;
2318
2319     gcmkHEADER_ARG("Event=0x%x IDs=0x%x", Event, IDs);
2320
2321     /* Verify the arguments. */
2322     gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT);
2323
2324     gcmDEBUG_ONLY(
2325         if (IDs != 0)
2326         {
2327             for (i = 0; i < gcmCOUNTOF(Event->queues); ++i)
2328             {
2329                 if (Event->queues[i].head != gcvNULL)
2330                 {
2331                     gcmkTRACE_ZONE(gcvLEVEL_VERBOSE, gcvZONE_EVENT,
2332                                    "Queue(%d): stamp=%llu source=%d",
2333                                    i,
2334                                    Event->queues[i].stamp,
2335                                    Event->queues[i].source);
2336                 }
2337             }
2338         }
2339     );
2340
2341 #if gcdMULTI_GPU
2342     /* Set busy flag. */
2343     gckOS_AtomicExchange(Event->os, &Event->busy, 1, &busy);
2344     if (busy)
2345     {
2346         /* Another thread is already busy - abort. */
2347         goto OnSuccess;
2348     }
2349 #endif
2350
2351     for (;;)
2352     {
2353         gcsEVENT_PTR record;
2354 #if gcdMULTI_GPU
2355         gctUINT32 pend[gcdMULTI_GPU];
2356         gctUINT32 pendMask[gcdMULTI_GPU];
2357 #endif
2358
2359         /* Grab the mutex queue. */
2360         gcmkONERROR(gckOS_AcquireMutex(Event->os,
2361                                        Event->eventQueueMutex,
2362                                        gcvINFINITE));
2363         acquired = gcvTRUE;
2364
2365 #if gcdSMP
2366 #if gcdMULTI_GPU
2367         if (core == gcvCORE_MAJOR)
2368         {
2369             /* Get current interrupts. */
2370             for (i = 0; i < gcdMULTI_GPU; i++)
2371             {
2372                 gckOS_AtomGet(Event->os, Event->pending3D[i], (gctINT32_PTR)&pend[i]);
2373                 gckOS_AtomGet(Event->os, Event->pending3DMask[i], (gctINT32_PTR)&pendMask[i]);
2374             }
2375
2376             gckOS_AtomGet(Event->os, Event->pendingMask, (gctINT32_PTR)&pendingMask);
2377         }
2378         else
2379 #endif
2380         {
2381             gckOS_AtomGet(Event->os, Event->pending, (gctINT32_PTR)&pending);
2382         }
2383 #else
2384         /* Suspend interrupts. */
2385         gcmkONERROR(gckOS_SuspendInterruptEx(Event->os, Event->kernel->core));
2386         suspended = gcvTRUE;
2387
2388 #if gcdMULTI_GPU
2389         if (core == gcvCORE_MAJOR)
2390         {
2391             for (i = 0; i < gcdMULTI_GPU; i++)
2392             {
2393                 /* Get current interrupts. */
2394                 pend[i] = Event->pending3D[i];
2395                 pendMask[i] = Event->pending3DMask[i];
2396             }
2397
2398             pendingMask = Event->pendingMask;
2399         }
2400         else
2401 #endif
2402         {
2403             pending = Event->pending;
2404         }
2405
2406         /* Resume interrupts. */
2407         gcmkONERROR(gckOS_ResumeInterruptEx(Event->os, Event->kernel->core));
2408         suspended = gcvFALSE;
2409 #endif
2410
2411 #if gcdMULTI_GPU
2412         if (core == gcvCORE_MAJOR)
2413         {
2414             for (i = 0; i < gcdMULTI_GPU; i++)
2415             {
2416                 gctUINT32 bad_pend = (pend[i] & ~pendMask[i]);
2417
2418                 if (bad_pend != 0)
2419                 {
2420                     gcmkTRACE_ZONE_N(
2421                         gcvLEVEL_ERROR, gcvZONE_EVENT,
2422                         gcmSIZEOF(bad_pend) + gcmSIZEOF(i),
2423                         "Interrupts 0x%x are not unexpected for Core%d.",
2424                         bad_pend, i
2425                         );
2426
2427                     gckOS_AtomClearMask(Event->pending3D[i], bad_pend);
2428
2429                     pend[i] &= pendMask[i];
2430                 }
2431             }
2432
2433             pending = (pend[0] & pend[1] & pendingMask) /* Check combined events on both GPUs */
2434                     | (pend[0] & ~pendingMask)          /* Check individual events on GPU 0   */
2435                     | (pend[1] & ~pendingMask);         /* Check individual events on GPU 1   */
2436         }
2437 #endif
2438
2439         if (pending == 0)
2440         {
2441             /* Release the mutex queue. */
2442             gcmkONERROR(gckOS_ReleaseMutex(Event->os, Event->eventQueueMutex));
2443             acquired = gcvFALSE;
2444
2445             /* No more pending interrupts - done. */
2446             break;
2447         }
2448
2449         if (pending & 0x80000000)
2450         {
2451             gcmkPRINT("AXI BUS ERROR");
2452             pending &= 0x7FFFFFFF;
2453         }
2454
2455         if (pending & 0x40000000)
2456         {
2457             gckHARDWARE_DumpMMUException(Event->kernel->hardware);
2458
2459             pending &= 0xBFFFFFFF;
2460         }
2461
2462         gcmkTRACE_ZONE_N(
2463             gcvLEVEL_INFO, gcvZONE_EVENT,
2464             gcmSIZEOF(pending),
2465             "Pending interrupts 0x%x",
2466             pending
2467             );
2468
2469         queue = gcvNULL;
2470
2471         gcmDEBUG_ONLY(
2472             if (IDs == 0)
2473             {
2474                 for (i = 0; i < gcmCOUNTOF(Event->queues); ++i)
2475                 {
2476                     if (Event->queues[i].head != gcvNULL)
2477                     {
2478                         gcmkTRACE_ZONE(gcvLEVEL_VERBOSE, gcvZONE_EVENT,
2479                                        "Queue(%d): stamp=%llu source=%d",
2480                                        i,
2481                                        Event->queues[i].stamp,
2482                                        Event->queues[i].source);
2483                     }
2484                 }
2485             }
2486         );
2487
2488         /* Find the oldest pending interrupt. */
2489         for (i = 0; i < gcmCOUNTOF(Event->queues); ++i)
2490         {
2491             if ((Event->queues[i].head != gcvNULL)
2492             &&  (pending & (1 << i))
2493             )
2494             {
2495                 if ((queue == gcvNULL)
2496                 ||  (Event->queues[i].stamp < queue->stamp)
2497                 )
2498                 {
2499                     queue = &Event->queues[i];
2500                     mask  = 1 << i;
2501 #if gcmIS_DEBUG(gcdDEBUG_TRACE)
2502                     eventNumber = i;
2503 #endif
2504                 }
2505             }
2506         }
2507
2508         if (queue == gcvNULL)
2509         {
2510             gcmkTRACE_ZONE_N(
2511                 gcvLEVEL_ERROR, gcvZONE_EVENT,
2512                 gcmSIZEOF(pending),
2513                 "Interrupts 0x%x are not pending.",
2514                 pending
2515                 );
2516
2517 #if gcdSMP
2518 #if gcdMULTI_GPU
2519             if (core == gcvCORE_MAJOR)
2520             {
2521                 /* Mark pending interrupts as handled. */
2522                 for (i = 0; i < gcdMULTI_GPU; i++)
2523                 {
2524                     gckOS_AtomClearMask(Event->pending3D[i], pending);
2525                     gckOS_AtomClearMask(Event->pending3DMask[i], pending);
2526                 }
2527
2528                 gckOS_AtomClearMask(Event->pendingMask, pending);
2529             }
2530             else
2531 #endif
2532             {
2533                 gckOS_AtomClearMask(Event->pending, pending);
2534             }
2535
2536 #elif defined(__QNXNTO__)
2537 #if gcdMULTI_GPU
2538             if (core == gcvCORE_MAJOR)
2539             {
2540                 for (i = 0; i < gcdMULTI_GPU; i++)
2541                 {
2542                     atomic_clr((gctUINT32_PTR)&Event->pending3D[i], pending);
2543                     atomic_clr((gctUINT32_PTR)&Event->pending3DMask[i], pending);
2544                 }
2545
2546                 atomic_clr((gctUINT32_PTR)&Event->pendingMask, pending);
2547             }
2548             else
2549 #endif
2550             {
2551                 atomic_clr((gctUINT32_PTR)&Event->pending, pending);
2552             }
2553 #else
2554             /* Suspend interrupts. */
2555             gcmkONERROR(gckOS_SuspendInterruptEx(Event->os, Event->kernel->core));
2556             suspended = gcvTRUE;
2557
2558 #if gcdMULTI_GPU
2559             if (core == gcvCORE_MAJOR)
2560             {
2561                 for (i = 0; i < gcdMULTI_GPU; i++)
2562                 {
2563                     /* Mark pending interrupts as handled. */
2564                     Event->pending3D[i] &= ~pending;
2565                     Event->pending3DMask[i] &= ~pending;
2566                 }
2567             }
2568             else
2569 #endif
2570             {
2571                 Event->pending &= ~pending;
2572             }
2573
2574             /* Resume interrupts. */
2575             gcmkONERROR(gckOS_ResumeInterruptEx(Event->os, Event->kernel->core));
2576             suspended = gcvFALSE;
2577 #endif
2578
2579             /* Release the mutex queue. */
2580             gcmkONERROR(gckOS_ReleaseMutex(Event->os, Event->eventQueueMutex));
2581             acquired = gcvFALSE;
2582             break;
2583         }
2584
2585         /* Check whether there is a missed interrupt. */
2586         for (i = 0; i < gcmCOUNTOF(Event->queues); ++i)
2587         {
2588             if ((Event->queues[i].head != gcvNULL)
2589             &&  (Event->queues[i].stamp < queue->stamp)
2590             &&  (Event->queues[i].source <= queue->source)
2591 #if gcdMULTI_GPU
2592             &&  (Event->queues[i].chipEnable == queue->chipEnable)
2593 #endif
2594             )
2595             {
2596                 gcmkTRACE_N(
2597                     gcvLEVEL_ERROR,
2598                     gcmSIZEOF(i) + gcmSIZEOF(Event->queues[i].stamp),
2599                     "Event %d lost (stamp %llu)",
2600                     i, Event->queues[i].stamp
2601                     );
2602
2603                 /* Use this event instead. */
2604                 queue = &Event->queues[i];
2605                 mask  = 0;
2606             }
2607         }
2608
2609         if (mask != 0)
2610         {
2611 #if gcmIS_DEBUG(gcdDEBUG_TRACE)
2612             gcmkTRACE_ZONE_N(
2613                 gcvLEVEL_INFO, gcvZONE_EVENT,
2614                 gcmSIZEOF(eventNumber),
2615                 "Processing interrupt %d",
2616                 eventNumber
2617                 );
2618 #endif
2619         }
2620
2621 #if gcdSMP
2622 #if gcdMULTI_GPU
2623         if (core == gcvCORE_MAJOR)
2624         {
2625             for (i = 0; i < gcdMULTI_GPU; i++)
2626             {
2627                 /* Mark pending interrupt as handled. */
2628                 gckOS_AtomClearMask(Event->pending3D[i], mask);
2629                 gckOS_AtomClearMask(Event->pending3DMask[i], mask);
2630             }
2631
2632             gckOS_AtomClearMask(Event->pendingMask, mask);
2633         }
2634         else
2635 #endif
2636         {
2637             gckOS_AtomClearMask(Event->pending, mask);
2638         }
2639
2640 #elif defined(__QNXNTO__)
2641 #if gcdMULTI_GPU
2642         if (core == gcvCORE_MAJOR)
2643         {
2644             for (i = 0; i < gcdMULTI_GPU; i++)
2645             {
2646                 atomic_clr(&Event->pending3D[i], mask);
2647                 atomic_clr(&Event->pending3DMask[i], mask);
2648             }
2649
2650             atomic_clr(&Event->pendingMask, mask);
2651         }
2652         else
2653 #endif
2654         {
2655             atomic_clr(&Event->pending, mask);
2656         }
2657 #else
2658         /* Suspend interrupts. */
2659         gcmkONERROR(gckOS_SuspendInterruptEx(Event->os, Event->kernel->core));
2660         suspended = gcvTRUE;
2661
2662 #if gcdMULTI_GPU
2663         if (core == gcvCORE_MAJOR)
2664         {
2665             for (i = 0; i < gcdMULTI_GPU; i++)
2666             {
2667                 /* Mark pending interrupt as handled. */
2668                 Event->pending3D[i] &= ~mask;
2669                 Event->pending3DMask[i] &= ~mask;
2670             }
2671
2672             Event->pendingMask &= ~mask;
2673         }
2674         else
2675 #endif
2676         {
2677             Event->pending &= ~mask;
2678         }
2679
2680         /* Resume interrupts. */
2681         gcmkONERROR(gckOS_ResumeInterruptEx(Event->os, Event->kernel->core));
2682         suspended = gcvFALSE;
2683 #endif
2684
2685         /* Grab the event head. */
2686         record = queue->head;
2687
2688         /* Now quickly clear its event list. */
2689         queue->head = gcvNULL;
2690
2691         /* Release the mutex queue. */
2692         gcmkONERROR(gckOS_ReleaseMutex(Event->os, Event->eventQueueMutex));
2693         acquired = gcvFALSE;
2694
2695         /* Increase the number of free events. */
2696         gcmkONERROR(gckOS_AtomIncrement(Event->os, Event->freeAtom, &free));
2697
2698         /* Walk all events for this interrupt. */
2699         while (record != gcvNULL)
2700         {
2701             gcsEVENT_PTR recordNext;
2702 #ifndef __QNXNTO__
2703             gctPOINTER logical;
2704 #endif
2705 #if gcdSECURE_USER
2706             gctSIZE_T bytes;
2707 #endif
2708
2709             /* Grab next record. */
2710             recordNext = record->next;
2711
2712 #ifdef __QNXNTO__
2713             /* Assign record->processID as the pid for this galcore thread.
2714              * Used in OS calls like gckOS_UnlockMemory() which do not take a pid.
2715              */
2716             drv_thread_specific_key_assign(record->processID, 0, Event->kernel->core);
2717 #endif
2718
2719 #if gcdSECURE_USER
2720             /* Get the cache that belongs to this process. */
2721             gcmkONERROR(gckKERNEL_GetProcessDBCache(Event->kernel,
2722                         record->processID,
2723                         &cache));
2724 #endif
2725
2726             gcmkTRACE_ZONE_N(
2727                 gcvLEVEL_INFO, gcvZONE_EVENT,
2728                 gcmSIZEOF(record->info.command),
2729                 "Processing event type: %d",
2730                 record->info.command
2731                 );
2732
2733             switch (record->info.command)
2734             {
2735             case gcvHAL_FREE_NON_PAGED_MEMORY:
2736                 gcmkTRACE_ZONE(gcvLEVEL_VERBOSE, gcvZONE_EVENT,
2737                                "gcvHAL_FREE_NON_PAGED_MEMORY: 0x%x",
2738                                gcmNAME_TO_PTR(record->info.u.FreeNonPagedMemory.physical));
2739
2740                 /* Free non-paged memory. */
2741                 status = gckOS_FreeNonPagedMemory(
2742                             Event->os,
2743                             (gctSIZE_T) record->info.u.FreeNonPagedMemory.bytes,
2744                             gcmNAME_TO_PTR(record->info.u.FreeNonPagedMemory.physical),
2745                             gcmUINT64_TO_PTR(record->info.u.FreeNonPagedMemory.logical));
2746
2747                 if (gcmIS_SUCCESS(status))
2748                 {
2749 #if gcdSECURE_USER
2750                     gcmkVERIFY_OK(gckKERNEL_FlushTranslationCache(
2751                         Event->kernel,
2752                         cache,
2753                         gcmUINT64_TO_PTR(record->record.u.FreeNonPagedMemory.logical),
2754                         (gctSIZE_T) record->record.u.FreeNonPagedMemory.bytes));
2755 #endif
2756                 }
2757                 gcmRELEASE_NAME(record->info.u.FreeNonPagedMemory.physical);
2758                 break;
2759
2760             case gcvHAL_FREE_CONTIGUOUS_MEMORY:
2761                 gcmkTRACE_ZONE(
2762                     gcvLEVEL_VERBOSE, gcvZONE_EVENT,
2763                     "gcvHAL_FREE_CONTIGUOUS_MEMORY: 0x%x",
2764                     gcmNAME_TO_PTR(record->info.u.FreeContiguousMemory.physical));
2765
2766                 /* Unmap the user memory. */
2767                 status = gckOS_FreeContiguous(
2768                             Event->os,
2769                             gcmNAME_TO_PTR(record->info.u.FreeContiguousMemory.physical),
2770                             gcmUINT64_TO_PTR(record->info.u.FreeContiguousMemory.logical),
2771                             (gctSIZE_T) record->info.u.FreeContiguousMemory.bytes);
2772
2773                 if (gcmIS_SUCCESS(status))
2774                 {
2775 #if gcdSECURE_USER
2776                     gcmkVERIFY_OK(gckKERNEL_FlushTranslationCache(
2777                         Event->kernel,
2778                         cache,
2779                         gcmUINT64_TO_PTR(event->event.u.FreeContiguousMemory.logical),
2780                         (gctSIZE_T) event->event.u.FreeContiguousMemory.bytes));
2781 #endif
2782                 }
2783                 gcmRELEASE_NAME(record->info.u.FreeContiguousMemory.physical);
2784                 break;
2785
2786             case gcvHAL_WRITE_DATA:
2787 #ifndef __QNXNTO__
2788                 /* Convert physical into logical address. */
2789                 gcmkERR_BREAK(
2790                     gckOS_MapPhysical(Event->os,
2791                                       record->info.u.WriteData.address,
2792                                       gcmSIZEOF(gctUINT32),
2793                                       &logical));
2794
2795                 /* Write data. */
2796                 gcmkERR_BREAK(
2797                     gckOS_WriteMemory(Event->os,
2798                                       logical,
2799                                       record->info.u.WriteData.data));
2800
2801                 /* Unmap the physical memory. */
2802                 gcmkERR_BREAK(
2803                     gckOS_UnmapPhysical(Event->os,
2804                                         logical,
2805                                         gcmSIZEOF(gctUINT32)));
2806 #else
2807                 /* Write data. */
2808                 gcmkERR_BREAK(
2809                     gckOS_WriteMemory(Event->os,
2810                                       (gctPOINTER)
2811                                           record->info.u.WriteData.address,
2812                                       record->info.u.WriteData.data));
2813 #endif
2814                 break;
2815
2816             case gcvHAL_UNLOCK_VIDEO_MEMORY:
2817                 gcmkTRACE_ZONE(gcvLEVEL_VERBOSE, gcvZONE_EVENT,
2818                                "gcvHAL_UNLOCK_VIDEO_MEMORY: 0x%x",
2819                                record->info.u.UnlockVideoMemory.node);
2820
2821                 nodeObject = gcmUINT64_TO_PTR(record->info.u.UnlockVideoMemory.node);
2822
2823                 node = nodeObject->node;
2824
2825                 /* Save node information before it disappears. */
2826 #if gcdSECURE_USER
2827                 node = event->event.u.UnlockVideoMemory.node;
2828                 if (node->VidMem.memory->object.type == gcvOBJ_VIDMEM)
2829                 {
2830                     logical = gcvNULL;
2831                     bytes   = 0;
2832                 }
2833                 else
2834                 {
2835                     logical = node->Virtual.logical;
2836                     bytes   = node->Virtual.bytes;
2837                 }
2838 #endif
2839
2840                 /* Unlock. */
2841                 status = gckVIDMEM_Unlock(
2842                     Event->kernel,
2843                     nodeObject,
2844                     record->info.u.UnlockVideoMemory.type,
2845                     gcvNULL);
2846
2847 #if gcdSECURE_USER
2848                 if (gcmIS_SUCCESS(status) && (logical != gcvNULL))
2849                 {
2850                     gcmkVERIFY_OK(gckKERNEL_FlushTranslationCache(
2851                         Event->kernel,
2852                         cache,
2853                         logical,
2854                         bytes));
2855                 }
2856 #endif
2857
2858 #if gcdPROCESS_ADDRESS_SPACE
2859                 gcmkVERIFY_OK(gckVIDMEM_NODE_Unlock(
2860                     Event->kernel,
2861                     nodeObject,
2862                     record->processID
2863                     ));
2864 #endif
2865
2866                 status = gckVIDMEM_NODE_Dereference(Event->kernel, nodeObject);
2867                 break;
2868
2869             case gcvHAL_SIGNAL:
2870                 signal = gcmUINT64_TO_PTR(record->info.u.Signal.signal);
2871                 gcmkTRACE_ZONE(gcvLEVEL_VERBOSE, gcvZONE_EVENT,
2872                                "gcvHAL_SIGNAL: 0x%x",
2873                                signal);
2874
2875 #ifdef __QNXNTO__
2876                 if ((record->info.u.Signal.coid == 0)
2877                 &&  (record->info.u.Signal.rcvid == 0)
2878                 )
2879                 {
2880                     /* Kernel signal. */
2881                     gcmkERR_BREAK(
2882                         gckOS_Signal(Event->os,
2883                                      signal,
2884                                      gcvTRUE));
2885                 }
2886                 else
2887                 {
2888                     /* User signal. */
2889                     gcmkERR_BREAK(
2890                         gckOS_UserSignal(Event->os,
2891                                          signal,
2892                                          record->info.u.Signal.rcvid,
2893                                          record->info.u.Signal.coid));
2894                 }
2895 #else
2896                 /* Set signal. */
2897                 if (gcmUINT64_TO_PTR(record->info.u.Signal.process) == gcvNULL)
2898                 {
2899                     /* Kernel signal. */
2900                     gcmkERR_BREAK(
2901                         gckOS_Signal(Event->os,
2902                                      signal,
2903                                      gcvTRUE));
2904                 }
2905                 else
2906                 {
2907                     /* User signal. */
2908                     gcmkERR_BREAK(
2909                         gckOS_UserSignal(Event->os,
2910                                          signal,
2911                                          gcmUINT64_TO_PTR(record->info.u.Signal.process)));
2912                 }
2913
2914                 gcmkASSERT(record->info.u.Signal.auxSignal == 0);
2915 #endif
2916                 break;
2917
2918             case gcvHAL_UNMAP_USER_MEMORY:
2919                 info = gcmNAME_TO_PTR(record->info.u.UnmapUserMemory.info);
2920                 gcmkTRACE_ZONE(gcvLEVEL_VERBOSE, gcvZONE_EVENT,
2921                                "gcvHAL_UNMAP_USER_MEMORY: 0x%x",
2922                                info);
2923
2924                 /* Unmap the user memory. */
2925                 status = gckOS_UnmapUserMemory(
2926                     Event->os,
2927                     Event->kernel->core,
2928                     gcmUINT64_TO_PTR(record->info.u.UnmapUserMemory.memory),
2929                     (gctSIZE_T) record->info.u.UnmapUserMemory.size,
2930                     info,
2931                     record->info.u.UnmapUserMemory.address);
2932
2933 #if gcdSECURE_USER
2934                 if (gcmIS_SUCCESS(status))
2935                 {
2936                     gcmkVERIFY_OK(gckKERNEL_FlushTranslationCache(
2937                         Event->kernel,
2938                         cache,
2939                         gcmUINT64_TO_PTR(record->info.u.UnmapUserMemory.memory),
2940                         (gctSIZE_T) record->info.u.UnmapUserMemory.size));
2941                 }
2942 #endif
2943                 gcmRELEASE_NAME(record->info.u.UnmapUserMemory.info);
2944                 break;
2945
2946             case gcvHAL_TIMESTAMP:
2947                 gcmkTRACE_ZONE(gcvLEVEL_VERBOSE, gcvZONE_EVENT,
2948                                "gcvHAL_TIMESTAMP: %d %d",
2949                                record->info.u.TimeStamp.timer,
2950                                record->info.u.TimeStamp.request);
2951
2952                 /* Process the timestamp. */
2953                 switch (record->info.u.TimeStamp.request)
2954                 {
2955                 case 0:
2956                     status = gckOS_GetTime(&Event->kernel->timers[
2957                                            record->info.u.TimeStamp.timer].
2958                                            stopTime);
2959                     break;
2960
2961                 case 1:
2962                     status = gckOS_GetTime(&Event->kernel->timers[
2963                                            record->info.u.TimeStamp.timer].
2964                                            startTime);
2965                     break;
2966
2967                 default:
2968                     gcmkTRACE_ZONE_N(
2969                         gcvLEVEL_ERROR, gcvZONE_EVENT,
2970                         gcmSIZEOF(record->info.u.TimeStamp.request),
2971                         "Invalid timestamp request: %d",
2972                         record->info.u.TimeStamp.request
2973                         );
2974
2975                     status = gcvSTATUS_INVALID_ARGUMENT;
2976                     break;
2977                 }
2978                 break;
2979
2980              case gcvHAL_FREE_VIRTUAL_COMMAND_BUFFER:
2981                  gcmkVERIFY_OK(
2982                      gckKERNEL_DestroyVirtualCommandBuffer(Event->kernel,
2983                          (gctSIZE_T) record->info.u.FreeVirtualCommandBuffer.bytes,
2984                          gcmNAME_TO_PTR(record->info.u.FreeVirtualCommandBuffer.physical),
2985                          gcmUINT64_TO_PTR(record->info.u.FreeVirtualCommandBuffer.logical)
2986                          ));
2987                  gcmRELEASE_NAME(record->info.u.FreeVirtualCommandBuffer.physical);
2988                  break;
2989
2990 #if gcdANDROID_NATIVE_FENCE_SYNC
2991             case gcvHAL_SYNC_POINT:
2992                 {
2993                     gctSYNC_POINT syncPoint;
2994
2995                     syncPoint = gcmUINT64_TO_PTR(record->info.u.SyncPoint.syncPoint);
2996                     status = gckOS_SignalSyncPoint(Event->os, syncPoint);
2997                 }
2998                 break;
2999 #endif
3000
3001 #if gcdPROCESS_ADDRESS_SPACE
3002             case gcvHAL_DESTROY_MMU:
3003                 status = gckMMU_Destroy(gcmUINT64_TO_PTR(record->info.u.DestroyMmu.mmu));
3004                 break;
3005 #endif
3006
3007             case gcvHAL_COMMIT_DONE:
3008                 break;
3009
3010             default:
3011                 /* Invalid argument. */
3012                 gcmkTRACE_ZONE_N(
3013                     gcvLEVEL_ERROR, gcvZONE_EVENT,
3014                     gcmSIZEOF(record->info.command),
3015                     "Unknown event type: %d",
3016                     record->info.command
3017                     );
3018
3019                 status = gcvSTATUS_INVALID_ARGUMENT;
3020                 break;
3021             }
3022
3023             /* Make sure there are no errors generated. */
3024             if (gcmIS_ERROR(status))
3025             {
3026                 gcmkTRACE_ZONE_N(
3027                     gcvLEVEL_WARNING, gcvZONE_EVENT,
3028                     gcmSIZEOF(status),
3029                     "Event produced status: %d(%s)",
3030                     status, gckOS_DebugStatus2Name(status));
3031             }
3032
3033             /* Free the event. */
3034             gcmkVERIFY_OK(gckEVENT_FreeRecord(Event, record));
3035
3036             /* Advance to next record. */
3037             record = recordNext;
3038         }
3039
3040         gcmkTRACE_ZONE(gcvLEVEL_VERBOSE, gcvZONE_EVENT,
3041                        "Handled interrupt 0x%x", mask);
3042     }
3043
3044 #if gcdMULTI_GPU
3045     /* Clear busy flag. */
3046     gckOS_AtomicExchange(Event->os, &Event->busy, 0, &oldValue);
3047 #endif
3048
3049     if (IDs == 0)
3050     {
3051         gcmkONERROR(_TryToIdleGPU(Event));
3052     }
3053
3054 #if gcdMULTI_GPU
3055 OnSuccess:
3056 #endif
3057     /* Success. */
3058     gcmkFOOTER_NO();
3059     return gcvSTATUS_OK;
3060
3061 OnError:
3062     if (acquired)
3063     {
3064         /* Release mutex. */
3065         gcmkVERIFY_OK(gckOS_ReleaseMutex(Event->os, Event->eventQueueMutex));
3066     }
3067
3068 #if !gcdSMP
3069     if (suspended)
3070     {
3071         /* Resume interrupts. */
3072         gcmkVERIFY_OK(gckOS_ResumeInterruptEx(Event->os, Event->kernel->core));
3073     }
3074 #endif
3075
3076     /* Return the status. */
3077     gcmkFOOTER();
3078     return status;
3079 }
3080
3081 /*******************************************************************************
3082 **  gckEVENT_FreeProcess
3083 **
3084 **  Free all events owned by a particular process ID.
3085 **
3086 **  INPUT:
3087 **
3088 **      gckEVENT Event
3089 **          Pointer to an gckEVENT object.
3090 **
3091 **      gctUINT32 ProcessID
3092 **          Process ID of the process to be freed up.
3093 **
3094 **  OUTPUT:
3095 **
3096 **      Nothing.
3097 */
3098 gceSTATUS
3099 gckEVENT_FreeProcess(
3100     IN gckEVENT Event,
3101     IN gctUINT32 ProcessID
3102     )
3103 {
3104     gctSIZE_T i;
3105     gctBOOL acquired = gcvFALSE;
3106     gcsEVENT_PTR record, next;
3107     gceSTATUS status;
3108     gcsEVENT_PTR deleteHead, deleteTail;
3109
3110     gcmkHEADER_ARG("Event=0x%x ProcessID=%d", Event, ProcessID);
3111
3112     /* Verify the arguments. */
3113     gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT);
3114
3115     /* Walk through all queues. */
3116     for (i = 0; i < gcmCOUNTOF(Event->queues); ++i)
3117     {
3118         if (Event->queues[i].head != gcvNULL)
3119         {
3120             /* Grab the event queue mutex. */
3121             gcmkONERROR(gckOS_AcquireMutex(Event->os,
3122                                            Event->eventQueueMutex,
3123                                            gcvINFINITE));
3124             acquired = gcvTRUE;
3125
3126             /* Grab the mutex head. */
3127             record                = Event->queues[i].head;
3128             Event->queues[i].head = gcvNULL;
3129             Event->queues[i].tail = gcvNULL;
3130             deleteHead            = gcvNULL;
3131             deleteTail            = gcvNULL;
3132
3133             while (record != gcvNULL)
3134             {
3135                 next = record->next;
3136                 if (record->processID == ProcessID)
3137                 {
3138                     if (deleteHead == gcvNULL)
3139                     {
3140                         deleteHead = record;
3141                     }
3142                     else
3143                     {
3144                         deleteTail->next = record;
3145                     }
3146
3147                     deleteTail = record;
3148                 }
3149                 else
3150                 {
3151                     if (Event->queues[i].head == gcvNULL)
3152                     {
3153                         Event->queues[i].head = record;
3154                     }
3155                     else
3156                     {
3157                         Event->queues[i].tail->next = record;
3158                     }
3159
3160                     Event->queues[i].tail = record;
3161                 }
3162
3163                 record->next = gcvNULL;
3164                 record = next;
3165             }
3166
3167             /* Release the mutex queue. */
3168             gcmkONERROR(gckOS_ReleaseMutex(Event->os, Event->eventQueueMutex));
3169             acquired = gcvFALSE;
3170
3171             /* Loop through the entire list of events. */
3172             for (record = deleteHead; record != gcvNULL; record = next)
3173             {
3174                 /* Get the next event record. */
3175                 next = record->next;
3176
3177                 /* Free the event record. */
3178                 gcmkONERROR(gckEVENT_FreeRecord(Event, record));
3179             }
3180         }
3181     }
3182
3183     gcmkONERROR(_TryToIdleGPU(Event));
3184
3185     /* Success. */
3186     gcmkFOOTER_NO();
3187     return gcvSTATUS_OK;
3188
3189 OnError:
3190     /* Release the event queue mutex. */
3191     if (acquired)
3192     {
3193         gcmkVERIFY_OK(gckOS_ReleaseMutex(Event->os, Event->eventQueueMutex));
3194     }
3195
3196     /* Return the status. */
3197     gcmkFOOTER();
3198     return status;
3199 }
3200
3201 /*******************************************************************************
3202 **  gckEVENT_Stop
3203 **
3204 **  Stop the hardware using the End event mechanism.
3205 **
3206 **  INPUT:
3207 **
3208 **      gckEVENT Event
3209 **          Pointer to an gckEVENT object.
3210 **
3211 **      gctUINT32 ProcessID
3212 **          Process ID Logical belongs.
3213 **
3214 **      gctPHYS_ADDR Handle
3215 **          Physical address handle.  If gcvNULL it is video memory.
3216 **
3217 **      gctPOINTER Logical
3218 **          Logical address to flush.
3219 **
3220 **      gctSIGNAL Signal
3221 **          Pointer to the signal to trigger.
3222 **
3223 **  OUTPUT:
3224 **
3225 **      Nothing.
3226 */
3227 gceSTATUS
3228 gckEVENT_Stop(
3229     IN gckEVENT Event,
3230     IN gctUINT32 ProcessID,
3231     IN gctPHYS_ADDR Handle,
3232     IN gctPOINTER Logical,
3233     IN gctSIGNAL Signal,
3234     IN OUT gctUINT32 * waitSize
3235     )
3236 {
3237     gceSTATUS status;
3238    /* gctSIZE_T waitSize;*/
3239     gcsEVENT_PTR record;
3240     gctUINT8 id = 0xFF;
3241
3242     gcmkHEADER_ARG("Event=0x%x ProcessID=%u Handle=0x%x Logical=0x%x "
3243                    "Signal=0x%x",
3244                    Event, ProcessID, Handle, Logical, Signal);
3245
3246     /* Verify the arguments. */
3247     gcmkVERIFY_OBJECT(Event, gcvOBJ_EVENT);
3248
3249     /* Submit the current event queue. */
3250 #if gcdMULTI_GPU
3251     gcmkONERROR(gckEVENT_Submit(Event, gcvTRUE, gcvFALSE, gcvCORE_3D_ALL_MASK));
3252 #else
3253     gcmkONERROR(gckEVENT_Submit(Event, gcvTRUE, gcvFALSE));
3254 #endif
3255 #if gcdMULTI_GPU
3256     gcmkONERROR(gckEVENT_GetEvent(Event, gcvTRUE, &id, gcvKERNEL_PIXEL, gcvCORE_3D_ALL_MASK));
3257 #else
3258     gcmkONERROR(gckEVENT_GetEvent(Event, gcvTRUE, &id, gcvKERNEL_PIXEL));
3259 #endif
3260
3261     /* Allocate a record. */
3262     gcmkONERROR(gckEVENT_AllocateRecord(Event, gcvTRUE, &record));
3263
3264     /* Initialize the record. */
3265     record->next = gcvNULL;
3266     record->processID               = ProcessID;
3267     record->info.command            = gcvHAL_SIGNAL;
3268     record->info.u.Signal.signal    = gcmPTR_TO_UINT64(Signal);
3269 #ifdef __QNXNTO__
3270     record->info.u.Signal.coid      = 0;
3271     record->info.u.Signal.rcvid     = 0;
3272 #endif
3273     record->info.u.Signal.auxSignal = 0;
3274     record->info.u.Signal.process   = 0;
3275
3276     /* Append the record. */
3277     Event->queues[id].head      = record;
3278
3279     /* Replace last WAIT with END. */
3280     gcmkONERROR(gckHARDWARE_End(
3281         Event->kernel->hardware, Logical, waitSize
3282         ));
3283
3284 #if gcdNONPAGED_MEMORY_CACHEABLE
3285     /* Flush the cache for the END. */
3286     gcmkONERROR(gckOS_CacheClean(
3287         Event->os,
3288         ProcessID,
3289         gcvNULL,
3290         (gctUINT32)Handle,
3291         Logical,
3292         *waitSize
3293         ));
3294 #endif
3295
3296     /* Wait for the signal. */
3297     gcmkONERROR(gckOS_WaitSignal(Event->os, Signal, gcvINFINITE));
3298
3299     /* Success. */
3300     gcmkFOOTER_NO();
3301     return gcvSTATUS_OK;
3302
3303 OnError:
3304
3305     /* Return the status. */
3306     gcmkFOOTER();
3307     return status;
3308 }
3309
3310 static void
3311 _PrintRecord(
3312     gcsEVENT_PTR record
3313     )
3314 {
3315     switch (record->info.command)
3316     {
3317     case gcvHAL_FREE_NON_PAGED_MEMORY:
3318         gcmkPRINT("      gcvHAL_FREE_NON_PAGED_MEMORY");
3319             break;
3320
3321     case gcvHAL_FREE_CONTIGUOUS_MEMORY:
3322         gcmkPRINT("      gcvHAL_FREE_CONTIGUOUS_MEMORY");
3323             break;
3324
3325     case gcvHAL_WRITE_DATA:
3326         gcmkPRINT("      gcvHAL_WRITE_DATA");
3327        break;
3328
3329     case gcvHAL_UNLOCK_VIDEO_MEMORY:
3330         gcmkPRINT("      gcvHAL_UNLOCK_VIDEO_MEMORY");
3331         break;
3332
3333     case gcvHAL_SIGNAL:
3334         gcmkPRINT("      gcvHAL_SIGNAL process=%d signal=0x%x",
3335                   record->info.u.Signal.process,
3336                   record->info.u.Signal.signal);
3337         break;
3338
3339     case gcvHAL_UNMAP_USER_MEMORY:
3340         gcmkPRINT("      gcvHAL_UNMAP_USER_MEMORY");
3341        break;
3342
3343     case gcvHAL_TIMESTAMP:
3344         gcmkPRINT("      gcvHAL_TIMESTAMP");
3345         break;
3346
3347     case gcvHAL_COMMIT_DONE:
3348         gcmkPRINT("      gcvHAL_COMMIT_DONE");
3349         break;
3350
3351     case gcvHAL_FREE_VIRTUAL_COMMAND_BUFFER:
3352         gcmkPRINT("      gcvHAL_FREE_VIRTUAL_COMMAND_BUFFER logical=0x%08x",
3353                   record->info.u.FreeVirtualCommandBuffer.logical);
3354         break;
3355
3356     case gcvHAL_SYNC_POINT:
3357         gcmkPRINT("      gcvHAL_SYNC_POINT syncPoint=0x%08x",
3358                   gcmUINT64_TO_PTR(record->info.u.SyncPoint.syncPoint));
3359
3360         break;
3361
3362     case gcvHAL_DESTROY_MMU:
3363         gcmkPRINT("      gcvHAL_DESTORY_MMU mmu=0x%08x",
3364                   gcmUINT64_TO_PTR(record->info.u.DestroyMmu.mmu));
3365
3366         break;
3367     default:
3368         gcmkPRINT("      Illegal Event %d", record->info.command);
3369         break;
3370     }
3371 }
3372
3373 /*******************************************************************************
3374 ** gckEVENT_Dump
3375 **
3376 ** Dump record in event queue when stuck happens.
3377 ** No protection for the event queue.
3378 **/
3379 gceSTATUS
3380 gckEVENT_Dump(
3381     IN gckEVENT Event
3382     )
3383 {
3384     gcsEVENT_QUEUE_PTR queueHead = Event->queueHead;
3385     gcsEVENT_QUEUE_PTR queue;
3386     gcsEVENT_PTR record = gcvNULL;
3387     gctINT i;
3388 #if gcdINTERRUPT_STATISTIC
3389     gctINT32 pendingInterrupt;
3390     gctUINT32 intrAcknowledge;
3391 #endif
3392
3393     gcmkHEADER_ARG("Event=0x%x", Event);
3394
3395     gcmkPRINT("**************************\n");
3396     gcmkPRINT("***  EVENT STATE DUMP  ***\n");
3397     gcmkPRINT("**************************\n");
3398
3399     gcmkPRINT("  Unsumbitted Event:");
3400     while(queueHead)
3401     {
3402         queue = queueHead;
3403         record = queueHead->head;
3404
3405         gcmkPRINT("    [%x]:", queue);
3406         while(record)
3407         {
3408             _PrintRecord(record);
3409             record = record->next;
3410         }
3411
3412         if (queueHead == Event->queueTail)
3413         {
3414             queueHead = gcvNULL;
3415         }
3416         else
3417         {
3418             queueHead = queueHead->next;
3419         }
3420     }
3421
3422     gcmkPRINT("  Untriggered Event:");
3423     for (i = 0; i < gcmCOUNTOF(Event->queues); i++)
3424     {
3425         queue = &Event->queues[i];
3426         record = queue->head;
3427
3428         gcmkPRINT("    [%d]:", i);
3429         while(record)
3430         {
3431             _PrintRecord(record);
3432             record = record->next;
3433         }
3434     }
3435
3436 #if gcdINTERRUPT_STATISTIC
3437     gckOS_AtomGet(Event->os, Event->interruptCount, &pendingInterrupt);
3438     gcmkPRINT("  Number of Pending Interrupt: %d", pendingInterrupt);
3439
3440     if (Event->kernel->recovery == 0)
3441     {
3442         gckOS_ReadRegisterEx(
3443             Event->os,
3444             Event->kernel->core,
3445             0x10,
3446             &intrAcknowledge
3447             );
3448
3449         gcmkPRINT("  INTR_ACKNOWLEDGE=0x%x", intrAcknowledge);
3450     }
3451 #endif
3452
3453     gcmkFOOTER_NO();
3454     return gcvSTATUS_OK;
3455 }
3456