drivers/mxc/gpu-viv/hal/kernel/gc_hal_kernel.c

   1 /****************************************************************************
   2 *
   3 *    Copyright (C) 2005 - 2014 by Vivante Corp.
   4 *
   5 *    This program is free software; you can redistribute it and/or modify
   6 *    it under the terms of the GNU General Public License as published by
   7 *    the Free Software Foundation; either version 2 of the license, or
   8 *    (at your option) any later version.
   9 *
  10 *    This program is distributed in the hope that it will be useful,
  11 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13 *    GNU General Public License for more details.
  14 *
  15 *    You should have received a copy of the GNU General Public License
  16 *    along with this program; if not write to the Free Software
  17 *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  18 *
  19 *****************************************************************************/
  20
  21
  22 #include "gc_hal_kernel_precomp.h"
  23
  24 #define _GC_OBJ_ZONE    gcvZONE_KERNEL
  25
  26 /*******************************************************************************
  27 ***** Version Signature *******************************************************/
  28
  29 #define _gcmTXT2STR(t) #t
  30 #define gcmTXT2STR(t) _gcmTXT2STR(t)
  31 const char * _VERSION = "\n\0$VERSION$"
  32                         gcmTXT2STR(gcvVERSION_MAJOR) "."
  33                         gcmTXT2STR(gcvVERSION_MINOR) "."
  34                         gcmTXT2STR(gcvVERSION_PATCH) ":"
  35                         gcmTXT2STR(gcvVERSION_BUILD) "$\n";
  36
  37 /******************************************************************************\
  38 ******************************* gckKERNEL API Code ******************************
  39 \******************************************************************************/
  40
  41 #if gcmIS_DEBUG(gcdDEBUG_TRACE)
  42 #define gcmDEFINE2TEXT(d) #d
  43 gctCONST_STRING _DispatchText[] =
  44 {
  45     gcmDEFINE2TEXT(gcvHAL_QUERY_VIDEO_MEMORY),
  46     gcmDEFINE2TEXT(gcvHAL_QUERY_CHIP_IDENTITY),
  47     gcmDEFINE2TEXT(gcvHAL_ALLOCATE_NON_PAGED_MEMORY),
  48     gcmDEFINE2TEXT(gcvHAL_FREE_NON_PAGED_MEMORY),
  49     gcmDEFINE2TEXT(gcvHAL_ALLOCATE_CONTIGUOUS_MEMORY),
  50     gcmDEFINE2TEXT(gcvHAL_FREE_CONTIGUOUS_MEMORY),
  51     gcmDEFINE2TEXT(gcvHAL_ALLOCATE_VIDEO_MEMORY),
  52     gcmDEFINE2TEXT(gcvHAL_ALLOCATE_LINEAR_VIDEO_MEMORY),
  53     gcmDEFINE2TEXT(gcvHAL_RELEASE_VIDEO_MEMORY),
  54     gcmDEFINE2TEXT(gcvHAL_MAP_MEMORY),
  55     gcmDEFINE2TEXT(gcvHAL_UNMAP_MEMORY),
  56     gcmDEFINE2TEXT(gcvHAL_MAP_USER_MEMORY),
  57     gcmDEFINE2TEXT(gcvHAL_UNMAP_USER_MEMORY),
  58     gcmDEFINE2TEXT(gcvHAL_LOCK_VIDEO_MEMORY),
  59     gcmDEFINE2TEXT(gcvHAL_UNLOCK_VIDEO_MEMORY),
  60     gcmDEFINE2TEXT(gcvHAL_EVENT_COMMIT),
  61     gcmDEFINE2TEXT(gcvHAL_USER_SIGNAL),
  62     gcmDEFINE2TEXT(gcvHAL_SIGNAL),
  63     gcmDEFINE2TEXT(gcvHAL_WRITE_DATA),
  64     gcmDEFINE2TEXT(gcvHAL_COMMIT),
  65     gcmDEFINE2TEXT(gcvHAL_STALL),
  66     gcmDEFINE2TEXT(gcvHAL_READ_REGISTER),
  67     gcmDEFINE2TEXT(gcvHAL_WRITE_REGISTER),
  68     gcmDEFINE2TEXT(gcvHAL_GET_PROFILE_SETTING),
  69     gcmDEFINE2TEXT(gcvHAL_SET_PROFILE_SETTING),
  70     gcmDEFINE2TEXT(gcvHAL_READ_ALL_PROFILE_REGISTERS),
  71     gcmDEFINE2TEXT(gcvHAL_PROFILE_REGISTERS_2D),
  72 #if VIVANTE_PROFILER_PERDRAW
  73     gcvHAL_READ_PROFILER_REGISTER_SETTING,
  74 #endif
  75     gcmDEFINE2TEXT(gcvHAL_SET_POWER_MANAGEMENT_STATE),
  76     gcmDEFINE2TEXT(gcvHAL_QUERY_POWER_MANAGEMENT_STATE),
  77     gcmDEFINE2TEXT(gcvHAL_GET_BASE_ADDRESS),
  78     gcmDEFINE2TEXT(gcvHAL_SET_IDLE),
  79     gcmDEFINE2TEXT(gcvHAL_QUERY_KERNEL_SETTINGS),
  80     gcmDEFINE2TEXT(gcvHAL_RESET),
  81     gcmDEFINE2TEXT(gcvHAL_MAP_PHYSICAL),
  82     gcmDEFINE2TEXT(gcvHAL_DEBUG),
  83     gcmDEFINE2TEXT(gcvHAL_CACHE),
  84     gcmDEFINE2TEXT(gcvHAL_TIMESTAMP),
  85     gcmDEFINE2TEXT(gcvHAL_DATABASE),
  86     gcmDEFINE2TEXT(gcvHAL_VERSION),
  87     gcmDEFINE2TEXT(gcvHAL_CHIP_INFO),
  88     gcmDEFINE2TEXT(gcvHAL_ATTACH),
  89     gcmDEFINE2TEXT(gcvHAL_DETACH),
  90     gcmDEFINE2TEXT(gcvHAL_COMPOSE),
  91     gcmDEFINE2TEXT(gcvHAL_SET_TIMEOUT),
  92     gcmDEFINE2TEXT(gcvHAL_GET_FRAME_INFO),
  93     gcmDEFINE2TEXT(gcvHAL_QUERY_COMMAND_BUFFER),
  94     gcmDEFINE2TEXT(gcvHAL_COMMIT_DONE),
  95     gcmDEFINE2TEXT(gcvHAL_DUMP_GPU_STATE),
  96     gcmDEFINE2TEXT(gcvHAL_DUMP_EVENT),
  97     gcmDEFINE2TEXT(gcvHAL_ALLOCATE_VIRTUAL_COMMAND_BUFFER),
  98     gcmDEFINE2TEXT(gcvHAL_FREE_VIRTUAL_COMMAND_BUFFER),
  99     gcmDEFINE2TEXT(gcvHAL_SET_FSCALE_VALUE),
 100     gcmDEFINE2TEXT(gcvHAL_GET_FSCALE_VALUE),
 101     gcmDEFINE2TEXT(gcvHAL_NAME_VIDEO_MEMORY),
 102     gcmDEFINE2TEXT(gcvHAL_IMPORT_VIDEO_MEMORY),
 103     gcmDEFINE2TEXT(gcvHAL_QUERY_RESET_TIME_STAMP),
 104     gcmDEFINE2TEXT(gcvHAL_READ_REGISTER_EX),
 105     gcmDEFINE2TEXT(gcvHAL_WRITE_REGISTER_EX),
 106     gcmDEFINE2TEXT(gcvHAL_SYNC_POINT),
 107     gcmDEFINE2TEXT(gcvHAL_CREATE_NATIVE_FENCE),
 108     gcmDEFINE2TEXT(gcvHAL_DESTROY_MMU),
 109     gcmDEFINE2TEXT(gcvHAL_SHBUF),
 110 };
 111 #endif
 112
 113 #if gcdGPU_TIMEOUT && gcdINTERRUPT_STATISTIC
 114 void
 115 _MonitorTimerFunction(
 116     gctPOINTER Data
 117     )
 118 {
 119     gckKERNEL kernel = (gckKERNEL)Data;
 120     gctUINT32 pendingInterrupt;
 121     gctBOOL reset = gcvFALSE;
 122     gctUINT32 mask;
 123     gctUINT32 advance = kernel->timeOut/2;
 124
 125 #if gcdENABLE_VG
 126     if (kernel->core == gcvCORE_VG)
 127     {
 128         return;
 129     }
 130 #endif
 131
 132     if (kernel->monitorTimerStop)
 133     {
 134         /* Stop. */
 135         return;
 136     }
 137
 138     gckOS_AtomGet(kernel->os, kernel->eventObj->interruptCount, &pendingInterrupt);
 139
 140     if (kernel->monitoring == gcvFALSE)
 141     {
 142         if (pendingInterrupt)
 143         {
 144             /* Begin to mointor GPU state. */
 145             kernel->monitoring = gcvTRUE;
 146
 147             /* Record current state. */
 148             kernel->lastCommitStamp = kernel->eventObj->lastCommitStamp;
 149             kernel->restoreAddress  = kernel->hardware->lastWaitLink;
 150             gcmkVERIFY_OK(gckOS_AtomGet(
 151                 kernel->os,
 152                 kernel->hardware->pendingEvent,
 153                 &kernel->restoreMask
 154                 ));
 155
 156             /* Clear timeout. */
 157             kernel->timer = 0;
 158         }
 159     }
 160     else
 161     {
 162         if (pendingInterrupt)
 163         {
 164             gcmkVERIFY_OK(gckOS_AtomGet(
 165                 kernel->os,
 166                 kernel->hardware->pendingEvent,
 167                 &mask
 168                 ));
 169
 170             if (kernel->eventObj->lastCommitStamp == kernel->lastCommitStamp
 171              && kernel->hardware->lastWaitLink    == kernel->restoreAddress
 172              && mask                              == kernel->restoreMask
 173             )
 174             {
 175                 /* GPU state is not changed, accumlate timeout. */
 176                 kernel->timer += advance;
 177
 178                 if (kernel->timer >= kernel->timeOut)
 179                 {
 180                     /* GPU stuck, trigger reset. */
 181                     reset = gcvTRUE;
 182                 }
 183             }
 184             else
 185             {
 186                 /* GPU state changed, cancel current timeout.*/
 187                 kernel->monitoring = gcvFALSE;
 188             }
 189         }
 190         else
 191         {
 192             /* GPU finish all jobs, cancel current timeout*/
 193             kernel->monitoring = gcvFALSE;
 194         }
 195     }
 196
 197     if (reset)
 198     {
 199         gckKERNEL_Recovery(kernel);
 200
 201         /* Work in this timeout is done. */
 202         kernel->monitoring = gcvFALSE;
 203     }
 204
 205     gcmkVERIFY_OK(gckOS_StartTimer(kernel->os, kernel->monitorTimer, advance));
 206 }
 207 #endif
 208
 209 #if gcdPROCESS_ADDRESS_SPACE
 210 gceSTATUS
 211 _MapCommandBuffer(
 212     IN gckKERNEL Kernel
 213     )
 214 {
 215     gceSTATUS status;
 216     gctUINT32 i;
 217     gctUINT32 physical;
 218     gckMMU mmu;
 219
 220     gcmkONERROR(gckKERNEL_GetProcessMMU(Kernel, &mmu));
 221
 222     for (i = 0; i < gcdCOMMAND_QUEUES; i++)
 223     {
 224         gcmkONERROR(gckOS_GetPhysicalAddress(
 225             Kernel->os,
 226             Kernel->command->queues[i].logical,
 227             &physical
 228             ));
 229
 230         gcmkONERROR(gckMMU_FlatMapping(mmu, physical));
 231     }
 232
 233     return gcvSTATUS_OK;
 234
 235 OnError:
 236     return status;
 237 }
 238 #endif
 239
 240 void
 241 _DumpDriverConfigure(
 242     IN gckKERNEL Kernel
 243     )
 244 {
 245     gcmkPRINT_N(0, "**************************\n");
 246     gcmkPRINT_N(0, "***   GPU DRV CONFIG   ***\n");
 247     gcmkPRINT_N(0, "**************************\n");
 248
 249     gcmkPRINT("Galcore version %d.%d.%d.%d\n",
 250               gcvVERSION_MAJOR, gcvVERSION_MINOR, gcvVERSION_PATCH, gcvVERSION_BUILD);
 251
 252     gckOS_DumpParam();
 253 }
 254
 255 void
 256 _DumpState(
 257     IN gckKERNEL Kernel
 258     )
 259 {
 260     /* Dump GPU Debug registers. */
 261     gcmkVERIFY_OK(gckHARDWARE_DumpGPUState(Kernel->hardware));
 262
 263     if (Kernel->virtualCommandBuffer)
 264     {
 265         gcmkVERIFY_OK(gckCOMMAND_DumpExecutingBuffer(Kernel->command));
 266     }
 267
 268     /* Dump Pending event. */
 269     gcmkVERIFY_OK(gckEVENT_Dump(Kernel->eventObj));
 270
 271     /* Dump Process DB. */
 272     gcmkVERIFY_OK(gckKERNEL_DumpProcessDB(Kernel));
 273
 274 #if gcdRECORD_COMMAND
 275     /* Dump record. */
 276     gckRECORDER_Dump(Kernel->command->recorder);
 277 #endif
 278 }
 279
 280 /*******************************************************************************
 281 **
 282 **  gckKERNEL_Construct
 283 **
 284 **  Construct a new gckKERNEL object.
 285 **
 286 **  INPUT:
 287 **
 288 **      gckOS Os
 289 **          Pointer to an gckOS object.
 290 **
 291 **      gceCORE Core
 292 **          Specified core.
 293 **
 294 **      IN gctPOINTER Context
 295 **          Pointer to a driver defined context.
 296 **
 297 **      IN gckDB SharedDB,
 298 **          Pointer to a shared DB.
 299 **
 300 **  OUTPUT:
 301 **
 302 **      gckKERNEL * Kernel
 303 **          Pointer to a variable that will hold the pointer to the gckKERNEL
 304 **          object.
 305 */
 306
 307 gceSTATUS
 308 gckKERNEL_Construct(
 309     IN gckOS Os,
 310     IN gceCORE Core,
 311     IN gctPOINTER Context,
 312     IN gckDB SharedDB,
 313     OUT gckKERNEL * Kernel
 314     )
 315 {
 316     gckKERNEL kernel = gcvNULL;
 317     gceSTATUS status;
 318     gctSIZE_T i;
 319     gctPOINTER pointer = gcvNULL;
 320
 321     gcmkHEADER_ARG("Os=0x%x Context=0x%x", Os, Context);
 322
 323     /* Verify the arguments. */
 324     gcmkVERIFY_OBJECT(Os, gcvOBJ_OS);
 325     gcmkVERIFY_ARGUMENT(Kernel != gcvNULL);
 326
 327     /* Allocate the gckKERNEL object. */
 328     gcmkONERROR(gckOS_Allocate(Os,
 329                                gcmSIZEOF(struct _gckKERNEL),
 330                                &pointer));
 331
 332     kernel = pointer;
 333
 334     /* Zero the object pointers. */
 335     kernel->hardware     = gcvNULL;
 336     kernel->command      = gcvNULL;
 337     kernel->eventObj     = gcvNULL;
 338     kernel->mmu          = gcvNULL;
 339 #if gcdDVFS
 340     kernel->dvfs         = gcvNULL;
 341 #endif
 342     kernel->monitorTimer = gcvNULL;
 343
 344     /* Initialize the gckKERNEL object. */
 345     kernel->object.type = gcvOBJ_KERNEL;
 346     kernel->os          = Os;
 347     kernel->core        = Core;
 348
 349     if (SharedDB == gcvNULL)
 350     {
 351         gcmkONERROR(gckOS_Allocate(Os,
 352                                    gcmSIZEOF(struct _gckDB),
 353                                    &pointer));
 354
 355         kernel->db               = pointer;
 356         kernel->dbCreated        = gcvTRUE;
 357         kernel->db->freeDatabase = gcvNULL;
 358         kernel->db->freeRecord   = gcvNULL;
 359         kernel->db->dbMutex      = gcvNULL;
 360         kernel->db->lastDatabase = gcvNULL;
 361         kernel->db->idleTime     = 0;
 362         kernel->db->lastIdle     = 0;
 363         kernel->db->lastSlowdown = 0;
 364
 365         for (i = 0; i < gcmCOUNTOF(kernel->db->db); ++i)
 366         {
 367             kernel->db->db[i] = gcvNULL;
 368         }
 369
 370         /* Construct a database mutex. */
 371         gcmkONERROR(gckOS_CreateMutex(Os, &kernel->db->dbMutex));
 372
 373         /* Construct a video memory name database. */
 374         gcmkONERROR(gckKERNEL_CreateIntegerDatabase(kernel, &kernel->db->nameDatabase));
 375
 376         /* Construct a video memory name database mutex. */
 377         gcmkONERROR(gckOS_CreateMutex(Os, &kernel->db->nameDatabaseMutex));
 378
 379         /* Construct a pointer name database. */
 380         gcmkONERROR(gckKERNEL_CreateIntegerDatabase(kernel, &kernel->db->pointerDatabase));
 381
 382         /* Construct a pointer name database mutex. */
 383         gcmkONERROR(gckOS_CreateMutex(Os, &kernel->db->pointerDatabaseMutex));
 384     }
 385     else
 386     {
 387         kernel->db               = SharedDB;
 388         kernel->dbCreated        = gcvFALSE;
 389     }
 390
 391     for (i = 0; i < gcmCOUNTOF(kernel->timers); ++i)
 392     {
 393         kernel->timers[i].startTime = 0;
 394         kernel->timers[i].stopTime = 0;
 395     }
 396
 397     /* Save context. */
 398     kernel->context = Context;
 399
 400     /* Construct atom holding number of clients. */
 401     kernel->atomClients = gcvNULL;
 402     gcmkONERROR(gckOS_AtomConstruct(Os, &kernel->atomClients));
 403
 404 #if gcdENABLE_VG
 405     kernel->vg = gcvNULL;
 406
 407     if (Core == gcvCORE_VG)
 408     {
 409         /* Construct the gckMMU object. */
 410         gcmkONERROR(
 411             gckVGKERNEL_Construct(Os, Context, kernel, &kernel->vg));
 412
 413         kernel->timeOut = gcdGPU_TIMEOUT;
 414     }
 415     else
 416 #endif
 417     {
 418         /* Construct the gckHARDWARE object. */
 419         gcmkONERROR(
 420             gckHARDWARE_Construct(Os, kernel->core, &kernel->hardware));
 421
 422         /* Set pointer to gckKERNEL object in gckHARDWARE object. */
 423         kernel->hardware->kernel = kernel;
 424
 425         kernel->timeOut = kernel->hardware->type == gcvHARDWARE_2D
 426                         ? gcdGPU_2D_TIMEOUT
 427                         : gcdGPU_TIMEOUT
 428                         ;
 429
 430         /* Initialize virtual command buffer. */
 431         /* TODO: Remove platform limitation after porting. */
 432 #if (defined(LINUX) || defined(__QNXNTO__))
 433         kernel->virtualCommandBuffer = gcvTRUE;
 434 #else
 435         kernel->virtualCommandBuffer = gcvFALSE;
 436 #endif
 437
 438 #if gcdSECURITY
 439         kernel->virtualCommandBuffer = gcvFALSE;
 440 #endif
 441
 442         /* Construct the gckCOMMAND object. */
 443         gcmkONERROR(
 444             gckCOMMAND_Construct(kernel, &kernel->command));
 445
 446         /* Construct the gckEVENT object. */
 447         gcmkONERROR(
 448             gckEVENT_Construct(kernel, &kernel->eventObj));
 449
 450         /* Construct the gckMMU object. */
 451         gcmkONERROR(
 452             gckMMU_Construct(kernel, gcdMMU_SIZE, &kernel->mmu));
 453
 454         gcmkVERIFY_OK(gckOS_GetTime(&kernel->resetTimeStamp));
 455
 456         gcmkONERROR(gckHARDWARE_PrepareFunctions(kernel->hardware));
 457
 458         /* Initialize the hardware. */
 459         gcmkONERROR(
 460             gckHARDWARE_InitializeHardware(kernel->hardware));
 461
 462 #if gcdDVFS
 463         if (gckHARDWARE_IsFeatureAvailable(kernel->hardware,
 464                                            gcvFEATURE_DYNAMIC_FREQUENCY_SCALING))
 465         {
 466             gcmkONERROR(gckDVFS_Construct(kernel->hardware, &kernel->dvfs));
 467             gcmkONERROR(gckDVFS_Start(kernel->dvfs));
 468         }
 469 #endif
 470     }
 471
 472 #if VIVANTE_PROFILER
 473     /* Initialize profile setting */
 474     kernel->profileEnable = gcvFALSE;
 475     kernel->profileCleanRegister = gcvTRUE;
 476 #endif
 477
 478 #if gcdANDROID_NATIVE_FENCE_SYNC
 479     gcmkONERROR(gckOS_CreateSyncTimeline(Os, &kernel->timeline));
 480 #endif
 481
 482     kernel->recovery      = gcvTRUE;
 483     kernel->stuckDump     = 1;
 484
 485     kernel->virtualBufferHead =
 486     kernel->virtualBufferTail = gcvNULL;
 487
 488     gcmkONERROR(
 489         gckOS_CreateMutex(Os, (gctPOINTER)&kernel->virtualBufferLock));
 490
 491 #if gcdSECURITY
 492     /* Connect to security service for this GPU. */
 493     gcmkONERROR(gckKERNEL_SecurityOpen(kernel, kernel->core, &kernel->securityChannel));
 494 #endif
 495
 496 #if gcdGPU_TIMEOUT && gcdINTERRUPT_STATISTIC
 497     if (kernel->timeOut)
 498     {
 499         gcmkVERIFY_OK(gckOS_CreateTimer(
 500             Os,
 501             (gctTIMERFUNCTION)_MonitorTimerFunction,
 502             (gctPOINTER)kernel,
 503             &kernel->monitorTimer
 504             ));
 505
 506         kernel->monitoring  = gcvFALSE;
 507
 508         kernel->monitorTimerStop = gcvFALSE;
 509
 510         gcmkVERIFY_OK(gckOS_StartTimer(
 511             Os,
 512             kernel->monitorTimer,
 513             100
 514             ));
 515     }
 516 #endif
 517
 518     /* Return pointer to the gckKERNEL object. */
 519     *Kernel = kernel;
 520
 521     /* Success. */
 522     gcmkFOOTER_ARG("*Kernel=0x%x", *Kernel);
 523     return gcvSTATUS_OK;
 524
 525 OnError:
 526     if (kernel != gcvNULL)
 527     {
 528 #if gcdENABLE_VG
 529         if (Core != gcvCORE_VG)
 530 #endif
 531         {
 532             if (kernel->eventObj != gcvNULL)
 533             {
 534                 gcmkVERIFY_OK(gckEVENT_Destroy(kernel->eventObj));
 535             }
 536
 537             if (kernel->command != gcvNULL)
 538             {
 539             gcmkVERIFY_OK(gckCOMMAND_Destroy(kernel->command));
 540             }
 541
 542             if (kernel->hardware != gcvNULL)
 543             {
 544                 /* Turn off the power. */
 545                 gcmkVERIFY_OK(gckOS_SetGPUPower(kernel->hardware->os,
 546                                                 kernel->hardware->core,
 547                                                 gcvFALSE,
 548                                                 gcvFALSE));
 549                 gcmkVERIFY_OK(gckHARDWARE_Destroy(kernel->hardware));
 550             }
 551         }
 552
 553         if (kernel->atomClients != gcvNULL)
 554         {
 555             gcmkVERIFY_OK(gckOS_AtomDestroy(Os, kernel->atomClients));
 556         }
 557
 558         if (kernel->dbCreated && kernel->db != gcvNULL)
 559         {
 560             if (kernel->db->dbMutex != gcvNULL)
 561             {
 562                 /* Destroy the database mutex. */
 563                 gcmkVERIFY_OK(gckOS_DeleteMutex(Os, kernel->db->dbMutex));
 564             }
 565
 566             gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Os, kernel->db));
 567         }
 568
 569         if (kernel->virtualBufferLock != gcvNULL)
 570         {
 571             /* Destroy the virtual command buffer mutex. */
 572             gcmkVERIFY_OK(gckOS_DeleteMutex(Os, kernel->virtualBufferLock));
 573         }
 574
 575 #if gcdDVFS
 576         if (kernel->dvfs)
 577         {
 578             gcmkVERIFY_OK(gckDVFS_Stop(kernel->dvfs));
 579             gcmkVERIFY_OK(gckDVFS_Destroy(kernel->dvfs));
 580         }
 581 #endif
 582
 583 #if gcdANDROID_NATIVE_FENCE_SYNC
 584         if (kernel->timeline)
 585         {
 586             gcmkVERIFY_OK(gckOS_DestroySyncTimeline(Os, kernel->timeline));
 587         }
 588 #endif
 589
 590         if (kernel->monitorTimer)
 591         {
 592             gcmkVERIFY_OK(gckOS_StopTimer(Os, kernel->monitorTimer));
 593             gcmkVERIFY_OK(gckOS_DestroyTimer(Os, kernel->monitorTimer));
 594         }
 595
 596         gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Os, kernel));
 597     }
 598
 599     /* Return the error. */
 600     gcmkFOOTER();
 601     return status;
 602 }
 603
 604 /*******************************************************************************
 605 **
 606 **  gckKERNEL_Destroy
 607 **
 608 **  Destroy an gckKERNEL object.
 609 **
 610 **  INPUT:
 611 **
 612 **      gckKERNEL Kernel
 613 **          Pointer to an gckKERNEL object to destroy.
 614 **
 615 **  OUTPUT:
 616 **
 617 **      Nothing.
 618 */
 619 gceSTATUS
 620 gckKERNEL_Destroy(
 621     IN gckKERNEL Kernel
 622     )
 623 {
 624     gctSIZE_T i;
 625     gcsDATABASE_PTR database, databaseNext;
 626     gcsDATABASE_RECORD_PTR record, recordNext;
 627
 628     gcmkHEADER_ARG("Kernel=0x%x", Kernel);
 629
 630     /* Verify the arguments. */
 631     gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL);
 632 #if QNX_SINGLE_THREADED_DEBUGGING
 633     gcmkVERIFY_OK(gckOS_DeleteMutex(Kernel->os, Kernel->debugMutex));
 634 #endif
 635
 636     /* Destroy the database. */
 637     if (Kernel->dbCreated)
 638     {
 639         for (i = 0; i < gcmCOUNTOF(Kernel->db->db); ++i)
 640         {
 641             if (Kernel->db->db[i] != gcvNULL)
 642             {
 643                 gcmkVERIFY_OK(
 644                     gckKERNEL_DestroyProcessDB(Kernel, Kernel->db->db[i]->processID));
 645             }
 646         }
 647
 648         /* Free all databases. */
 649         for (database = Kernel->db->freeDatabase;
 650              database != gcvNULL;
 651              database = databaseNext)
 652         {
 653             databaseNext = database->next;
 654
 655             gcmkVERIFY_OK(gckOS_DeleteMutex(Kernel->os, database->counterMutex));
 656             gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Kernel->os, database));
 657         }
 658
 659         if (Kernel->db->lastDatabase != gcvNULL)
 660         {
 661             gcmkVERIFY_OK(gckOS_DeleteMutex(Kernel->os, Kernel->db->lastDatabase->counterMutex));
 662             gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Kernel->os, Kernel->db->lastDatabase));
 663         }
 664
 665         /* Free all database records. */
 666         for (record = Kernel->db->freeRecord; record != gcvNULL; record = recordNext)
 667         {
 668             recordNext = record->next;
 669             gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Kernel->os, record));
 670         }
 671
 672         /* Destroy the database mutex. */
 673         gcmkVERIFY_OK(gckOS_DeleteMutex(Kernel->os, Kernel->db->dbMutex));
 674
 675         /* Destroy video memory name database. */
 676         gcmkVERIFY_OK(gckKERNEL_DestroyIntegerDatabase(Kernel, Kernel->db->nameDatabase));
 677
 678         /* Destroy video memory name database mutex. */
 679         gcmkVERIFY_OK(gckOS_DeleteMutex(Kernel->os, Kernel->db->nameDatabaseMutex));
 680
 681
 682         /* Destroy id-pointer database. */
 683         gcmkVERIFY_OK(gckKERNEL_DestroyIntegerDatabase(Kernel, Kernel->db->pointerDatabase));
 684
 685         /* Destroy id-pointer database mutex. */
 686         gcmkVERIFY_OK(gckOS_DeleteMutex(Kernel->os, Kernel->db->pointerDatabaseMutex));
 687
 688         /* Destroy the database. */
 689         gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Kernel->os, Kernel->db));
 690
 691         /* Notify stuck timer to quit. */
 692         Kernel->monitorTimerStop = gcvTRUE;
 693     }
 694
 695 #if gcdENABLE_VG
 696     if (Kernel->vg)
 697     {
 698         gcmkVERIFY_OK(gckVGKERNEL_Destroy(Kernel->vg));
 699     }
 700     else
 701 #endif
 702     {
 703         /* Destroy the gckMMU object. */
 704         gcmkVERIFY_OK(gckMMU_Destroy(Kernel->mmu));
 705
 706         /* Destroy the gckCOMMNAND object. */
 707         gcmkVERIFY_OK(gckCOMMAND_Destroy(Kernel->command));
 708
 709         /* Destroy the gckEVENT object. */
 710         gcmkVERIFY_OK(gckEVENT_Destroy(Kernel->eventObj));
 711
 712         /* Destroy the gckHARDWARE object. */
 713         gcmkVERIFY_OK(gckHARDWARE_Destroy(Kernel->hardware));
 714     }
 715
 716     /* Detsroy the client atom. */
 717     gcmkVERIFY_OK(gckOS_AtomDestroy(Kernel->os, Kernel->atomClients));
 718
 719     gcmkVERIFY_OK(gckOS_DeleteMutex(Kernel->os, Kernel->virtualBufferLock));
 720
 721 #if gcdDVFS
 722     if (Kernel->dvfs)
 723     {
 724         gcmkVERIFY_OK(gckDVFS_Stop(Kernel->dvfs));
 725         gcmkVERIFY_OK(gckDVFS_Destroy(Kernel->dvfs));
 726     }
 727 #endif
 728
 729 #if gcdANDROID_NATIVE_FENCE_SYNC
 730     gcmkVERIFY_OK(gckOS_DestroySyncTimeline(Kernel->os, Kernel->timeline));
 731 #endif
 732
 733 #if gcdSECURITY
 734     gcmkVERIFY_OK(gckKERNEL_SecurityClose(Kernel->securityChannel));
 735 #endif
 736
 737     if (Kernel->monitorTimer)
 738     {
 739         gcmkVERIFY_OK(gckOS_StopTimer(Kernel->os, Kernel->monitorTimer));
 740         gcmkVERIFY_OK(gckOS_DestroyTimer(Kernel->os, Kernel->monitorTimer));
 741     }
 742
 743     /* Mark the gckKERNEL object as unknown. */
 744     Kernel->object.type = gcvOBJ_UNKNOWN;
 745
 746     /* Free the gckKERNEL object. */
 747     gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Kernel->os, Kernel));
 748
 749     /* Success. */
 750     gcmkFOOTER_NO();
 751     return gcvSTATUS_OK;
 752 }
 753
 754 /*******************************************************************************
 755 **
 756 **  _AllocateMemory
 757 **
 758 **  Private function to walk all required memory pools to allocate the requested
 759 **  amount of video memory.
 760 **
 761 **  INPUT:
 762 **
 763 **      gckKERNEL Kernel
 764 **          Pointer to an gckKERNEL object.
 765 **
 766 **      gcsHAL_INTERFACE * Interface
 767 **          Pointer to a gcsHAL_INTERFACE structure that defines the command to
 768 **          be dispatched.
 769 **
 770 **  OUTPUT:
 771 **
 772 **      gcsHAL_INTERFACE * Interface
 773 **          Pointer to a gcsHAL_INTERFACE structure that receives any data to be
 774 **          returned.
 775 */
 776 gceSTATUS
 777 gckKERNEL_AllocateLinearMemory(
 778     IN gckKERNEL Kernel,
 779     IN gctUINT32 ProcessID,
 780     IN OUT gcePOOL * Pool,
 781     IN gctSIZE_T Bytes,
 782     IN gctUINT32 Alignment,
 783     IN gceSURF_TYPE Type,
 784     IN gctUINT32 Flag,
 785     OUT gctUINT32 * Node
 786     )
 787 {
 788     gcePOOL pool;
 789     gceSTATUS status;
 790     gckVIDMEM videoMemory;
 791     gctINT loopCount;
 792     gcuVIDMEM_NODE_PTR node = gcvNULL;
 793     gctBOOL tileStatusInVirtual;
 794     gctBOOL contiguous = gcvFALSE;
 795     gctBOOL cacheable = gcvFALSE;
 796     gctSIZE_T bytes = Bytes;
 797     gctUINT32 handle = 0;
 798     gceDATABASE_TYPE type;
 799
 800     gcmkHEADER_ARG("Kernel=0x%x *Pool=%d Bytes=%lu Alignment=%lu Type=%d",
 801                    Kernel, *Pool, Bytes, Alignment, Type);
 802
 803     gcmkVERIFY_ARGUMENT(Pool != gcvNULL);
 804     gcmkVERIFY_ARGUMENT(Bytes != 0);
 805
 806      /* Get basic type. */
 807      Type &= 0xFF;
 808
 809     /* Check flags. */
 810     contiguous = Flag & gcvALLOC_FLAG_CONTIGUOUS;
 811     cacheable  = Flag & gcvALLOC_FLAG_CACHEABLE;
 812
 813 AllocateMemory:
 814
 815     /* Get initial pool. */
 816     switch (pool = *Pool)
 817     {
 818     case gcvPOOL_DEFAULT:
 819     case gcvPOOL_LOCAL:
 820         pool      = gcvPOOL_LOCAL_INTERNAL;
 821         loopCount = (gctINT) gcvPOOL_NUMBER_OF_POOLS;
 822         break;
 823
 824     case gcvPOOL_UNIFIED:
 825         pool      = gcvPOOL_SYSTEM;
 826         loopCount = (gctINT) gcvPOOL_NUMBER_OF_POOLS;
 827         break;
 828
 829     case gcvPOOL_CONTIGUOUS:
 830         loopCount = (gctINT) gcvPOOL_NUMBER_OF_POOLS;
 831         break;
 832
 833     default:
 834         loopCount = 1;
 835         break;
 836     }
 837
 838     while (loopCount-- > 0)
 839     {
 840         if (pool == gcvPOOL_VIRTUAL)
 841         {
 842             /* Create a gcuVIDMEM_NODE for virtual memory. */
 843             gcmkONERROR(
 844                 gckVIDMEM_ConstructVirtual(Kernel, Flag | gcvALLOC_FLAG_NON_CONTIGUOUS, Bytes, &node));
 845
 846             bytes = node->Virtual.bytes;
 847             node->Virtual.type = Type;
 848
 849             /* Success. */
 850             break;
 851         }
 852
 853         else
 854         if (pool == gcvPOOL_CONTIGUOUS)
 855         {
 856 #if gcdCONTIGUOUS_SIZE_LIMIT
 857             if (Bytes > gcdCONTIGUOUS_SIZE_LIMIT && contiguous == gcvFALSE)
 858             {
 859                 status = gcvSTATUS_OUT_OF_MEMORY;
 860             }
 861             else
 862 #endif
 863             {
 864                 /* Create a gcuVIDMEM_NODE from contiguous memory. */
 865                 status = gckVIDMEM_ConstructVirtual(
 866                             Kernel,
 867                             Flag | gcvALLOC_FLAG_CONTIGUOUS,
 868                             Bytes,
 869                             &node);
 870             }
 871
 872             if (gcmIS_SUCCESS(status))
 873             {
 874                 bytes = node->Virtual.bytes;
 875                 node->Virtual.type = Type;
 876
 877                 /* Memory allocated. */
 878                 break;
 879             }
 880         }
 881
 882         else
 883         /* gcvPOOL_SYSTEM can't be cacheable. */
 884         if (cacheable == gcvFALSE)
 885         {
 886             /* Get pointer to gckVIDMEM object for pool. */
 887             status = gckKERNEL_GetVideoMemoryPool(Kernel, pool, &videoMemory);
 888
 889             if (gcmIS_SUCCESS(status))
 890             {
 891                 /* Allocate memory. */
 892 #if defined(gcdLINEAR_SIZE_LIMIT)
 893                 /* 512 KB */
 894                 if (Bytes > gcdLINEAR_SIZE_LIMIT)
 895                 {
 896                     status = gcvSTATUS_OUT_OF_MEMORY;
 897                 }
 898                 else
 899 #endif
 900                 {
 901                     status = gckVIDMEM_AllocateLinear(Kernel,
 902                                                       videoMemory,
 903                                                       Bytes,
 904                                                       Alignment,
 905                                                       Type,
 906                                                       (*Pool == gcvPOOL_SYSTEM),
 907                                                       &node);
 908                 }
 909
 910                 if (gcmIS_SUCCESS(status))
 911                 {
 912                     /* Memory allocated. */
 913                     node->VidMem.pool = pool;
 914                     bytes = node->VidMem.bytes;
 915                     break;
 916                 }
 917             }
 918         }
 919
 920         if (pool == gcvPOOL_LOCAL_INTERNAL)
 921         {
 922             /* Advance to external memory. */
 923             pool = gcvPOOL_LOCAL_EXTERNAL;
 924         }
 925
 926         else
 927         if (pool == gcvPOOL_LOCAL_EXTERNAL)
 928         {
 929             /* Advance to contiguous system memory. */
 930             pool = gcvPOOL_SYSTEM;
 931         }
 932
 933         else
 934         if (pool == gcvPOOL_SYSTEM)
 935         {
 936             /* Advance to contiguous memory. */
 937             pool = gcvPOOL_CONTIGUOUS;
 938         }
 939
 940         else
 941         if (pool == gcvPOOL_CONTIGUOUS)
 942         {
 943 #if gcdENABLE_VG
 944             if (Kernel->vg)
 945             {
 946                 tileStatusInVirtual = gcvFALSE;
 947             }
 948             else
 949 #endif
 950             {
 951                 tileStatusInVirtual =
 952                     gckHARDWARE_IsFeatureAvailable(Kernel->hardware,
 953                                                    gcvFEATURE_MC20);
 954             }
 955
 956             if (Type == gcvSURF_TILE_STATUS && tileStatusInVirtual != gcvTRUE)
 957             {
 958                 gcmkONERROR(gcvSTATUS_OUT_OF_MEMORY);
 959             }
 960
 961             if (contiguous)
 962             {
 963                 break;
 964             }
 965
 966             /* Advance to virtual memory. */
 967             pool = gcvPOOL_VIRTUAL;
 968         }
 969
 970         else
 971         {
 972             /* Out of pools. */
 973             gcmkONERROR(gcvSTATUS_OUT_OF_MEMORY);
 974         }
 975     }
 976
 977     if (node == gcvNULL)
 978     {
 979         if (contiguous)
 980         {
 981             /* Broadcast OOM message. */
 982             status = gckOS_Broadcast(Kernel->os, Kernel->hardware, gcvBROADCAST_OUT_OF_MEMORY);
 983
 984             if (gcmIS_SUCCESS(status))
 985             {
 986                 /* Get some memory. */
 987                 gckOS_Delay(gcvNULL, 1);
 988                 goto AllocateMemory;
 989             }
 990         }
 991
 992         /* Nothing allocated. */
 993         gcmkONERROR(gcvSTATUS_OUT_OF_MEMORY);
 994     }
 995
 996     /* Allocate handle for this video memory. */
 997     gcmkONERROR(
 998         gckVIDMEM_NODE_Allocate(Kernel, node, Type, pool, &handle));
 999
1000     /* Return node and pool used for allocation. */
1001     *Node = handle;
1002     *Pool = pool;
1003
1004     /* Encode surface type and pool to database type. */
1005     type = gcvDB_VIDEO_MEMORY
1006          | (Type << gcdDB_VIDEO_MEMORY_TYPE_SHIFT)
1007          | (pool << gcdDB_VIDEO_MEMORY_POOL_SHIFT);
1008
1009     /* Record in process db. */
1010     gcmkONERROR(
1011             gckKERNEL_AddProcessDB(Kernel,
1012                                    ProcessID,
1013                                    type,
1014                                    gcmINT2PTR(handle),
1015                                    gcvNULL,
1016                                    bytes));
1017
1018     /* Return status. */
1019     gcmkFOOTER_ARG("*Pool=%d *Node=0x%x", *Pool, *Node);
1020     return gcvSTATUS_OK;
1021
1022 OnError:
1023     if (handle)
1024     {
1025         /* Destroy handle allocated. */
1026         gcmkVERIFY_OK(gckVIDMEM_HANDLE_Dereference(Kernel, ProcessID, handle));
1027     }
1028
1029     if (node)
1030     {
1031         /* Free video memory allocated. */
1032         gcmkVERIFY_OK(gckVIDMEM_Free(Kernel, node));
1033     }
1034
1035     /* For some case like chrome with webgl test, it needs too much memory so that it invokes oom_killer
1036     * And the case is killed by oom_killer, the user wants not to see the crash and hope the case iteself handles the condition
1037     * So the patch reports the out_of_memory to the case */
1038     if ( status == gcvSTATUS_OUT_OF_MEMORY && (Flag & gcvALLOC_FLAG_MEMLIMIT) )
1039         gcmkPRINT("The running case is out_of_memory");
1040
1041     /* Return the status. */
1042     gcmkFOOTER();
1043     return status;
1044 }
1045
1046 /*******************************************************************************
1047 **
1048 **  gckKERNEL_ReleaseVideoMemory
1049 **
1050 **  Release handle of a video memory.
1051 **
1052 **  INPUT:
1053 **
1054 **      gckKERNEL Kernel
1055 **          Pointer to an gckKERNEL object.
1056 **
1057 **      gctUINT32 ProcessID
1058 **          ProcessID of current process.
1059 **
1060 **      gctUINT32 Handle
1061 **          Handle of video memory.
1062 **
1063 **  OUTPUT:
1064 **
1065 **          Nothing.
1066 */
1067 gceSTATUS
1068 gckKERNEL_ReleaseVideoMemory(
1069     IN gckKERNEL Kernel,
1070     IN gctUINT32 ProcessID,
1071     IN gctUINT32 Handle
1072     )
1073 {
1074     gceSTATUS status;
1075     gckVIDMEM_NODE nodeObject;
1076     gceDATABASE_TYPE type;
1077
1078     gcmkHEADER_ARG("Kernel=0x%08X ProcessID=%d Handle=%d",
1079                    Kernel, ProcessID, Handle);
1080
1081     gcmkONERROR(
1082         gckVIDMEM_HANDLE_Lookup(Kernel, ProcessID, Handle, &nodeObject));
1083
1084     type = gcvDB_VIDEO_MEMORY
1085          | (nodeObject->type << gcdDB_VIDEO_MEMORY_TYPE_SHIFT)
1086          | (nodeObject->pool << gcdDB_VIDEO_MEMORY_POOL_SHIFT);
1087
1088     gcmkONERROR(
1089         gckKERNEL_RemoveProcessDB(Kernel,
1090             ProcessID,
1091             type,
1092             gcmINT2PTR(Handle)));
1093
1094     gckVIDMEM_HANDLE_Dereference(Kernel, ProcessID, Handle);
1095
1096     gckVIDMEM_NODE_Dereference(Kernel, nodeObject);
1097
1098     gcmkFOOTER_NO();
1099     return gcvSTATUS_OK;
1100
1101 OnError:
1102     gcmkFOOTER();
1103     return status;
1104 }
1105
1106 /*******************************************************************************
1107 **
1108 **  gckKERNEL_LockVideoMemory
1109 **
1110 **      Lock a video memory node. It will generate a cpu virtual address used
1111 **      by software and a GPU address used by GPU.
1112 **
1113 **  INPUT:
1114 **
1115 **      gckKERNEL Kernel
1116 **          Pointer to an gckKERNEL object.
1117 **
1118 **      gceCORE Core
1119 **          GPU to which video memory is locked.
1120 **
1121 **      gcsHAL_INTERFACE * Interface
1122 **          Pointer to a gcsHAL_INTERFACE structure that defines the command to
1123 **          be dispatched.
1124 **
1125 **  OUTPUT:
1126 **
1127 **      gcsHAL_INTERFACE * Interface
1128 **          Pointer to a gcsHAL_INTERFACE structure that receives any data to be
1129 **          returned.
1130 */
1131 gceSTATUS
1132 gckKERNEL_LockVideoMemory(
1133     IN gckKERNEL Kernel,
1134     IN gceCORE Core,
1135     IN gctUINT32 ProcessID,
1136     IN gctBOOL FromUser,
1137     IN OUT gcsHAL_INTERFACE * Interface
1138     )
1139 {
1140     gceSTATUS status;
1141     gckVIDMEM_NODE nodeObject = gcvNULL;
1142     gcuVIDMEM_NODE_PTR node   = gcvNULL;
1143     gctBOOL locked            = gcvFALSE;
1144     gctBOOL asynchronous      = gcvFALSE;
1145 #ifndef __QNXNTO__
1146     gctPOINTER pointer        = gcvNULL;
1147 #endif
1148
1149     gcmkHEADER_ARG("Kernel=0x%08X ProcessID=%d",
1150                    Kernel, ProcessID);
1151
1152     gcmkONERROR(
1153         gckVIDMEM_HANDLE_LookupAndReference(Kernel,
1154                 Interface->u.LockVideoMemory.node,
1155                 &nodeObject));
1156
1157     node = nodeObject->node;
1158
1159     Interface->u.LockVideoMemory.gid = 0;
1160
1161     /* Lock video memory. */
1162     gcmkONERROR(
1163             gckVIDMEM_Lock(Kernel,
1164                 nodeObject,
1165                 Interface->u.LockVideoMemory.cacheable,
1166                 &Interface->u.LockVideoMemory.address,
1167                 &Interface->u.LockVideoMemory.gid,
1168                 &Interface->u.LockVideoMemory.physicalAddress));
1169
1170     locked = gcvTRUE;
1171
1172     if (node->VidMem.memory->object.type == gcvOBJ_VIDMEM)
1173     {
1174         /* Map video memory address into user space. */
1175 #ifdef __QNXNTO__
1176         if (node->VidMem.logical == gcvNULL)
1177         {
1178             gcmkONERROR(
1179                     gckKERNEL_MapVideoMemory(Kernel,
1180                         FromUser,
1181                         Interface->u.LockVideoMemory.address,
1182                         ProcessID,
1183                         node->VidMem.bytes,
1184                         &node->VidMem.logical));
1185         }
1186         gcmkASSERT(node->VidMem.logical != gcvNULL);
1187
1188         Interface->u.LockVideoMemory.memory = gcmPTR_TO_UINT64(node->VidMem.logical);
1189 #else
1190         gcmkONERROR(
1191                 gckKERNEL_MapVideoMemoryEx(Kernel,
1192                     Core,
1193                     FromUser,
1194                     Interface->u.LockVideoMemory.address,
1195                     &pointer));
1196
1197         Interface->u.LockVideoMemory.memory = gcmPTR_TO_UINT64(pointer);
1198 #endif
1199     }
1200     else
1201     {
1202         Interface->u.LockVideoMemory.memory = gcmPTR_TO_UINT64(node->Virtual.logical);
1203
1204         /* Success. */
1205         status = gcvSTATUS_OK;
1206     }
1207
1208 #if gcdPROCESS_ADDRESS_SPACE
1209     gcmkONERROR(gckVIDMEM_Node_Lock(
1210         Kernel,
1211         nodeObject,
1212         &Interface->u.LockVideoMemory.address
1213         ));
1214 #endif
1215
1216
1217 #if gcdSECURE_USER
1218     /* Return logical address as physical address. */
1219     Interface->u.LockVideoMemory.address =
1220         (gctUINT32)(Interface->u.LockVideoMemory.memory);
1221 #endif
1222     gcmkONERROR(
1223         gckKERNEL_AddProcessDB(Kernel,
1224             ProcessID, gcvDB_VIDEO_MEMORY_LOCKED,
1225             gcmINT2PTR(Interface->u.LockVideoMemory.node),
1226             gcvNULL,
1227             0));
1228
1229     gckVIDMEM_HANDLE_Reference(
1230         Kernel, ProcessID, (gctUINT32)Interface->u.LockVideoMemory.node);
1231
1232     gcmkFOOTER_NO();
1233     return gcvSTATUS_OK;
1234
1235 OnError:
1236     if (locked)
1237     {
1238         /* Roll back the lock. */
1239         gcmkVERIFY_OK(gckVIDMEM_Unlock(Kernel,
1240                     nodeObject,
1241                     gcvSURF_TYPE_UNKNOWN,
1242                     &asynchronous));
1243
1244         if (gcvTRUE == asynchronous)
1245         {
1246             /* Bottom Half */
1247             gcmkVERIFY_OK(gckVIDMEM_Unlock(Kernel,
1248                         nodeObject,
1249                         gcvSURF_TYPE_UNKNOWN,
1250                         gcvNULL));
1251         }
1252     }
1253
1254     if (nodeObject != gcvNULL)
1255     {
1256         gckVIDMEM_NODE_Dereference(Kernel, nodeObject);
1257     }
1258
1259     gcmkFOOTER();
1260     return status;
1261 }
1262
1263 /*******************************************************************************
1264 **
1265 **  gckKERNEL_UnlockVideoMemory
1266 **
1267 **      Unlock a video memory node.
1268 **
1269 **  INPUT:
1270 **
1271 **      gckKERNEL Kernel
1272 **          Pointer to an gckKERNEL object.
1273 **
1274 **      gctUINT32 ProcessID
1275 **          ProcessID of current process.
1276 **
1277 **      gcsHAL_INTERFACE * Interface
1278 **          Pointer to a gcsHAL_INTERFACE structure that defines the command to
1279 **          be dispatched.
1280 **
1281 **  OUTPUT:
1282 **
1283 **      gcsHAL_INTERFACE * Interface
1284 **          Pointer to a gcsHAL_INTERFACE structure that receives any data to be
1285 **          returned.
1286 */
1287 gceSTATUS
1288 gckKERNEL_UnlockVideoMemory(
1289     IN gckKERNEL Kernel,
1290     IN gctUINT32 ProcessID,
1291     IN OUT gcsHAL_INTERFACE * Interface
1292     )
1293 {
1294     gceSTATUS status;
1295     gckVIDMEM_NODE nodeObject;
1296     gcuVIDMEM_NODE_PTR node;
1297
1298     gcmkHEADER_ARG("Kernel=0x%08X ProcessID=%d",
1299                    Kernel, ProcessID);
1300
1301     gcmkONERROR(gckVIDMEM_HANDLE_Lookup(
1302         Kernel,
1303         ProcessID,
1304         (gctUINT32)Interface->u.UnlockVideoMemory.node,
1305         &nodeObject));
1306
1307     node = nodeObject->node;
1308
1309     /* Unlock video memory. */
1310 #if gcdSECURE_USER
1311     /* Save node information before it disappears. */
1312     if (node->VidMem.memory->object.type == gcvOBJ_VIDMEM)
1313     {
1314         logical = gcvNULL;
1315         bytes   = 0;
1316     }
1317     else
1318     {
1319         logical = node->Virtual.logical;
1320         bytes   = node->Virtual.bytes;
1321     }
1322 #endif
1323
1324     /* Unlock video memory. */
1325     gcmkONERROR(gckVIDMEM_Unlock(
1326         Kernel,
1327         nodeObject,
1328         Interface->u.UnlockVideoMemory.type,
1329         &Interface->u.UnlockVideoMemory.asynchroneous));
1330
1331 #if gcdSECURE_USER
1332     /* Flush the translation cache for virtual surfaces. */
1333     if (logical != gcvNULL)
1334     {
1335         gcmkVERIFY_OK(gckKERNEL_FlushTranslationCache(Kernel,
1336                     cache,
1337                     logical,
1338                     bytes));
1339     }
1340 #endif
1341
1342     gcmkFOOTER_NO();
1343     return gcvSTATUS_OK;
1344
1345 OnError:
1346     gcmkFOOTER();
1347     return status;
1348 }
1349
1350 gceSTATUS
1351 gckKERNEL_QueryDatabase(
1352     IN gckKERNEL Kernel,
1353     IN gctUINT32 ProcessID,
1354     IN OUT gcsHAL_INTERFACE * Interface
1355     )
1356 {
1357     gceSTATUS status;
1358     gctINT i;
1359     gcuDATABASE_INFO tmp;
1360
1361     gceDATABASE_TYPE type[3] = {
1362         gcvDB_VIDEO_MEMORY | (gcvPOOL_SYSTEM << gcdDB_VIDEO_MEMORY_POOL_SHIFT),
1363         gcvDB_VIDEO_MEMORY | (gcvPOOL_CONTIGUOUS << gcdDB_VIDEO_MEMORY_POOL_SHIFT),
1364         gcvDB_VIDEO_MEMORY | (gcvPOOL_VIRTUAL << gcdDB_VIDEO_MEMORY_POOL_SHIFT),
1365     };
1366
1367     gcmkHEADER();
1368
1369     /* Query video memory. */
1370     gcmkONERROR(
1371         gckKERNEL_QueryProcessDB(Kernel,
1372                                  Interface->u.Database.processID,
1373                                  !Interface->u.Database.validProcessID,
1374                                  gcvDB_VIDEO_MEMORY,
1375                                  &Interface->u.Database.vidMem));
1376
1377     /* Query non-paged memory. */
1378     gcmkONERROR(
1379         gckKERNEL_QueryProcessDB(Kernel,
1380                                  Interface->u.Database.processID,
1381                                  !Interface->u.Database.validProcessID,
1382                                  gcvDB_NON_PAGED,
1383                                  &Interface->u.Database.nonPaged));
1384
1385     /* Query contiguous memory. */
1386     gcmkONERROR(
1387         gckKERNEL_QueryProcessDB(Kernel,
1388                                  Interface->u.Database.processID,
1389                                  !Interface->u.Database.validProcessID,
1390                                  gcvDB_CONTIGUOUS,
1391                                  &Interface->u.Database.contiguous));
1392
1393     /* Query GPU idle time. */
1394     gcmkONERROR(
1395         gckKERNEL_QueryProcessDB(Kernel,
1396                                  Interface->u.Database.processID,
1397                                  !Interface->u.Database.validProcessID,
1398                                  gcvDB_IDLE,
1399                                  &Interface->u.Database.gpuIdle));
1400     for (i = 0; i < 3; i++)
1401     {
1402         /* Query each video memory pool. */
1403         gcmkONERROR(
1404             gckKERNEL_QueryProcessDB(Kernel,
1405                                      Interface->u.Database.processID,
1406                                      !Interface->u.Database.validProcessID,
1407                                      type[i],
1408                                      &Interface->u.Database.vidMemPool[i]));
1409     }
1410
1411     /* Query virtual command buffer pool. */
1412     gcmkONERROR(
1413         gckKERNEL_QueryProcessDB(Kernel,
1414                                  Interface->u.Database.processID,
1415                                  !Interface->u.Database.validProcessID,
1416                                  gcvDB_COMMAND_BUFFER,
1417                                  &tmp));
1418
1419     Interface->u.Database.vidMemPool[2].counters.bytes += tmp.counters.bytes;
1420     Interface->u.Database.vidMemPool[2].counters.maxBytes += tmp.counters.maxBytes;
1421     Interface->u.Database.vidMemPool[2].counters.totalBytes += tmp.counters.totalBytes;
1422
1423     Interface->u.Database.vidMem.counters.bytes += tmp.counters.bytes;
1424     Interface->u.Database.vidMem.counters.maxBytes += tmp.counters.maxBytes;
1425     Interface->u.Database.vidMem.counters.totalBytes += tmp.counters.totalBytes;
1426
1427 #if gcmIS_DEBUG(gcdDEBUG_TRACE)
1428     gckKERNEL_DumpVidMemUsage(Kernel, Interface->u.Database.processID);
1429 #endif
1430
1431     gcmkFOOTER_NO();
1432     return gcvSTATUS_OK;
1433
1434 OnError:
1435     gcmkFOOTER();
1436     return status;
1437 }
1438
1439 gceSTATUS
1440 gckKERNEL_ConfigPowerManagement(
1441     IN gckKERNEL Kernel,
1442     IN OUT gcsHAL_INTERFACE * Interface
1443 )
1444 {
1445     gceSTATUS status;
1446     gctBOOL enable = Interface->u.ConfigPowerManagement.enable;
1447
1448     gcmkHEADER();
1449
1450     gcmkONERROR(gckHARDWARE_SetPowerManagement(Kernel->hardware, enable));
1451
1452     if (enable == gcvTRUE)
1453     {
1454         gcmkONERROR(
1455             gckHARDWARE_SetPowerManagementState(Kernel->hardware, gcvPOWER_ON));
1456     }
1457
1458     gcmkFOOTER_NO();
1459     return gcvSTATUS_OK;
1460
1461 OnError:
1462     gcmkFOOTER();
1463     return status;
1464 }
1465
1466 /*******************************************************************************
1467 **
1468 **  gckKERNEL_Dispatch
1469 **
1470 **  Dispatch a command received from the user HAL layer.
1471 **
1472 **  INPUT:
1473 **
1474 **      gckKERNEL Kernel
1475 **          Pointer to an gckKERNEL object.
1476 **
1477 **      gctBOOL FromUser
1478 **          whether the call is from the user space.
1479 **
1480 **      gcsHAL_INTERFACE * Interface
1481 **          Pointer to a gcsHAL_INTERFACE structure that defines the command to
1482 **          be dispatched.
1483 **
1484 **  OUTPUT:
1485 **
1486 **      gcsHAL_INTERFACE * Interface
1487 **          Pointer to a gcsHAL_INTERFACE structure that receives any data to be
1488 **          returned.
1489 */
1490 gceSTATUS
1491 gckKERNEL_Dispatch(
1492     IN gckKERNEL Kernel,
1493     IN gctBOOL FromUser,
1494     IN OUT gcsHAL_INTERFACE * Interface
1495     )
1496 {
1497     gceSTATUS status = gcvSTATUS_OK;
1498     gctPHYS_ADDR physical = gcvNULL;
1499     gctSIZE_T bytes;
1500     gctPOINTER logical = gcvNULL;
1501     gctPOINTER info = gcvNULL;
1502 #if (gcdENABLE_3D || gcdENABLE_2D)
1503     gckCONTEXT context = gcvNULL;
1504 #endif
1505     gckKERNEL kernel = Kernel;
1506     gctUINT32 address;
1507     gctUINT32 processID;
1508 #if gcdSECURE_USER
1509     gcskSECURE_CACHE_PTR cache;
1510     gctPOINTER logical;
1511 #endif
1512     gctUINT32 paddr = gcvINVALID_ADDRESS;
1513 #if !USE_NEW_LINUX_SIGNAL
1514     gctSIGNAL   signal;
1515 #endif
1516     gckVIRTUAL_COMMAND_BUFFER_PTR buffer;
1517
1518     gckVIDMEM_NODE nodeObject;
1519     gctBOOL powerMutexAcquired = gcvFALSE;
1520
1521     gcmkHEADER_ARG("Kernel=0x%x FromUser=%d Interface=0x%x",
1522                    Kernel, FromUser, Interface);
1523
1524     /* Verify the arguments. */
1525     gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL);
1526     gcmkVERIFY_ARGUMENT(Interface != gcvNULL);
1527
1528 #if gcmIS_DEBUG(gcdDEBUG_TRACE)
1529     gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_KERNEL,
1530                    "Dispatching command %d (%s)",
1531                    Interface->command, _DispatchText[Interface->command]);
1532 #endif
1533 #if QNX_SINGLE_THREADED_DEBUGGING
1534     gckOS_AcquireMutex(Kernel->os, Kernel->debugMutex, gcvINFINITE);
1535 #endif
1536
1537     /* Get the current process ID. */
1538     gcmkONERROR(gckOS_GetProcessID(&processID));
1539
1540 #if gcdSECURE_USER
1541     gcmkONERROR(gckKERNEL_GetProcessDBCache(Kernel, processID, &cache));
1542 #endif
1543
1544     /* Dispatch on command. */
1545     switch (Interface->command)
1546     {
1547     case gcvHAL_GET_BASE_ADDRESS:
1548         /* Get base address. */
1549         gcmkONERROR(
1550             gckOS_GetBaseAddress(Kernel->os,
1551                                  &Interface->u.GetBaseAddress.baseAddress));
1552         break;
1553
1554     case gcvHAL_QUERY_VIDEO_MEMORY:
1555         /* Query video memory size. */
1556         gcmkONERROR(gckKERNEL_QueryVideoMemory(Kernel, Interface));
1557         break;
1558
1559     case gcvHAL_QUERY_CHIP_IDENTITY:
1560         /* Query chip identity. */
1561         gcmkONERROR(
1562             gckHARDWARE_QueryChipIdentity(
1563                 Kernel->hardware,
1564                 &Interface->u.QueryChipIdentity));
1565         break;
1566
1567     case gcvHAL_MAP_MEMORY:
1568         physical = gcmINT2PTR(Interface->u.MapMemory.physical);
1569
1570         /* Map memory. */
1571         gcmkONERROR(
1572             gckKERNEL_MapMemory(Kernel,
1573                                 physical,
1574                                 (gctSIZE_T) Interface->u.MapMemory.bytes,
1575                                 &logical));
1576
1577         Interface->u.MapMemory.logical = gcmPTR_TO_UINT64(logical);
1578
1579         gcmkVERIFY_OK(
1580             gckKERNEL_AddProcessDB(Kernel,
1581                                    processID, gcvDB_MAP_MEMORY,
1582                                    logical,
1583                                    physical,
1584                                    (gctSIZE_T) Interface->u.MapMemory.bytes));
1585         break;
1586
1587     case gcvHAL_UNMAP_MEMORY:
1588         physical = gcmINT2PTR(Interface->u.UnmapMemory.physical);
1589
1590         gcmkVERIFY_OK(
1591             gckKERNEL_RemoveProcessDB(Kernel,
1592                                       processID, gcvDB_MAP_MEMORY,
1593                                       gcmUINT64_TO_PTR(Interface->u.UnmapMemory.logical)));
1594
1595         /* Unmap memory. */
1596         gcmkONERROR(
1597             gckKERNEL_UnmapMemory(Kernel,
1598                                   physical,
1599                                   (gctSIZE_T) Interface->u.UnmapMemory.bytes,
1600                                   gcmUINT64_TO_PTR(Interface->u.UnmapMemory.logical)));
1601         break;
1602
1603     case gcvHAL_ALLOCATE_NON_PAGED_MEMORY:
1604         bytes = (gctSIZE_T) Interface->u.AllocateNonPagedMemory.bytes;
1605
1606         /* Allocate non-paged memory. */
1607         gcmkONERROR(
1608             gckOS_AllocateNonPagedMemory(
1609                 Kernel->os,
1610                 FromUser,
1611                 &bytes,
1612                 &physical,
1613                 &logical));
1614
1615         Interface->u.AllocateNonPagedMemory.bytes    = bytes;
1616         Interface->u.AllocateNonPagedMemory.logical  = gcmPTR_TO_UINT64(logical);
1617         Interface->u.AllocateNonPagedMemory.physical = gcmPTR_TO_NAME(physical);
1618
1619         gcmkVERIFY_OK(
1620             gckKERNEL_AddProcessDB(Kernel,
1621                                    processID, gcvDB_NON_PAGED,
1622                                    logical,
1623                                    gcmINT2PTR(Interface->u.AllocateNonPagedMemory.physical),
1624                                    bytes));
1625         break;
1626
1627     case gcvHAL_ALLOCATE_VIRTUAL_COMMAND_BUFFER:
1628         bytes = (gctSIZE_T) Interface->u.AllocateVirtualCommandBuffer.bytes;
1629
1630         gcmkONERROR(
1631             gckKERNEL_AllocateVirtualCommandBuffer(
1632                 Kernel,
1633                 FromUser,
1634                 &bytes,
1635                 &physical,
1636                 &logical));
1637
1638         Interface->u.AllocateVirtualCommandBuffer.bytes    = bytes;
1639         Interface->u.AllocateVirtualCommandBuffer.logical  = gcmPTR_TO_UINT64(logical);
1640         Interface->u.AllocateVirtualCommandBuffer.physical = gcmPTR_TO_NAME(physical);
1641
1642         gcmkVERIFY_OK(
1643             gckKERNEL_AddProcessDB(Kernel,
1644                                    processID, gcvDB_COMMAND_BUFFER,
1645                                    logical,
1646                                    gcmINT2PTR(Interface->u.AllocateVirtualCommandBuffer.physical),
1647                                    bytes));
1648         break;
1649
1650     case gcvHAL_FREE_NON_PAGED_MEMORY:
1651         physical = gcmNAME_TO_PTR(Interface->u.FreeNonPagedMemory.physical);
1652
1653         gcmkVERIFY_OK(
1654             gckKERNEL_RemoveProcessDB(Kernel,
1655                                       processID, gcvDB_NON_PAGED,
1656                                       gcmUINT64_TO_PTR(Interface->u.FreeNonPagedMemory.logical)));
1657
1658         /* Unmap user logical out of physical memory first. */
1659         gcmkONERROR(gckOS_UnmapUserLogical(Kernel->os,
1660                                            physical,
1661                                            (gctSIZE_T) Interface->u.FreeNonPagedMemory.bytes,
1662                                            gcmUINT64_TO_PTR(Interface->u.FreeNonPagedMemory.logical)));
1663
1664         /* Free non-paged memory. */
1665         gcmkONERROR(
1666             gckOS_FreeNonPagedMemory(Kernel->os,
1667                                      (gctSIZE_T) Interface->u.FreeNonPagedMemory.bytes,
1668                                      physical,
1669                                      gcmUINT64_TO_PTR(Interface->u.FreeNonPagedMemory.logical)));
1670
1671 #if gcdSECURE_USER
1672         gcmkVERIFY_OK(gckKERNEL_FlushTranslationCache(
1673             Kernel,
1674             cache,
1675             gcmUINT64_TO_PTR(Interface->u.FreeNonPagedMemory.logical),
1676             (gctSIZE_T) Interface->u.FreeNonPagedMemory.bytes));
1677 #endif
1678
1679         gcmRELEASE_NAME(Interface->u.FreeNonPagedMemory.physical);
1680         break;
1681
1682     case gcvHAL_ALLOCATE_CONTIGUOUS_MEMORY:
1683         bytes = (gctSIZE_T) Interface->u.AllocateContiguousMemory.bytes;
1684
1685         /* Allocate contiguous memory. */
1686         gcmkONERROR(gckOS_AllocateContiguous(
1687             Kernel->os,
1688             FromUser,
1689             &bytes,
1690             &physical,
1691             &logical));
1692
1693         Interface->u.AllocateContiguousMemory.bytes    = bytes;
1694         Interface->u.AllocateContiguousMemory.logical  = gcmPTR_TO_UINT64(logical);
1695         Interface->u.AllocateContiguousMemory.physical = gcmPTR_TO_NAME(physical);
1696
1697         gcmkONERROR(gckHARDWARE_ConvertLogical(
1698             Kernel->hardware,
1699             logical,
1700             gcvTRUE,
1701             &Interface->u.AllocateContiguousMemory.address));
1702
1703         gcmkVERIFY_OK(gckKERNEL_AddProcessDB(
1704             Kernel,
1705             processID, gcvDB_CONTIGUOUS,
1706             logical,
1707             gcmINT2PTR(Interface->u.AllocateContiguousMemory.physical),
1708             bytes));
1709         break;
1710
1711     case gcvHAL_FREE_CONTIGUOUS_MEMORY:
1712         physical = gcmNAME_TO_PTR(Interface->u.FreeContiguousMemory.physical);
1713
1714         gcmkVERIFY_OK(
1715             gckKERNEL_RemoveProcessDB(Kernel,
1716                                       processID, gcvDB_CONTIGUOUS,
1717                                       gcmUINT64_TO_PTR(Interface->u.FreeNonPagedMemory.logical)));
1718
1719         /* Unmap user logical out of physical memory first. */
1720         gcmkONERROR(gckOS_UnmapUserLogical(Kernel->os,
1721                                            physical,
1722                                            (gctSIZE_T) Interface->u.FreeContiguousMemory.bytes,
1723                                            gcmUINT64_TO_PTR(Interface->u.FreeContiguousMemory.logical)));
1724
1725         /* Free contiguous memory. */
1726         gcmkONERROR(
1727             gckOS_FreeContiguous(Kernel->os,
1728                                  physical,
1729                                  gcmUINT64_TO_PTR(Interface->u.FreeContiguousMemory.logical),
1730                                  (gctSIZE_T) Interface->u.FreeContiguousMemory.bytes));
1731
1732 #if gcdSECURE_USER
1733         gcmkVERIFY_OK(gckKERNEL_FlushTranslationCache(
1734             Kernel,
1735             cache,
1736             gcmUINT64_TO_PTR(Interface->u.FreeContiguousMemory.logical),
1737             (gctSIZE_T) Interface->u.FreeContiguousMemory.bytes));
1738 #endif
1739
1740         gcmRELEASE_NAME(Interface->u.FreeContiguousMemory.physical);
1741         break;
1742
1743     case gcvHAL_ALLOCATE_VIDEO_MEMORY:
1744
1745         gcmkONERROR(gcvSTATUS_NOT_SUPPORTED);
1746
1747         break;
1748
1749     case gcvHAL_ALLOCATE_LINEAR_VIDEO_MEMORY:
1750         /* Allocate memory. */
1751         gcmkONERROR(
1752             gckKERNEL_AllocateLinearMemory(Kernel, processID,
1753                             &Interface->u.AllocateLinearVideoMemory.pool,
1754                             Interface->u.AllocateLinearVideoMemory.bytes,
1755                             Interface->u.AllocateLinearVideoMemory.alignment,
1756                             Interface->u.AllocateLinearVideoMemory.type,
1757                             Interface->u.AllocateLinearVideoMemory.flag,
1758                             &Interface->u.AllocateLinearVideoMemory.node));
1759         break;
1760
1761     case gcvHAL_RELEASE_VIDEO_MEMORY:
1762         /* Release video memory. */
1763         gcmkONERROR(gckKERNEL_ReleaseVideoMemory(
1764             Kernel, processID,
1765             (gctUINT32)Interface->u.ReleaseVideoMemory.node
1766             ));
1767         break;
1768
1769     case gcvHAL_LOCK_VIDEO_MEMORY:
1770         /* Lock video memory. */
1771         gcmkONERROR(gckKERNEL_LockVideoMemory(Kernel, Kernel->core, processID, FromUser, Interface));
1772         break;
1773
1774     case gcvHAL_UNLOCK_VIDEO_MEMORY:
1775         /* Unlock video memory. */
1776         gcmkONERROR(gckKERNEL_UnlockVideoMemory(Kernel, processID, Interface));
1777         break;
1778
1779     case gcvHAL_EVENT_COMMIT:
1780         /* Commit an event queue. */
1781 #if gcdMULTI_GPU
1782         if (Interface->u.Event.gpuMode == gcvMULTI_GPU_MODE_INDEPENDENT)
1783         {
1784             gcmkONERROR(
1785                 gckEVENT_Commit(Kernel->eventObj,
1786                                 gcmUINT64_TO_PTR(Interface->u.Event.queue),
1787                                 Interface->u.Event.chipEnable));
1788         }
1789         else
1790         {
1791             gcmkONERROR(
1792                 gckEVENT_Commit(Kernel->eventObj,
1793                                 gcmUINT64_TO_PTR(Interface->u.Event.queue),
1794                                 gcvCORE_3D_ALL_MASK));
1795         }
1796 #else
1797         gcmkONERROR(
1798             gckEVENT_Commit(Kernel->eventObj,
1799                             gcmUINT64_TO_PTR(Interface->u.Event.queue)));
1800 #endif
1801         break;
1802
1803     case gcvHAL_COMMIT:
1804         /* Commit a command and context buffer. */
1805 #if gcdMULTI_GPU
1806         if (Interface->u.Commit.gpuMode == gcvMULTI_GPU_MODE_INDEPENDENT)
1807         {
1808             gcmkONERROR(
1809                 gckCOMMAND_Commit(Kernel->command,
1810                                   Interface->u.Commit.context ?
1811                                       gcmNAME_TO_PTR(Interface->u.Commit.context) : gcvNULL,
1812                                   gcmUINT64_TO_PTR(Interface->u.Commit.commandBuffer),
1813                                   gcmUINT64_TO_PTR(Interface->u.Commit.delta),
1814                                   gcmUINT64_TO_PTR(Interface->u.Commit.queue),
1815                                   processID,
1816                                   Interface->u.Commit.chipEnable));
1817         }
1818         else
1819         {
1820             gcmkONERROR(
1821                 gckCOMMAND_Commit(Kernel->command,
1822                                   Interface->u.Commit.context ?
1823                                       gcmNAME_TO_PTR(Interface->u.Commit.context) : gcvNULL,
1824                                   gcmUINT64_TO_PTR(Interface->u.Commit.commandBuffer),
1825                                   gcmUINT64_TO_PTR(Interface->u.Commit.delta),
1826                                   gcmUINT64_TO_PTR(Interface->u.Commit.queue),
1827                                   processID,
1828                                   gcvCORE_3D_ALL_MASK));
1829         }
1830 #else
1831         gcmkONERROR(
1832             gckCOMMAND_Commit(Kernel->command,
1833                               Interface->u.Commit.context ?
1834                                   gcmNAME_TO_PTR(Interface->u.Commit.context) : gcvNULL,
1835                               gcmUINT64_TO_PTR(Interface->u.Commit.commandBuffer),
1836                               gcmUINT64_TO_PTR(Interface->u.Commit.delta),
1837                               gcmUINT64_TO_PTR(Interface->u.Commit.queue),
1838                               processID));
1839 #endif
1840
1841         break;
1842
1843     case gcvHAL_STALL:
1844         /* Stall the command queue. */
1845 #if gcdMULTI_GPU
1846         gcmkONERROR(gckCOMMAND_Stall(Kernel->command, gcvFALSE, gcvCORE_3D_ALL_MASK));
1847 #else
1848         gcmkONERROR(gckCOMMAND_Stall(Kernel->command, gcvFALSE));
1849 #endif
1850         break;
1851
1852     case gcvHAL_MAP_USER_MEMORY:
1853         /* Map user memory to DMA. */
1854         gcmkONERROR(
1855             gckOS_MapUserMemory(Kernel->os,
1856                                 Kernel->core,
1857                                 gcmUINT64_TO_PTR(Interface->u.MapUserMemory.memory),
1858                                 Interface->u.MapUserMemory.physical,
1859                                 (gctSIZE_T) Interface->u.MapUserMemory.size,
1860                                 &info,
1861                                 &Interface->u.MapUserMemory.address));
1862
1863         Interface->u.MapUserMemory.info = gcmPTR_TO_NAME(info);
1864
1865         gcmkVERIFY_OK(
1866             gckKERNEL_AddProcessDB(Kernel,
1867                                    processID, gcvDB_MAP_USER_MEMORY,
1868                                    gcmINT2PTR(Interface->u.MapUserMemory.info),
1869                                    gcmUINT64_TO_PTR(Interface->u.MapUserMemory.memory),
1870                                    (gctSIZE_T) Interface->u.MapUserMemory.size));
1871         break;
1872
1873     case gcvHAL_UNMAP_USER_MEMORY:
1874         address = Interface->u.UnmapUserMemory.address;
1875         info = gcmNAME_TO_PTR(Interface->u.UnmapUserMemory.info);
1876
1877         gcmkVERIFY_OK(
1878             gckKERNEL_RemoveProcessDB(Kernel,
1879                                       processID, gcvDB_MAP_USER_MEMORY,
1880                                       gcmINT2PTR(Interface->u.UnmapUserMemory.info)));
1881         /* Unmap user memory. */
1882         gcmkONERROR(
1883             gckOS_UnmapUserMemory(Kernel->os,
1884                                   Kernel->core,
1885                                   gcmUINT64_TO_PTR(Interface->u.UnmapUserMemory.memory),
1886                                   (gctSIZE_T) Interface->u.UnmapUserMemory.size,
1887                                   info,
1888                                   address));
1889
1890 #if gcdSECURE_USER
1891         gcmkVERIFY_OK(gckKERNEL_FlushTranslationCache(
1892             Kernel,
1893             cache,
1894             gcmUINT64_TO_PTR(Interface->u.UnmapUserMemory.memory),
1895             (gctSIZE_T) Interface->u.UnmapUserMemory.size));
1896 #endif
1897
1898         gcmRELEASE_NAME(Interface->u.UnmapUserMemory.info);
1899         break;
1900
1901 #if !USE_NEW_LINUX_SIGNAL
1902     case gcvHAL_USER_SIGNAL:
1903         /* Dispatch depends on the user signal subcommands. */
1904         switch(Interface->u.UserSignal.command)
1905         {
1906         case gcvUSER_SIGNAL_CREATE:
1907             /* Create a signal used in the user space. */
1908             gcmkONERROR(
1909                 gckOS_CreateUserSignal(Kernel->os,
1910                                        Interface->u.UserSignal.manualReset,
1911                                        &Interface->u.UserSignal.id));
1912
1913             gcmkVERIFY_OK(
1914                 gckKERNEL_AddProcessDB(Kernel,
1915                                        processID, gcvDB_SIGNAL,
1916                                        gcmINT2PTR(Interface->u.UserSignal.id),
1917                                        gcvNULL,
1918                                        0));
1919             break;
1920
1921         case gcvUSER_SIGNAL_DESTROY:
1922             gcmkVERIFY_OK(gckKERNEL_RemoveProcessDB(
1923                 Kernel,
1924                 processID, gcvDB_SIGNAL,
1925                 gcmINT2PTR(Interface->u.UserSignal.id)));
1926
1927             /* Destroy the signal. */
1928             gcmkONERROR(
1929                 gckOS_DestroyUserSignal(Kernel->os,
1930                                         Interface->u.UserSignal.id));
1931             break;
1932
1933         case gcvUSER_SIGNAL_SIGNAL:
1934             /* Signal the signal. */
1935             gcmkONERROR(
1936                 gckOS_SignalUserSignal(Kernel->os,
1937                                        Interface->u.UserSignal.id,
1938                                        Interface->u.UserSignal.state));
1939             break;
1940
1941         case gcvUSER_SIGNAL_WAIT:
1942             /* Wait on the signal. */
1943             status = gckOS_WaitUserSignal(Kernel->os,
1944                                           Interface->u.UserSignal.id,
1945                                           Interface->u.UserSignal.wait);
1946
1947             break;
1948
1949         case gcvUSER_SIGNAL_MAP:
1950             gcmkONERROR(
1951                 gckOS_MapSignal(Kernel->os,
1952                                (gctSIGNAL)(gctUINTPTR_T)Interface->u.UserSignal.id,
1953                                (gctHANDLE)(gctUINTPTR_T)processID,
1954                                &signal));
1955
1956             gcmkVERIFY_OK(
1957                 gckKERNEL_AddProcessDB(Kernel,
1958                                        processID, gcvDB_SIGNAL,
1959                                        gcmINT2PTR(Interface->u.UserSignal.id),
1960                                        gcvNULL,
1961                                        0));
1962             break;
1963
1964         case gcvUSER_SIGNAL_UNMAP:
1965             gcmkVERIFY_OK(gckKERNEL_RemoveProcessDB(
1966                 Kernel,
1967                 processID, gcvDB_SIGNAL,
1968                 gcmINT2PTR(Interface->u.UserSignal.id)));
1969
1970             /* Destroy the signal. */
1971             gcmkONERROR(
1972                 gckOS_DestroyUserSignal(Kernel->os,
1973                                         Interface->u.UserSignal.id));
1974             break;
1975
1976         default:
1977             /* Invalid user signal command. */
1978             gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT);
1979         }
1980         break;
1981 #endif
1982
1983     case gcvHAL_SET_POWER_MANAGEMENT_STATE:
1984         /* Set the power management state. */
1985         gcmkONERROR(
1986             gckHARDWARE_SetPowerManagementState(
1987                 Kernel->hardware,
1988                 Interface->u.SetPowerManagement.state));
1989         break;
1990
1991     case gcvHAL_QUERY_POWER_MANAGEMENT_STATE:
1992         /* Chip is not idle. */
1993         Interface->u.QueryPowerManagement.isIdle = gcvFALSE;
1994
1995         /* Query the power management state. */
1996         gcmkONERROR(gckHARDWARE_QueryPowerManagementState(
1997             Kernel->hardware,
1998             &Interface->u.QueryPowerManagement.state));
1999
2000         /* Query the idle state. */
2001         gcmkONERROR(
2002             gckHARDWARE_QueryIdle(Kernel->hardware,
2003                                   &Interface->u.QueryPowerManagement.isIdle));
2004         break;
2005
2006     case gcvHAL_READ_REGISTER:
2007 #if gcdREGISTER_ACCESS_FROM_USER
2008         {
2009             gceCHIPPOWERSTATE power;
2010
2011             gcmkONERROR(gckOS_AcquireMutex(Kernel->os, Kernel->hardware->powerMutex, gcvINFINITE));
2012             powerMutexAcquired = gcvTRUE;
2013             gcmkONERROR(gckHARDWARE_QueryPowerManagementState(Kernel->hardware,
2014                                                               &power));
2015             if (power == gcvPOWER_ON)
2016             {
2017                 /* Read a register. */
2018                 gcmkONERROR(gckOS_ReadRegisterEx(
2019                     Kernel->os,
2020                     Kernel->core,
2021                     Interface->u.ReadRegisterData.address,
2022                     &Interface->u.ReadRegisterData.data));
2023             }
2024             else
2025             {
2026                 /* Chip is in power-state. */
2027                 Interface->u.ReadRegisterData.data = 0;
2028                 status = gcvSTATUS_CHIP_NOT_READY;
2029             }
2030             gcmkONERROR(gckOS_ReleaseMutex(Kernel->os, Kernel->hardware->powerMutex));
2031             powerMutexAcquired = gcvFALSE;
2032         }
2033 #else
2034         /* No access from user land to read registers. */
2035         Interface->u.ReadRegisterData.data = 0;
2036         status = gcvSTATUS_NOT_SUPPORTED;
2037 #endif
2038         break;
2039
2040 #if gcdMULTI_GPU
2041     case gcvHAL_READ_REGISTER_EX:
2042 #if gcdREGISTER_ACCESS_FROM_USER
2043         {
2044             gceCHIPPOWERSTATE power;
2045             gctUINT32 coreId = 0;
2046             gctUINT32 coreSelect = Interface->u.ReadRegisterDataEx.coreSelect;
2047
2048             gckOS_AcquireMutex(Kernel->os, Kernel->hardware->powerMutex, gcvINFINITE);
2049             powerMutexAcquired = gcvTRUE;
2050             gcmkONERROR(gckHARDWARE_QueryPowerManagementState(Kernel->hardware,
2051                     &power));
2052             if (power == gcvPOWER_ON)
2053             {
2054                 for (; coreSelect != 0; coreSelect >>= 1, coreId++)
2055                 {
2056                     if (coreSelect & 1UL)
2057                     {
2058                         /* Read a register. */
2059                         gcmkONERROR(
2060                             gckOS_ReadRegisterByCoreId(
2061                                 Kernel->os,
2062                                 Kernel->core,
2063                                 coreId,
2064                                 Interface->u.ReadRegisterDataEx.address,
2065                                 &Interface->u.ReadRegisterDataEx.data[coreId]));
2066                     }
2067                 }
2068             }
2069             else
2070             {
2071                 for (coreId = 0; coreId < gcdMULTI_GPU; coreId++)
2072                 {
2073                     /* Chip is in power-state. */
2074                     Interface->u.ReadRegisterDataEx.data[coreId] = 0;
2075                 }
2076                 status = gcvSTATUS_CHIP_NOT_READY;
2077             }
2078             gcmkONERROR(gckOS_ReleaseMutex(Kernel->os, Kernel->hardware->powerMutex));
2079             powerMutexAcquired = gcvFALSE;
2080         }
2081 #else
2082         gctUINT32 coreId;
2083
2084         /* No access from user land to read registers. */
2085         for (coreId = 0; coreId < gcdMULTI_GPU; coreId++)
2086         {
2087             Interface->u.ReadRegisterDataEx.data[coreId] = 0;
2088         }
2089
2090         status = gcvSTATUS_NOT_SUPPORTED;
2091 #endif
2092         break;
2093
2094     case gcvHAL_WRITE_REGISTER_EX:
2095 #if gcdREGISTER_ACCESS_FROM_USER
2096         {
2097             gceCHIPPOWERSTATE power;
2098             gctUINT32 coreId = 0;
2099             gctUINT32 coreSelect = Interface->u.WriteRegisterDataEx.coreSelect;
2100
2101             gcmkONERROR(gckOS_AcquireMutex(Kernel->os, Kernel->hardware->powerMutex, gcvINFINITE));
2102             powerMutexAcquired = gcvTRUE;
2103             gcmkONERROR(gckHARDWARE_QueryPowerManagementState(Kernel->hardware,
2104                     &power));
2105             if (power == gcvPOWER_ON)
2106             {
2107                 for (; coreSelect != 0; coreSelect >>= 1, coreId++)
2108                 {
2109                     if (coreSelect & 1UL)
2110                     {
2111                         /* Write a register. */
2112                         gcmkONERROR(
2113                             gckOS_WriteRegisterByCoreId(
2114                                 Kernel->os,
2115                                 Kernel->core,
2116                                 coreId,
2117                                 Interface->u.WriteRegisterDataEx.address,
2118                                 Interface->u.WriteRegisterDataEx.data[coreId]));
2119                     }
2120                 }
2121             }
2122             else
2123             {
2124                 /* Chip is in power-state. */
2125                 for (coreId = 0; coreId < gcdMULTI_GPU; coreId++)
2126                 {
2127                     Interface->u.WriteRegisterDataEx.data[coreId] = 0;
2128                 }
2129                 status = gcvSTATUS_CHIP_NOT_READY;
2130             }
2131             gcmkONERROR(gckOS_ReleaseMutex(Kernel->os, Kernel->hardware->powerMutex));
2132             powerMutexAcquired = gcvFALSE;
2133         }
2134 #else
2135         status = gcvSTATUS_NOT_SUPPORTED;
2136 #endif
2137         break;
2138 #endif
2139
2140     case gcvHAL_WRITE_REGISTER:
2141 #if gcdREGISTER_ACCESS_FROM_USER
2142         {
2143             gceCHIPPOWERSTATE power;
2144
2145             gckOS_AcquireMutex(Kernel->os, Kernel->hardware->powerMutex, gcvINFINITE);
2146             gcmkONERROR(gckHARDWARE_QueryPowerManagementState(Kernel->hardware,
2147                                                                   &power));
2148             if (power == gcvPOWER_ON)
2149             {
2150                 /* Write a register. */
2151                 gcmkONERROR(
2152                     gckOS_WriteRegisterEx(Kernel->os,
2153                                           Kernel->core,
2154                                           Interface->u.WriteRegisterData.address,
2155                                           Interface->u.WriteRegisterData.data));
2156             }
2157             else
2158             {
2159                 /* Chip is in power-state. */
2160                 Interface->u.WriteRegisterData.data = 0;
2161                 status = gcvSTATUS_CHIP_NOT_READY;
2162             }
2163             gcmkONERROR(gckOS_ReleaseMutex(Kernel->os, Kernel->hardware->powerMutex));
2164         }
2165 #else
2166         /* No access from user land to write registers. */
2167         status = gcvSTATUS_NOT_SUPPORTED;
2168 #endif
2169         break;
2170
2171     case gcvHAL_READ_ALL_PROFILE_REGISTERS:
2172 #if VIVANTE_PROFILER && VIVANTE_PROFILER_CONTEXT
2173         /* Read profile data according to the context. */
2174         gcmkONERROR(
2175             gckHARDWARE_QueryContextProfile(
2176                 Kernel->hardware,
2177                 Kernel->profileCleanRegister,
2178                 gcmNAME_TO_PTR(Interface->u.RegisterProfileData.context),
2179                 &Interface->u.RegisterProfileData.counters));
2180 #elif VIVANTE_PROFILER
2181         /* Read all 3D profile registers. */
2182         gcmkONERROR(
2183             gckHARDWARE_QueryProfileRegisters(
2184                 Kernel->hardware,
2185                 Kernel->profileCleanRegister,
2186                 &Interface->u.RegisterProfileData.counters));
2187 #else
2188         status = gcvSTATUS_OK;
2189 #endif
2190         break;
2191
2192     case gcvHAL_PROFILE_REGISTERS_2D:
2193 #if VIVANTE_PROFILER
2194         /* Read all 2D profile registers. */
2195         gcmkONERROR(
2196             gckHARDWARE_ProfileEngine2D(
2197                 Kernel->hardware,
2198                 gcmUINT64_TO_PTR(Interface->u.RegisterProfileData2D.hwProfile2D)));
2199 #else
2200         status = gcvSTATUS_OK;
2201 #endif
2202         break;
2203
2204     case gcvHAL_GET_PROFILE_SETTING:
2205 #if VIVANTE_PROFILER
2206         /* Get profile setting */
2207         Interface->u.GetProfileSetting.enable = Kernel->profileEnable;
2208 #endif
2209
2210         status = gcvSTATUS_OK;
2211         break;
2212
2213     case gcvHAL_SET_PROFILE_SETTING:
2214 #if VIVANTE_PROFILER
2215         /* Set profile setting */
2216         if(Kernel->hardware->gpuProfiler)
2217         {
2218             Kernel->profileEnable = Interface->u.SetProfileSetting.enable;
2219 #if VIVANTE_PROFILER_NEW
2220             if (Kernel->profileEnable)
2221                 gckHARDWARE_InitProfiler(Kernel->hardware);
2222 #endif
2223         }
2224         else
2225         {
2226             status = gcvSTATUS_NOT_SUPPORTED;
2227             break;
2228         }
2229 #endif
2230
2231         status = gcvSTATUS_OK;
2232         break;
2233
2234 #if VIVANTE_PROFILER_PERDRAW
2235     case gcvHAL_READ_PROFILER_REGISTER_SETTING:
2236     #if VIVANTE_PROFILER
2237         Kernel->profileCleanRegister = Interface->u.SetProfilerRegisterClear.bclear;
2238     #endif
2239         status = gcvSTATUS_OK;
2240         break;
2241 #endif
2242
2243     case gcvHAL_QUERY_KERNEL_SETTINGS:
2244         /* Get kernel settings. */
2245         gcmkONERROR(
2246             gckKERNEL_QuerySettings(Kernel,
2247                                     &Interface->u.QueryKernelSettings.settings));
2248         break;
2249
2250     case gcvHAL_RESET:
2251         /* Reset the hardware. */
2252         gcmkONERROR(
2253             gckHARDWARE_Reset(Kernel->hardware));
2254         break;
2255
2256     case gcvHAL_DEBUG:
2257         /* Set debug level and zones. */
2258         if (Interface->u.Debug.set)
2259         {
2260             gckOS_SetDebugLevel(Interface->u.Debug.level);
2261             gckOS_SetDebugZones(Interface->u.Debug.zones,
2262                                 Interface->u.Debug.enable);
2263         }
2264
2265         if (Interface->u.Debug.message[0] != '\0')
2266         {
2267             /* Print a message to the debugger. */
2268             if (Interface->u.Debug.type == gcvMESSAGE_TEXT)
2269             {
2270                gckOS_CopyPrint(Interface->u.Debug.message);
2271             }
2272             else
2273             {
2274                gckOS_DumpBuffer(Kernel->os,
2275                                 Interface->u.Debug.message,
2276                                 Interface->u.Debug.messageSize,
2277                                 gceDUMP_BUFFER_FROM_USER,
2278                                 gcvTRUE);
2279             }
2280         }
2281         status = gcvSTATUS_OK;
2282         break;
2283
2284     case gcvHAL_DUMP_GPU_STATE:
2285         {
2286             gceCHIPPOWERSTATE power;
2287
2288             _DumpDriverConfigure(Kernel);
2289
2290             gcmkONERROR(gckHARDWARE_QueryPowerManagementState(
2291                 Kernel->hardware,
2292                 &power
2293                 ));
2294
2295             if (power == gcvPOWER_ON)
2296             {
2297                 Interface->u.ReadRegisterData.data = 1;
2298
2299                 _DumpState(Kernel);
2300             }
2301             else
2302             {
2303                 Interface->u.ReadRegisterData.data = 0;
2304                 status = gcvSTATUS_CHIP_NOT_READY;
2305
2306                 gcmkPRINT("[galcore]: Can't dump state if GPU isn't POWER ON.");
2307             }
2308         }
2309         break;
2310
2311     case gcvHAL_DUMP_EVENT:
2312         break;
2313
2314     case gcvHAL_CACHE:
2315
2316         logical = gcmUINT64_TO_PTR(Interface->u.Cache.logical);
2317
2318         if (Interface->u.Cache.node)
2319         {
2320             gcmkONERROR(gckVIDMEM_HANDLE_Lookup(
2321                 Kernel,
2322                 processID,
2323                 Interface->u.Cache.node,
2324                 &nodeObject));
2325
2326             if (nodeObject->node->VidMem.memory->object.type == gcvOBJ_VIDMEM
2327              || nodeObject->node->Virtual.contiguous
2328             )
2329             {
2330                 /* If memory is contiguous, get physical address. */
2331                 gcmkONERROR(gckOS_GetPhysicalAddress(
2332                     Kernel->os, logical, (gctUINT32*)&paddr));
2333             }
2334         }
2335
2336         bytes = (gctSIZE_T) Interface->u.Cache.bytes;
2337         switch(Interface->u.Cache.operation)
2338         {
2339         case gcvCACHE_FLUSH:
2340             /* Clean and invalidate the cache. */
2341             status = gckOS_CacheFlush(Kernel->os,
2342                                       processID,
2343                                       physical,
2344                                       paddr,
2345                                       logical,
2346                                       bytes);
2347             break;
2348         case gcvCACHE_CLEAN:
2349             /* Clean the cache. */
2350             status = gckOS_CacheClean(Kernel->os,
2351                                       processID,
2352                                       physical,
2353                                       paddr,
2354                                       logical,
2355                                       bytes);
2356             break;
2357         case gcvCACHE_INVALIDATE:
2358             /* Invalidate the cache. */
2359             status = gckOS_CacheInvalidate(Kernel->os,
2360                                            processID,
2361                                            physical,
2362                                            paddr,
2363                                            logical,
2364                                            bytes);
2365             break;
2366
2367         case gcvCACHE_MEMORY_BARRIER:
2368             status = gckOS_MemoryBarrier(Kernel->os,
2369                                          logical);
2370             break;
2371         default:
2372             status = gcvSTATUS_INVALID_ARGUMENT;
2373             break;
2374         }
2375         break;
2376
2377     case gcvHAL_TIMESTAMP:
2378         /* Check for invalid timer. */
2379         if ((Interface->u.TimeStamp.timer >= gcmCOUNTOF(Kernel->timers))
2380         ||  (Interface->u.TimeStamp.request != 2))
2381         {
2382             Interface->u.TimeStamp.timeDelta = 0;
2383             gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT);
2384         }
2385
2386         /* Return timer results and reset timer. */
2387         {
2388             gcsTIMER_PTR timer = &(Kernel->timers[Interface->u.TimeStamp.timer]);
2389             gctUINT64 timeDelta = 0;
2390
2391             if (timer->stopTime < timer->startTime )
2392             {
2393                 Interface->u.TimeStamp.timeDelta = 0;
2394                 gcmkONERROR(gcvSTATUS_TIMER_OVERFLOW);
2395             }
2396
2397             timeDelta = timer->stopTime - timer->startTime;
2398
2399             /* Check truncation overflow. */
2400             Interface->u.TimeStamp.timeDelta = (gctINT32) timeDelta;
2401             /*bit0~bit30 is available*/
2402             if (timeDelta>>31)
2403             {
2404                 Interface->u.TimeStamp.timeDelta = 0;
2405                 gcmkONERROR(gcvSTATUS_TIMER_OVERFLOW);
2406             }
2407
2408             status = gcvSTATUS_OK;
2409         }
2410         break;
2411
2412     case gcvHAL_DATABASE:
2413         gcmkONERROR(gckKERNEL_QueryDatabase(Kernel, processID, Interface));
2414         break;
2415
2416     case gcvHAL_VERSION:
2417         Interface->u.Version.major = gcvVERSION_MAJOR;
2418         Interface->u.Version.minor = gcvVERSION_MINOR;
2419         Interface->u.Version.patch = gcvVERSION_PATCH;
2420         Interface->u.Version.build = gcvVERSION_BUILD;
2421 #if gcmIS_DEBUG(gcdDEBUG_TRACE)
2422         gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_KERNEL,
2423                        "KERNEL version %d.%d.%d build %u %s %s",
2424                        gcvVERSION_MAJOR, gcvVERSION_MINOR, gcvVERSION_PATCH,
2425                        gcvVERSION_BUILD, gcvVERSION_DATE, gcvVERSION_TIME);
2426 #endif
2427         break;
2428
2429     case gcvHAL_CHIP_INFO:
2430         /* Only if not support multi-core */
2431         Interface->u.ChipInfo.count = 1;
2432         Interface->u.ChipInfo.types[0] = Kernel->hardware->type;
2433         break;
2434
2435 #if (gcdENABLE_3D || gcdENABLE_2D)
2436     case gcvHAL_ATTACH:
2437         /* Attach user process. */
2438         gcmkONERROR(
2439             gckCOMMAND_Attach(Kernel->command,
2440                               &context,
2441                               &bytes,
2442                               processID));
2443
2444         Interface->u.Attach.stateCount = bytes;
2445         Interface->u.Attach.context = gcmPTR_TO_NAME(context);
2446
2447         if (Interface->u.Attach.map == gcvTRUE)
2448         {
2449             gcmkVERIFY_OK(
2450                 gckCONTEXT_MapBuffer(context,
2451                                      Interface->u.Attach.physicals,
2452                                      Interface->u.Attach.logicals,
2453                                      &Interface->u.Attach.bytes));
2454         }
2455
2456         gcmkVERIFY_OK(
2457             gckKERNEL_AddProcessDB(Kernel,
2458                                    processID, gcvDB_CONTEXT,
2459                                    gcmINT2PTR(Interface->u.Attach.context),
2460                                    gcvNULL,
2461                                    0));
2462         break;
2463 #endif
2464
2465     case gcvHAL_DETACH:
2466         gcmkVERIFY_OK(
2467             gckKERNEL_RemoveProcessDB(Kernel,
2468                               processID, gcvDB_CONTEXT,
2469                               gcmINT2PTR(Interface->u.Detach.context)));
2470
2471         /* Detach user process. */
2472         gcmkONERROR(
2473             gckCOMMAND_Detach(Kernel->command,
2474                               gcmNAME_TO_PTR(Interface->u.Detach.context)));
2475
2476         gcmRELEASE_NAME(Interface->u.Detach.context);
2477         break;
2478
2479     case gcvHAL_COMPOSE:
2480         Interface->u.Compose.physical = gcmPTR_TO_UINT64(gcmNAME_TO_PTR(Interface->u.Compose.physical));
2481         /* Start composition. */
2482         gcmkONERROR(
2483             gckEVENT_Compose(Kernel->eventObj,
2484                              &Interface->u.Compose));
2485         break;
2486
2487     case gcvHAL_SET_TIMEOUT:
2488          /* set timeOut value from user */
2489          gckKERNEL_SetTimeOut(Kernel, Interface->u.SetTimeOut.timeOut);
2490         break;
2491
2492     case gcvHAL_GET_FRAME_INFO:
2493         gcmkONERROR(gckHARDWARE_GetFrameInfo(
2494                     Kernel->hardware,
2495                     gcmUINT64_TO_PTR(Interface->u.GetFrameInfo.frameInfo)));
2496         break;
2497
2498     case gcvHAL_SET_FSCALE_VALUE:
2499 #if gcdENABLE_FSCALE_VAL_ADJUST
2500         status = gckHARDWARE_SetFscaleValue(Kernel->hardware,
2501                                             Interface->u.SetFscaleValue.value);
2502 #else
2503         status = gcvSTATUS_NOT_SUPPORTED;
2504 #endif
2505         break;
2506     case gcvHAL_GET_FSCALE_VALUE:
2507 #if gcdENABLE_FSCALE_VAL_ADJUST
2508         status = gckHARDWARE_GetFscaleValue(Kernel->hardware,
2509                                             &Interface->u.GetFscaleValue.value,
2510                                             &Interface->u.GetFscaleValue.minValue,
2511                                             &Interface->u.GetFscaleValue.maxValue);
2512 #else
2513         status = gcvSTATUS_NOT_SUPPORTED;
2514 #endif
2515         break;
2516
2517     case gcvHAL_NAME_VIDEO_MEMORY:
2518         gcmkONERROR(gckVIDMEM_NODE_Name(Kernel,
2519                                         Interface->u.NameVideoMemory.handle,
2520                                         &Interface->u.NameVideoMemory.name));
2521         break;
2522
2523     case gcvHAL_IMPORT_VIDEO_MEMORY:
2524         gcmkONERROR(gckVIDMEM_NODE_Import(Kernel,
2525                                           Interface->u.ImportVideoMemory.name,
2526                                           &Interface->u.ImportVideoMemory.handle));
2527
2528         gcmkONERROR(
2529             gckKERNEL_AddProcessDB(Kernel,
2530                                    processID, gcvDB_VIDEO_MEMORY,
2531                                    gcmINT2PTR(Interface->u.ImportVideoMemory.handle),
2532                                    gcvNULL,
2533                                    0));
2534         break;
2535
2536     case gcvHAL_GET_VIDEO_MEMORY_FD:
2537         gcmkONERROR(gckVIDMEM_NODE_GetFd(
2538             Kernel,
2539             Interface->u.GetVideoMemoryFd.handle,
2540             &Interface->u.GetVideoMemoryFd.fd
2541             ));
2542
2543         /* No need to add it to processDB because OS will release all fds when
2544         ** process quits.
2545         */
2546         break;
2547
2548     case gcvHAL_QUERY_RESET_TIME_STAMP:
2549         Interface->u.QueryResetTimeStamp.timeStamp = Kernel->resetTimeStamp;
2550         break;
2551
2552     case gcvHAL_FREE_VIRTUAL_COMMAND_BUFFER:
2553         buffer = (gckVIRTUAL_COMMAND_BUFFER_PTR)gcmNAME_TO_PTR(Interface->u.FreeVirtualCommandBuffer.physical);
2554
2555         gcmkVERIFY_OK(gckKERNEL_RemoveProcessDB(
2556             Kernel,
2557             processID,
2558             gcvDB_COMMAND_BUFFER,
2559             gcmUINT64_TO_PTR(Interface->u.FreeVirtualCommandBuffer.logical)));
2560
2561         gcmkONERROR(gckOS_DestroyUserVirtualMapping(
2562             Kernel->os,
2563             buffer->physical,
2564             (gctSIZE_T)Interface->u.FreeVirtualCommandBuffer.bytes,
2565             gcmUINT64_TO_PTR(Interface->u.FreeVirtualCommandBuffer.logical)));
2566
2567         gcmkONERROR(gckKERNEL_DestroyVirtualCommandBuffer(
2568             Kernel,
2569             (gctSIZE_T)Interface->u.FreeVirtualCommandBuffer.bytes,
2570             (gctPHYS_ADDR)buffer,
2571             gcmUINT64_TO_PTR(Interface->u.FreeVirtualCommandBuffer.logical)));
2572
2573         gcmRELEASE_NAME(Interface->u.FreeVirtualCommandBuffer.physical);
2574         break;
2575
2576 #if gcdANDROID_NATIVE_FENCE_SYNC
2577     case gcvHAL_SYNC_POINT:
2578         {
2579             gctSYNC_POINT syncPoint;
2580
2581             switch (Interface->u.SyncPoint.command)
2582             {
2583             case gcvSYNC_POINT_CREATE:
2584                 gcmkONERROR(gckOS_CreateSyncPoint(Kernel->os, &syncPoint));
2585
2586                 Interface->u.SyncPoint.syncPoint = gcmPTR_TO_UINT64(syncPoint);
2587
2588                 gcmkVERIFY_OK(
2589                     gckKERNEL_AddProcessDB(Kernel,
2590                                            processID, gcvDB_SYNC_POINT,
2591                                            syncPoint,
2592                                            gcvNULL,
2593                                            0));
2594                 break;
2595
2596             case gcvSYNC_POINT_DESTROY:
2597                 syncPoint = gcmUINT64_TO_PTR(Interface->u.SyncPoint.syncPoint);
2598
2599                 gcmkONERROR(gckOS_DestroySyncPoint(Kernel->os, syncPoint));
2600
2601                 gcmkVERIFY_OK(
2602                     gckKERNEL_RemoveProcessDB(Kernel,
2603                                               processID, gcvDB_SYNC_POINT,
2604                                               syncPoint));
2605                 break;
2606
2607             default:
2608                 gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT);
2609                 break;
2610             }
2611         }
2612         break;
2613
2614     case gcvHAL_CREATE_NATIVE_FENCE:
2615         {
2616             gctINT fenceFD;
2617             gctSYNC_POINT syncPoint =
2618                 gcmUINT64_TO_PTR(Interface->u.CreateNativeFence.syncPoint);
2619
2620             gcmkONERROR(
2621                 gckOS_CreateNativeFence(Kernel->os,
2622                                         Kernel->timeline,
2623                                         syncPoint,
2624                                         &fenceFD));
2625
2626             Interface->u.CreateNativeFence.fenceFD = fenceFD;
2627         }
2628         break;
2629 #endif
2630
2631     case gcvHAL_SHBUF:
2632         {
2633             gctSHBUF shBuf;
2634             gctPOINTER uData;
2635             gctUINT32 bytes;
2636
2637             switch (Interface->u.ShBuf.command)
2638             {
2639             case gcvSHBUF_CREATE:
2640                 bytes = Interface->u.ShBuf.bytes;
2641
2642                 /* Create. */
2643                 gcmkONERROR(gckKERNEL_CreateShBuffer(Kernel, bytes, &shBuf));
2644
2645                 Interface->u.ShBuf.id = gcmPTR_TO_UINT64(shBuf);
2646
2647                 gcmkVERIFY_OK(
2648                     gckKERNEL_AddProcessDB(Kernel,
2649                                            processID,
2650                                            gcvDB_SHBUF,
2651                                            shBuf,
2652                                            gcvNULL,
2653                                            0));
2654                 break;
2655
2656             case gcvSHBUF_DESTROY:
2657                 shBuf = gcmUINT64_TO_PTR(Interface->u.ShBuf.id);
2658
2659                 /* Check db first to avoid illegal destroy in the process. */
2660                 gcmkONERROR(
2661                     gckKERNEL_RemoveProcessDB(Kernel,
2662                                               processID,
2663                                               gcvDB_SHBUF,
2664                                               shBuf));
2665
2666                 gcmkONERROR(gckKERNEL_DestroyShBuffer(Kernel, shBuf));
2667                 break;
2668
2669             case gcvSHBUF_MAP:
2670                 shBuf = gcmUINT64_TO_PTR(Interface->u.ShBuf.id);
2671
2672                 /* Map for current process access. */
2673                 gcmkONERROR(gckKERNEL_MapShBuffer(Kernel, shBuf));
2674
2675                 gcmkVERIFY_OK(
2676                     gckKERNEL_AddProcessDB(Kernel,
2677                                            processID,
2678                                            gcvDB_SHBUF,
2679                                            shBuf,
2680                                            gcvNULL,
2681                                            0));
2682                 break;
2683
2684             case gcvSHBUF_WRITE:
2685                 shBuf = gcmUINT64_TO_PTR(Interface->u.ShBuf.id);
2686                 uData = gcmUINT64_TO_PTR(Interface->u.ShBuf.data);
2687                 bytes = Interface->u.ShBuf.bytes;
2688
2689                 /* Write. */
2690                 gcmkONERROR(
2691                     gckKERNEL_WriteShBuffer(Kernel, shBuf, uData, bytes));
2692                 break;
2693
2694             case gcvSHBUF_READ:
2695                 shBuf = gcmUINT64_TO_PTR(Interface->u.ShBuf.id);
2696                 uData = gcmUINT64_TO_PTR(Interface->u.ShBuf.data);
2697                 bytes = Interface->u.ShBuf.bytes;
2698
2699                 /* Read. */
2700                 gcmkONERROR(
2701                     gckKERNEL_ReadShBuffer(Kernel,
2702                                            shBuf,
2703                                            uData,
2704                                            bytes,
2705                                            &bytes));
2706
2707                 /* Return copied size. */
2708                 Interface->u.ShBuf.bytes = bytes;
2709                 break;
2710
2711             default:
2712                 gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT);
2713                 break;
2714             }
2715         }
2716         break;
2717
2718     case gcvHAL_CONFIG_POWER_MANAGEMENT:
2719         gcmkONERROR(gckKERNEL_ConfigPowerManagement(Kernel, Interface));
2720         break;
2721
2722     default:
2723         /* Invalid command. */
2724         gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT);
2725     }
2726
2727 OnError:
2728     /* Save status. */
2729     Interface->status = status;
2730
2731 #if QNX_SINGLE_THREADED_DEBUGGING
2732     gckOS_ReleaseMutex(Kernel->os, Kernel->debugMutex);
2733 #endif
2734
2735     if (powerMutexAcquired == gcvTRUE)
2736     {
2737         gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, Kernel->hardware->powerMutex));
2738     }
2739
2740     /* Return the status. */
2741     gcmkFOOTER();
2742     return status;
2743 }
2744
2745 /*******************************************************************************
2746 **  gckKERNEL_AttachProcess
2747 **
2748 **  Attach or detach a process.
2749 **
2750 **  INPUT:
2751 **
2752 **      gckKERNEL Kernel
2753 **          Pointer to an gckKERNEL object.
2754 **
2755 **      gctBOOL Attach
2756 **          gcvTRUE if a new process gets attached or gcFALSE when a process
2757 **          gets detatched.
2758 **
2759 **  OUTPUT:
2760 **
2761 **      Nothing.
2762 */
2763 gceSTATUS
2764 gckKERNEL_AttachProcess(
2765     IN gckKERNEL Kernel,
2766     IN gctBOOL Attach
2767     )
2768 {
2769     gceSTATUS status;
2770     gctUINT32 processID;
2771
2772     gcmkHEADER_ARG("Kernel=0x%x Attach=%d", Kernel, Attach);
2773
2774     /* Verify the arguments. */
2775     gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL);
2776
2777     /* Get current process ID. */
2778     gcmkONERROR(gckOS_GetProcessID(&processID));
2779
2780     gcmkONERROR(gckKERNEL_AttachProcessEx(Kernel, Attach, processID));
2781
2782     /* Success. */
2783     gcmkFOOTER_NO();
2784     return gcvSTATUS_OK;
2785
2786 OnError:
2787     /* Return the status. */
2788     gcmkFOOTER();
2789     return status;
2790 }
2791
2792 /*******************************************************************************
2793 **  gckKERNEL_AttachProcessEx
2794 **
2795 **  Attach or detach a process with the given PID. Can be paired with gckKERNEL_AttachProcess
2796 **     provided the programmer is aware of the consequences.
2797 **
2798 **  INPUT:
2799 **
2800 **      gckKERNEL Kernel
2801 **          Pointer to an gckKERNEL object.
2802 **
2803 **      gctBOOL Attach
2804 **          gcvTRUE if a new process gets attached or gcFALSE when a process
2805 **          gets detatched.
2806 **
2807 **      gctUINT32 PID
2808 **          PID of the process to attach or detach.
2809 **
2810 **  OUTPUT:
2811 **
2812 **      Nothing.
2813 */
2814 gceSTATUS
2815 gckKERNEL_AttachProcessEx(
2816     IN gckKERNEL Kernel,
2817     IN gctBOOL Attach,
2818     IN gctUINT32 PID
2819     )
2820 {
2821     gceSTATUS status;
2822     gctINT32 old;
2823
2824     gcmkHEADER_ARG("Kernel=0x%x Attach=%d PID=%d", Kernel, Attach, PID);
2825
2826     /* Verify the arguments. */
2827     gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL);
2828
2829     if (Attach)
2830     {
2831         /* Increment the number of clients attached. */
2832         gcmkONERROR(
2833             gckOS_AtomIncrement(Kernel->os, Kernel->atomClients, &old));
2834
2835         if (old == 0)
2836         {
2837 #if gcdENABLE_VG
2838             if (Kernel->vg == gcvNULL)
2839 #endif
2840             {
2841                 gcmkONERROR(gckOS_Broadcast(Kernel->os,
2842                                             Kernel->hardware,
2843                                             gcvBROADCAST_FIRST_PROCESS));
2844             }
2845         }
2846
2847         if (Kernel->dbCreated)
2848         {
2849             /* Create the process database. */
2850             gcmkONERROR(gckKERNEL_CreateProcessDB(Kernel, PID));
2851         }
2852
2853 #if gcdPROCESS_ADDRESS_SPACE
2854         /* Map kernel command buffer in the process's own MMU. */
2855         gcmkONERROR(_MapCommandBuffer(Kernel));
2856 #endif
2857     }
2858     else
2859     {
2860         if (Kernel->dbCreated)
2861         {
2862             /* Clean up the process database. */
2863             gcmkONERROR(gckKERNEL_DestroyProcessDB(Kernel, PID));
2864
2865             /* Save the last know process ID. */
2866             Kernel->db->lastProcessID = PID;
2867         }
2868
2869 #if gcdENABLE_VG
2870         if (Kernel->vg == gcvNULL)
2871 #endif
2872         {
2873 #if gcdMULTI_GPU
2874             status = gckEVENT_Submit(Kernel->eventObj, gcvTRUE, gcvFALSE, gcvCORE_3D_ALL_MASK);
2875 #else
2876             status = gckEVENT_Submit(Kernel->eventObj, gcvTRUE, gcvFALSE);
2877 #endif
2878
2879             if (status == gcvSTATUS_INTERRUPTED && Kernel->eventObj->submitTimer)
2880             {
2881                 gcmkONERROR(gckOS_StartTimer(Kernel->os,
2882                                              Kernel->eventObj->submitTimer,
2883                                              1));
2884             }
2885             else
2886             {
2887                 gcmkONERROR(status);
2888             }
2889         }
2890
2891         /* Decrement the number of clients attached. */
2892         gcmkONERROR(
2893             gckOS_AtomDecrement(Kernel->os, Kernel->atomClients, &old));
2894
2895         if (old == 1)
2896         {
2897 #if gcdENABLE_VG
2898             if (Kernel->vg == gcvNULL)
2899 #endif
2900             {
2901                 /* Last client detached, switch to SUSPEND power state. */
2902                 gcmkONERROR(gckOS_Broadcast(Kernel->os,
2903                                             Kernel->hardware,
2904                                             gcvBROADCAST_LAST_PROCESS));
2905             }
2906
2907             /* Flush the debug cache. */
2908             gcmkDEBUGFLUSH(~0U);
2909         }
2910     }
2911
2912     /* Success. */
2913     gcmkFOOTER_NO();
2914     return gcvSTATUS_OK;
2915
2916 OnError:
2917     /* Return the status. */
2918     gcmkFOOTER();
2919     return status;
2920 }
2921
2922 #if gcdSECURE_USER
2923 gceSTATUS
2924 gckKERNEL_MapLogicalToPhysical(
2925     IN gckKERNEL Kernel,
2926     IN gcskSECURE_CACHE_PTR Cache,
2927     IN OUT gctPOINTER * Data
2928     )
2929 {
2930     gceSTATUS status;
2931     static gctBOOL baseAddressValid = gcvFALSE;
2932     static gctUINT32 baseAddress;
2933     gctBOOL needBase;
2934     gcskLOGICAL_CACHE_PTR slot;
2935
2936     gcmkHEADER_ARG("Kernel=0x%x Cache=0x%x *Data=0x%x",
2937                    Kernel, Cache, gcmOPT_POINTER(Data));
2938
2939     /* Verify the arguments. */
2940     gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL);
2941
2942     if (!baseAddressValid)
2943     {
2944         /* Get base address. */
2945         gcmkONERROR(gckHARDWARE_GetBaseAddress(Kernel->hardware, &baseAddress));
2946
2947         baseAddressValid = gcvTRUE;
2948     }
2949
2950     /* Does this state load need a base address? */
2951     gcmkONERROR(gckHARDWARE_NeedBaseAddress(Kernel->hardware,
2952                                             ((gctUINT32_PTR) Data)[-1],
2953                                             &needBase));
2954
2955 #if gcdSECURE_CACHE_METHOD == gcdSECURE_CACHE_LRU
2956     {
2957         gcskLOGICAL_CACHE_PTR next;
2958         gctINT i;
2959
2960         /* Walk all used cache slots. */
2961         for (i = 1, slot = Cache->cache[0].next, next = gcvNULL;
2962              (i <= gcdSECURE_CACHE_SLOTS) && (slot->logical != gcvNULL);
2963              ++i, slot = slot->next
2964         )
2965         {
2966             if (slot->logical == *Data)
2967             {
2968                 /* Bail out. */
2969                 next = slot;
2970                 break;
2971             }
2972         }
2973
2974         /* See if we had a miss. */
2975         if (next == gcvNULL)
2976         {
2977             /* Use the tail of the cache. */
2978             slot = Cache->cache[0].prev;
2979
2980             /* Initialize the cache line. */
2981             slot->logical = *Data;
2982
2983             /* Map the logical address to a DMA address. */
2984             gcmkONERROR(
2985                 gckOS_GetPhysicalAddress(Kernel->os, *Data, &slot->dma));
2986         }
2987
2988         /* Move slot to head of list. */
2989         if (slot != Cache->cache[0].next)
2990         {
2991             /* Unlink. */
2992             slot->prev->next = slot->next;
2993             slot->next->prev = slot->prev;
2994
2995             /* Move to head of chain. */
2996             slot->prev       = &Cache->cache[0];
2997             slot->next       = Cache->cache[0].next;
2998             slot->prev->next = slot;
2999             slot->next->prev = slot;
3000         }
3001     }
3002 #elif gcdSECURE_CACHE_METHOD == gcdSECURE_CACHE_LINEAR
3003     {
3004         gctINT i;
3005         gcskLOGICAL_CACHE_PTR next = gcvNULL;
3006         gcskLOGICAL_CACHE_PTR oldestSlot = gcvNULL;
3007         slot = gcvNULL;
3008
3009         if (Cache->cacheIndex != gcvNULL)
3010         {
3011             /* Walk the cache forwards. */
3012             for (i = 1, slot = Cache->cacheIndex;
3013                  (i <= gcdSECURE_CACHE_SLOTS) && (slot->logical != gcvNULL);
3014                  ++i, slot = slot->next)
3015             {
3016                 if (slot->logical == *Data)
3017                 {
3018                     /* Bail out. */
3019                     next = slot;
3020                     break;
3021                 }
3022
3023                 /* Determine age of this slot. */
3024                 if ((oldestSlot       == gcvNULL)
3025                 ||  (oldestSlot->stamp > slot->stamp)
3026                 )
3027                 {
3028                     oldestSlot = slot;
3029                 }
3030             }
3031
3032             if (next == gcvNULL)
3033             {
3034                 /* Walk the cache backwards. */
3035                 for (slot = Cache->cacheIndex->prev;
3036                      (i <= gcdSECURE_CACHE_SLOTS) && (slot->logical != gcvNULL);
3037                      ++i, slot = slot->prev)
3038                 {
3039                     if (slot->logical == *Data)
3040                     {
3041                         /* Bail out. */
3042                         next = slot;
3043                         break;
3044                     }
3045
3046                     /* Determine age of this slot. */
3047                     if ((oldestSlot       == gcvNULL)
3048                     ||  (oldestSlot->stamp > slot->stamp)
3049                     )
3050                     {
3051                         oldestSlot = slot;
3052                     }
3053                 }
3054             }
3055         }
3056
3057         /* See if we had a miss. */
3058         if (next == gcvNULL)
3059         {
3060             if (Cache->cacheFree != 0)
3061             {
3062                 slot = &Cache->cache[Cache->cacheFree];
3063                 gcmkASSERT(slot->logical == gcvNULL);
3064
3065                 ++ Cache->cacheFree;
3066                 if (Cache->cacheFree >= gcmCOUNTOF(Cache->cache))
3067                 {
3068                     Cache->cacheFree = 0;
3069                 }
3070             }
3071             else
3072             {
3073                 /* Use the oldest cache slot. */
3074                 gcmkASSERT(oldestSlot != gcvNULL);
3075                 slot = oldestSlot;
3076
3077                 /* Unlink from the chain. */
3078                 slot->prev->next = slot->next;
3079                 slot->next->prev = slot->prev;
3080
3081                 /* Append to the end. */
3082                 slot->prev       = Cache->cache[0].prev;
3083                 slot->next       = &Cache->cache[0];
3084                 slot->prev->next = slot;
3085                 slot->next->prev = slot;
3086             }
3087
3088             /* Initialize the cache line. */
3089             slot->logical = *Data;
3090
3091             /* Map the logical address to a DMA address. */
3092             gcmkONERROR(
3093                 gckOS_GetPhysicalAddress(Kernel->os, *Data, &slot->dma));
3094         }
3095
3096         /* Save time stamp. */
3097         slot->stamp = ++ Cache->cacheStamp;
3098
3099         /* Save current slot for next lookup. */
3100         Cache->cacheIndex = slot;
3101     }
3102 #elif gcdSECURE_CACHE_METHOD == gcdSECURE_CACHE_HASH
3103     {
3104         gctINT i;
3105         gctUINT32 data = gcmPTR2INT32(*Data);
3106         gctUINT32 key, index;
3107         gcskLOGICAL_CACHE_PTR hash;
3108
3109         /* Generate a hash key. */
3110         key   = (data >> 24) + (data >> 16) + (data >> 8) + data;
3111         index = key % gcmCOUNTOF(Cache->hash);
3112
3113         /* Get the hash entry. */
3114         hash = &Cache->hash[index];
3115
3116         for (slot = hash->nextHash, i = 0;
3117              (slot != gcvNULL) && (i < gcdSECURE_CACHE_SLOTS);
3118              slot = slot->nextHash, ++i
3119         )
3120         {
3121             if (slot->logical == (*Data))
3122             {
3123                 break;
3124             }
3125         }
3126
3127         if (slot == gcvNULL)
3128         {
3129             /* Grab from the tail of the cache. */
3130             slot = Cache->cache[0].prev;
3131
3132             /* Unlink slot from any hash table it is part of. */
3133             if (slot->prevHash != gcvNULL)
3134             {
3135                 slot->prevHash->nextHash = slot->nextHash;
3136             }
3137             if (slot->nextHash != gcvNULL)
3138             {
3139                 slot->nextHash->prevHash = slot->prevHash;
3140             }
3141
3142             /* Initialize the cache line. */
3143             slot->logical = *Data;
3144
3145             /* Map the logical address to a DMA address. */
3146             gcmkONERROR(
3147                 gckOS_GetPhysicalAddress(Kernel->os, *Data, &slot->dma));
3148
3149             if (hash->nextHash != gcvNULL)
3150             {
3151                 gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_KERNEL,
3152                                "Hash Collision: logical=0x%x key=0x%08x",
3153                                *Data, key);
3154             }
3155
3156             /* Insert the slot at the head of the hash list. */
3157             slot->nextHash     = hash->nextHash;
3158             if (slot->nextHash != gcvNULL)
3159             {
3160                 slot->nextHash->prevHash = slot;
3161             }
3162             slot->prevHash     = hash;
3163             hash->nextHash     = slot;
3164         }
3165
3166         /* Move slot to head of list. */
3167         if (slot != Cache->cache[0].next)
3168         {
3169             /* Unlink. */
3170             slot->prev->next = slot->next;
3171             slot->next->prev = slot->prev;
3172
3173             /* Move to head of chain. */
3174             slot->prev       = &Cache->cache[0];
3175             slot->next       = Cache->cache[0].next;
3176             slot->prev->next = slot;
3177             slot->next->prev = slot;
3178         }
3179     }
3180 #elif gcdSECURE_CACHE_METHOD == gcdSECURE_CACHE_TABLE
3181     {
3182         gctUINT32 index = (gcmPTR2INT32(*Data) % gcdSECURE_CACHE_SLOTS) + 1;
3183
3184         /* Get cache slot. */
3185         slot = &Cache->cache[index];
3186
3187         /* Check for cache miss. */
3188         if (slot->logical != *Data)
3189         {
3190             /* Initialize the cache line. */
3191             slot->logical = *Data;
3192
3193             /* Map the logical address to a DMA address. */
3194             gcmkONERROR(
3195                 gckOS_GetPhysicalAddress(Kernel->os, *Data, &slot->dma));
3196         }
3197     }
3198 #endif
3199
3200     /* Return DMA address. */
3201     *Data = gcmINT2PTR(slot->dma + (needBase ? baseAddress : 0));
3202
3203     /* Success. */
3204     gcmkFOOTER_ARG("*Data=0x%08x", *Data);
3205     return gcvSTATUS_OK;
3206
3207 OnError:
3208     /* Return the status. */
3209     gcmkFOOTER();
3210     return status;
3211 }
3212
3213 gceSTATUS
3214 gckKERNEL_FlushTranslationCache(
3215     IN gckKERNEL Kernel,
3216     IN gcskSECURE_CACHE_PTR Cache,
3217     IN gctPOINTER Logical,
3218     IN gctSIZE_T Bytes
3219     )
3220 {
3221     gctINT i;
3222     gcskLOGICAL_CACHE_PTR slot;
3223     gctUINT8_PTR ptr;
3224
3225     gcmkHEADER_ARG("Kernel=0x%x Cache=0x%x Logical=0x%x Bytes=%lu",
3226                    Kernel, Cache, Logical, Bytes);
3227
3228     /* Do we need to flush the entire cache? */
3229     if (Logical == gcvNULL)
3230     {
3231         /* Clear all cache slots. */
3232         for (i = 1; i <= gcdSECURE_CACHE_SLOTS; ++i)
3233         {
3234             Cache->cache[i].logical  = gcvNULL;
3235
3236 #if gcdSECURE_CACHE_METHOD == gcdSECURE_CACHE_HASH
3237             Cache->cache[i].nextHash = gcvNULL;
3238             Cache->cache[i].prevHash = gcvNULL;
3239 #endif
3240 }
3241
3242 #if gcdSECURE_CACHE_METHOD == gcdSECURE_CACHE_HASH
3243         /* Zero the hash table. */
3244         for (i = 0; i < gcmCOUNTOF(Cache->hash); ++i)
3245         {
3246             Cache->hash[i].nextHash = gcvNULL;
3247         }
3248 #endif
3249
3250         /* Reset the cache functionality. */
3251         Cache->cacheIndex = gcvNULL;
3252         Cache->cacheFree  = 1;
3253         Cache->cacheStamp = 0;
3254     }
3255
3256     else
3257     {
3258         gctUINT8_PTR low  = (gctUINT8_PTR) Logical;
3259         gctUINT8_PTR high = low + Bytes;
3260
3261 #if gcdSECURE_CACHE_METHOD == gcdSECURE_CACHE_LRU
3262         gcskLOGICAL_CACHE_PTR next;
3263
3264         /* Walk all used cache slots. */
3265         for (i = 1, slot = Cache->cache[0].next;
3266              (i <= gcdSECURE_CACHE_SLOTS) && (slot->logical != gcvNULL);
3267              ++i, slot = next
3268         )
3269         {
3270             /* Save pointer to next slot. */
3271             next = slot->next;
3272
3273             /* Test if this slot falls within the range to flush. */
3274             ptr = (gctUINT8_PTR) slot->logical;
3275             if ((ptr >= low) && (ptr < high))
3276             {
3277                 /* Unlink slot. */
3278                 slot->prev->next = slot->next;
3279                 slot->next->prev = slot->prev;
3280
3281                 /* Append slot to tail of cache. */
3282                 slot->prev       = Cache->cache[0].prev;
3283                 slot->next       = &Cache->cache[0];
3284                 slot->prev->next = slot;
3285                 slot->next->prev = slot;
3286
3287                 /* Mark slot as empty. */
3288                 slot->logical = gcvNULL;
3289             }
3290         }
3291
3292 #elif gcdSECURE_CACHE_METHOD == gcdSECURE_CACHE_LINEAR
3293         gcskLOGICAL_CACHE_PTR next;
3294
3295         for (i = 1, slot = Cache->cache[0].next;
3296              (i <= gcdSECURE_CACHE_SLOTS) && (slot->logical != gcvNULL);
3297              ++i, slot = next)
3298         {
3299             /* Save pointer to next slot. */
3300             next = slot->next;
3301
3302             /* Test if this slot falls within the range to flush. */
3303             ptr = (gctUINT8_PTR) slot->logical;
3304             if ((ptr >= low) && (ptr < high))
3305             {
3306                 /* Test if this slot is the current slot. */
3307                 if (slot == Cache->cacheIndex)
3308                 {
3309                     /* Move to next or previous slot. */
3310                     Cache->cacheIndex = (slot->next->logical != gcvNULL)
3311                                       ? slot->next
3312                                       : (slot->prev->logical != gcvNULL)
3313                                       ? slot->prev
3314                                       : gcvNULL;
3315                 }
3316
3317                 /* Unlink slot from cache. */
3318                 slot->prev->next = slot->next;
3319                 slot->next->prev = slot->prev;
3320
3321                 /* Insert slot to head of cache. */
3322                 slot->prev       = &Cache->cache[0];
3323                 slot->next       = Cache->cache[0].next;
3324                 slot->prev->next = slot;
3325                 slot->next->prev = slot;
3326
3327                 /* Mark slot as empty. */
3328                 slot->logical = gcvNULL;
3329                 slot->stamp   = 0;
3330             }
3331         }
3332
3333 #elif gcdSECURE_CACHE_METHOD == gcdSECURE_CACHE_HASH
3334         gctINT j;
3335         gcskLOGICAL_CACHE_PTR hash, next;
3336
3337         /* Walk all hash tables. */
3338         for (i = 0, hash = Cache->hash;
3339              i < gcmCOUNTOF(Cache->hash);
3340              ++i, ++hash)
3341         {
3342             /* Walk all slots in the hash. */
3343             for (j = 0, slot = hash->nextHash;
3344                  (j < gcdSECURE_CACHE_SLOTS) && (slot != gcvNULL);
3345                  ++j, slot = next)
3346             {
3347                 /* Save pointer to next slot. */
3348                 next = slot->next;
3349
3350                 /* Test if this slot falls within the range to flush. */
3351                 ptr = (gctUINT8_PTR) slot->logical;
3352                 if ((ptr >= low) && (ptr < high))
3353                 {
3354                     /* Unlink slot from hash table. */
3355                     if (slot->prevHash == hash)
3356                     {
3357                         hash->nextHash = slot->nextHash;
3358                     }
3359                     else
3360                     {
3361                         slot->prevHash->nextHash = slot->nextHash;
3362                     }
3363
3364                     if (slot->nextHash != gcvNULL)
3365                     {
3366                         slot->nextHash->prevHash = slot->prevHash;
3367                     }
3368
3369                     /* Unlink slot from cache. */
3370                     slot->prev->next = slot->next;
3371                     slot->next->prev = slot->prev;
3372
3373                     /* Append slot to tail of cache. */
3374                     slot->prev       = Cache->cache[0].prev;
3375                     slot->next       = &Cache->cache[0];
3376                     slot->prev->next = slot;
3377                     slot->next->prev = slot;
3378
3379                     /* Mark slot as empty. */
3380                     slot->logical  = gcvNULL;
3381                     slot->prevHash = gcvNULL;
3382                     slot->nextHash = gcvNULL;
3383                 }
3384             }
3385         }
3386
3387 #elif gcdSECURE_CACHE_METHOD == gcdSECURE_CACHE_TABLE
3388         gctUINT32 index;
3389
3390         /* Loop while inside the range. */
3391         for (i = 1; (low < high) && (i <= gcdSECURE_CACHE_SLOTS); ++i)
3392         {
3393             /* Get index into cache for this range. */
3394             index = (gcmPTR2INT32(low) % gcdSECURE_CACHE_SLOTS) + 1;
3395             slot  = &Cache->cache[index];
3396
3397             /* Test if this slot falls within the range to flush. */
3398             ptr = (gctUINT8_PTR) slot->logical;
3399             if ((ptr >= low) && (ptr < high))
3400             {
3401                 /* Remove entry from cache. */
3402                 slot->logical = gcvNULL;
3403             }
3404
3405             /* Next block. */
3406             low += gcdSECURE_CACHE_SLOTS;
3407         }
3408 #endif
3409     }
3410
3411     /* Success. */
3412     gcmkFOOTER_NO();
3413     return gcvSTATUS_OK;
3414 }
3415 #endif
3416
3417 /*******************************************************************************
3418 **
3419 **  gckKERNEL_Recovery
3420 **
3421 **  Try to recover the GPU from a fatal error.
3422 **
3423 **  INPUT:
3424 **
3425 **      gckKERNEL Kernel
3426 **          Pointer to an gckKERNEL object.
3427 **
3428 **  OUTPUT:
3429 **
3430 **      Nothing.
3431 */
3432 gceSTATUS
3433 gckKERNEL_Recovery(
3434     IN gckKERNEL Kernel
3435     )
3436 {
3437     gceSTATUS status;
3438     gckEVENT eventObj;
3439     gckHARDWARE hardware;
3440 #if gcdSECURE_USER
3441     gctUINT32 processID;
3442     gcskSECURE_CACHE_PTR cache;
3443 #endif
3444     gctUINT32 mask = 0;
3445     gckCOMMAND command;
3446     gckENTRYDATA data;
3447     gctUINT32 i = 0, count = 0;
3448 #if gcdINTERRUPT_STATISTIC
3449     gctINT32 oldValue;
3450 #endif
3451
3452     gcmkHEADER_ARG("Kernel=0x%x", Kernel);
3453
3454     /* Validate the arguemnts. */
3455     gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL);
3456
3457     /* Grab gckEVENT object. */
3458     eventObj = Kernel->eventObj;
3459     gcmkVERIFY_OBJECT(eventObj, gcvOBJ_EVENT);
3460
3461     /* Grab gckHARDWARE object. */
3462     hardware = Kernel->hardware;
3463     gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE);
3464
3465     /* Grab gckCOMMAND object. */
3466     command = Kernel->command;
3467     gcmkVERIFY_OBJECT(command, gcvOBJ_COMMAND);
3468
3469 #if gcdSECURE_USER
3470     /* Flush the secure mapping cache. */
3471     gcmkONERROR(gckOS_GetProcessID(&processID));
3472     gcmkONERROR(gckKERNEL_GetProcessDBCache(Kernel, processID, &cache));
3473     gcmkONERROR(gckKERNEL_FlushTranslationCache(Kernel, cache, gcvNULL, 0));
3474 #endif
3475
3476     if (Kernel->stuckDump == gcdSTUCK_DUMP_MINIMAL)
3477     {
3478         gcmkPRINT("[galcore]: GPU[%d] hang, automatic recovery.", Kernel->core);
3479     }
3480     else
3481     {
3482         _DumpDriverConfigure(Kernel);
3483         _DumpState(Kernel);
3484     }
3485
3486     if (Kernel->recovery == gcvFALSE)
3487     {
3488         gcmkPRINT("[galcore]: Stop driver to keep scene.");
3489
3490         for (;;)
3491         {
3492             gckOS_Delay(Kernel->os, 10000);
3493         }
3494     }
3495
3496     /* Clear queue. */
3497     do
3498     {
3499         status = gckENTRYQUEUE_Dequeue(&command->queue, &data);
3500     }
3501     while (status == gcvSTATUS_OK);
3502
3503     /* Issuing a soft reset for the GPU. */
3504     gcmkONERROR(gckHARDWARE_Reset(hardware));
3505
3506     mask = Kernel->restoreMask;
3507
3508     for (i = 0; i < 32; i++)
3509     {
3510         if (mask & (1 << i))
3511         {
3512             count++;
3513         }
3514     }
3515
3516     /* Handle all outstanding events now. */
3517 #if gcdSMP
3518 #if gcdMULTI_GPU
3519     if (Kernel->core == gcvCORE_MAJOR)
3520     {
3521         for (i = 0; i < gcdMULTI_GPU; i++)
3522         {
3523             gcmkONERROR(gckOS_AtomSet(Kernel->os, eventObj->pending3D[i], mask));
3524         }
3525     }
3526     else
3527     {
3528         gcmkONERROR(gckOS_AtomSet(Kernel->os, eventObj->pending, mask));
3529     }
3530 #else
3531     gcmkONERROR(gckOS_AtomSet(Kernel->os, eventObj->pending, mask));
3532 #endif
3533 #else
3534 #if gcdMULTI_GPU
3535     if (Kernel->core == gcvCORE_MAJOR)
3536     {
3537         for (i = 0; i < gcdMULTI_GPU; i++)
3538         {
3539             eventObj->pending3D[i] = mask;
3540         }
3541     }
3542     else
3543     {
3544         eventObj->pending = mask;
3545     }
3546 #else
3547     eventObj->pending = mask;
3548 #endif
3549 #endif
3550
3551 #if gcdINTERRUPT_STATISTIC
3552     while (count--)
3553     {
3554         gcmkONERROR(gckOS_AtomDecrement(
3555             Kernel->os,
3556             eventObj->interruptCount,
3557             &oldValue
3558             ));
3559     }
3560
3561     gckOS_AtomClearMask(Kernel->hardware->pendingEvent, mask);
3562 #endif
3563
3564     gcmkONERROR(gckEVENT_Notify(eventObj, 1));
3565
3566     gcmkVERIFY_OK(gckOS_GetTime(&Kernel->resetTimeStamp));
3567
3568     /* Success. */
3569     gcmkFOOTER_NO();
3570     return gcvSTATUS_OK;
3571
3572 OnError:
3573     /* Return the status. */
3574     gcmkFOOTER();
3575     return status;
3576 }
3577
3578 /*******************************************************************************
3579 **
3580 **  gckKERNEL_OpenUserData
3581 **
3582 **  Get access to the user data.
3583 **
3584 **  INPUT:
3585 **
3586 **      gckKERNEL Kernel
3587 **          Pointer to an gckKERNEL object.
3588 **
3589 **      gctBOOL NeedCopy
3590 **          The flag indicating whether or not the data should be copied.
3591 **
3592 **      gctPOINTER StaticStorage
3593 **          Pointer to the kernel storage where the data is to be copied if
3594 **          NeedCopy is gcvTRUE.
3595 **
3596 **      gctPOINTER UserPointer
3597 **          User pointer to the data.
3598 **
3599 **      gctSIZE_T Size
3600 **          Size of the data.
3601 **
3602 **  OUTPUT:
3603 **
3604 **      gctPOINTER * KernelPointer
3605 **          Pointer to the kernel pointer that will be pointing to the data.
3606 */
3607 gceSTATUS
3608 gckKERNEL_OpenUserData(
3609     IN gckKERNEL Kernel,
3610     IN gctBOOL NeedCopy,
3611     IN gctPOINTER StaticStorage,
3612     IN gctPOINTER UserPointer,
3613     IN gctSIZE_T Size,
3614     OUT gctPOINTER * KernelPointer
3615     )
3616 {
3617     gceSTATUS status;
3618
3619     gcmkHEADER_ARG(
3620         "Kernel=0x%08X NeedCopy=%d StaticStorage=0x%08X "
3621         "UserPointer=0x%08X Size=%lu KernelPointer=0x%08X",
3622         Kernel, NeedCopy, StaticStorage, UserPointer, Size, KernelPointer
3623         );
3624
3625     /* Validate the arguemnts. */
3626     gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL);
3627     gcmkVERIFY_ARGUMENT(!NeedCopy || (StaticStorage != gcvNULL));
3628     gcmkVERIFY_ARGUMENT(UserPointer != gcvNULL);
3629     gcmkVERIFY_ARGUMENT(KernelPointer != gcvNULL);
3630     gcmkVERIFY_ARGUMENT(Size > 0);
3631
3632     if (NeedCopy)
3633     {
3634         /* Copy the user data to the static storage. */
3635         gcmkONERROR(gckOS_CopyFromUserData(
3636             Kernel->os, StaticStorage, UserPointer, Size
3637             ));
3638
3639         /* Set the kernel pointer. */
3640         * KernelPointer = StaticStorage;
3641     }
3642     else
3643     {
3644         gctPOINTER pointer = gcvNULL;
3645
3646         /* Map the user pointer. */
3647         gcmkONERROR(gckOS_MapUserPointer(
3648             Kernel->os, UserPointer, Size, &pointer
3649             ));
3650
3651         /* Set the kernel pointer. */
3652         * KernelPointer = pointer;
3653     }
3654
3655 OnError:
3656     /* Return the status. */
3657     gcmkFOOTER();
3658     return status;
3659 }
3660
3661 /*******************************************************************************
3662 **
3663 **  gckKERNEL_CloseUserData
3664 **
3665 **  Release resources associated with the user data connection opened by
3666 **  gckKERNEL_OpenUserData.
3667 **
3668 **  INPUT:
3669 **
3670 **      gckKERNEL Kernel
3671 **          Pointer to an gckKERNEL object.
3672 **
3673 **      gctBOOL NeedCopy
3674 **          The flag indicating whether or not the data should be copied.
3675 **
3676 **      gctBOOL FlushData
3677 **          If gcvTRUE, the data is written back to the user.
3678 **
3679 **      gctPOINTER UserPointer
3680 **          User pointer to the data.
3681 **
3682 **      gctSIZE_T Size
3683 **          Size of the data.
3684 **
3685 **  OUTPUT:
3686 **
3687 **      gctPOINTER * KernelPointer
3688 **          Kernel pointer to the data.
3689 */
3690 gceSTATUS
3691 gckKERNEL_CloseUserData(
3692     IN gckKERNEL Kernel,
3693     IN gctBOOL NeedCopy,
3694     IN gctBOOL FlushData,
3695     IN gctPOINTER UserPointer,
3696     IN gctSIZE_T Size,
3697     OUT gctPOINTER * KernelPointer
3698     )
3699 {
3700     gceSTATUS status = gcvSTATUS_OK;
3701     gctPOINTER pointer;
3702
3703     gcmkHEADER_ARG(
3704         "Kernel=0x%08X NeedCopy=%d FlushData=%d "
3705         "UserPointer=0x%08X Size=%lu KernelPointer=0x%08X",
3706         Kernel, NeedCopy, FlushData, UserPointer, Size, KernelPointer
3707         );
3708
3709     /* Validate the arguemnts. */
3710     gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL);
3711     gcmkVERIFY_ARGUMENT(UserPointer != gcvNULL);
3712     gcmkVERIFY_ARGUMENT(KernelPointer != gcvNULL);
3713     gcmkVERIFY_ARGUMENT(Size > 0);
3714
3715     /* Get a shortcut to the kernel pointer. */
3716     pointer = * KernelPointer;
3717
3718     if (pointer != gcvNULL)
3719     {
3720         if (NeedCopy)
3721         {
3722             if (FlushData)
3723             {
3724                 gcmkONERROR(gckOS_CopyToUserData(
3725                     Kernel->os, * KernelPointer, UserPointer, Size
3726                     ));
3727             }
3728         }
3729         else
3730         {
3731             /* Unmap record from kernel memory. */
3732             gcmkONERROR(gckOS_UnmapUserPointer(
3733                 Kernel->os,
3734                 UserPointer,
3735                 Size,
3736                 * KernelPointer
3737                 ));
3738         }
3739
3740         /* Reset the kernel pointer. */
3741         * KernelPointer = gcvNULL;
3742     }
3743
3744 OnError:
3745     /* Return the status. */
3746     gcmkFOOTER();
3747     return status;
3748 }
3749
3750 void
3751 gckKERNEL_SetTimeOut(
3752     IN gckKERNEL Kernel,
3753     IN gctUINT32 timeOut
3754     )
3755 {
3756     gcmkHEADER_ARG("Kernel=0x%x timeOut=%d", Kernel, timeOut);
3757 #if gcdGPU_TIMEOUT
3758     Kernel->timeOut = timeOut;
3759 #endif
3760     gcmkFOOTER_NO();
3761 }
3762
3763 gceSTATUS
3764 gckKERNEL_AllocateVirtualCommandBuffer(
3765     IN gckKERNEL Kernel,
3766     IN gctBOOL InUserSpace,
3767     IN OUT gctSIZE_T * Bytes,
3768     OUT gctPHYS_ADDR * Physical,
3769     OUT gctPOINTER * Logical
3770     )
3771 {
3772     gckOS os                             = Kernel->os;
3773     gceSTATUS status;
3774     gctPOINTER logical                   = gcvNULL;
3775     gctSIZE_T pageCount;
3776     gctSIZE_T bytes                      = *Bytes;
3777     gckVIRTUAL_COMMAND_BUFFER_PTR buffer = gcvNULL;
3778     gckMMU mmu;
3779     gctUINT32 flag = gcvALLOC_FLAG_NON_CONTIGUOUS;
3780
3781     gcmkHEADER_ARG("Os=0x%X InUserSpace=%d *Bytes=%lu",
3782                    os, InUserSpace, gcmOPT_VALUE(Bytes));
3783
3784     /* Verify the arguments. */
3785     gcmkVERIFY_OBJECT(os, gcvOBJ_OS);
3786     gcmkVERIFY_ARGUMENT(Bytes != gcvNULL);
3787     gcmkVERIFY_ARGUMENT(*Bytes > 0);
3788     gcmkVERIFY_ARGUMENT(Physical != gcvNULL);
3789     gcmkVERIFY_ARGUMENT(Logical != gcvNULL);
3790
3791     gcmkONERROR(gckOS_Allocate(os,
3792                                sizeof(gckVIRTUAL_COMMAND_BUFFER),
3793                                (gctPOINTER)&buffer));
3794
3795     gcmkONERROR(gckOS_ZeroMemory(buffer, sizeof(gckVIRTUAL_COMMAND_BUFFER)));
3796
3797     buffer->bytes = bytes;
3798
3799     gcmkONERROR(gckOS_AllocatePagedMemoryEx(os,
3800                                             flag,
3801                                             bytes,
3802                                             gcvNULL,
3803                                             &buffer->physical));
3804
3805     if (InUserSpace)
3806     {
3807         gcmkONERROR(gckOS_CreateUserVirtualMapping(os,
3808                                                    buffer->physical,
3809                                                    bytes,
3810                                                    &logical,
3811                                                    &pageCount));
3812
3813         *Logical =
3814         buffer->userLogical = logical;
3815     }
3816     else
3817     {
3818         gcmkONERROR(gckOS_CreateKernelVirtualMapping(os,
3819                                                      buffer->physical,
3820                                                      bytes,
3821                                                      &logical,
3822                                                      &pageCount));
3823
3824         *Logical =
3825         buffer->kernelLogical = logical;
3826     }
3827
3828     buffer->pageCount = pageCount;
3829     buffer->kernel = Kernel;
3830
3831     gcmkONERROR(gckOS_GetProcessID(&buffer->pid));
3832
3833 #if gcdPROCESS_ADDRESS_SPACE
3834     gcmkONERROR(gckKERNEL_GetProcessMMU(Kernel, &mmu));
3835     buffer->mmu = mmu;
3836 #else
3837     mmu = Kernel->mmu;
3838 #endif
3839
3840     gcmkONERROR(gckMMU_AllocatePages(mmu,
3841                                      pageCount,
3842                                      &buffer->pageTable,
3843                                      &buffer->gpuAddress));
3844
3845
3846     gcmkONERROR(gckOS_MapPagesEx(os,
3847                                  Kernel->core,
3848                                  buffer->physical,
3849                                  pageCount,
3850                                  buffer->gpuAddress,
3851                                  buffer->pageTable));
3852
3853     gcmkONERROR(gckMMU_Flush(mmu, gcvSURF_INDEX));
3854
3855     *Physical = buffer;
3856
3857     gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_KERNEL,
3858                    "gpuAddress = %x pageCount = %d kernelLogical = %x userLogical=%x",
3859                    buffer->gpuAddress, buffer->pageCount,
3860                    buffer->kernelLogical, buffer->userLogical);
3861
3862     gcmkVERIFY_OK(gckOS_AcquireMutex(os, Kernel->virtualBufferLock, gcvINFINITE));
3863
3864     if (Kernel->virtualBufferHead == gcvNULL)
3865     {
3866         Kernel->virtualBufferHead =
3867         Kernel->virtualBufferTail = buffer;
3868     }
3869     else
3870     {
3871         buffer->prev = Kernel->virtualBufferTail;
3872         Kernel->virtualBufferTail->next = buffer;
3873         Kernel->virtualBufferTail = buffer;
3874     }
3875
3876     gcmkVERIFY_OK(gckOS_ReleaseMutex(os, Kernel->virtualBufferLock));
3877
3878     gcmkFOOTER_NO();
3879     return gcvSTATUS_OK;
3880
3881 OnError:
3882     if (buffer->gpuAddress)
3883     {
3884 #if gcdPROCESS_ADDRESS_SPACE
3885         gcmkVERIFY_OK(
3886             gckMMU_FreePages(mmu, buffer->pageTable, buffer->pageCount));
3887 #else
3888         gcmkVERIFY_OK(
3889             gckMMU_FreePages(Kernel->mmu, buffer->pageTable, buffer->pageCount));
3890 #endif
3891     }
3892
3893     if (buffer->userLogical)
3894     {
3895         gcmkVERIFY_OK(
3896             gckOS_DestroyUserVirtualMapping(os,
3897                                             buffer->physical,
3898                                             bytes,
3899                                             buffer->userLogical));
3900     }
3901
3902     if (buffer->kernelLogical)
3903     {
3904         gcmkVERIFY_OK(
3905             gckOS_DestroyKernelVirtualMapping(os,
3906                                               buffer->physical,
3907                                               bytes,
3908                                               buffer->kernelLogical));
3909     }
3910
3911     if (buffer->physical)
3912     {
3913         gcmkVERIFY_OK(gckOS_FreePagedMemory(os, buffer->physical, bytes));
3914     }
3915
3916     gcmkVERIFY_OK(gckOS_Free(os, buffer));
3917
3918     /* Return the status. */
3919     gcmkFOOTER();
3920     return status;
3921 }
3922
3923 gceSTATUS
3924 gckKERNEL_DestroyVirtualCommandBuffer(
3925     IN gckKERNEL Kernel,
3926     IN gctSIZE_T Bytes,
3927     IN gctPHYS_ADDR Physical,
3928     IN gctPOINTER Logical
3929     )
3930 {
3931     gckOS os;
3932     gckKERNEL kernel;
3933     gckVIRTUAL_COMMAND_BUFFER_PTR buffer = (gckVIRTUAL_COMMAND_BUFFER_PTR)Physical;
3934
3935     gcmkHEADER();
3936     gcmkVERIFY_ARGUMENT(buffer != gcvNULL);
3937
3938     kernel = buffer->kernel;
3939     os = kernel->os;
3940
3941     if (!buffer->userLogical)
3942     {
3943         gcmkVERIFY_OK(gckOS_DestroyKernelVirtualMapping(os,
3944                                                         buffer->physical,
3945                                                         Bytes,
3946                                                         Logical));
3947     }
3948
3949 #if !gcdPROCESS_ADDRESS_SPACE
3950     gcmkVERIFY_OK(
3951         gckMMU_FreePages(kernel->mmu, buffer->pageTable, buffer->pageCount));
3952 #endif
3953
3954     gcmkVERIFY_OK(gckOS_UnmapPages(os, buffer->pageCount, buffer->gpuAddress));
3955
3956     gcmkVERIFY_OK(gckOS_FreePagedMemory(os, buffer->physical, Bytes));
3957
3958     gcmkVERIFY_OK(gckOS_AcquireMutex(os, kernel->virtualBufferLock, gcvINFINITE));
3959
3960     if (buffer == kernel->virtualBufferHead)
3961     {
3962         if ((kernel->virtualBufferHead = buffer->next) == gcvNULL)
3963         {
3964             kernel->virtualBufferTail = gcvNULL;
3965         }
3966     }
3967     else
3968     {
3969         buffer->prev->next = buffer->next;
3970
3971         if (buffer == kernel->virtualBufferTail)
3972         {
3973             kernel->virtualBufferTail = buffer->prev;
3974         }
3975         else
3976         {
3977             buffer->next->prev = buffer->prev;
3978         }
3979     }
3980
3981     gcmkVERIFY_OK(gckOS_ReleaseMutex(os, kernel->virtualBufferLock));
3982
3983     gcmkVERIFY_OK(gckOS_Free(os, buffer));
3984
3985     gcmkFOOTER_NO();
3986     return gcvSTATUS_OK;
3987 }
3988
3989 gceSTATUS
3990 gckKERNEL_GetGPUAddress(
3991     IN gckKERNEL Kernel,
3992     IN gctPOINTER Logical,
3993     IN gctBOOL InUserSpace,
3994     OUT gctUINT32 * Address
3995     )
3996 {
3997     gceSTATUS status;
3998     gckVIRTUAL_COMMAND_BUFFER_PTR buffer;
3999     gctPOINTER start;
4000     gctUINT32 pid;
4001
4002     gcmkHEADER_ARG("Logical = %x InUserSpace=%d.", Logical, InUserSpace);
4003
4004     gcmkVERIFY_OK(gckOS_GetProcessID(&pid));
4005
4006     status = gcvSTATUS_INVALID_ADDRESS;
4007
4008     gcmkVERIFY_OK(gckOS_AcquireMutex(Kernel->os, Kernel->virtualBufferLock, gcvINFINITE));
4009
4010     /* Walk all command buffer. */
4011     for (buffer = Kernel->virtualBufferHead; buffer != gcvNULL; buffer = buffer->next)
4012     {
4013         if (InUserSpace)
4014         {
4015             start = buffer->userLogical;
4016         }
4017         else
4018         {
4019             start = buffer->kernelLogical;
4020         }
4021
4022         if (start == gcvNULL)
4023         {
4024             continue;
4025         }
4026
4027         if (Logical >= start
4028         && (Logical < (gctPOINTER)((gctUINT8_PTR)start + buffer->pageCount * 4096))
4029         && pid == buffer->pid
4030         )
4031         {
4032             * Address = buffer->gpuAddress + (gctUINT32)((gctUINT8_PTR)Logical - (gctUINT8_PTR)start);
4033             status = gcvSTATUS_OK;
4034             break;
4035         }
4036     }
4037
4038     gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, Kernel->virtualBufferLock));
4039
4040     gcmkFOOTER_NO();
4041     return status;
4042 }
4043
4044 gceSTATUS
4045 gckKERNEL_QueryGPUAddress(
4046     IN gckKERNEL Kernel,
4047     IN gctUINT32 GpuAddress,
4048     OUT gckVIRTUAL_COMMAND_BUFFER_PTR * Buffer
4049     )
4050 {
4051     gckVIRTUAL_COMMAND_BUFFER_PTR buffer;
4052     gctUINT32 start;
4053     gceSTATUS status = gcvSTATUS_NOT_SUPPORTED;
4054
4055     gcmkVERIFY_OK(gckOS_AcquireMutex(Kernel->os, Kernel->virtualBufferLock, gcvINFINITE));
4056
4057     /* Walk all command buffers. */
4058     for (buffer = Kernel->virtualBufferHead; buffer != gcvNULL; buffer = buffer->next)
4059     {
4060         start = (gctUINT32)buffer->gpuAddress;
4061
4062         if (GpuAddress >= start && GpuAddress < (start + buffer->pageCount * 4096))
4063         {
4064             /* Find a range matched. */
4065             *Buffer = buffer;
4066             status = gcvSTATUS_OK;
4067             break;
4068         }
4069     }
4070
4071     gcmkVERIFY_OK(gckOS_ReleaseMutex(Kernel->os, Kernel->virtualBufferLock));
4072
4073     return status;
4074 }
4075
4076 #if gcdLINK_QUEUE_SIZE
4077 static void
4078 gckLINKQUEUE_Dequeue(
4079     IN gckLINKQUEUE LinkQueue
4080     )
4081 {
4082     gcmkASSERT(LinkQueue->count == gcdLINK_QUEUE_SIZE);
4083
4084     LinkQueue->count--;
4085     LinkQueue->front = (LinkQueue->front + 1) % gcdLINK_QUEUE_SIZE;
4086 }
4087
4088 void
4089 gckLINKQUEUE_Enqueue(
4090     IN gckLINKQUEUE LinkQueue,
4091     IN gctUINT32 start,
4092     IN gctUINT32 end
4093     )
4094 {
4095     if (LinkQueue->count == gcdLINK_QUEUE_SIZE)
4096     {
4097         gckLINKQUEUE_Dequeue(LinkQueue);
4098     }
4099
4100     gcmkASSERT(LinkQueue->count < gcdLINK_QUEUE_SIZE);
4101
4102     LinkQueue->count++;
4103
4104     LinkQueue->data[LinkQueue->rear].start = start;
4105     LinkQueue->data[LinkQueue->rear].end = end;
4106
4107     gcmkVERIFY_OK(
4108         gckOS_GetProcessID(&LinkQueue->data[LinkQueue->rear].pid));
4109
4110     LinkQueue->rear = (LinkQueue->rear + 1) % gcdLINK_QUEUE_SIZE;
4111 }
4112
4113 void
4114 gckLINKQUEUE_GetData(
4115     IN gckLINKQUEUE LinkQueue,
4116     IN gctUINT32 Index,
4117     OUT gckLINKDATA * Data
4118     )
4119 {
4120     gcmkASSERT(Index >= 0 && Index < gcdLINK_QUEUE_SIZE);
4121
4122     *Data = &LinkQueue->data[(Index + LinkQueue->front) % gcdLINK_QUEUE_SIZE];
4123 }
4124 #endif
4125
4126 /*
4127 * gckENTRYQUEUE_Enqueue is called with Command->mutexQueue acquired.
4128 */
4129 gceSTATUS
4130 gckENTRYQUEUE_Enqueue(
4131     IN gckKERNEL Kernel,
4132     IN gckENTRYQUEUE Queue,
4133     IN gctUINT32 physical,
4134     IN gctUINT32 bytes
4135     )
4136 {
4137     gctUINT32 next = (Queue->rear + 1) % gcdENTRY_QUEUE_SIZE;
4138
4139     if (next == Queue->front)
4140     {
4141         /* Queue is full. */
4142         return gcvSTATUS_INVALID_REQUEST;
4143     }
4144
4145     /* Copy data. */
4146     Queue->data[Queue->rear].physical = physical;
4147     Queue->data[Queue->rear].bytes = bytes;
4148
4149     gcmkVERIFY_OK(gckOS_MemoryBarrier(Kernel->os, &Queue->rear));
4150
4151     /* Update rear. */
4152     Queue->rear = next;
4153
4154     return gcvSTATUS_OK;
4155 }
4156
4157 gceSTATUS
4158 gckENTRYQUEUE_Dequeue(
4159     IN gckENTRYQUEUE Queue,
4160     OUT gckENTRYDATA * Data
4161     )
4162 {
4163     if (Queue->front == Queue->rear)
4164     {
4165         /* Queue is empty. */
4166         return gcvSTATUS_INVALID_REQUEST;
4167     }
4168
4169     /* Copy data. */
4170     *Data = &Queue->data[Queue->front];
4171
4172     /* Update front. */
4173     Queue->front = (Queue->front + 1) % gcdENTRY_QUEUE_SIZE;
4174
4175     return gcvSTATUS_OK;
4176 }
4177
4178 /******************************************************************************\
4179 *************************** Pointer - ID translation ***************************
4180 \******************************************************************************/
4181 #define gcdID_TABLE_LENGTH 1024
4182 typedef struct _gcsINTEGERDB * gckINTEGERDB;
4183 typedef struct _gcsINTEGERDB
4184 {
4185     gckOS                       os;
4186     gctPOINTER*                 table;
4187     gctPOINTER                  mutex;
4188     gctUINT32                   tableLen;
4189     gctUINT32                   currentID;
4190     gctUINT32                   unused;
4191 }
4192 gcsINTEGERDB;
4193
4194 gceSTATUS
4195 gckKERNEL_CreateIntegerDatabase(
4196     IN gckKERNEL Kernel,
4197     OUT gctPOINTER * Database
4198     )
4199 {
4200     gceSTATUS status;
4201     gckINTEGERDB database = gcvNULL;
4202
4203     gcmkHEADER_ARG("Kernel=0x%08X Datbase=0x%08X", Kernel, Database);
4204
4205     gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL);
4206     gcmkVERIFY_ARGUMENT(Database != gcvNULL);
4207
4208     /* Allocate a database. */
4209     gcmkONERROR(gckOS_Allocate(
4210         Kernel->os, gcmSIZEOF(gcsINTEGERDB), (gctPOINTER *)&database));
4211
4212     gcmkONERROR(gckOS_ZeroMemory(database, gcmSIZEOF(gcsINTEGERDB)));
4213
4214     /* Allocate a pointer table. */
4215     gcmkONERROR(gckOS_Allocate(
4216         Kernel->os, gcmSIZEOF(gctPOINTER) * gcdID_TABLE_LENGTH, (gctPOINTER *)&database->table));
4217
4218     gcmkONERROR(gckOS_ZeroMemory(database->table, gcmSIZEOF(gctPOINTER) * gcdID_TABLE_LENGTH));
4219
4220     /* Allocate a database mutex. */
4221     gcmkONERROR(gckOS_CreateMutex(Kernel->os, &database->mutex));
4222
4223     /* Initialize. */
4224     database->currentID = 0;
4225     database->unused = gcdID_TABLE_LENGTH;
4226     database->os = Kernel->os;
4227     database->tableLen = gcdID_TABLE_LENGTH;
4228
4229     *Database = database;
4230
4231     gcmkFOOTER_ARG("*Database=0x%08X", *Database);
4232     return gcvSTATUS_OK;
4233
4234 OnError:
4235     /* Rollback. */
4236     if (database)
4237     {
4238         if (database->table)
4239         {
4240             gcmkOS_SAFE_FREE(Kernel->os, database->table);
4241         }
4242
4243         gcmkOS_SAFE_FREE(Kernel->os, database);
4244     }
4245
4246     gcmkFOOTER();
4247     return status;
4248 }
4249
4250 gceSTATUS
4251 gckKERNEL_DestroyIntegerDatabase(
4252     IN gckKERNEL Kernel,
4253     IN gctPOINTER Database
4254     )
4255 {
4256     gckINTEGERDB database = Database;
4257
4258     gcmkHEADER_ARG("Kernel=0x%08X Datbase=0x%08X", Kernel, Database);
4259
4260     gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL);
4261     gcmkVERIFY_ARGUMENT(Database != gcvNULL);
4262
4263     /* Destroy pointer table. */
4264     gcmkOS_SAFE_FREE(Kernel->os, database->table);
4265
4266     /* Destroy database mutex. */
4267     gcmkVERIFY_OK(gckOS_DeleteMutex(Kernel->os, database->mutex));
4268
4269     /* Destroy database. */
4270     gcmkOS_SAFE_FREE(Kernel->os, database);
4271
4272     gcmkFOOTER_NO();
4273     return gcvSTATUS_OK;
4274 }
4275
4276 gceSTATUS
4277 gckKERNEL_AllocateIntegerId(
4278     IN gctPOINTER Database,
4279     IN gctPOINTER Pointer,
4280     OUT gctUINT32 * Id
4281     )
4282 {
4283     gceSTATUS status;
4284     gckINTEGERDB database = Database;
4285     gctUINT32 i, unused, currentID, tableLen;
4286     gctPOINTER * table;
4287     gckOS os = database->os;
4288     gctBOOL acquired = gcvFALSE;
4289
4290     gcmkHEADER_ARG("Database=0x%08X Pointer=0x%08X", Database, Pointer);
4291
4292     gcmkVERIFY_ARGUMENT(Id != gcvNULL);
4293
4294     gcmkVERIFY_OK(gckOS_AcquireMutex(os, database->mutex, gcvINFINITE));
4295     acquired = gcvTRUE;
4296
4297     if (database->unused < 1)
4298     {
4299         /* Extend table. */
4300         gcmkONERROR(
4301             gckOS_Allocate(os,
4302                            gcmSIZEOF(gctPOINTER) * (database->tableLen + gcdID_TABLE_LENGTH),
4303                            (gctPOINTER *)&table));
4304
4305         gcmkONERROR(gckOS_ZeroMemory(table + database->tableLen,
4306                                      gcmSIZEOF(gctPOINTER) * gcdID_TABLE_LENGTH));
4307
4308         /* Copy data from old table. */
4309         gckOS_MemCopy(table,
4310                       database->table,
4311                       database->tableLen * gcmSIZEOF(gctPOINTER));
4312
4313         gcmkOS_SAFE_FREE(os, database->table);
4314
4315         /* Update databse with new allocated table. */
4316         database->table = table;
4317         database->currentID = database->tableLen;
4318         database->tableLen += gcdID_TABLE_LENGTH;
4319         database->unused += gcdID_TABLE_LENGTH;
4320     }
4321
4322     table = database->table;
4323     currentID = database->currentID;
4324     tableLen = database->tableLen;
4325     unused = database->unused;
4326
4327     /* Connect id with pointer. */
4328     table[currentID] = Pointer;
4329
4330     *Id = currentID + 1;
4331
4332     /* Update the currentID. */
4333     if (--unused > 0)
4334     {
4335         for (i = 0; i < tableLen; i++)
4336         {
4337             if (++currentID >= tableLen)
4338             {
4339                 /* Wrap to the begin. */
4340                 currentID = 0;
4341             }
4342
4343             if (table[currentID] == gcvNULL)
4344             {
4345                 break;
4346             }
4347         }
4348     }
4349
4350     database->table = table;
4351     database->currentID = currentID;
4352     database->tableLen = tableLen;
4353     database->unused = unused;
4354
4355     gcmkVERIFY_OK(gckOS_ReleaseMutex(os, database->mutex));
4356     acquired = gcvFALSE;
4357
4358     gcmkFOOTER_ARG("*Id=%d", *Id);
4359     return gcvSTATUS_OK;
4360
4361 OnError:
4362     if (acquired)
4363     {
4364         gcmkVERIFY_OK(gckOS_ReleaseMutex(os, database->mutex));
4365     }
4366
4367     gcmkFOOTER();
4368     return status;
4369 }
4370
4371 gceSTATUS
4372 gckKERNEL_FreeIntegerId(
4373     IN gctPOINTER Database,
4374     IN gctUINT32 Id
4375     )
4376 {
4377     gceSTATUS status;
4378     gckINTEGERDB database = Database;
4379     gckOS os = database->os;
4380     gctBOOL acquired = gcvFALSE;
4381
4382     gcmkHEADER_ARG("Database=0x%08X Id=%d", Database, Id);
4383
4384     gcmkVERIFY_OK(gckOS_AcquireMutex(os, database->mutex, gcvINFINITE));
4385     acquired = gcvTRUE;
4386
4387     if (!(Id > 0 && Id <= database->tableLen))
4388     {
4389         gcmkONERROR(gcvSTATUS_NOT_FOUND);
4390     }
4391
4392     Id -= 1;
4393
4394     database->table[Id] = gcvNULL;
4395
4396     if (database->unused++ == 0)
4397     {
4398         database->currentID = Id;
4399     }
4400
4401     gcmkVERIFY_OK(gckOS_ReleaseMutex(os, database->mutex));
4402     acquired = gcvFALSE;
4403
4404     gcmkFOOTER_NO();
4405     return gcvSTATUS_OK;
4406
4407 OnError:
4408     if (acquired)
4409     {
4410         gcmkVERIFY_OK(gckOS_ReleaseMutex(os, database->mutex));
4411     }
4412
4413     gcmkFOOTER();
4414     return status;
4415 }
4416
4417 gceSTATUS
4418 gckKERNEL_QueryIntegerId(
4419     IN gctPOINTER Database,
4420     IN gctUINT32 Id,
4421     OUT gctPOINTER * Pointer
4422     )
4423 {
4424     gceSTATUS status;
4425     gckINTEGERDB database = Database;
4426     gctPOINTER pointer;
4427     gckOS os = database->os;
4428     gctBOOL acquired = gcvFALSE;
4429
4430     gcmkHEADER_ARG("Database=0x%08X Id=%d", Database, Id);
4431     gcmkVERIFY_ARGUMENT(Pointer != gcvNULL);
4432
4433     gcmkVERIFY_OK(gckOS_AcquireMutex(os, database->mutex, gcvINFINITE));
4434     acquired = gcvTRUE;
4435
4436     if (!(Id > 0 && Id <= database->tableLen))
4437     {
4438         gcmkONERROR(gcvSTATUS_NOT_FOUND);
4439     }
4440
4441     Id -= 1;
4442
4443     pointer = database->table[Id];
4444
4445     gcmkVERIFY_OK(gckOS_ReleaseMutex(os, database->mutex));
4446     acquired = gcvFALSE;
4447
4448     if (pointer)
4449     {
4450         *Pointer = pointer;
4451     }
4452     else
4453     {
4454         gcmkONERROR(gcvSTATUS_NOT_FOUND);
4455     }
4456
4457     gcmkFOOTER_ARG("*Pointer=0x%08X", *Pointer);
4458     return gcvSTATUS_OK;
4459
4460 OnError:
4461     if (acquired)
4462     {
4463         gcmkVERIFY_OK(gckOS_ReleaseMutex(os, database->mutex));
4464     }
4465
4466     gcmkFOOTER();
4467     return status;
4468 }
4469
4470
4471 gctUINT32
4472 gckKERNEL_AllocateNameFromPointer(
4473     IN gckKERNEL Kernel,
4474     IN gctPOINTER Pointer
4475     )
4476 {
4477     gceSTATUS status;
4478     gctUINT32 name;
4479     gctPOINTER database = Kernel->db->pointerDatabase;
4480
4481     gcmkHEADER_ARG("Kernel=0x%X Pointer=0x%X", Kernel, Pointer);
4482
4483     gcmkONERROR(
4484         gckKERNEL_AllocateIntegerId(database, Pointer, &name));
4485
4486     gcmkFOOTER_ARG("name=%d", name);
4487     return name;
4488
4489 OnError:
4490     gcmkFOOTER();
4491     return 0;
4492 }
4493
4494 gctPOINTER
4495 gckKERNEL_QueryPointerFromName(
4496     IN gckKERNEL Kernel,
4497     IN gctUINT32 Name
4498     )
4499 {
4500     gceSTATUS status;
4501     gctPOINTER pointer = gcvNULL;
4502     gctPOINTER database = Kernel->db->pointerDatabase;
4503
4504     gcmkHEADER_ARG("Kernel=0x%X Name=%d", Kernel, Name);
4505
4506     /* Lookup in database to get pointer. */
4507     gcmkONERROR(gckKERNEL_QueryIntegerId(database, Name, &pointer));
4508
4509     gcmkFOOTER_ARG("pointer=0x%X", pointer);
4510     return pointer;
4511
4512 OnError:
4513     gcmkFOOTER();
4514     return gcvNULL;
4515 }
4516
4517 gceSTATUS
4518 gckKERNEL_DeleteName(
4519     IN gckKERNEL Kernel,
4520     IN gctUINT32 Name
4521     )
4522 {
4523     gctPOINTER database = Kernel->db->pointerDatabase;
4524
4525     gcmkHEADER_ARG("Kernel=0x%X Name=0x%X", Kernel, Name);
4526
4527     /* Free name if exists. */
4528     gcmkVERIFY_OK(gckKERNEL_FreeIntegerId(database, Name));
4529
4530     gcmkFOOTER_NO();
4531     return gcvSTATUS_OK;
4532 }
4533
4534 gceSTATUS
4535 gckKERNEL_SetRecovery(
4536     IN gckKERNEL Kernel,
4537     IN gctBOOL  Recovery,
4538     IN gctUINT32 StuckDump
4539     )
4540 {
4541     Kernel->recovery = Recovery;
4542
4543     if (Recovery == gcvFALSE)
4544     {
4545         /* Dump stuck information if Recovery is disabled. */
4546         Kernel->stuckDump = gcmMAX(StuckDump, gcdSTUCK_DUMP_MIDDLE);
4547     }
4548
4549     return gcvSTATUS_OK;
4550 }
4551
4552
4553 /*******************************************************************************
4554 ***** Shared Buffer ************************************************************
4555 *******************************************************************************/
4556
4557 /*******************************************************************************
4558 **
4559 **  gckKERNEL_CreateShBuffer
4560 **
4561 **  Create shared buffer.
4562 **  The shared buffer can be used across processes. Other process needs call
4563 **  gckKERNEL_MapShBuffer before use it.
4564 **
4565 **  INPUT:
4566 **
4567 **      gckKERNEL Kernel
4568 **          Pointer to an gckKERNEL object.
4569 **
4570 **      gctUINT32 Size
4571 **          Specify the shared buffer size.
4572 **
4573 **  OUTPUT:
4574 **
4575 **      gctSHBUF * ShBuf
4576 **          Pointer to hold return shared buffer handle.
4577 */
4578 gceSTATUS
4579 gckKERNEL_CreateShBuffer(
4580     IN gckKERNEL Kernel,
4581     IN gctUINT32 Size,
4582     OUT gctSHBUF * ShBuf
4583     )
4584 {
4585     gceSTATUS status;
4586     gcsSHBUF_PTR shBuf = gcvNULL;
4587
4588     gcmkHEADER_ARG("Kernel=0x%X, Size=%u", Kernel, Size);
4589
4590     /* Verify the arguments. */
4591     gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL);
4592
4593     if (Size == 0)
4594     {
4595         /* Invalid size. */
4596         gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT);
4597     }
4598     else if (Size > 1024)
4599     {
4600         /* Limite shared buffer size. */
4601         gcmkONERROR(gcvSTATUS_OUT_OF_RESOURCES);
4602     }
4603
4604     /* Create a shared buffer structure. */
4605     gcmkONERROR(
4606         gckOS_Allocate(Kernel->os,
4607                        sizeof (gcsSHBUF),
4608                        (gctPOINTER *)&shBuf));
4609
4610     /* Initialize shared buffer. */
4611     shBuf->id        = 0;
4612     shBuf->reference = gcvNULL;
4613     shBuf->size      = Size;
4614     shBuf->data      = gcvNULL;
4615
4616     /* Allocate integer id for this shared buffer. */
4617     gcmkONERROR(
4618         gckKERNEL_AllocateIntegerId(Kernel->db->pointerDatabase,
4619                                     shBuf,
4620                                     &shBuf->id));
4621
4622     /* Allocate atom. */
4623     gcmkONERROR(gckOS_AtomConstruct(Kernel->os, &shBuf->reference));
4624
4625     /* Set default reference count to 1. */
4626     gcmkVERIFY_OK(gckOS_AtomSet(Kernel->os, shBuf->reference, 1));
4627
4628     /* Return integer id. */
4629     *ShBuf = (gctSHBUF)(gctUINTPTR_T)shBuf->id;
4630
4631     gcmkFOOTER_ARG("*ShBuf=%u", shBuf->id);
4632     return gcvSTATUS_OK;
4633
4634 OnError:
4635     /* Error roll back. */
4636     if (shBuf != gcvNULL)
4637     {
4638         if (shBuf->id != 0)
4639         {
4640             gcmkVERIFY_OK(
4641                 gckKERNEL_FreeIntegerId(Kernel->db->pointerDatabase,
4642                                         shBuf->id));
4643         }
4644
4645         gcmkOS_SAFE_FREE(Kernel->os, shBuf);
4646     }
4647
4648     gcmkFOOTER();
4649     return status;
4650 }
4651
4652 /*******************************************************************************
4653 **
4654 **  gckKERNEL_DestroyShBuffer
4655 **
4656 **  Destroy shared buffer.
4657 **  This will decrease reference of specified shared buffer and do actual
4658 **  destroy when no reference on it.
4659 **
4660 **  INPUT:
4661 **
4662 **      gckKERNEL Kernel
4663 **          Pointer to an gckKERNEL object.
4664 **
4665 **      gctSHBUF ShBuf
4666 **          Specify the shared buffer to be destroyed.
4667 **
4668 **  OUTPUT:
4669 **
4670 **      Nothing.
4671 */
4672 gceSTATUS
4673 gckKERNEL_DestroyShBuffer(
4674     IN gckKERNEL Kernel,
4675     IN gctSHBUF ShBuf
4676     )
4677 {
4678     gceSTATUS status;
4679     gcsSHBUF_PTR shBuf;
4680     gctINT32 oldValue = 0;
4681     gctBOOL acquired = gcvFALSE;
4682
4683     gcmkHEADER_ARG("Kernel=0x%X ShBuf=%u",
4684                    Kernel, (gctUINT32)(gctUINTPTR_T) ShBuf);
4685
4686     /* Verify the arguments. */
4687     gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL);
4688     gcmkVERIFY_ARGUMENT(ShBuf != gcvNULL);
4689
4690     /* Acquire mutex. */
4691     gcmkONERROR(
4692         gckOS_AcquireMutex(Kernel->os,
4693                            Kernel->db->pointerDatabaseMutex,
4694                            gcvINFINITE));
4695     acquired = gcvTRUE;
4696
4697     /* Find shared buffer structure. */
4698     gcmkONERROR(
4699         gckKERNEL_QueryIntegerId(Kernel->db->pointerDatabase,
4700                                  (gctUINT32)(gctUINTPTR_T)ShBuf,
4701                                  (gctPOINTER)&shBuf));
4702
4703     gcmkASSERT(shBuf->id == (gctUINT32)(gctUINTPTR_T)ShBuf);
4704
4705     /* Decrease the reference count. */
4706     gckOS_AtomDecrement(Kernel->os, shBuf->reference, &oldValue);
4707
4708     if (oldValue == 1)
4709     {
4710         /* Free integer id. */
4711         gcmkVERIFY_OK(
4712             gckKERNEL_FreeIntegerId(Kernel->db->pointerDatabase,
4713                                     shBuf->id));
4714
4715         /* Free atom. */
4716         gcmkVERIFY_OK(gckOS_AtomDestroy(Kernel->os, shBuf->reference));
4717
4718         if (shBuf->data)
4719         {
4720             gcmkOS_SAFE_FREE(Kernel->os, shBuf->data);
4721             shBuf->data = gcvNULL;
4722         }
4723
4724         /* Free the shared buffer. */
4725         gcmkOS_SAFE_FREE(Kernel->os, shBuf);
4726     }
4727
4728     /* Release the mutex. */
4729     gcmkVERIFY_OK(
4730         gckOS_ReleaseMutex(Kernel->os, Kernel->db->pointerDatabaseMutex));
4731     acquired = gcvFALSE;
4732
4733     gcmkFOOTER_NO();
4734     return gcvSTATUS_OK;
4735
4736 OnError:
4737     if (acquired)
4738     {
4739         /* Release the mutex. */
4740         gcmkVERIFY_OK(
4741             gckOS_ReleaseMutex(Kernel->os, Kernel->db->pointerDatabaseMutex));
4742     }
4743
4744     gcmkFOOTER();
4745     return status;
4746 }
4747
4748 /*******************************************************************************
4749 **
4750 **  gckKERNEL_MapShBuffer
4751 **
4752 **  Map shared buffer into this process so that it can be used in this process.
4753 **  This will increase reference count on the specified shared buffer.
4754 **  Call gckKERNEL_DestroyShBuffer to dereference.
4755 **
4756 **  INPUT:
4757 **
4758 **      gckKERNEL Kernel
4759 **          Pointer to an gckKERNEL object.
4760 **
4761 **      gctSHBUF ShBuf
4762 **          Specify the shared buffer to be mapped.
4763 **
4764 **  OUTPUT:
4765 **
4766 **      Nothing.
4767 */
4768 gceSTATUS
4769 gckKERNEL_MapShBuffer(
4770     IN gckKERNEL Kernel,
4771     IN gctSHBUF ShBuf
4772     )
4773 {
4774     gceSTATUS status;
4775     gcsSHBUF_PTR shBuf;
4776     gctINT32 oldValue = 0;
4777     gctBOOL acquired = gcvFALSE;
4778
4779     gcmkHEADER_ARG("Kernel=0x%X ShBuf=%u",
4780                    Kernel, (gctUINT32)(gctUINTPTR_T) ShBuf);
4781
4782     /* Verify the arguments. */
4783     gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL);
4784     gcmkVERIFY_ARGUMENT(ShBuf != gcvNULL);
4785
4786     /* Acquire mutex. */
4787     gcmkONERROR(
4788         gckOS_AcquireMutex(Kernel->os,
4789                            Kernel->db->pointerDatabaseMutex,
4790                            gcvINFINITE));
4791     acquired = gcvTRUE;
4792
4793     /* Find shared buffer structure. */
4794     gcmkONERROR(
4795         gckKERNEL_QueryIntegerId(Kernel->db->pointerDatabase,
4796                                  (gctUINT32)(gctUINTPTR_T)ShBuf,
4797                                  (gctPOINTER)&shBuf));
4798
4799     gcmkASSERT(shBuf->id == (gctUINT32)(gctUINTPTR_T)ShBuf);
4800
4801     /* Increase the reference count. */
4802     gckOS_AtomIncrement(Kernel->os, shBuf->reference, &oldValue);
4803
4804     /* Release the mutex. */
4805     gcmkVERIFY_OK(
4806         gckOS_ReleaseMutex(Kernel->os, Kernel->db->pointerDatabaseMutex));
4807     acquired = gcvFALSE;
4808
4809     gcmkFOOTER_NO();
4810     return gcvSTATUS_OK;
4811
4812 OnError:
4813     if (acquired)
4814     {
4815         /* Release the mutex. */
4816         gcmkVERIFY_OK(
4817             gckOS_ReleaseMutex(Kernel->os, Kernel->db->pointerDatabaseMutex));
4818     }
4819
4820     gcmkFOOTER();
4821     return status;
4822 }
4823
4824 /*******************************************************************************
4825 **
4826 **  gckKERNEL_WriteShBuffer
4827 **
4828 **  Write user data into shared buffer.
4829 **
4830 **  INPUT:
4831 **
4832 **      gckKERNEL Kernel
4833 **          Pointer to an gckKERNEL object.
4834 **
4835 **      gctSHBUF ShBuf
4836 **          Specify the shared buffer to be written to.
4837 **
4838 **      gctPOINTER UserData
4839 **          User mode pointer to hold the source data.
4840 **
4841 **      gctUINT32 ByteCount
4842 **          Specify number of bytes to write. If this is larger than
4843 **          shared buffer size, gcvSTATUS_INVALID_ARGUMENT is returned.
4844 **
4845 **  OUTPUT:
4846 **
4847 **      Nothing.
4848 */
4849 gceSTATUS
4850 gckKERNEL_WriteShBuffer(
4851     IN gckKERNEL Kernel,
4852     IN gctSHBUF ShBuf,
4853     IN gctPOINTER UserData,
4854     IN gctUINT32 ByteCount
4855     )
4856 {
4857     gceSTATUS status;
4858     gcsSHBUF_PTR shBuf;
4859     gctBOOL acquired = gcvFALSE;
4860
4861     gcmkHEADER_ARG("Kernel=0x%X ShBuf=%u UserData=0x%X ByteCount=%u",
4862                    Kernel, (gctUINT32)(gctUINTPTR_T) ShBuf, UserData, ByteCount);
4863
4864     /* Verify the arguments. */
4865     gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL);
4866     gcmkVERIFY_ARGUMENT(ShBuf != gcvNULL);
4867
4868     /* Acquire mutex. */
4869     gcmkONERROR(
4870         gckOS_AcquireMutex(Kernel->os,
4871                            Kernel->db->pointerDatabaseMutex,
4872                            gcvINFINITE));
4873     acquired = gcvTRUE;
4874
4875     /* Find shared buffer structure. */
4876     gcmkONERROR(
4877         gckKERNEL_QueryIntegerId(Kernel->db->pointerDatabase,
4878                                  (gctUINT32)(gctUINTPTR_T)ShBuf,
4879                                  (gctPOINTER)&shBuf));
4880
4881     gcmkASSERT(shBuf->id == (gctUINT32)(gctUINTPTR_T)ShBuf);
4882
4883     if ((ByteCount > shBuf->size) ||
4884         (ByteCount == 0) ||
4885         (UserData  == gcvNULL))
4886     {
4887         /* Exceeds buffer max size or invalid. */
4888         gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT);
4889     }
4890
4891     if (shBuf->data == gcvNULL)
4892     {
4893         /* Allocate buffer data when first time write. */
4894         gcmkONERROR(gckOS_Allocate(Kernel->os, ByteCount, &shBuf->data));
4895     }
4896
4897     /* Copy data from user. */
4898     gcmkONERROR(
4899         gckOS_CopyFromUserData(Kernel->os,
4900                                shBuf->data,
4901                                UserData,
4902                                ByteCount));
4903
4904     /* Release the mutex. */
4905     gcmkVERIFY_OK(
4906         gckOS_ReleaseMutex(Kernel->os, Kernel->db->pointerDatabaseMutex));
4907     acquired = gcvFALSE;
4908
4909     gcmkFOOTER_NO();
4910     return gcvSTATUS_OK;
4911
4912 OnError:
4913     if (acquired)
4914     {
4915         /* Release the mutex. */
4916         gcmkVERIFY_OK(
4917             gckOS_ReleaseMutex(Kernel->os, Kernel->db->pointerDatabaseMutex));
4918     }
4919
4920     gcmkFOOTER();
4921     return status;
4922 }
4923
4924 /*******************************************************************************
4925 **
4926 **  gckKERNEL_ReadShBuffer
4927 **
4928 **  Read data from shared buffer and copy to user pointer.
4929 **
4930 **  INPUT:
4931 **
4932 **      gckKERNEL Kernel
4933 **          Pointer to an gckKERNEL object.
4934 **
4935 **      gctSHBUF ShBuf
4936 **          Specify the shared buffer to be read from.
4937 **
4938 **      gctPOINTER UserData
4939 **          User mode pointer to save output data.
4940 **
4941 **      gctUINT32 ByteCount
4942 **          Specify number of bytes to read.
4943 **          If this is larger than shared buffer size, only avaiable bytes are
4944 **          copied. If smaller, copy requested size.
4945 **
4946 **  OUTPUT:
4947 **
4948 **      gctUINT32 * BytesRead
4949 **          Pointer to hold how many bytes actually read from shared buffer.
4950 */
4951 gceSTATUS
4952 gckKERNEL_ReadShBuffer(
4953     IN gckKERNEL Kernel,
4954     IN gctSHBUF ShBuf,
4955     IN gctPOINTER UserData,
4956     IN gctUINT32 ByteCount,
4957     OUT gctUINT32 * BytesRead
4958     )
4959 {
4960     gceSTATUS status;
4961     gcsSHBUF_PTR shBuf;
4962     gctUINT32 bytes;
4963     gctBOOL acquired = gcvFALSE;
4964
4965     gcmkHEADER_ARG("Kernel=0x%X ShBuf=%u UserData=0x%X ByteCount=%u",
4966                    Kernel, (gctUINT32)(gctUINTPTR_T) ShBuf, UserData, ByteCount);
4967
4968     /* Verify the arguments. */
4969     gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL);
4970     gcmkVERIFY_ARGUMENT(ShBuf != gcvNULL);
4971
4972     /* Acquire mutex. */
4973     gcmkONERROR(
4974         gckOS_AcquireMutex(Kernel->os,
4975                            Kernel->db->pointerDatabaseMutex,
4976                            gcvINFINITE));
4977     acquired = gcvTRUE;
4978
4979     /* Find shared buffer structure. */
4980     gcmkONERROR(
4981         gckKERNEL_QueryIntegerId(Kernel->db->pointerDatabase,
4982                                  (gctUINT32)(gctUINTPTR_T)ShBuf,
4983                                  (gctPOINTER)&shBuf));
4984
4985     gcmkASSERT(shBuf->id == (gctUINT32)(gctUINTPTR_T)ShBuf);
4986
4987     if (shBuf->data == gcvNULL)
4988     {
4989         *BytesRead = 0;
4990
4991         /* No data in shared buffer, skip copy. */
4992         status = gcvSTATUS_SKIP;
4993         goto OnError;
4994     }
4995     else if (ByteCount == 0)
4996     {
4997         /* Invalid size to read. */
4998         gcmkONERROR(gcvSTATUS_INVALID_ARGUMENT);
4999     }
5000
5001     /* Determine bytes to copy. */
5002     bytes = (ByteCount < shBuf->size) ? ByteCount : shBuf->size;
5003
5004     /* Copy data to user. */
5005     gcmkONERROR(
5006         gckOS_CopyToUserData(Kernel->os,
5007                              shBuf->data,
5008                              UserData,
5009                              bytes));
5010
5011     /* Return copied size. */
5012     *BytesRead = bytes;
5013
5014     /* Release the mutex. */
5015     gcmkVERIFY_OK(
5016         gckOS_ReleaseMutex(Kernel->os, Kernel->db->pointerDatabaseMutex));
5017     acquired = gcvFALSE;
5018
5019     gcmkFOOTER_ARG("*BytesRead=%u", bytes);
5020     return gcvSTATUS_OK;
5021
5022 OnError:
5023     if (acquired)
5024     {
5025         /* Release the mutex. */
5026         gcmkVERIFY_OK(
5027             gckOS_ReleaseMutex(Kernel->os, Kernel->db->pointerDatabaseMutex));
5028     }
5029
5030     gcmkFOOTER();
5031     return status;
5032 }
5033
5034
5035 /*******************************************************************************
5036 ***** Test Code ****************************************************************
5037 *******************************************************************************/
5038