1 /****************************************************************************
3 * Copyright (C) 2005 - 2013 by Vivante Corp.
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the license, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not write to the Free Software
17 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
19 *****************************************************************************/
22 #include "gc_hal_kernel_precomp.h"
24 #define _GC_OBJ_ZONE gcvZONE_MMU
26 typedef enum _gceMMU_TYPE
28 gcvMMU_USED = (0 << 4),
29 gcvMMU_SINGLE = (1 << 4),
30 gcvMMU_FREE = (2 << 4),
34 #define gcmENTRY_TYPE(x) (x & 0xF0)
36 #define gcdMMU_TABLE_DUMP 0
38 #define gcdUSE_MMU_EXCEPTION 0
43 The clear value for the entry of the old MMU.
45 #ifndef gcdMMU_CLEAR_VALUE
46 # define gcdMMU_CLEAR_VALUE 0x00000ABC
49 typedef struct _gcsMMU_STLB *gcsMMU_STLB_PTR;
51 typedef struct _gcsMMU_STLB
53 gctPHYS_ADDR physical;
54 gctUINT32_PTR logical;
59 gctUINT32 mtlbEntryNum;
63 #if gcdSHARED_PAGETABLE
64 typedef struct _gcsSharedPageTable * gcsSharedPageTable_PTR;
65 typedef struct _gcsSharedPageTable
67 /* Shared gckMMU object. */
70 /* Hardwares which use this shared pagetable. */
71 gckHARDWARE hardwares[gcdMAX_GPU_COUNT];
73 /* Number of cores use this shared pagetable. */
78 static gcsSharedPageTable_PTR sharedPageTable = gcvNULL;
81 #if gcdMIRROR_PAGETABLE
82 typedef struct _gcsMirrorPageTable * gcsMirrorPageTable_PTR;
83 typedef struct _gcsMirrorPageTable
86 gckMMU mmus[gcdMAX_GPU_COUNT];
88 /* Hardwares which use this shared pagetable. */
89 gckHARDWARE hardwares[gcdMAX_GPU_COUNT];
91 /* Number of cores use this shared pagetable. */
96 static gcsMirrorPageTable_PTR mirrorPageTable = gcvNULL;
97 static gctPOINTER mirrorPageTableMutex = gcvNULL;
100 typedef struct _gcsDynamicSpaceNode * gcsDynamicSpaceNode_PTR;
101 typedef struct _gcsDynamicSpaceNode
110 IN gctUINT32_PTR PageEntry,
111 IN gctUINT32 EntryValue
114 static gctUINT16 data = 0xff00;
116 if (*(gctUINT8 *)&data == 0xff)
118 *PageEntry = gcmSWAB32(EntryValue);
122 *PageEntry = EntryValue;
128 IN gctUINT32_PTR PageEntry
131 static gctUINT16 data = 0xff00;
132 gctUINT32 entryValue;
134 if (*(gctUINT8 *)&data == 0xff)
136 entryValue = *PageEntry;
137 return gcmSWAB32(entryValue);
147 IN gctUINT32_PTR PageTable,
148 IN gctUINT32 PageCount,
149 IN gctUINT32 EntryValue
154 for (i = 0; i < PageCount; i++)
156 _WritePageEntry(PageTable + i, EntryValue);
169 if (Index >= Mmu->pageTableEntries)
171 /* Just move heap pointer. */
172 Mmu->heapList = Next;
176 /* Address page table. */
177 gctUINT32_PTR pageTable = Mmu->pageTableLogical;
179 /* Dispatch on node type. */
180 switch (gcmENTRY_TYPE(_ReadPageEntry(&pageTable[Index])))
183 /* Set single index. */
184 _WritePageEntry(&pageTable[Index], (Next << 8) | gcvMMU_SINGLE);
189 _WritePageEntry(&pageTable[Index + 1], Next);
193 gcmkFATAL("MMU table correcupted at index %u!", Index);
194 return gcvSTATUS_HEAP_CORRUPTED;
210 gctUINT32_PTR pageTable = Mmu->pageTableLogical;
214 /* Initialize a single page node. */
215 _WritePageEntry(pageTable + Node, (~((1U<<8)-1)) | gcvMMU_SINGLE);
219 /* Initialize the node. */
220 _WritePageEntry(pageTable + Node + 0, (Count << 8) | gcvMMU_FREE);
221 _WritePageEntry(pageTable + Node + 1, ~0U);
224 /* Append the node. */
225 return _Link(Mmu, Index, Node);
233 gctUINT32_PTR pageTable = Mmu->pageTableLogical;
235 gctUINT32 i, previous, start = 0, count = 0;
237 previous = Mmu->heapList = ~0U;
238 Mmu->freeNodes = gcvFALSE;
240 /* Walk the entire page table. */
241 for (i = 0; i < Mmu->pageTableEntries; ++i)
243 /* Dispatch based on type of page. */
244 switch (gcmENTRY_TYPE(_ReadPageEntry(&pageTable[i])))
247 /* Used page, so close any open node. */
251 gcmkONERROR(_AddFree(Mmu, previous, start, count));
253 /* Reset the node. */
260 /* Single free node. */
263 /* Start a new node. */
272 /* Start a new node. */
276 /* Advance the count. */
277 count += _ReadPageEntry(&pageTable[i]) >> 8;
279 /* Advance the index into the page table. */
280 i += (_ReadPageEntry(&pageTable[i]) >> 8) - 1;
284 gcmkFATAL("MMU page table correcupted at index %u!", i);
285 return gcvSTATUS_HEAP_CORRUPTED;
289 /* See if we have an open node left. */
292 /* Add the node to the list. */
293 gcmkONERROR(_AddFree(Mmu, previous, start, count));
296 gcmkTRACE_ZONE(gcvLEVEL_INFO, gcvZONE_MMU,
297 "Performed a garbage collection of the MMU heap.");
303 /* Return the staus. */
308 _SetPage(gctUINT32 PageAddress)
313 /* Ignore exception */
322 IN gctUINT32 PhysBase,
327 gctBOOL mutex = gcvFALSE;
328 gcsMMU_STLB_PTR head = gcvNULL, pre = gcvNULL;
329 gctUINT32 start = PhysBase & (~gcdMMU_PAGE_64K_MASK);
330 gctUINT32 end = (PhysBase + Size - 1) & (~gcdMMU_PAGE_64K_MASK);
331 gctUINT32 mStart = start >> gcdMMU_MTLB_SHIFT;
332 gctUINT32 mEnd = end >> gcdMMU_MTLB_SHIFT;
333 gctUINT32 sStart = (start & gcdMMU_STLB_64K_MASK) >> gcdMMU_STLB_64K_SHIFT;
334 gctUINT32 sEnd = (end & gcdMMU_STLB_64K_MASK) >> gcdMMU_STLB_64K_SHIFT;
336 /* Grab the mutex. */
337 gcmkONERROR(gckOS_AcquireMutex(Mmu->os, Mmu->pageTableMutex, gcvINFINITE));
340 while (mStart <= mEnd)
342 gcmkASSERT(mStart < gcdMMU_MTLB_ENTRY_NUM);
343 if (*(Mmu->mtlbLogical + mStart) == 0)
345 gcsMMU_STLB_PTR stlb;
346 gctPOINTER pointer = gcvNULL;
347 gctUINT32 last = (mStart == mEnd) ? sEnd : (gcdMMU_STLB_64K_ENTRY_NUM - 1);
349 gcmkONERROR(gckOS_Allocate(Mmu->os, sizeof(struct _gcsMMU_STLB), &pointer));
352 stlb->mtlbEntryNum = 0;
353 stlb->next = gcvNULL;
354 stlb->physical = gcvNULL;
355 stlb->logical = gcvNULL;
356 stlb->size = gcdMMU_STLB_64K_SIZE;
365 gcmkASSERT(pre->next == gcvNULL);
371 gckOS_AllocateContiguous(Mmu->os,
375 (gctPOINTER)&stlb->logical));
377 gcmkONERROR(gckOS_ZeroMemory(stlb->logical, stlb->size));
379 gcmkONERROR(gckOS_GetPhysicalAddress(
384 if (stlb->physBase & (gcdMMU_STLB_64K_SIZE - 1))
386 gcmkONERROR(gcvSTATUS_NOT_ALIGNED);
389 _WritePageEntry(Mmu->mtlbLogical + mStart,
393 /* Ignore exception */
398 #if gcdMMU_TABLE_DUMP
399 gckOS_Print("%s(%d): insert MTLB[%d]: %08x\n",
400 __FUNCTION__, __LINE__,
402 _ReadPageEntry(Mmu->mtlbLogical + mStart));
405 stlb->mtlbIndex = mStart;
406 stlb->mtlbEntryNum = 1;
407 #if gcdMMU_TABLE_DUMP
408 gckOS_Print("%s(%d): STLB: logical:%08x -> physical:%08x\n",
409 __FUNCTION__, __LINE__,
414 while (sStart <= last)
416 gcmkASSERT(!(start & gcdMMU_PAGE_64K_MASK));
417 _WritePageEntry(stlb->logical + sStart, _SetPage(start));
418 #if gcdMMU_TABLE_DUMP
419 gckOS_Print("%s(%d): insert STLB[%d]: %08x\n",
420 __FUNCTION__, __LINE__,
422 _ReadPageEntry(stlb->logical + sStart));
425 start += gcdMMU_PAGE_64K_SIZE;
435 gcmkONERROR(gcvSTATUS_INVALID_REQUEST);
439 /* Insert the stlb into staticSTLB. */
440 if (Mmu->staticSTLB == gcvNULL)
442 Mmu->staticSTLB = head;
446 gcmkASSERT(pre == gcvNULL);
447 gcmkASSERT(pre->next == gcvNULL);
448 pre->next = Mmu->staticSTLB;
449 Mmu->staticSTLB = head;
452 /* Release the mutex. */
453 gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex));
460 while (head != gcvNULL)
465 if (pre->physical != gcvNULL)
468 gckOS_FreeContiguous(Mmu->os,
474 if (pre->mtlbEntryNum != 0)
476 gcmkASSERT(pre->mtlbEntryNum == 1);
477 _WritePageEntry(Mmu->mtlbLogical + pre->mtlbIndex, 0);
480 gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Mmu->os, pre));
485 /* Release the mutex. */
486 gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex));
495 OUT gcsDynamicSpaceNode_PTR *Array,
499 gceSTATUS status = gcvSTATUS_OK;
500 gctPOINTER pointer = gcvNULL;
501 gcsDynamicSpaceNode_PTR array = gcvNULL;
503 gctINT i = 0, nodeStart = -1, nodeEntries = 0;
505 /* Allocate memory for the array. */
506 gcmkONERROR(gckOS_Allocate(Mmu->os,
507 gcmSIZEOF(*array) * (gcdMMU_MTLB_ENTRY_NUM / 2),
510 array = (gcsDynamicSpaceNode_PTR)pointer;
512 /* Loop all the entries. */
513 while (i < gcdMMU_MTLB_ENTRY_NUM)
515 if (!Mmu->mtlbLogical[i])
519 /* This is the first entry of the dynamic space. */
525 /* Other entries of the dynamic space. */
529 else if (nodeStart >= 0)
531 /* Save the previous node. */
532 array[size].start = nodeStart;
533 array[size].entries = nodeEntries;
536 /* Reset the start. */
544 /* Save the previous node. */
547 array[size].start = nodeStart;
548 array[size].entries = nodeEntries;
552 #if gcdMMU_TABLE_DUMP
553 for (i = 0; i < size; i++)
555 gckOS_Print("%s(%d): [%d]: start=%d, entries=%d.\n",
556 __FUNCTION__, __LINE__,
569 if (pointer != gcvNULL)
571 gckOS_Free(Mmu->os, pointer);
583 gcsDynamicSpaceNode_PTR nodeArray = gcvNULL;
584 gctINT i, nodeArraySize = 0;
586 gctINT numEntries = 0;
587 gctUINT32_PTR pageTable;
588 gctBOOL acquired = gcvFALSE;
590 /* Find all the dynamic address space. */
591 gcmkONERROR(_FindDynamicSpace(Mmu, &nodeArray, &nodeArraySize));
593 /* TODO: We only use the largest one for now. */
594 for (i = 0; i < nodeArraySize; i++)
596 if (nodeArray[i].entries > numEntries)
598 Mmu->dynamicMappingStart = nodeArray[i].start;
599 numEntries = nodeArray[i].entries;
603 gckOS_Free(Mmu->os, (gctPOINTER)nodeArray);
605 Mmu->pageTableSize = numEntries * 4096;
607 Mmu->pageTableEntries = Mmu->pageTableSize / gcmSIZEOF(gctUINT32);
609 /* Construct Slave TLB. */
610 gcmkONERROR(gckOS_AllocateContiguous(Mmu->os,
613 &Mmu->pageTablePhysical,
614 (gctPOINTER)&Mmu->pageTableLogical));
616 #if gcdUSE_MMU_EXCEPTION
617 gcmkONERROR(_FillPageTable(Mmu->pageTableLogical,
618 Mmu->pageTableEntries,
619 /* Enable exception */
622 /* Invalidate all entries. */
623 gcmkONERROR(gckOS_ZeroMemory(Mmu->pageTableLogical,
624 Mmu->pageTableSize));
628 pageTable = Mmu->pageTableLogical;
629 _WritePageEntry(pageTable, (Mmu->pageTableEntries << 8) | gcvMMU_FREE);
630 _WritePageEntry(pageTable + 1, ~0U);
632 Mmu->freeNodes = gcvFALSE;
634 gcmkONERROR(gckOS_GetPhysicalAddress(Mmu->os,
635 Mmu->pageTableLogical,
638 /* Grab the mutex. */
639 gcmkONERROR(gckOS_AcquireMutex(Mmu->os, Mmu->pageTableMutex, gcvINFINITE));
642 /* Map to Master TLB. */
643 for (i = (gctINT)Mmu->dynamicMappingStart;
644 i < (gctINT)Mmu->dynamicMappingStart + numEntries;
647 _WritePageEntry(Mmu->mtlbLogical + i,
651 /* Ignore exception */
656 #if gcdMMU_TABLE_DUMP
657 gckOS_Print("%s(%d): insert MTLB[%d]: %08x\n",
658 __FUNCTION__, __LINE__,
660 _ReadPageEntry(Mmu->mtlbLogical + i));
662 physical += gcdMMU_STLB_4K_SIZE;
665 /* Release the mutex. */
666 gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex));
671 if (Mmu->pageTableLogical)
673 /* Free the page table. */
675 gckOS_FreeContiguous(Mmu->os,
676 Mmu->pageTablePhysical,
677 (gctPOINTER) Mmu->pageTableLogical,
678 Mmu->pageTableSize));
683 /* Release the mutex. */
684 gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex));
690 /*******************************************************************************
694 ** Construct a new gckMMU object.
699 ** Pointer to an gckKERNEL object.
702 ** Number of bytes for the page table.
707 ** Pointer to a variable that receives the gckMMU object pointer.
712 IN gctSIZE_T MmuSize,
717 gckHARDWARE hardware;
719 gckMMU mmu = gcvNULL;
720 gctUINT32_PTR pageTable;
721 gctPOINTER pointer = gcvNULL;
723 gcmkHEADER_ARG("Kernel=0x%x MmuSize=%lu", Kernel, MmuSize);
725 /* Verify the arguments. */
726 gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL);
727 gcmkVERIFY_ARGUMENT(MmuSize > 0);
728 gcmkVERIFY_ARGUMENT(Mmu != gcvNULL);
730 /* Extract the gckOS object pointer. */
732 gcmkVERIFY_OBJECT(os, gcvOBJ_OS);
734 /* Extract the gckHARDWARE object pointer. */
735 hardware = Kernel->hardware;
736 gcmkVERIFY_OBJECT(hardware, gcvOBJ_HARDWARE);
738 /* Allocate memory for the gckMMU object. */
739 gcmkONERROR(gckOS_Allocate(os, sizeof(struct _gckMMU), &pointer));
743 /* Initialize the gckMMU object. */
744 mmu->object.type = gcvOBJ_MMU;
746 mmu->hardware = hardware;
747 mmu->pageTableMutex = gcvNULL;
748 mmu->pageTableLogical = gcvNULL;
749 mmu->mtlbLogical = gcvNULL;
750 mmu->staticSTLB = gcvNULL;
751 mmu->enabled = gcvFALSE;
753 mmu->nodeList = gcvNULL;
754 mmu->nodeMutex = gcvNULL;
757 /* Create the page table mutex. */
758 gcmkONERROR(gckOS_CreateMutex(os, &mmu->pageTableMutex));
761 /* Create the node list mutex. */
762 gcmkONERROR(gckOS_CreateMutex(os, &mmu->nodeMutex));
765 if (hardware->mmuVersion == 0)
767 mmu->pageTableSize = MmuSize;
770 gckOS_AllocateContiguous(os,
773 &mmu->pageTablePhysical,
776 mmu->pageTableLogical = pointer;
778 /* Compute number of entries in page table. */
779 mmu->pageTableEntries = mmu->pageTableSize / sizeof(gctUINT32);
781 /* Mark all pages as free. */
782 pageTable = mmu->pageTableLogical;
784 #if gcdMMU_CLEAR_VALUE
785 _FillPageTable(pageTable, mmu->pageTableEntries, gcdMMU_CLEAR_VALUE);
788 _WritePageEntry(pageTable, (mmu->pageTableEntries << 8) | gcvMMU_FREE);
789 _WritePageEntry(pageTable + 1, ~0U);
791 mmu->freeNodes = gcvFALSE;
793 /* Set page table address. */
795 gckHARDWARE_SetMMU(hardware, (gctPOINTER) mmu->pageTableLogical));
799 /* Allocate the 4K mode MTLB table. */
800 mmu->mtlbSize = gcdMMU_MTLB_SIZE + 64;
803 gckOS_AllocateContiguous(os,
809 mmu->mtlbLogical = pointer;
811 /* Invalid all the entries. */
813 gckOS_ZeroMemory(pointer, mmu->mtlbSize));
816 /* Return the gckMMU object pointer. */
820 gcmkFOOTER_ARG("*Mmu=0x%x", *Mmu);
827 if (mmu->pageTableLogical != gcvNULL)
829 /* Free the page table. */
831 gckOS_FreeContiguous(os,
832 mmu->pageTablePhysical,
833 (gctPOINTER) mmu->pageTableLogical,
834 mmu->pageTableSize));
838 if (mmu->mtlbLogical != gcvNULL)
841 gckOS_FreeContiguous(os,
843 (gctPOINTER) mmu->mtlbLogical,
847 if (mmu->pageTableMutex != gcvNULL)
849 /* Delete the mutex. */
851 gckOS_DeleteMutex(os, mmu->pageTableMutex));
855 if (mmu->nodeMutex != gcvNULL)
857 /* Delete the mutex. */
859 gckOS_DeleteMutex(os, mmu->nodeMutex));
863 /* Mark the gckMMU object as unknown. */
864 mmu->object.type = gcvOBJ_UNKNOWN;
866 /* Free the allocates memory. */
867 gcmkVERIFY_OK(gcmkOS_SAFE_FREE(os, mmu));
870 /* Return the status. */
875 /*******************************************************************************
879 ** Destroy a gckMMU object.
884 ** Pointer to an gckMMU object.
896 gcuVIDMEM_NODE_PTR node, next;
899 gcmkHEADER_ARG("Mmu=0x%x", Mmu);
901 /* Verify the arguments. */
902 gcmkVERIFY_OBJECT(Mmu, gcvOBJ_MMU);
905 /* Free all associated virtual memory. */
906 for (node = Mmu->nodeList; node != gcvNULL; node = next)
908 next = node->Virtual.next;
909 gcmkVERIFY_OK(gckVIDMEM_Free(node));
913 while (Mmu->staticSTLB != gcvNULL)
915 gcsMMU_STLB_PTR pre = Mmu->staticSTLB;
916 Mmu->staticSTLB = pre->next;
918 if (pre->physical != gcvNULL)
921 gckOS_FreeContiguous(Mmu->os,
927 if (pre->mtlbEntryNum != 0)
929 gcmkASSERT(pre->mtlbEntryNum == 1);
930 _WritePageEntry(Mmu->mtlbLogical + pre->mtlbIndex, 0);
931 #if gcdMMU_TABLE_DUMP
932 gckOS_Print("%s(%d): clean MTLB[%d]\n",
933 __FUNCTION__, __LINE__,
938 gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Mmu->os, pre));
941 if (Mmu->hardware->mmuVersion != 0)
944 gckOS_FreeContiguous(Mmu->os,
946 (gctPOINTER) Mmu->mtlbLogical,
950 /* Free the page table. */
952 gckOS_FreeContiguous(Mmu->os,
953 Mmu->pageTablePhysical,
954 (gctPOINTER) Mmu->pageTableLogical,
955 Mmu->pageTableSize));
958 /* Delete the node list mutex. */
959 gcmkVERIFY_OK(gckOS_DeleteMutex(Mmu->os, Mmu->nodeMutex));
962 /* Delete the page table mutex. */
963 gcmkVERIFY_OK(gckOS_DeleteMutex(Mmu->os, Mmu->pageTableMutex));
965 /* Mark the gckMMU object as unknown. */
966 Mmu->object.type = gcvOBJ_UNKNOWN;
968 /* Free the gckMMU object. */
969 gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Mmu->os, Mmu));
979 IN gctSIZE_T MmuSize,
983 #if gcdSHARED_PAGETABLE
987 gcmkHEADER_ARG("Kernel=0x%08x", Kernel);
989 if (sharedPageTable == gcvNULL)
992 gckOS_Allocate(Kernel->os,
993 sizeof(struct _gcsSharedPageTable),
995 sharedPageTable = pointer;
998 gckOS_ZeroMemory(sharedPageTable,
999 sizeof(struct _gcsSharedPageTable)));
1001 gcmkONERROR(_Construct(Kernel, MmuSize, &sharedPageTable->mmu));
1003 else if (Kernel->hardware->mmuVersion == 0)
1005 /* Set page table address. */
1007 gckHARDWARE_SetMMU(Kernel->hardware, (gctPOINTER) sharedPageTable->mmu->pageTableLogical));
1010 *Mmu = sharedPageTable->mmu;
1012 sharedPageTable->hardwares[sharedPageTable->reference] = Kernel->hardware;
1014 sharedPageTable->reference++;
1016 gcmkFOOTER_ARG("sharedPageTable->reference=%lu", sharedPageTable->reference);
1017 return gcvSTATUS_OK;
1020 if (sharedPageTable)
1022 if (sharedPageTable->mmu)
1024 gcmkVERIFY_OK(gckMMU_Destroy(sharedPageTable->mmu));
1027 gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Kernel->os, sharedPageTable));
1032 #elif gcdMIRROR_PAGETABLE
1036 gcmkHEADER_ARG("Kernel=0x%08x", Kernel);
1038 if (mirrorPageTable == gcvNULL)
1041 gckOS_Allocate(Kernel->os,
1042 sizeof(struct _gcsMirrorPageTable),
1044 mirrorPageTable = pointer;
1047 gckOS_ZeroMemory(mirrorPageTable,
1048 sizeof(struct _gcsMirrorPageTable)));
1051 gckOS_CreateMutex(Kernel->os, &mirrorPageTableMutex));
1054 gcmkONERROR(_Construct(Kernel, MmuSize, Mmu));
1056 mirrorPageTable->mmus[mirrorPageTable->reference] = *Mmu;
1058 mirrorPageTable->hardwares[mirrorPageTable->reference] = Kernel->hardware;
1060 mirrorPageTable->reference++;
1062 gcmkFOOTER_ARG("mirrorPageTable->reference=%lu", mirrorPageTable->reference);
1063 return gcvSTATUS_OK;
1066 if (mirrorPageTable && mirrorPageTable->reference == 0)
1068 gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Kernel->os, mirrorPageTable));
1074 return _Construct(Kernel, MmuSize, Mmu);
1083 #if gcdSHARED_PAGETABLE
1084 sharedPageTable->reference--;
1086 if (sharedPageTable->reference == 0)
1088 if (sharedPageTable->mmu)
1090 gcmkVERIFY_OK(_Destroy(Mmu));
1093 gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Mmu->os, sharedPageTable));
1096 return gcvSTATUS_OK;
1097 #elif gcdMIRROR_PAGETABLE
1098 mirrorPageTable->reference--;
1100 if (mirrorPageTable->reference == 0)
1102 gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Mmu->os, mirrorPageTable));
1103 gcmkVERIFY_OK(gcmkOS_SAFE_FREE(Mmu->os, mirrorPageTableMutex));
1106 return _Destroy(Mmu);
1108 return _Destroy(Mmu);
1112 /*******************************************************************************
1114 ** gckMMU_AllocatePages
1116 ** Allocate pages inside the page table.
1121 ** Pointer to an gckMMU object.
1123 ** gctSIZE_T PageCount
1124 ** Number of pages to allocate.
1128 ** gctPOINTER * PageTable
1129 ** Pointer to a variable that receives the base address of the page
1132 ** gctUINT32 * Address
1133 ** Pointer to a variable that receives the hardware specific address.
1138 IN gctSIZE_T PageCount,
1139 OUT gctPOINTER * PageTable,
1140 OUT gctUINT32 * Address
1144 gctBOOL mutex = gcvFALSE;
1145 gctUINT32 index = 0, previous = ~0U, left;
1146 gctUINT32_PTR pageTable;
1150 gcmkHEADER_ARG("Mmu=0x%x PageCount=%lu", Mmu, PageCount);
1152 /* Verify the arguments. */
1153 gcmkVERIFY_OBJECT(Mmu, gcvOBJ_MMU);
1154 gcmkVERIFY_ARGUMENT(PageCount > 0);
1155 gcmkVERIFY_ARGUMENT(PageTable != gcvNULL);
1157 if (PageCount > Mmu->pageTableEntries)
1159 gcmkPRINT("[galcore]: %s(%d): Run out of free page entry.",
1160 __FUNCTION__, __LINE__);
1162 /* Not enough pages avaiable. */
1163 gcmkONERROR(gcvSTATUS_OUT_OF_RESOURCES);
1166 /* Grab the mutex. */
1167 gcmkONERROR(gckOS_AcquireMutex(Mmu->os, Mmu->pageTableMutex, gcvINFINITE));
1170 /* Cast pointer to page table. */
1171 for (pageTable = Mmu->pageTableLogical, gotIt = gcvFALSE; !gotIt;)
1173 /* Walk the heap list. */
1174 for (index = Mmu->heapList; !gotIt && (index < Mmu->pageTableEntries);)
1176 /* Check the node type. */
1177 switch (gcmENTRY_TYPE(_ReadPageEntry(&pageTable[index])))
1180 /* Single odes are valid if we only need 1 page. */
1187 /* Move to next node. */
1189 index = _ReadPageEntry(&pageTable[index]) >> 8;
1194 /* Test if the node has enough space. */
1195 if (PageCount <= (_ReadPageEntry(&pageTable[index]) >> 8))
1201 /* Move to next node. */
1203 index = _ReadPageEntry(&pageTable[index + 1]);
1208 gcmkFATAL("MMU table correcupted at index %u!", index);
1209 gcmkONERROR(gcvSTATUS_OUT_OF_RESOURCES);
1213 /* Test if we are out of memory. */
1214 if (index >= Mmu->pageTableEntries)
1218 /* Time to move out the trash! */
1219 gcmkONERROR(_Collect(Mmu));
1223 gcmkPRINT("[galcore]: %s(%d): Run out of free page entry.",
1224 __FUNCTION__, __LINE__);
1226 /* Out of resources. */
1227 gcmkONERROR(gcvSTATUS_OUT_OF_RESOURCES);
1232 switch (gcmENTRY_TYPE(_ReadPageEntry(&pageTable[index])))
1235 /* Unlink single node from free list. */
1237 _Link(Mmu, previous, _ReadPageEntry(&pageTable[index]) >> 8));
1241 /* Check how many pages will be left. */
1242 left = (_ReadPageEntry(&pageTable[index]) >> 8) - PageCount;
1246 /* The entire node is consumed, just unlink it. */
1248 _Link(Mmu, previous, _ReadPageEntry(&pageTable[index + 1])));
1252 /* One page will remain. Convert the node to a single node and
1253 ** advance the index. */
1254 _WritePageEntry(&pageTable[index], (_ReadPageEntry(&pageTable[index + 1]) << 8) | gcvMMU_SINGLE);
1259 /* Enough pages remain for a new node. However, we will just adjust
1260 ** the size of the current node and advance the index. */
1261 _WritePageEntry(&pageTable[index], (left << 8) | gcvMMU_FREE);
1268 /* Mark node as used. */
1269 gcmkONERROR(_FillPageTable(&pageTable[index], PageCount, gcvMMU_USED));
1271 /* Return pointer to page table. */
1272 *PageTable = &pageTable[index];
1274 /* Build virtual address. */
1275 if (Mmu->hardware->mmuVersion == 0)
1278 gckHARDWARE_BuildVirtualAddress(Mmu->hardware, index, 0, &address));
1282 gctUINT32 masterOffset = index / gcdMMU_STLB_4K_ENTRY_NUM
1283 + Mmu->dynamicMappingStart;
1284 gctUINT32 slaveOffset = index % gcdMMU_STLB_4K_ENTRY_NUM;
1286 address = (masterOffset << gcdMMU_MTLB_SHIFT)
1287 | (slaveOffset << gcdMMU_STLB_4K_SHIFT);
1290 if (Address != gcvNULL)
1295 /* Release the mutex. */
1296 gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex));
1299 gcmkFOOTER_ARG("*PageTable=0x%x *Address=%08x",
1300 *PageTable, gcmOPT_VALUE(Address));
1301 return gcvSTATUS_OK;
1307 /* Release the mutex. */
1308 gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex));
1311 /* Return the status. */
1316 /*******************************************************************************
1320 ** Free pages inside the page table.
1325 ** Pointer to an gckMMU object.
1327 ** gctPOINTER PageTable
1328 ** Base address of the page table to free.
1330 ** gctSIZE_T PageCount
1331 ** Number of pages to free.
1340 IN gctPOINTER PageTable,
1341 IN gctSIZE_T PageCount
1344 gctUINT32_PTR pageTable;
1346 gctBOOL acquired = gcvFALSE;
1348 gcmkHEADER_ARG("Mmu=0x%x PageTable=0x%x PageCount=%lu",
1349 Mmu, PageTable, PageCount);
1351 /* Verify the arguments. */
1352 gcmkVERIFY_OBJECT(Mmu, gcvOBJ_MMU);
1353 gcmkVERIFY_ARGUMENT(PageTable != gcvNULL);
1354 gcmkVERIFY_ARGUMENT(PageCount > 0);
1356 /* Convert the pointer. */
1357 pageTable = (gctUINT32_PTR) PageTable;
1359 gcmkONERROR(gckOS_AcquireMutex(Mmu->os, Mmu->pageTableMutex, gcvINFINITE));
1362 #if gcdMMU_CLEAR_VALUE
1363 if (Mmu->hardware->mmuVersion == 0)
1365 _FillPageTable(pageTable, PageCount, gcdMMU_CLEAR_VALUE);
1371 /* Single page node. */
1372 _WritePageEntry(pageTable,
1373 (~((1U<<8)-1)) | gcvMMU_SINGLE
1374 #if gcdUSE_MMU_EXCEPTION
1375 /* Enable exception */
1382 /* Mark the node as free. */
1383 _WritePageEntry(pageTable,
1384 (PageCount << 8) | gcvMMU_FREE
1385 #if gcdUSE_MMU_EXCEPTION
1386 /* Enable exception */
1390 _WritePageEntry(pageTable + 1, ~0U);
1392 #if gcdUSE_MMU_EXCEPTION
1393 /* Enable exception */
1394 gcmkVERIFY_OK(_FillPageTable(pageTable + 2, PageCount - 2, 1 << 1));
1398 /* We have free nodes. */
1399 Mmu->freeNodes = gcvTRUE;
1401 gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex));
1402 acquired = gcvFALSE;
1406 return gcvSTATUS_OK;
1411 gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->pageTableMutex));
1419 gckMMU_AllocatePages(
1421 IN gctSIZE_T PageCount,
1422 OUT gctPOINTER * PageTable,
1423 OUT gctUINT32 * Address
1426 #if gcdMIRROR_PAGETABLE
1428 gctPOINTER pageTable;
1432 gctBOOL acquired = gcvFALSE;
1433 gctBOOL allocated = gcvFALSE;
1435 gckOS_AcquireMutex(Mmu->os, mirrorPageTableMutex, gcvINFINITE);
1438 /* Allocate page table for current MMU. */
1439 for (i = 0; i < mirrorPageTable->reference; i++)
1441 if (Mmu == mirrorPageTable->mmus[i])
1443 gcmkONERROR(_AllocatePages(Mmu, PageCount, PageTable, Address));
1444 allocated = gcvTRUE;
1448 /* Allocate page table for other MMUs. */
1449 for (i = 0; i < mirrorPageTable->reference; i++)
1451 mmu = mirrorPageTable->mmus[i];
1455 gcmkONERROR(_AllocatePages(mmu, PageCount, &pageTable, &address));
1456 gcmkASSERT(address == *Address);
1460 gckOS_ReleaseMutex(Mmu->os, mirrorPageTableMutex);
1461 acquired = gcvFALSE;
1463 return gcvSTATUS_OK;
1468 /* Page tables for multiple GPU always keep the same. So it is impossible
1469 * the fist one allocates successfully but others fail.
1476 gckOS_ReleaseMutex(Mmu->os, mirrorPageTableMutex);
1481 return _AllocatePages(Mmu, PageCount, PageTable, Address);
1488 IN gctPOINTER PageTable,
1489 IN gctSIZE_T PageCount
1492 #if gcdMIRROR_PAGETABLE
1497 gckOS_AcquireMutex(Mmu->os, mirrorPageTableMutex, gcvINFINITE);
1499 gcmkVERIFY_OK(_FreePages(Mmu, PageTable, PageCount));
1501 offset = (gctUINT32)PageTable - (gctUINT32)Mmu->pageTableLogical;
1503 for (i = 0; i < mirrorPageTable->reference; i++)
1505 mmu = mirrorPageTable->mmus[i];
1509 gcmkVERIFY_OK(_FreePages(mmu, mmu->pageTableLogical + offset/4, PageCount));
1513 gckOS_ReleaseMutex(Mmu->os, mirrorPageTableMutex);
1515 return gcvSTATUS_OK;
1517 return _FreePages(Mmu, PageTable, PageCount);
1524 IN gctUINT32 PhysBaseAddr,
1525 IN gctUINT32 PhysSize
1529 #if gcdSHARED_PAGETABLE
1530 gckHARDWARE hardware;
1534 gcmkHEADER_ARG("Mmu=0x%x", Mmu);
1536 /* Verify the arguments. */
1537 gcmkVERIFY_OBJECT(Mmu, gcvOBJ_MMU);
1539 #if gcdSHARED_PAGETABLE
1542 gcmkFOOTER_ARG("Status=%d", gcvSTATUS_SKIP);
1543 return gcvSTATUS_SKIP;
1547 if (Mmu->hardware->mmuVersion == 0)
1550 gcmkFOOTER_ARG("Status=%d", gcvSTATUS_SKIP);
1551 return gcvSTATUS_SKIP;
1557 gcmkONERROR(_FillFlatMapping(
1564 gcmkONERROR(_SetupDynamicSpace(Mmu));
1566 #if gcdSHARED_PAGETABLE
1567 for(i = 0; i < gcdMAX_GPU_COUNT; i++)
1569 hardware = sharedPageTable->hardwares[i];
1570 if (hardware != gcvNULL)
1573 gckHARDWARE_SetMMUv2(
1578 (gctUINT8_PTR)Mmu->mtlbLogical + gcdMMU_MTLB_SIZE,
1585 gckHARDWARE_SetMMUv2(
1590 (gctUINT8_PTR)Mmu->mtlbLogical + gcdMMU_MTLB_SIZE,
1595 Mmu->enabled = gcvTRUE;
1599 return gcvSTATUS_OK;
1603 /* Return the status. */
1611 IN gctUINT32 PageAddress,
1612 IN gctUINT32 *PageEntry
1615 #if gcdMIRROR_PAGETABLE
1616 gctUINT32_PTR pageEntry;
1619 gctUINT32 offset = (gctUINT32)PageEntry - (gctUINT32)Mmu->pageTableLogical;
1623 gcmkHEADER_ARG("Mmu=0x%x", Mmu);
1625 /* Verify the arguments. */
1626 gcmkVERIFY_OBJECT(Mmu, gcvOBJ_MMU);
1627 gcmkVERIFY_ARGUMENT(PageEntry != gcvNULL);
1628 gcmkVERIFY_ARGUMENT(!(PageAddress & 0xFFF));
1630 if (Mmu->hardware->mmuVersion == 0)
1636 data = _SetPage(PageAddress);
1639 _WritePageEntry(PageEntry, data);
1641 #if gcdMIRROR_PAGETABLE
1642 for (i = 0; i < mirrorPageTable->reference; i++)
1644 mmu = mirrorPageTable->mmus[i];
1648 pageEntry = mmu->pageTableLogical + offset / 4;
1650 if (mmu->hardware->mmuVersion == 0)
1652 _WritePageEntry(pageEntry, PageAddress);
1656 _WritePageEntry(pageEntry, _SetPage(PageAddress));
1664 return gcvSTATUS_OK;
1671 IN gcuVIDMEM_NODE_PTR Node)
1674 gctBOOL mutex = gcvFALSE;
1676 gcmkHEADER_ARG("Mmu=0x%x Node=0x%x", Mmu, Node);
1678 gcmkVERIFY_OBJECT(Mmu, gcvOBJ_MMU);
1680 gcmkONERROR(gckOS_AcquireMutex(Mmu->os, Mmu->nodeMutex, gcvINFINITE));
1683 Node->Virtual.next = Mmu->nodeList;
1684 Mmu->nodeList = Node;
1686 gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->nodeMutex));
1689 return gcvSTATUS_OK;
1694 gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->nodeMutex));
1704 IN gcuVIDMEM_NODE_PTR Node)
1707 gctBOOL mutex = gcvFALSE;
1708 gcuVIDMEM_NODE_PTR *iter;
1710 gcmkHEADER_ARG("Mmu=0x%x Node=0x%x", Mmu, Node);
1712 gcmkVERIFY_OBJECT(Mmu, gcvOBJ_MMU);
1714 gcmkONERROR(gckOS_AcquireMutex(Mmu->os, Mmu->nodeMutex, gcvINFINITE));
1717 for (iter = &Mmu->nodeList; *iter; iter = &(*iter)->Virtual.next)
1721 *iter = Node->Virtual.next;
1726 gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->nodeMutex));
1729 return gcvSTATUS_OK;
1734 gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->nodeMutex));
1742 gckMMU_FreeHandleMemory(
1743 IN gckKERNEL Kernel,
1749 gctBOOL acquired = gcvFALSE;
1750 gcuVIDMEM_NODE_PTR curr, next;
1752 gcmkHEADER_ARG("Kernel=0x%x, Mmu=0x%x Pid=%u", Kernel, Mmu, Pid);
1754 gcmkVERIFY_OBJECT(Kernel, gcvOBJ_KERNEL);
1755 gcmkVERIFY_OBJECT(Mmu, gcvOBJ_MMU);
1757 gcmkONERROR(gckOS_AcquireMutex(Mmu->os, Mmu->nodeMutex, gcvINFINITE));
1760 for (curr = Mmu->nodeList; curr != gcvNULL; curr = next)
1762 next = curr->Virtual.next;
1764 if (curr->Virtual.processID == Pid)
1766 while (curr->Virtual.unlockPendings[Kernel->core] == 0 && curr->Virtual.lockeds[Kernel->core] > 0)
1768 gcmkONERROR(gckVIDMEM_Unlock(Kernel, curr, gcvSURF_TYPE_UNKNOWN, gcvNULL));
1771 gcmkVERIFY_OK(gckVIDMEM_Free(curr));
1775 gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->nodeMutex));
1778 return gcvSTATUS_OK;
1783 gcmkVERIFY_OK(gckOS_ReleaseMutex(Mmu->os, Mmu->nodeMutex));
1796 gckHARDWARE hardware;
1797 #if gcdSHARED_PAGETABLE
1799 for (i = 0; i < gcdMAX_GPU_COUNT; i++)
1802 if (i == gcvCORE_VG)
1807 hardware = sharedPageTable->hardwares[i];
1810 /* Notify cores who use this page table. */
1812 gckOS_AtomSet(hardware->os, hardware->pageTableDirty, 1));
1815 #elif gcdMIRROR_PAGETABLE
1817 for (i = 0; i < mirrorPageTable->reference; i++)
1819 hardware = mirrorPageTable->hardwares[i];
1821 /* Notify cores who use this page table. */
1823 gckOS_AtomSet(hardware->os, hardware->pageTableDirty, 1));
1826 hardware = Mmu->hardware;
1828 gckOS_AtomSet(hardware->os, hardware->pageTableDirty, 1));
1831 return gcvSTATUS_OK;
1835 gckMMU_DumpPageTableEntry(
1837 IN gctUINT32 Address
1840 gctUINT32_PTR pageTable;
1842 gctUINT32 mtlb, stlb;
1844 gcmkHEADER_ARG("Mmu=0x%08X Address=0x%08X", Mmu, Address);
1845 gcmkVERIFY_OBJECT(Mmu, gcvOBJ_MMU);
1847 gcmkASSERT(Mmu->hardware->mmuVersion > 0);
1849 mtlb = (Address & gcdMMU_MTLB_MASK) >> gcdMMU_MTLB_SHIFT;
1850 stlb = (Address & gcdMMU_STLB_4K_MASK) >> gcdMMU_STLB_4K_SHIFT;
1852 if (Address >= 0x80000000)
1854 pageTable = Mmu->pageTableLogical;
1856 index = (mtlb - Mmu->dynamicMappingStart)
1857 * gcdMMU_STLB_4K_ENTRY_NUM
1860 gcmkPRINT(" Page table entry = 0x%08X", _ReadPageEntry(pageTable + index));
1864 return gcvSTATUS_OK;
1867 /******************************************************************************
1868 ****************************** T E S T C O D E ******************************
1869 ******************************************************************************/