1 /* Copyright (c) 2016-2017 The Linux Foundation. All rights reserved.
3 * This program is free software; you can redistribute it and/or modify
4 * it under the terms of the GNU General Public License version 2 and
5 * only version 2 as published by the Free Software Foundation.
7 * This program is distributed in the hope that it will be useful,
8 * but WITHOUT ANY WARRANTY; without even the implied warranty of
9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 * GNU General Public License for more details.
14 #include <linux/types.h>
15 #include <linux/cpumask.h>
16 #include <linux/qcom_scm.h>
17 #include <linux/dma-mapping.h>
18 #include <linux/of_address.h>
19 #include <linux/soc/qcom/mdt_loader.h>
24 extern bool hang_debug;
25 static void a5xx_dump(struct msm_gpu *gpu);
29 static int zap_shader_load_mdt(struct device *dev, const char *fwname)
31 const struct firmware *fw;
32 struct device_node *np;
36 void *mem_region = NULL;
39 if (!IS_ENABLED(CONFIG_ARCH_QCOM))
42 np = of_get_child_by_name(dev->of_node, "zap-shader");
46 np = of_parse_phandle(np, "memory-region", 0);
50 ret = of_address_to_resource(np, 0, &r);
55 mem_size = resource_size(&r);
57 /* Request the MDT file for the firmware */
58 ret = request_firmware(&fw, fwname, dev);
60 DRM_DEV_ERROR(dev, "Unable to load %s\n", fwname);
64 /* Figure out how much memory we need */
65 mem_size = qcom_mdt_get_size(fw);
71 /* Allocate memory for the firmware image */
72 mem_region = memremap(mem_phys, mem_size, MEMREMAP_WC);
78 /* Load the rest of the MDT */
79 ret = qcom_mdt_load(dev, fw, fwname, GPU_PAS_ID, mem_region, mem_phys,
84 /* Send the image to the secure world */
85 ret = qcom_scm_pas_auth_and_reset(GPU_PAS_ID);
87 DRM_DEV_ERROR(dev, "Unable to authorize the image\n");
98 static void a5xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit,
99 struct msm_file_private *ctx)
101 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
102 struct msm_drm_private *priv = gpu->dev->dev_private;
103 struct msm_ringbuffer *ring = gpu->rb;
104 unsigned int i, ibs = 0;
106 for (i = 0; i < submit->nr_cmds; i++) {
107 switch (submit->cmd[i].type) {
108 case MSM_SUBMIT_CMD_IB_TARGET_BUF:
110 case MSM_SUBMIT_CMD_CTX_RESTORE_BUF:
111 if (priv->lastctx == ctx)
113 case MSM_SUBMIT_CMD_BUF:
114 OUT_PKT7(ring, CP_INDIRECT_BUFFER_PFE, 3);
115 OUT_RING(ring, lower_32_bits(submit->cmd[i].iova));
116 OUT_RING(ring, upper_32_bits(submit->cmd[i].iova));
117 OUT_RING(ring, submit->cmd[i].size);
123 OUT_PKT4(ring, REG_A5XX_CP_SCRATCH_REG(2), 1);
124 OUT_RING(ring, submit->fence->seqno);
126 OUT_PKT7(ring, CP_EVENT_WRITE, 4);
127 OUT_RING(ring, CACHE_FLUSH_TS | (1 << 31));
128 OUT_RING(ring, lower_32_bits(rbmemptr(adreno_gpu, fence)));
129 OUT_RING(ring, upper_32_bits(rbmemptr(adreno_gpu, fence)));
130 OUT_RING(ring, submit->fence->seqno);
132 gpu->funcs->flush(gpu);
135 static const struct {
139 {REG_A5XX_RBBM_CLOCK_CNTL_SP0, 0x02222222},
140 {REG_A5XX_RBBM_CLOCK_CNTL_SP1, 0x02222222},
141 {REG_A5XX_RBBM_CLOCK_CNTL_SP2, 0x02222222},
142 {REG_A5XX_RBBM_CLOCK_CNTL_SP3, 0x02222222},
143 {REG_A5XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220},
144 {REG_A5XX_RBBM_CLOCK_CNTL2_SP1, 0x02222220},
145 {REG_A5XX_RBBM_CLOCK_CNTL2_SP2, 0x02222220},
146 {REG_A5XX_RBBM_CLOCK_CNTL2_SP3, 0x02222220},
147 {REG_A5XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF},
148 {REG_A5XX_RBBM_CLOCK_HYST_SP1, 0x0000F3CF},
149 {REG_A5XX_RBBM_CLOCK_HYST_SP2, 0x0000F3CF},
150 {REG_A5XX_RBBM_CLOCK_HYST_SP3, 0x0000F3CF},
151 {REG_A5XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
152 {REG_A5XX_RBBM_CLOCK_DELAY_SP1, 0x00000080},
153 {REG_A5XX_RBBM_CLOCK_DELAY_SP2, 0x00000080},
154 {REG_A5XX_RBBM_CLOCK_DELAY_SP3, 0x00000080},
155 {REG_A5XX_RBBM_CLOCK_CNTL_TP0, 0x22222222},
156 {REG_A5XX_RBBM_CLOCK_CNTL_TP1, 0x22222222},
157 {REG_A5XX_RBBM_CLOCK_CNTL_TP2, 0x22222222},
158 {REG_A5XX_RBBM_CLOCK_CNTL_TP3, 0x22222222},
159 {REG_A5XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
160 {REG_A5XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222},
161 {REG_A5XX_RBBM_CLOCK_CNTL2_TP2, 0x22222222},
162 {REG_A5XX_RBBM_CLOCK_CNTL2_TP3, 0x22222222},
163 {REG_A5XX_RBBM_CLOCK_CNTL3_TP0, 0x00002222},
164 {REG_A5XX_RBBM_CLOCK_CNTL3_TP1, 0x00002222},
165 {REG_A5XX_RBBM_CLOCK_CNTL3_TP2, 0x00002222},
166 {REG_A5XX_RBBM_CLOCK_CNTL3_TP3, 0x00002222},
167 {REG_A5XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
168 {REG_A5XX_RBBM_CLOCK_HYST_TP1, 0x77777777},
169 {REG_A5XX_RBBM_CLOCK_HYST_TP2, 0x77777777},
170 {REG_A5XX_RBBM_CLOCK_HYST_TP3, 0x77777777},
171 {REG_A5XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
172 {REG_A5XX_RBBM_CLOCK_HYST2_TP1, 0x77777777},
173 {REG_A5XX_RBBM_CLOCK_HYST2_TP2, 0x77777777},
174 {REG_A5XX_RBBM_CLOCK_HYST2_TP3, 0x77777777},
175 {REG_A5XX_RBBM_CLOCK_HYST3_TP0, 0x00007777},
176 {REG_A5XX_RBBM_CLOCK_HYST3_TP1, 0x00007777},
177 {REG_A5XX_RBBM_CLOCK_HYST3_TP2, 0x00007777},
178 {REG_A5XX_RBBM_CLOCK_HYST3_TP3, 0x00007777},
179 {REG_A5XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
180 {REG_A5XX_RBBM_CLOCK_DELAY_TP1, 0x11111111},
181 {REG_A5XX_RBBM_CLOCK_DELAY_TP2, 0x11111111},
182 {REG_A5XX_RBBM_CLOCK_DELAY_TP3, 0x11111111},
183 {REG_A5XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
184 {REG_A5XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111},
185 {REG_A5XX_RBBM_CLOCK_DELAY2_TP2, 0x11111111},
186 {REG_A5XX_RBBM_CLOCK_DELAY2_TP3, 0x11111111},
187 {REG_A5XX_RBBM_CLOCK_DELAY3_TP0, 0x00001111},
188 {REG_A5XX_RBBM_CLOCK_DELAY3_TP1, 0x00001111},
189 {REG_A5XX_RBBM_CLOCK_DELAY3_TP2, 0x00001111},
190 {REG_A5XX_RBBM_CLOCK_DELAY3_TP3, 0x00001111},
191 {REG_A5XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
192 {REG_A5XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222},
193 {REG_A5XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222},
194 {REG_A5XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222},
195 {REG_A5XX_RBBM_CLOCK_HYST_UCHE, 0x00444444},
196 {REG_A5XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
197 {REG_A5XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
198 {REG_A5XX_RBBM_CLOCK_CNTL_RB1, 0x22222222},
199 {REG_A5XX_RBBM_CLOCK_CNTL_RB2, 0x22222222},
200 {REG_A5XX_RBBM_CLOCK_CNTL_RB3, 0x22222222},
201 {REG_A5XX_RBBM_CLOCK_CNTL2_RB0, 0x00222222},
202 {REG_A5XX_RBBM_CLOCK_CNTL2_RB1, 0x00222222},
203 {REG_A5XX_RBBM_CLOCK_CNTL2_RB2, 0x00222222},
204 {REG_A5XX_RBBM_CLOCK_CNTL2_RB3, 0x00222222},
205 {REG_A5XX_RBBM_CLOCK_CNTL_CCU0, 0x00022220},
206 {REG_A5XX_RBBM_CLOCK_CNTL_CCU1, 0x00022220},
207 {REG_A5XX_RBBM_CLOCK_CNTL_CCU2, 0x00022220},
208 {REG_A5XX_RBBM_CLOCK_CNTL_CCU3, 0x00022220},
209 {REG_A5XX_RBBM_CLOCK_CNTL_RAC, 0x05522222},
210 {REG_A5XX_RBBM_CLOCK_CNTL2_RAC, 0x00505555},
211 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU0, 0x04040404},
212 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU1, 0x04040404},
213 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU2, 0x04040404},
214 {REG_A5XX_RBBM_CLOCK_HYST_RB_CCU3, 0x04040404},
215 {REG_A5XX_RBBM_CLOCK_HYST_RAC, 0x07444044},
216 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_0, 0x00000002},
217 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_1, 0x00000002},
218 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_2, 0x00000002},
219 {REG_A5XX_RBBM_CLOCK_DELAY_RB_CCU_L1_3, 0x00000002},
220 {REG_A5XX_RBBM_CLOCK_DELAY_RAC, 0x00010011},
221 {REG_A5XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
222 {REG_A5XX_RBBM_CLOCK_MODE_GPC, 0x02222222},
223 {REG_A5XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
224 {REG_A5XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
225 {REG_A5XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
226 {REG_A5XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
227 {REG_A5XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
228 {REG_A5XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
229 {REG_A5XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
230 {REG_A5XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}
233 void a5xx_set_hwcg(struct msm_gpu *gpu, bool state)
237 for (i = 0; i < ARRAY_SIZE(a5xx_hwcg); i++)
238 gpu_write(gpu, a5xx_hwcg[i].offset,
239 state ? a5xx_hwcg[i].value : 0);
241 gpu_write(gpu, REG_A5XX_RBBM_CLOCK_CNTL, state ? 0xAAA8AA00 : 0);
242 gpu_write(gpu, REG_A5XX_RBBM_ISDB_CNT, state ? 0x182 : 0x180);
245 static int a5xx_me_init(struct msm_gpu *gpu)
247 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
248 struct msm_ringbuffer *ring = gpu->rb;
250 OUT_PKT7(ring, CP_ME_INIT, 8);
252 OUT_RING(ring, 0x0000002F);
254 /* Enable multiple hardware contexts */
255 OUT_RING(ring, 0x00000003);
257 /* Enable error detection */
258 OUT_RING(ring, 0x20000000);
260 /* Don't enable header dump */
261 OUT_RING(ring, 0x00000000);
262 OUT_RING(ring, 0x00000000);
264 /* Specify workarounds for various microcode issues */
265 if (adreno_is_a530(adreno_gpu)) {
266 /* Workaround for token end syncs
267 * Force a WFI after every direct-render 3D mode draw and every
270 OUT_RING(ring, 0x0000000B);
272 /* No workarounds enabled */
273 OUT_RING(ring, 0x00000000);
276 OUT_RING(ring, 0x00000000);
277 OUT_RING(ring, 0x00000000);
279 gpu->funcs->flush(gpu);
281 return a5xx_idle(gpu) ? 0 : -EINVAL;
284 static struct drm_gem_object *a5xx_ucode_load_bo(struct msm_gpu *gpu,
285 const struct firmware *fw, u64 *iova)
287 struct drm_device *drm = gpu->dev;
288 struct drm_gem_object *bo;
291 bo = msm_gem_new_locked(drm, fw->size - 4, MSM_BO_UNCACHED);
295 ptr = msm_gem_get_vaddr(bo);
297 drm_gem_object_unreference(bo);
298 return ERR_PTR(-ENOMEM);
302 int ret = msm_gem_get_iova(bo, gpu->aspace, iova);
305 drm_gem_object_unreference(bo);
310 memcpy(ptr, &fw->data[4], fw->size - 4);
312 msm_gem_put_vaddr(bo);
316 static int a5xx_ucode_init(struct msm_gpu *gpu)
318 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
319 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
322 if (!a5xx_gpu->pm4_bo) {
323 a5xx_gpu->pm4_bo = a5xx_ucode_load_bo(gpu, adreno_gpu->pm4,
324 &a5xx_gpu->pm4_iova);
326 if (IS_ERR(a5xx_gpu->pm4_bo)) {
327 ret = PTR_ERR(a5xx_gpu->pm4_bo);
328 a5xx_gpu->pm4_bo = NULL;
329 dev_err(gpu->dev->dev, "could not allocate PM4: %d\n",
335 if (!a5xx_gpu->pfp_bo) {
336 a5xx_gpu->pfp_bo = a5xx_ucode_load_bo(gpu, adreno_gpu->pfp,
337 &a5xx_gpu->pfp_iova);
339 if (IS_ERR(a5xx_gpu->pfp_bo)) {
340 ret = PTR_ERR(a5xx_gpu->pfp_bo);
341 a5xx_gpu->pfp_bo = NULL;
342 dev_err(gpu->dev->dev, "could not allocate PFP: %d\n",
348 gpu_write64(gpu, REG_A5XX_CP_ME_INSTR_BASE_LO,
349 REG_A5XX_CP_ME_INSTR_BASE_HI, a5xx_gpu->pm4_iova);
351 gpu_write64(gpu, REG_A5XX_CP_PFP_INSTR_BASE_LO,
352 REG_A5XX_CP_PFP_INSTR_BASE_HI, a5xx_gpu->pfp_iova);
357 #define SCM_GPU_ZAP_SHADER_RESUME 0
359 static int a5xx_zap_shader_resume(struct msm_gpu *gpu)
363 ret = qcom_scm_set_remote_state(SCM_GPU_ZAP_SHADER_RESUME, GPU_PAS_ID);
365 DRM_ERROR("%s: zap-shader resume failed: %d\n",
371 static int a5xx_zap_shader_init(struct msm_gpu *gpu)
374 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
375 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
376 struct platform_device *pdev = a5xx_gpu->pdev;
380 * If the zap shader is already loaded into memory we just need to kick
381 * the remote processor to reinitialize it
384 return a5xx_zap_shader_resume(gpu);
386 /* We need SCM to be able to load the firmware */
387 if (!qcom_scm_is_available()) {
388 DRM_DEV_ERROR(&pdev->dev, "SCM is not available\n");
389 return -EPROBE_DEFER;
392 /* Each GPU has a target specific zap shader firmware name to use */
393 if (!adreno_gpu->info->zapfw) {
394 DRM_DEV_ERROR(&pdev->dev,
395 "Zap shader firmware file not specified for this target\n");
399 ret = zap_shader_load_mdt(&pdev->dev, adreno_gpu->info->zapfw);
406 #define A5XX_INT_MASK (A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR | \
407 A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT | \
408 A5XX_RBBM_INT_0_MASK_RBBM_ME_MS_TIMEOUT | \
409 A5XX_RBBM_INT_0_MASK_RBBM_PFP_MS_TIMEOUT | \
410 A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT | \
411 A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW | \
412 A5XX_RBBM_INT_0_MASK_CP_HW_ERROR | \
413 A5XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS | \
414 A5XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS | \
415 A5XX_RBBM_INT_0_MASK_GPMU_VOLTAGE_DROOP)
417 static int a5xx_hw_init(struct msm_gpu *gpu)
419 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
422 gpu_write(gpu, REG_A5XX_VBIF_ROUND_ROBIN_QOS_ARB, 0x00000003);
424 /* Make all blocks contribute to the GPU BUSY perf counter */
425 gpu_write(gpu, REG_A5XX_RBBM_PERFCTR_GPU_BUSY_MASKED, 0xFFFFFFFF);
427 /* Enable RBBM error reporting bits */
428 gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL0, 0x00000001);
430 if (adreno_gpu->info->quirks & ADRENO_QUIRK_FAULT_DETECT_MASK) {
432 * Mask out the activity signals from RB1-3 to avoid false
436 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL11,
438 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL12,
440 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL13,
442 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL14,
444 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL15,
446 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL16,
448 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL17,
450 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_MASK_CNTL18,
454 /* Enable fault detection */
455 gpu_write(gpu, REG_A5XX_RBBM_INTERFACE_HANG_INT_CNTL,
458 /* Turn on performance counters */
459 gpu_write(gpu, REG_A5XX_RBBM_PERFCTR_CNTL, 0x01);
461 /* Increase VFD cache access so LRZ and other data gets evicted less */
462 gpu_write(gpu, REG_A5XX_UCHE_CACHE_WAYS, 0x02);
464 /* Disable L2 bypass in the UCHE */
465 gpu_write(gpu, REG_A5XX_UCHE_TRAP_BASE_LO, 0xFFFF0000);
466 gpu_write(gpu, REG_A5XX_UCHE_TRAP_BASE_HI, 0x0001FFFF);
467 gpu_write(gpu, REG_A5XX_UCHE_WRITE_THRU_BASE_LO, 0xFFFF0000);
468 gpu_write(gpu, REG_A5XX_UCHE_WRITE_THRU_BASE_HI, 0x0001FFFF);
470 /* Set the GMEM VA range (0 to gpu->gmem) */
471 gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MIN_LO, 0x00100000);
472 gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MIN_HI, 0x00000000);
473 gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MAX_LO,
474 0x00100000 + adreno_gpu->gmem - 1);
475 gpu_write(gpu, REG_A5XX_UCHE_GMEM_RANGE_MAX_HI, 0x00000000);
477 gpu_write(gpu, REG_A5XX_CP_MEQ_THRESHOLDS, 0x40);
478 gpu_write(gpu, REG_A5XX_CP_MERCIU_SIZE, 0x40);
479 gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_2, 0x80000060);
480 gpu_write(gpu, REG_A5XX_CP_ROQ_THRESHOLDS_1, 0x40201B16);
482 gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL, (0x400 << 11 | 0x300 << 22));
484 if (adreno_gpu->info->quirks & ADRENO_QUIRK_TWO_PASS_USE_WFI)
485 gpu_rmw(gpu, REG_A5XX_PC_DBG_ECO_CNTL, 0, (1 << 8));
487 gpu_write(gpu, REG_A5XX_PC_DBG_ECO_CNTL, 0xc0200100);
489 /* Enable USE_RETENTION_FLOPS */
490 gpu_write(gpu, REG_A5XX_CP_CHICKEN_DBG, 0x02000000);
492 /* Enable ME/PFP split notification */
493 gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL1, 0xA6FFFFFF);
496 a5xx_set_hwcg(gpu, true);
498 gpu_write(gpu, REG_A5XX_RBBM_AHB_CNTL2, 0x0000003F);
500 /* Set the highest bank bit */
501 gpu_write(gpu, REG_A5XX_TPL1_MODE_CNTL, 2 << 7);
502 gpu_write(gpu, REG_A5XX_RB_MODE_CNTL, 2 << 1);
504 /* Protect registers from the CP */
505 gpu_write(gpu, REG_A5XX_CP_PROTECT_CNTL, 0x00000007);
508 gpu_write(gpu, REG_A5XX_CP_PROTECT(0), ADRENO_PROTECT_RW(0x04, 4));
509 gpu_write(gpu, REG_A5XX_CP_PROTECT(1), ADRENO_PROTECT_RW(0x08, 8));
510 gpu_write(gpu, REG_A5XX_CP_PROTECT(2), ADRENO_PROTECT_RW(0x10, 16));
511 gpu_write(gpu, REG_A5XX_CP_PROTECT(3), ADRENO_PROTECT_RW(0x20, 32));
512 gpu_write(gpu, REG_A5XX_CP_PROTECT(4), ADRENO_PROTECT_RW(0x40, 64));
513 gpu_write(gpu, REG_A5XX_CP_PROTECT(5), ADRENO_PROTECT_RW(0x80, 64));
515 /* Content protect */
516 gpu_write(gpu, REG_A5XX_CP_PROTECT(6),
517 ADRENO_PROTECT_RW(REG_A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO,
519 gpu_write(gpu, REG_A5XX_CP_PROTECT(7),
520 ADRENO_PROTECT_RW(REG_A5XX_RBBM_SECVID_TRUST_CNTL, 2));
523 gpu_write(gpu, REG_A5XX_CP_PROTECT(8), ADRENO_PROTECT_RW(0x800, 64));
524 gpu_write(gpu, REG_A5XX_CP_PROTECT(9), ADRENO_PROTECT_RW(0x840, 8));
525 gpu_write(gpu, REG_A5XX_CP_PROTECT(10), ADRENO_PROTECT_RW(0x880, 32));
526 gpu_write(gpu, REG_A5XX_CP_PROTECT(11), ADRENO_PROTECT_RW(0xAA0, 1));
529 gpu_write(gpu, REG_A5XX_CP_PROTECT(12), ADRENO_PROTECT_RW(0xCC0, 1));
530 gpu_write(gpu, REG_A5XX_CP_PROTECT(13), ADRENO_PROTECT_RW(0xCF0, 2));
533 gpu_write(gpu, REG_A5XX_CP_PROTECT(14), ADRENO_PROTECT_RW(0xE68, 8));
534 gpu_write(gpu, REG_A5XX_CP_PROTECT(15), ADRENO_PROTECT_RW(0xE70, 4));
537 gpu_write(gpu, REG_A5XX_CP_PROTECT(16), ADRENO_PROTECT_RW(0xE80, 16));
539 if (adreno_is_a530(adreno_gpu))
540 gpu_write(gpu, REG_A5XX_CP_PROTECT(17),
541 ADRENO_PROTECT_RW(0x10000, 0x8000));
543 gpu_write(gpu, REG_A5XX_RBBM_SECVID_TSB_CNTL, 0);
545 * Disable the trusted memory range - we don't actually supported secure
546 * memory rendering at this point in time and we don't want to block off
547 * part of the virtual memory space.
549 gpu_write64(gpu, REG_A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO,
550 REG_A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_HI, 0x00000000);
551 gpu_write(gpu, REG_A5XX_RBBM_SECVID_TSB_TRUSTED_SIZE, 0x00000000);
553 /* Load the GPMU firmware before starting the HW init */
554 a5xx_gpmu_ucode_init(gpu);
556 ret = adreno_hw_init(gpu);
560 ret = a5xx_ucode_init(gpu);
564 /* Disable the interrupts through the initial bringup stage */
565 gpu_write(gpu, REG_A5XX_RBBM_INT_0_MASK, A5XX_INT_MASK);
567 /* Clear ME_HALT to start the micro engine */
568 gpu_write(gpu, REG_A5XX_CP_PFP_ME_CNTL, 0);
569 ret = a5xx_me_init(gpu);
573 ret = a5xx_power_init(gpu);
578 * Send a pipeline event stat to get misbehaving counters to start
581 if (adreno_is_a530(adreno_gpu)) {
582 OUT_PKT7(gpu->rb, CP_EVENT_WRITE, 1);
583 OUT_RING(gpu->rb, 0x0F);
585 gpu->funcs->flush(gpu);
591 * Try to load a zap shader into the secure world. If successful
592 * we can use the CP to switch out of secure mode. If not then we
593 * have no resource but to try to switch ourselves out manually. If we
594 * guessed wrong then access to the RBBM_SECVID_TRUST_CNTL register will
595 * be blocked and a permissions violation will soon follow.
597 ret = a5xx_zap_shader_init(gpu);
599 OUT_PKT7(gpu->rb, CP_SET_SECURE_MODE, 1);
600 OUT_RING(gpu->rb, 0x00000000);
602 gpu->funcs->flush(gpu);
606 /* Print a warning so if we die, we know why */
607 dev_warn_once(gpu->dev->dev,
608 "Zap shader not enabled - using SECVID_TRUST_CNTL instead\n");
609 gpu_write(gpu, REG_A5XX_RBBM_SECVID_TRUST_CNTL, 0x0);
615 static void a5xx_recover(struct msm_gpu *gpu)
619 adreno_dump_info(gpu);
621 for (i = 0; i < 8; i++) {
622 printk("CP_SCRATCH_REG%d: %u\n", i,
623 gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(i)));
629 gpu_write(gpu, REG_A5XX_RBBM_SW_RESET_CMD, 1);
630 gpu_read(gpu, REG_A5XX_RBBM_SW_RESET_CMD);
631 gpu_write(gpu, REG_A5XX_RBBM_SW_RESET_CMD, 0);
635 static void a5xx_destroy(struct msm_gpu *gpu)
637 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
638 struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
640 DBG("%s", gpu->name);
642 if (a5xx_gpu->pm4_bo) {
643 if (a5xx_gpu->pm4_iova)
644 msm_gem_put_iova(a5xx_gpu->pm4_bo, gpu->aspace);
645 drm_gem_object_unreference_unlocked(a5xx_gpu->pm4_bo);
648 if (a5xx_gpu->pfp_bo) {
649 if (a5xx_gpu->pfp_iova)
650 msm_gem_put_iova(a5xx_gpu->pfp_bo, gpu->aspace);
651 drm_gem_object_unreference_unlocked(a5xx_gpu->pfp_bo);
654 if (a5xx_gpu->gpmu_bo) {
655 if (a5xx_gpu->gpmu_iova)
656 msm_gem_put_iova(a5xx_gpu->gpmu_bo, gpu->aspace);
657 drm_gem_object_unreference_unlocked(a5xx_gpu->gpmu_bo);
660 adreno_gpu_cleanup(adreno_gpu);
664 static inline bool _a5xx_check_idle(struct msm_gpu *gpu)
666 if (gpu_read(gpu, REG_A5XX_RBBM_STATUS) & ~A5XX_RBBM_STATUS_HI_BUSY)
670 * Nearly every abnormality ends up pausing the GPU and triggering a
671 * fault so we can safely just watch for this one interrupt to fire
673 return !(gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS) &
674 A5XX_RBBM_INT_0_MASK_MISC_HANG_DETECT);
677 bool a5xx_idle(struct msm_gpu *gpu)
679 /* wait for CP to drain ringbuffer: */
680 if (!adreno_idle(gpu))
683 if (spin_until(_a5xx_check_idle(gpu))) {
684 DRM_ERROR("%s: %ps: timeout waiting for GPU to idle: status %8.8X irq %8.8X\n",
685 gpu->name, __builtin_return_address(0),
686 gpu_read(gpu, REG_A5XX_RBBM_STATUS),
687 gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS));
695 static int a5xx_fault_handler(void *arg, unsigned long iova, int flags)
697 struct msm_gpu *gpu = arg;
698 pr_warn_ratelimited("*** gpu fault: iova=%08lx, flags=%d (%u,%u,%u,%u)\n",
700 gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(4)),
701 gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(5)),
702 gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(6)),
703 gpu_read(gpu, REG_A5XX_CP_SCRATCH_REG(7)));
708 static void a5xx_cp_err_irq(struct msm_gpu *gpu)
710 u32 status = gpu_read(gpu, REG_A5XX_CP_INTERRUPT_STATUS);
712 if (status & A5XX_CP_INT_CP_OPCODE_ERROR) {
715 gpu_write(gpu, REG_A5XX_CP_PFP_STAT_ADDR, 0);
718 * REG_A5XX_CP_PFP_STAT_DATA is indexed, and we want index 1 so
722 gpu_read(gpu, REG_A5XX_CP_PFP_STAT_DATA);
723 val = gpu_read(gpu, REG_A5XX_CP_PFP_STAT_DATA);
725 dev_err_ratelimited(gpu->dev->dev, "CP | opcode error | possible opcode=0x%8.8X\n",
729 if (status & A5XX_CP_INT_CP_HW_FAULT_ERROR)
730 dev_err_ratelimited(gpu->dev->dev, "CP | HW fault | status=0x%8.8X\n",
731 gpu_read(gpu, REG_A5XX_CP_HW_FAULT));
733 if (status & A5XX_CP_INT_CP_DMA_ERROR)
734 dev_err_ratelimited(gpu->dev->dev, "CP | DMA error\n");
736 if (status & A5XX_CP_INT_CP_REGISTER_PROTECTION_ERROR) {
737 u32 val = gpu_read(gpu, REG_A5XX_CP_PROTECT_STATUS);
739 dev_err_ratelimited(gpu->dev->dev,
740 "CP | protected mode error | %s | addr=0x%8.8X | status=0x%8.8X\n",
741 val & (1 << 24) ? "WRITE" : "READ",
742 (val & 0xFFFFF) >> 2, val);
745 if (status & A5XX_CP_INT_CP_AHB_ERROR) {
746 u32 status = gpu_read(gpu, REG_A5XX_CP_AHB_FAULT);
747 const char *access[16] = { "reserved", "reserved",
748 "timestamp lo", "timestamp hi", "pfp read", "pfp write",
749 "", "", "me read", "me write", "", "", "crashdump read",
752 dev_err_ratelimited(gpu->dev->dev,
753 "CP | AHB error | addr=%X access=%s error=%d | status=0x%8.8X\n",
754 status & 0xFFFFF, access[(status >> 24) & 0xF],
755 (status & (1 << 31)), status);
759 static void a5xx_rbbm_err_irq(struct msm_gpu *gpu, u32 status)
761 if (status & A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR) {
762 u32 val = gpu_read(gpu, REG_A5XX_RBBM_AHB_ERROR_STATUS);
764 dev_err_ratelimited(gpu->dev->dev,
765 "RBBM | AHB bus error | %s | addr=0x%X | ports=0x%X:0x%X\n",
766 val & (1 << 28) ? "WRITE" : "READ",
767 (val & 0xFFFFF) >> 2, (val >> 20) & 0x3,
770 /* Clear the error */
771 gpu_write(gpu, REG_A5XX_RBBM_AHB_CMD, (1 << 4));
773 /* Clear the interrupt */
774 gpu_write(gpu, REG_A5XX_RBBM_INT_CLEAR_CMD,
775 A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR);
778 if (status & A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT)
779 dev_err_ratelimited(gpu->dev->dev, "RBBM | AHB transfer timeout\n");
781 if (status & A5XX_RBBM_INT_0_MASK_RBBM_ME_MS_TIMEOUT)
782 dev_err_ratelimited(gpu->dev->dev, "RBBM | ME master split | status=0x%X\n",
783 gpu_read(gpu, REG_A5XX_RBBM_AHB_ME_SPLIT_STATUS));
785 if (status & A5XX_RBBM_INT_0_MASK_RBBM_PFP_MS_TIMEOUT)
786 dev_err_ratelimited(gpu->dev->dev, "RBBM | PFP master split | status=0x%X\n",
787 gpu_read(gpu, REG_A5XX_RBBM_AHB_PFP_SPLIT_STATUS));
789 if (status & A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT)
790 dev_err_ratelimited(gpu->dev->dev, "RBBM | ETS master split | status=0x%X\n",
791 gpu_read(gpu, REG_A5XX_RBBM_AHB_ETS_SPLIT_STATUS));
793 if (status & A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW)
794 dev_err_ratelimited(gpu->dev->dev, "RBBM | ATB ASYNC overflow\n");
796 if (status & A5XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW)
797 dev_err_ratelimited(gpu->dev->dev, "RBBM | ATB bus overflow\n");
800 static void a5xx_uche_err_irq(struct msm_gpu *gpu)
802 uint64_t addr = (uint64_t) gpu_read(gpu, REG_A5XX_UCHE_TRAP_LOG_HI);
804 addr |= gpu_read(gpu, REG_A5XX_UCHE_TRAP_LOG_LO);
806 dev_err_ratelimited(gpu->dev->dev, "UCHE | Out of bounds access | addr=0x%llX\n",
810 static void a5xx_gpmu_err_irq(struct msm_gpu *gpu)
812 dev_err_ratelimited(gpu->dev->dev, "GPMU | voltage droop\n");
815 #define RBBM_ERROR_MASK \
816 (A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR | \
817 A5XX_RBBM_INT_0_MASK_RBBM_TRANSFER_TIMEOUT | \
818 A5XX_RBBM_INT_0_MASK_RBBM_ME_MS_TIMEOUT | \
819 A5XX_RBBM_INT_0_MASK_RBBM_PFP_MS_TIMEOUT | \
820 A5XX_RBBM_INT_0_MASK_RBBM_ETS_MS_TIMEOUT | \
821 A5XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNC_OVERFLOW)
823 static irqreturn_t a5xx_irq(struct msm_gpu *gpu)
825 u32 status = gpu_read(gpu, REG_A5XX_RBBM_INT_0_STATUS);
828 * Clear all the interrupts except RBBM_AHB_ERROR - if we clear it
829 * before the source is cleared the interrupt will storm.
831 gpu_write(gpu, REG_A5XX_RBBM_INT_CLEAR_CMD,
832 status & ~A5XX_RBBM_INT_0_MASK_RBBM_AHB_ERROR);
834 /* Pass status to a5xx_rbbm_err_irq because we've already cleared it */
835 if (status & RBBM_ERROR_MASK)
836 a5xx_rbbm_err_irq(gpu, status);
838 if (status & A5XX_RBBM_INT_0_MASK_CP_HW_ERROR)
839 a5xx_cp_err_irq(gpu);
841 if (status & A5XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS)
842 a5xx_uche_err_irq(gpu);
844 if (status & A5XX_RBBM_INT_0_MASK_GPMU_VOLTAGE_DROOP)
845 a5xx_gpmu_err_irq(gpu);
847 if (status & A5XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS)
853 static const u32 a5xx_register_offsets[REG_ADRENO_REGISTER_MAX] = {
854 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_BASE, REG_A5XX_CP_RB_BASE),
855 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_BASE_HI, REG_A5XX_CP_RB_BASE_HI),
856 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_RPTR_ADDR, REG_A5XX_CP_RB_RPTR_ADDR),
857 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_RPTR_ADDR_HI,
858 REG_A5XX_CP_RB_RPTR_ADDR_HI),
859 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_RPTR, REG_A5XX_CP_RB_RPTR),
860 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_WPTR, REG_A5XX_CP_RB_WPTR),
861 REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_CNTL, REG_A5XX_CP_RB_CNTL),
864 static const u32 a5xx_registers[] = {
865 0x0000, 0x0002, 0x0004, 0x0020, 0x0022, 0x0026, 0x0029, 0x002B,
866 0x002E, 0x0035, 0x0038, 0x0042, 0x0044, 0x0044, 0x0047, 0x0095,
867 0x0097, 0x00BB, 0x03A0, 0x0464, 0x0469, 0x046F, 0x04D2, 0x04D3,
868 0x04E0, 0x0533, 0x0540, 0x0555, 0x0800, 0x081A, 0x081F, 0x0841,
869 0x0860, 0x0860, 0x0880, 0x08A0, 0x0B00, 0x0B12, 0x0B15, 0x0B28,
870 0x0B78, 0x0B7F, 0x0BB0, 0x0BBD, 0x0BC0, 0x0BC6, 0x0BD0, 0x0C53,
871 0x0C60, 0x0C61, 0x0C80, 0x0C82, 0x0C84, 0x0C85, 0x0C90, 0x0C98,
872 0x0CA0, 0x0CA0, 0x0CB0, 0x0CB2, 0x2180, 0x2185, 0x2580, 0x2585,
873 0x0CC1, 0x0CC1, 0x0CC4, 0x0CC7, 0x0CCC, 0x0CCC, 0x0CD0, 0x0CD8,
874 0x0CE0, 0x0CE5, 0x0CE8, 0x0CE8, 0x0CEC, 0x0CF1, 0x0CFB, 0x0D0E,
875 0x2100, 0x211E, 0x2140, 0x2145, 0x2500, 0x251E, 0x2540, 0x2545,
876 0x0D10, 0x0D17, 0x0D20, 0x0D23, 0x0D30, 0x0D30, 0x20C0, 0x20C0,
877 0x24C0, 0x24C0, 0x0E40, 0x0E43, 0x0E4A, 0x0E4A, 0x0E50, 0x0E57,
878 0x0E60, 0x0E7C, 0x0E80, 0x0E8E, 0x0E90, 0x0E96, 0x0EA0, 0x0EA8,
879 0x0EB0, 0x0EB2, 0xE140, 0xE147, 0xE150, 0xE187, 0xE1A0, 0xE1A9,
880 0xE1B0, 0xE1B6, 0xE1C0, 0xE1C7, 0xE1D0, 0xE1D1, 0xE200, 0xE201,
881 0xE210, 0xE21C, 0xE240, 0xE268, 0xE000, 0xE006, 0xE010, 0xE09A,
882 0xE0A0, 0xE0A4, 0xE0AA, 0xE0EB, 0xE100, 0xE105, 0xE380, 0xE38F,
883 0xE3B0, 0xE3B0, 0xE400, 0xE405, 0xE408, 0xE4E9, 0xE4F0, 0xE4F0,
884 0xE280, 0xE280, 0xE282, 0xE2A3, 0xE2A5, 0xE2C2, 0xE940, 0xE947,
885 0xE950, 0xE987, 0xE9A0, 0xE9A9, 0xE9B0, 0xE9B6, 0xE9C0, 0xE9C7,
886 0xE9D0, 0xE9D1, 0xEA00, 0xEA01, 0xEA10, 0xEA1C, 0xEA40, 0xEA68,
887 0xE800, 0xE806, 0xE810, 0xE89A, 0xE8A0, 0xE8A4, 0xE8AA, 0xE8EB,
888 0xE900, 0xE905, 0xEB80, 0xEB8F, 0xEBB0, 0xEBB0, 0xEC00, 0xEC05,
889 0xEC08, 0xECE9, 0xECF0, 0xECF0, 0xEA80, 0xEA80, 0xEA82, 0xEAA3,
890 0xEAA5, 0xEAC2, 0xA800, 0xA8FF, 0xAC60, 0xAC60, 0xB000, 0xB97F,
894 static void a5xx_dump(struct msm_gpu *gpu)
896 dev_info(gpu->dev->dev, "status: %08x\n",
897 gpu_read(gpu, REG_A5XX_RBBM_STATUS));
901 static int a5xx_pm_resume(struct msm_gpu *gpu)
905 /* Turn on the core power */
906 ret = msm_gpu_pm_resume(gpu);
910 /* Turn the RBCCU domain first to limit the chances of voltage droop */
911 gpu_write(gpu, REG_A5XX_GPMU_RBCCU_POWER_CNTL, 0x778000);
913 /* Wait 3 usecs before polling */
916 ret = spin_usecs(gpu, 20, REG_A5XX_GPMU_RBCCU_PWR_CLK_STATUS,
917 (1 << 20), (1 << 20));
919 DRM_ERROR("%s: timeout waiting for RBCCU GDSC enable: %X\n",
921 gpu_read(gpu, REG_A5XX_GPMU_RBCCU_PWR_CLK_STATUS));
925 /* Turn on the SP domain */
926 gpu_write(gpu, REG_A5XX_GPMU_SP_POWER_CNTL, 0x778000);
927 ret = spin_usecs(gpu, 20, REG_A5XX_GPMU_SP_PWR_CLK_STATUS,
928 (1 << 20), (1 << 20));
930 DRM_ERROR("%s: timeout waiting for SP GDSC enable\n",
936 static int a5xx_pm_suspend(struct msm_gpu *gpu)
938 /* Clear the VBIF pipe before shutting down */
939 gpu_write(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL0, 0xF);
940 spin_until((gpu_read(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL1) & 0xF) == 0xF);
942 gpu_write(gpu, REG_A5XX_VBIF_XIN_HALT_CTRL0, 0);
945 * Reset the VBIF before power collapse to avoid issue with FIFO
948 gpu_write(gpu, REG_A5XX_RBBM_BLOCK_SW_RESET_CMD, 0x003C0000);
949 gpu_write(gpu, REG_A5XX_RBBM_BLOCK_SW_RESET_CMD, 0x00000000);
951 return msm_gpu_pm_suspend(gpu);
954 static int a5xx_get_timestamp(struct msm_gpu *gpu, uint64_t *value)
956 *value = gpu_read64(gpu, REG_A5XX_RBBM_PERFCTR_CP_0_LO,
957 REG_A5XX_RBBM_PERFCTR_CP_0_HI);
962 #ifdef CONFIG_DEBUG_FS
963 static void a5xx_show(struct msm_gpu *gpu, struct seq_file *m)
965 seq_printf(m, "status: %08x\n",
966 gpu_read(gpu, REG_A5XX_RBBM_STATUS));
969 * Temporarily disable hardware clock gating before going into
970 * adreno_show to avoid issues while reading the registers
972 a5xx_set_hwcg(gpu, false);
974 a5xx_set_hwcg(gpu, true);
978 static const struct adreno_gpu_funcs funcs = {
980 .get_param = adreno_get_param,
981 .hw_init = a5xx_hw_init,
982 .pm_suspend = a5xx_pm_suspend,
983 .pm_resume = a5xx_pm_resume,
984 .recover = a5xx_recover,
985 .last_fence = adreno_last_fence,
986 .submit = a5xx_submit,
987 .flush = adreno_flush,
989 .destroy = a5xx_destroy,
990 #ifdef CONFIG_DEBUG_FS
994 .get_timestamp = a5xx_get_timestamp,
997 struct msm_gpu *a5xx_gpu_init(struct drm_device *dev)
999 struct msm_drm_private *priv = dev->dev_private;
1000 struct platform_device *pdev = priv->gpu_pdev;
1001 struct a5xx_gpu *a5xx_gpu = NULL;
1002 struct adreno_gpu *adreno_gpu;
1003 struct msm_gpu *gpu;
1007 dev_err(dev->dev, "No A5XX device is defined\n");
1008 return ERR_PTR(-ENXIO);
1011 a5xx_gpu = kzalloc(sizeof(*a5xx_gpu), GFP_KERNEL);
1013 return ERR_PTR(-ENOMEM);
1015 adreno_gpu = &a5xx_gpu->base;
1016 gpu = &adreno_gpu->base;
1018 a5xx_gpu->pdev = pdev;
1019 adreno_gpu->registers = a5xx_registers;
1020 adreno_gpu->reg_offsets = a5xx_register_offsets;
1022 a5xx_gpu->lm_leakage = 0x4E001A;
1024 ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs);
1026 a5xx_destroy(&(a5xx_gpu->base.base));
1027 return ERR_PTR(ret);
1031 msm_mmu_set_fault_handler(gpu->aspace->mmu, gpu, a5xx_fault_handler);