2 * Copyright © 2010 Daniel Vetter
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
26 #include <drm/i915_drm.h>
28 #include "i915_trace.h"
29 #include "intel_drv.h"
31 /* PPGTT support for Sandybdrige/Gen6 and later */
32 static void i915_ppgtt_clear_range(struct i915_hw_ppgtt *ppgtt,
38 unsigned act_pd = first_entry / I915_PPGTT_PT_ENTRIES;
39 unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
42 scratch_pte = GEN6_PTE_ADDR_ENCODE(ppgtt->scratch_page_dma_addr);
43 scratch_pte |= GEN6_PTE_VALID | GEN6_PTE_CACHE_LLC;
46 last_pte = first_pte + num_entries;
47 if (last_pte > I915_PPGTT_PT_ENTRIES)
48 last_pte = I915_PPGTT_PT_ENTRIES;
50 pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pd]);
52 for (i = first_pte; i < last_pte; i++)
53 pt_vaddr[i] = scratch_pte;
55 kunmap_atomic(pt_vaddr);
57 num_entries -= last_pte - first_pte;
63 int i915_gem_init_aliasing_ppgtt(struct drm_device *dev)
65 struct drm_i915_private *dev_priv = dev->dev_private;
66 struct i915_hw_ppgtt *ppgtt;
67 unsigned first_pd_entry_in_global_pt;
71 /* ppgtt PDEs reside in the global gtt pagetable, which has 512*1024
72 * entries. For aliasing ppgtt support we just steal them at the end for
74 first_pd_entry_in_global_pt = dev_priv->mm.gtt->gtt_total_entries - I915_PPGTT_PD_ENTRIES;
76 ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
80 ppgtt->num_pd_entries = I915_PPGTT_PD_ENTRIES;
81 ppgtt->pt_pages = kzalloc(sizeof(struct page *)*ppgtt->num_pd_entries,
86 for (i = 0; i < ppgtt->num_pd_entries; i++) {
87 ppgtt->pt_pages[i] = alloc_page(GFP_KERNEL);
88 if (!ppgtt->pt_pages[i])
92 if (dev_priv->mm.gtt->needs_dmar) {
93 ppgtt->pt_dma_addr = kzalloc(sizeof(dma_addr_t)
94 *ppgtt->num_pd_entries,
96 if (!ppgtt->pt_dma_addr)
99 for (i = 0; i < ppgtt->num_pd_entries; i++) {
102 pt_addr = pci_map_page(dev->pdev, ppgtt->pt_pages[i],
104 PCI_DMA_BIDIRECTIONAL);
106 if (pci_dma_mapping_error(dev->pdev,
112 ppgtt->pt_dma_addr[i] = pt_addr;
116 ppgtt->scratch_page_dma_addr = dev_priv->mm.gtt->scratch_page_dma;
118 i915_ppgtt_clear_range(ppgtt, 0,
119 ppgtt->num_pd_entries*I915_PPGTT_PT_ENTRIES);
121 ppgtt->pd_offset = (first_pd_entry_in_global_pt)*sizeof(uint32_t);
123 dev_priv->mm.aliasing_ppgtt = ppgtt;
128 if (ppgtt->pt_dma_addr) {
129 for (i--; i >= 0; i--)
130 pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i],
131 4096, PCI_DMA_BIDIRECTIONAL);
134 kfree(ppgtt->pt_dma_addr);
135 for (i = 0; i < ppgtt->num_pd_entries; i++) {
136 if (ppgtt->pt_pages[i])
137 __free_page(ppgtt->pt_pages[i]);
139 kfree(ppgtt->pt_pages);
146 void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev)
148 struct drm_i915_private *dev_priv = dev->dev_private;
149 struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
155 if (ppgtt->pt_dma_addr) {
156 for (i = 0; i < ppgtt->num_pd_entries; i++)
157 pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i],
158 4096, PCI_DMA_BIDIRECTIONAL);
161 kfree(ppgtt->pt_dma_addr);
162 for (i = 0; i < ppgtt->num_pd_entries; i++)
163 __free_page(ppgtt->pt_pages[i]);
164 kfree(ppgtt->pt_pages);
168 static void i915_ppgtt_insert_sg_entries(struct i915_hw_ppgtt *ppgtt,
169 struct scatterlist *sg_list,
171 unsigned first_entry,
174 uint32_t *pt_vaddr, pte;
175 unsigned act_pd = first_entry / I915_PPGTT_PT_ENTRIES;
176 unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
177 unsigned i, j, m, segment_len;
178 dma_addr_t page_addr;
179 struct scatterlist *sg;
181 /* init sg walking */
184 segment_len = sg_dma_len(sg) >> PAGE_SHIFT;
188 pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pd]);
190 for (j = first_pte; j < I915_PPGTT_PT_ENTRIES; j++) {
191 page_addr = sg_dma_address(sg) + (m << PAGE_SHIFT);
192 pte = GEN6_PTE_ADDR_ENCODE(page_addr);
193 pt_vaddr[j] = pte | pte_flags;
195 /* grab the next page */
197 if (m == segment_len) {
203 segment_len = sg_dma_len(sg) >> PAGE_SHIFT;
208 kunmap_atomic(pt_vaddr);
215 static void i915_ppgtt_insert_pages(struct i915_hw_ppgtt *ppgtt,
216 unsigned first_entry, unsigned num_entries,
217 struct page **pages, uint32_t pte_flags)
219 uint32_t *pt_vaddr, pte;
220 unsigned act_pd = first_entry / I915_PPGTT_PT_ENTRIES;
221 unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
222 unsigned last_pte, i;
223 dma_addr_t page_addr;
225 while (num_entries) {
226 last_pte = first_pte + num_entries;
227 last_pte = min_t(unsigned, last_pte, I915_PPGTT_PT_ENTRIES);
229 pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pd]);
231 for (i = first_pte; i < last_pte; i++) {
232 page_addr = page_to_phys(*pages);
233 pte = GEN6_PTE_ADDR_ENCODE(page_addr);
234 pt_vaddr[i] = pte | pte_flags;
239 kunmap_atomic(pt_vaddr);
241 num_entries -= last_pte - first_pte;
247 void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt,
248 struct drm_i915_gem_object *obj,
249 enum i915_cache_level cache_level)
251 struct drm_device *dev = obj->base.dev;
252 struct drm_i915_private *dev_priv = dev->dev_private;
253 uint32_t pte_flags = GEN6_PTE_VALID;
255 switch (cache_level) {
256 case I915_CACHE_LLC_MLC:
257 pte_flags |= GEN6_PTE_CACHE_LLC_MLC;
260 pte_flags |= GEN6_PTE_CACHE_LLC;
262 case I915_CACHE_NONE:
264 pte_flags |= HSW_PTE_UNCACHED;
266 pte_flags |= GEN6_PTE_UNCACHED;
273 i915_ppgtt_insert_sg_entries(ppgtt,
275 obj->sg_table->nents,
276 obj->gtt_space->start >> PAGE_SHIFT,
278 } else if (dev_priv->mm.gtt->needs_dmar) {
279 BUG_ON(!obj->sg_list);
281 i915_ppgtt_insert_sg_entries(ppgtt,
284 obj->gtt_space->start >> PAGE_SHIFT,
287 i915_ppgtt_insert_pages(ppgtt,
288 obj->gtt_space->start >> PAGE_SHIFT,
289 obj->base.size >> PAGE_SHIFT,
294 void i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt,
295 struct drm_i915_gem_object *obj)
297 i915_ppgtt_clear_range(ppgtt,
298 obj->gtt_space->start >> PAGE_SHIFT,
299 obj->base.size >> PAGE_SHIFT);
302 /* XXX kill agp_type! */
303 static unsigned int cache_level_to_agp_type(struct drm_device *dev,
304 enum i915_cache_level cache_level)
306 switch (cache_level) {
307 case I915_CACHE_LLC_MLC:
308 if (INTEL_INFO(dev)->gen >= 6)
309 return AGP_USER_CACHED_MEMORY_LLC_MLC;
310 /* Older chipsets do not have this extra level of CPU
311 * cacheing, so fallthrough and request the PTE simply
315 return AGP_USER_CACHED_MEMORY;
317 case I915_CACHE_NONE:
318 return AGP_USER_MEMORY;
322 static bool do_idling(struct drm_i915_private *dev_priv)
324 bool ret = dev_priv->mm.interruptible;
326 if (unlikely(dev_priv->mm.gtt->do_idle_maps)) {
327 dev_priv->mm.interruptible = false;
328 if (i915_gpu_idle(dev_priv->dev)) {
329 DRM_ERROR("Couldn't idle GPU\n");
330 /* Wait a bit, in hopes it avoids the hang */
338 static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible)
340 if (unlikely(dev_priv->mm.gtt->do_idle_maps))
341 dev_priv->mm.interruptible = interruptible;
344 void i915_gem_restore_gtt_mappings(struct drm_device *dev)
346 struct drm_i915_private *dev_priv = dev->dev_private;
347 struct drm_i915_gem_object *obj;
349 /* First fill our portion of the GTT with scratch pages */
350 intel_gtt_clear_range(dev_priv->mm.gtt_start / PAGE_SIZE,
351 (dev_priv->mm.gtt_end - dev_priv->mm.gtt_start) / PAGE_SIZE);
353 list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
354 i915_gem_clflush_object(obj);
355 i915_gem_gtt_bind_object(obj, obj->cache_level);
358 intel_gtt_chipset_flush();
361 int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj)
363 struct drm_device *dev = obj->base.dev;
364 struct drm_i915_private *dev_priv = dev->dev_private;
366 /* don't map imported dma buf objects */
367 if (dev_priv->mm.gtt->needs_dmar && !obj->sg_table)
368 return intel_gtt_map_memory(obj->pages,
369 obj->base.size >> PAGE_SHIFT,
376 void i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj,
377 enum i915_cache_level cache_level)
379 struct drm_device *dev = obj->base.dev;
380 struct drm_i915_private *dev_priv = dev->dev_private;
381 unsigned int agp_type = cache_level_to_agp_type(dev, cache_level);
384 intel_gtt_insert_sg_entries(obj->sg_table->sgl,
385 obj->sg_table->nents,
386 obj->gtt_space->start >> PAGE_SHIFT,
388 } else if (dev_priv->mm.gtt->needs_dmar) {
389 BUG_ON(!obj->sg_list);
391 intel_gtt_insert_sg_entries(obj->sg_list,
393 obj->gtt_space->start >> PAGE_SHIFT,
396 intel_gtt_insert_pages(obj->gtt_space->start >> PAGE_SHIFT,
397 obj->base.size >> PAGE_SHIFT,
401 obj->has_global_gtt_mapping = 1;
404 void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj)
406 intel_gtt_clear_range(obj->gtt_space->start >> PAGE_SHIFT,
407 obj->base.size >> PAGE_SHIFT);
409 obj->has_global_gtt_mapping = 0;
412 void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj)
414 struct drm_device *dev = obj->base.dev;
415 struct drm_i915_private *dev_priv = dev->dev_private;
418 interruptible = do_idling(dev_priv);
421 intel_gtt_unmap_memory(obj->sg_list, obj->num_sg);
425 undo_idling(dev_priv, interruptible);
428 void i915_gem_init_global_gtt(struct drm_device *dev,
430 unsigned long mappable_end,
433 drm_i915_private_t *dev_priv = dev->dev_private;
435 /* Substract the guard page ... */
436 drm_mm_init(&dev_priv->mm.gtt_space, start, end - start - PAGE_SIZE);
438 dev_priv->mm.gtt_start = start;
439 dev_priv->mm.gtt_mappable_end = mappable_end;
440 dev_priv->mm.gtt_end = end;
441 dev_priv->mm.gtt_total = end - start;
442 dev_priv->mm.mappable_gtt_total = min(end, mappable_end) - start;
444 /* ... but ensure that we clear the entire range. */
445 intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE);