1 /* 2 * Copyright © 2010 Daniel Vetter 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 */ 24 25 #include "drmP.h" 26 #include "drm.h" 27 #include "i915_drm.h" 28 #include "i915_drv.h" 29 #include "i915_trace.h" 30 #include "intel_drv.h" 31 32 /* PPGTT support for Sandybdrige/Gen6 and later */ 33 static void i915_ppgtt_clear_range(struct i915_hw_ppgtt *ppgtt, 34 unsigned first_entry, 35 unsigned num_entries) 36 { 37 uint32_t *pt_vaddr; 38 uint32_t scratch_pte; 39 unsigned act_pd = first_entry / I915_PPGTT_PT_ENTRIES; 40 unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES; 41 unsigned last_pte, i; 42 43 scratch_pte = GEN6_PTE_ADDR_ENCODE(ppgtt->scratch_page_dma_addr); 44 scratch_pte |= GEN6_PTE_VALID | GEN6_PTE_CACHE_LLC; 45 46 while (num_entries) { 47 last_pte = first_pte + num_entries; 48 if (last_pte > I915_PPGTT_PT_ENTRIES) 49 last_pte = I915_PPGTT_PT_ENTRIES; 50 51 pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pd]); 52 53 for (i = first_pte; i < last_pte; i++) 54 pt_vaddr[i] = scratch_pte; 55 56 kunmap_atomic(pt_vaddr); 57 58 num_entries -= last_pte - first_pte; 59 first_pte = 0; 60 act_pd++; 61 } 62 } 63 64 int i915_gem_init_aliasing_ppgtt(struct drm_device *dev) 65 { 66 struct drm_i915_private *dev_priv = dev->dev_private; 67 struct i915_hw_ppgtt *ppgtt; 68 unsigned first_pd_entry_in_global_pt; 69 int i; 70 int ret = -ENOMEM; 71 72 /* ppgtt PDEs reside in the global gtt pagetable, which has 512*1024 73 * entries. For aliasing ppgtt support we just steal them at the end for 74 * now. */ 75 first_pd_entry_in_global_pt = 512*1024 - I915_PPGTT_PD_ENTRIES; 76 77 ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL); 78 if (!ppgtt) 79 return ret; 80 81 ppgtt->num_pd_entries = I915_PPGTT_PD_ENTRIES; 82 ppgtt->pt_pages = kzalloc(sizeof(struct page *)*ppgtt->num_pd_entries, 83 GFP_KERNEL); 84 if (!ppgtt->pt_pages) 85 goto err_ppgtt; 86 87 for (i = 0; i < ppgtt->num_pd_entries; i++) { 88 ppgtt->pt_pages[i] = alloc_page(GFP_KERNEL); 89 if (!ppgtt->pt_pages[i]) 90 goto err_pt_alloc; 91 } 92 93 if (dev_priv->mm.gtt->needs_dmar) { 94 ppgtt->pt_dma_addr = kzalloc(sizeof(dma_addr_t) 95 *ppgtt->num_pd_entries, 96 GFP_KERNEL); 97 if (!ppgtt->pt_dma_addr) 98 goto err_pt_alloc; 99 } 100 101 for (i = 0; i < ppgtt->num_pd_entries; i++) { 102 dma_addr_t pt_addr; 103 if (dev_priv->mm.gtt->needs_dmar) { 104 pt_addr = pci_map_page(dev->pdev, ppgtt->pt_pages[i], 105 0, 4096, 106 PCI_DMA_BIDIRECTIONAL); 107 108 if (pci_dma_mapping_error(dev->pdev, 109 pt_addr)) { 110 ret = -EIO; 111 goto err_pd_pin; 112 113 } 114 ppgtt->pt_dma_addr[i] = pt_addr; 115 } else 116 pt_addr = page_to_phys(ppgtt->pt_pages[i]); 117 } 118 119 ppgtt->scratch_page_dma_addr = dev_priv->mm.gtt->scratch_page_dma; 120 121 i915_ppgtt_clear_range(ppgtt, 0, 122 ppgtt->num_pd_entries*I915_PPGTT_PT_ENTRIES); 123 124 ppgtt->pd_offset = (first_pd_entry_in_global_pt)*sizeof(uint32_t); 125 126 dev_priv->mm.aliasing_ppgtt = ppgtt; 127 128 return 0; 129 130 err_pd_pin: 131 if (ppgtt->pt_dma_addr) { 132 for (i--; i >= 0; i--) 133 pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i], 134 4096, PCI_DMA_BIDIRECTIONAL); 135 } 136 err_pt_alloc: 137 kfree(ppgtt->pt_dma_addr); 138 for (i = 0; i < ppgtt->num_pd_entries; i++) { 139 if (ppgtt->pt_pages[i]) 140 __free_page(ppgtt->pt_pages[i]); 141 } 142 kfree(ppgtt->pt_pages); 143 err_ppgtt: 144 kfree(ppgtt); 145 146 return ret; 147 } 148 149 void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev) 150 { 151 struct drm_i915_private *dev_priv = dev->dev_private; 152 struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; 153 int i; 154 155 if (!ppgtt) 156 return; 157 158 if (ppgtt->pt_dma_addr) { 159 for (i = 0; i < ppgtt->num_pd_entries; i++) 160 pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i], 161 4096, PCI_DMA_BIDIRECTIONAL); 162 } 163 164 kfree(ppgtt->pt_dma_addr); 165 for (i = 0; i < ppgtt->num_pd_entries; i++) 166 __free_page(ppgtt->pt_pages[i]); 167 kfree(ppgtt->pt_pages); 168 kfree(ppgtt); 169 } 170 171 static void i915_ppgtt_insert_sg_entries(struct i915_hw_ppgtt *ppgtt, 172 struct scatterlist *sg_list, 173 unsigned sg_len, 174 unsigned first_entry, 175 uint32_t pte_flags) 176 { 177 uint32_t *pt_vaddr, pte; 178 unsigned act_pd = first_entry / I915_PPGTT_PT_ENTRIES; 179 unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES; 180 unsigned i, j, m, segment_len; 181 dma_addr_t page_addr; 182 struct scatterlist *sg; 183 184 /* init sg walking */ 185 sg = sg_list; 186 i = 0; 187 segment_len = sg_dma_len(sg) >> PAGE_SHIFT; 188 m = 0; 189 190 while (i < sg_len) { 191 pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pd]); 192 193 for (j = first_pte; j < I915_PPGTT_PT_ENTRIES; j++) { 194 page_addr = sg_dma_address(sg) + (m << PAGE_SHIFT); 195 pte = GEN6_PTE_ADDR_ENCODE(page_addr); 196 pt_vaddr[j] = pte | pte_flags; 197 198 /* grab the next page */ 199 m++; 200 if (m == segment_len) { 201 sg = sg_next(sg); 202 i++; 203 if (i == sg_len) 204 break; 205 206 segment_len = sg_dma_len(sg) >> PAGE_SHIFT; 207 m = 0; 208 } 209 } 210 211 kunmap_atomic(pt_vaddr); 212 213 first_pte = 0; 214 act_pd++; 215 } 216 } 217 218 static void i915_ppgtt_insert_pages(struct i915_hw_ppgtt *ppgtt, 219 unsigned first_entry, unsigned num_entries, 220 struct page **pages, uint32_t pte_flags) 221 { 222 uint32_t *pt_vaddr, pte; 223 unsigned act_pd = first_entry / I915_PPGTT_PT_ENTRIES; 224 unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES; 225 unsigned last_pte, i; 226 dma_addr_t page_addr; 227 228 while (num_entries) { 229 last_pte = first_pte + num_entries; 230 last_pte = min_t(unsigned, last_pte, I915_PPGTT_PT_ENTRIES); 231 232 pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pd]); 233 234 for (i = first_pte; i < last_pte; i++) { 235 page_addr = page_to_phys(*pages); 236 pte = GEN6_PTE_ADDR_ENCODE(page_addr); 237 pt_vaddr[i] = pte | pte_flags; 238 239 pages++; 240 } 241 242 kunmap_atomic(pt_vaddr); 243 244 num_entries -= last_pte - first_pte; 245 first_pte = 0; 246 act_pd++; 247 } 248 } 249 250 void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt, 251 struct drm_i915_gem_object *obj, 252 enum i915_cache_level cache_level) 253 { 254 struct drm_device *dev = obj->base.dev; 255 struct drm_i915_private *dev_priv = dev->dev_private; 256 uint32_t pte_flags = GEN6_PTE_VALID; 257 258 switch (cache_level) { 259 case I915_CACHE_LLC_MLC: 260 pte_flags |= GEN6_PTE_CACHE_LLC_MLC; 261 break; 262 case I915_CACHE_LLC: 263 pte_flags |= GEN6_PTE_CACHE_LLC; 264 break; 265 case I915_CACHE_NONE: 266 pte_flags |= GEN6_PTE_UNCACHED; 267 break; 268 default: 269 BUG(); 270 } 271 272 if (dev_priv->mm.gtt->needs_dmar) { 273 BUG_ON(!obj->sg_list); 274 275 i915_ppgtt_insert_sg_entries(ppgtt, 276 obj->sg_list, 277 obj->num_sg, 278 obj->gtt_space->start >> PAGE_SHIFT, 279 pte_flags); 280 } else 281 i915_ppgtt_insert_pages(ppgtt, 282 obj->gtt_space->start >> PAGE_SHIFT, 283 obj->base.size >> PAGE_SHIFT, 284 obj->pages, 285 pte_flags); 286 } 287 288 void i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt, 289 struct drm_i915_gem_object *obj) 290 { 291 i915_ppgtt_clear_range(ppgtt, 292 obj->gtt_space->start >> PAGE_SHIFT, 293 obj->base.size >> PAGE_SHIFT); 294 } 295 296 /* XXX kill agp_type! */ 297 static unsigned int cache_level_to_agp_type(struct drm_device *dev, 298 enum i915_cache_level cache_level) 299 { 300 switch (cache_level) { 301 case I915_CACHE_LLC_MLC: 302 if (INTEL_INFO(dev)->gen >= 6) 303 return AGP_USER_CACHED_MEMORY_LLC_MLC; 304 /* Older chipsets do not have this extra level of CPU 305 * cacheing, so fallthrough and request the PTE simply 306 * as cached. 307 */ 308 case I915_CACHE_LLC: 309 return AGP_USER_CACHED_MEMORY; 310 default: 311 case I915_CACHE_NONE: 312 return AGP_USER_MEMORY; 313 } 314 } 315 316 static bool do_idling(struct drm_i915_private *dev_priv) 317 { 318 bool ret = dev_priv->mm.interruptible; 319 320 if (unlikely(dev_priv->mm.gtt->do_idle_maps)) { 321 dev_priv->mm.interruptible = false; 322 if (i915_gpu_idle(dev_priv->dev, false)) { 323 DRM_ERROR("Couldn't idle GPU\n"); 324 /* Wait a bit, in hopes it avoids the hang */ 325 udelay(10); 326 } 327 } 328 329 return ret; 330 } 331 332 static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible) 333 { 334 if (unlikely(dev_priv->mm.gtt->do_idle_maps)) 335 dev_priv->mm.interruptible = interruptible; 336 } 337 338 void i915_gem_restore_gtt_mappings(struct drm_device *dev) 339 { 340 struct drm_i915_private *dev_priv = dev->dev_private; 341 struct drm_i915_gem_object *obj; 342 343 /* First fill our portion of the GTT with scratch pages */ 344 intel_gtt_clear_range(dev_priv->mm.gtt_start / PAGE_SIZE, 345 (dev_priv->mm.gtt_end - dev_priv->mm.gtt_start) / PAGE_SIZE); 346 347 list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) { 348 i915_gem_clflush_object(obj); 349 i915_gem_gtt_rebind_object(obj, obj->cache_level); 350 } 351 352 intel_gtt_chipset_flush(); 353 } 354 355 int i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj) 356 { 357 struct drm_device *dev = obj->base.dev; 358 struct drm_i915_private *dev_priv = dev->dev_private; 359 unsigned int agp_type = cache_level_to_agp_type(dev, obj->cache_level); 360 int ret; 361 362 if (dev_priv->mm.gtt->needs_dmar) { 363 ret = intel_gtt_map_memory(obj->pages, 364 obj->base.size >> PAGE_SHIFT, 365 &obj->sg_list, 366 &obj->num_sg); 367 if (ret != 0) 368 return ret; 369 370 intel_gtt_insert_sg_entries(obj->sg_list, 371 obj->num_sg, 372 obj->gtt_space->start >> PAGE_SHIFT, 373 agp_type); 374 } else 375 intel_gtt_insert_pages(obj->gtt_space->start >> PAGE_SHIFT, 376 obj->base.size >> PAGE_SHIFT, 377 obj->pages, 378 agp_type); 379 380 return 0; 381 } 382 383 void i915_gem_gtt_rebind_object(struct drm_i915_gem_object *obj, 384 enum i915_cache_level cache_level) 385 { 386 struct drm_device *dev = obj->base.dev; 387 struct drm_i915_private *dev_priv = dev->dev_private; 388 unsigned int agp_type = cache_level_to_agp_type(dev, cache_level); 389 390 if (dev_priv->mm.gtt->needs_dmar) { 391 BUG_ON(!obj->sg_list); 392 393 intel_gtt_insert_sg_entries(obj->sg_list, 394 obj->num_sg, 395 obj->gtt_space->start >> PAGE_SHIFT, 396 agp_type); 397 } else 398 intel_gtt_insert_pages(obj->gtt_space->start >> PAGE_SHIFT, 399 obj->base.size >> PAGE_SHIFT, 400 obj->pages, 401 agp_type); 402 } 403 404 void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj) 405 { 406 struct drm_device *dev = obj->base.dev; 407 struct drm_i915_private *dev_priv = dev->dev_private; 408 bool interruptible; 409 410 interruptible = do_idling(dev_priv); 411 412 intel_gtt_clear_range(obj->gtt_space->start >> PAGE_SHIFT, 413 obj->base.size >> PAGE_SHIFT); 414 415 if (obj->sg_list) { 416 intel_gtt_unmap_memory(obj->sg_list, obj->num_sg); 417 obj->sg_list = NULL; 418 } 419 420 undo_idling(dev_priv, interruptible); 421 } 422