1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2014-2016 Intel Corporation 5 */ 6 7 #include <linux/highmem.h> 8 #include <linux/shmem_fs.h> 9 #include <linux/swap.h> 10 11 #include <drm/drm.h> /* for drm_legacy.h! */ 12 #include <drm/drm_cache.h> 13 14 #include "gt/intel_gt.h" 15 #include "i915_drv.h" 16 #include "i915_gem_object.h" 17 #include "i915_gem_region.h" 18 #include "i915_scatterlist.h" 19 20 static int i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj) 21 { 22 struct address_space *mapping = obj->base.filp->f_mapping; 23 struct scatterlist *sg; 24 struct sg_table *st; 25 dma_addr_t dma; 26 void *vaddr; 27 void *dst; 28 int i; 29 30 if (GEM_WARN_ON(i915_gem_object_needs_bit17_swizzle(obj))) 31 return -EINVAL; 32 33 /* 34 * Always aligning to the object size, allows a single allocation 35 * to handle all possible callers, and given typical object sizes, 36 * the alignment of the buddy allocation will naturally match. 37 */ 38 vaddr = dma_alloc_coherent(&obj->base.dev->pdev->dev, 39 roundup_pow_of_two(obj->base.size), 40 &dma, GFP_KERNEL); 41 if (!vaddr) 42 return -ENOMEM; 43 44 st = kmalloc(sizeof(*st), GFP_KERNEL); 45 if (!st) 46 goto err_pci; 47 48 if (sg_alloc_table(st, 1, GFP_KERNEL)) 49 goto err_st; 50 51 sg = st->sgl; 52 sg->offset = 0; 53 sg->length = obj->base.size; 54 55 sg_assign_page(sg, (struct page *)vaddr); 56 sg_dma_address(sg) = dma; 57 sg_dma_len(sg) = obj->base.size; 58 59 dst = vaddr; 60 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) { 61 struct page *page; 62 void *src; 63 64 page = shmem_read_mapping_page(mapping, i); 65 if (IS_ERR(page)) 66 goto err_st; 67 68 src = kmap_atomic(page); 69 memcpy(dst, src, PAGE_SIZE); 70 drm_clflush_virt_range(dst, PAGE_SIZE); 71 kunmap_atomic(src); 72 73 put_page(page); 74 dst += PAGE_SIZE; 75 } 76 77 intel_gt_chipset_flush(&to_i915(obj->base.dev)->gt); 78 79 __i915_gem_object_set_pages(obj, st, sg->length); 80 81 return 0; 82 83 err_st: 84 kfree(st); 85 err_pci: 86 dma_free_coherent(&obj->base.dev->pdev->dev, 87 roundup_pow_of_two(obj->base.size), 88 vaddr, dma); 89 return -ENOMEM; 90 } 91 92 static void 93 i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj, 94 struct sg_table *pages) 95 { 96 dma_addr_t dma = sg_dma_address(pages->sgl); 97 void *vaddr = sg_page(pages->sgl); 98 99 __i915_gem_object_release_shmem(obj, pages, false); 100 101 if (obj->mm.dirty) { 102 struct address_space *mapping = obj->base.filp->f_mapping; 103 void *src = vaddr; 104 int i; 105 106 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) { 107 struct page *page; 108 char *dst; 109 110 page = shmem_read_mapping_page(mapping, i); 111 if (IS_ERR(page)) 112 continue; 113 114 dst = kmap_atomic(page); 115 drm_clflush_virt_range(src, PAGE_SIZE); 116 memcpy(dst, src, PAGE_SIZE); 117 kunmap_atomic(dst); 118 119 set_page_dirty(page); 120 if (obj->mm.madv == I915_MADV_WILLNEED) 121 mark_page_accessed(page); 122 put_page(page); 123 124 src += PAGE_SIZE; 125 } 126 obj->mm.dirty = false; 127 } 128 129 sg_free_table(pages); 130 kfree(pages); 131 132 dma_free_coherent(&obj->base.dev->pdev->dev, 133 roundup_pow_of_two(obj->base.size), 134 vaddr, dma); 135 } 136 137 static void phys_release(struct drm_i915_gem_object *obj) 138 { 139 fput(obj->base.filp); 140 } 141 142 static const struct drm_i915_gem_object_ops i915_gem_phys_ops = { 143 .name = "i915_gem_object_phys", 144 .get_pages = i915_gem_object_get_pages_phys, 145 .put_pages = i915_gem_object_put_pages_phys, 146 147 .release = phys_release, 148 }; 149 150 int i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, int align) 151 { 152 struct sg_table *pages; 153 int err; 154 155 if (align > obj->base.size) 156 return -EINVAL; 157 158 if (obj->ops == &i915_gem_phys_ops) 159 return 0; 160 161 if (obj->ops != &i915_gem_shmem_ops) 162 return -EINVAL; 163 164 err = i915_gem_object_unbind(obj, I915_GEM_OBJECT_UNBIND_ACTIVE); 165 if (err) 166 return err; 167 168 mutex_lock_nested(&obj->mm.lock, I915_MM_GET_PAGES); 169 170 if (obj->mm.madv != I915_MADV_WILLNEED) { 171 err = -EFAULT; 172 goto err_unlock; 173 } 174 175 if (obj->mm.quirked) { 176 err = -EFAULT; 177 goto err_unlock; 178 } 179 180 if (obj->mm.mapping) { 181 err = -EBUSY; 182 goto err_unlock; 183 } 184 185 pages = __i915_gem_object_unset_pages(obj); 186 187 obj->ops = &i915_gem_phys_ops; 188 189 err = ____i915_gem_object_get_pages(obj); 190 if (err) 191 goto err_xfer; 192 193 /* Perma-pin (until release) the physical set of pages */ 194 __i915_gem_object_pin_pages(obj); 195 196 if (!IS_ERR_OR_NULL(pages)) 197 i915_gem_shmem_ops.put_pages(obj, pages); 198 199 i915_gem_object_release_memory_region(obj); 200 201 mutex_unlock(&obj->mm.lock); 202 return 0; 203 204 err_xfer: 205 obj->ops = &i915_gem_shmem_ops; 206 if (!IS_ERR_OR_NULL(pages)) { 207 unsigned int sg_page_sizes = i915_sg_page_sizes(pages->sgl); 208 209 __i915_gem_object_set_pages(obj, pages, sg_page_sizes); 210 } 211 err_unlock: 212 mutex_unlock(&obj->mm.lock); 213 return err; 214 } 215 216 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 217 #include "selftests/i915_gem_phys.c" 218 #endif 219