1 /************************************************************************** 2 * 3 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA 4 * All Rights Reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sub license, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the 15 * next paragraph) shall be included in all copies or substantial portions 16 * of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 24 * USE OR OTHER DEALINGS IN THE SOFTWARE. 25 * 26 **************************************************************************/ 27 /* 28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> 29 */ 30 31 #define pr_fmt(fmt) "[TTM] " fmt 32 33 #include <ttm/ttm_module.h> 34 #include <ttm/ttm_bo_driver.h> 35 #include <ttm/ttm_placement.h> 36 #include <drm/drm_vma_manager.h> 37 #include <linux/mm.h> 38 #include <linux/rbtree.h> 39 #include <linux/module.h> 40 #include <linux/uaccess.h> 41 42 #define TTM_BO_VM_NUM_PREFAULT 16 43 44 static int ttm_bo_vm_fault_idle(struct ttm_buffer_object *bo, 45 struct vm_area_struct *vma, 46 struct vm_fault *vmf) 47 { 48 struct ttm_bo_device *bdev = bo->bdev; 49 int ret = 0; 50 51 spin_lock(&bdev->fence_lock); 52 if (likely(!test_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags))) 53 goto out_unlock; 54 55 /* 56 * Quick non-stalling check for idle. 57 */ 58 ret = ttm_bo_wait(bo, false, false, true); 59 if (likely(ret == 0)) 60 goto out_unlock; 61 62 /* 63 * If possible, avoid waiting for GPU with mmap_sem 64 * held. 65 */ 66 if (vmf->flags & FAULT_FLAG_ALLOW_RETRY) { 67 ret = VM_FAULT_RETRY; 68 if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT) 69 goto out_unlock; 70 71 up_read(&vma->vm_mm->mmap_sem); 72 (void) ttm_bo_wait(bo, false, true, false); 73 goto out_unlock; 74 } 75 76 /* 77 * Ordinary wait. 78 */ 79 ret = ttm_bo_wait(bo, false, true, false); 80 if (unlikely(ret != 0)) 81 ret = (ret != -ERESTARTSYS) ? VM_FAULT_SIGBUS : 82 VM_FAULT_NOPAGE; 83 84 out_unlock: 85 spin_unlock(&bdev->fence_lock); 86 return ret; 87 } 88 89 static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 90 { 91 struct ttm_buffer_object *bo = (struct ttm_buffer_object *) 92 vma->vm_private_data; 93 struct ttm_bo_device *bdev = bo->bdev; 94 unsigned long page_offset; 95 unsigned long page_last; 96 unsigned long pfn; 97 struct ttm_tt *ttm = NULL; 98 struct page *page; 99 int ret; 100 int i; 101 unsigned long address = (unsigned long)vmf->virtual_address; 102 int retval = VM_FAULT_NOPAGE; 103 struct ttm_mem_type_manager *man = 104 &bdev->man[bo->mem.mem_type]; 105 struct vm_area_struct cvma; 106 107 /* 108 * Work around locking order reversal in fault / nopfn 109 * between mmap_sem and bo_reserve: Perform a trylock operation 110 * for reserve, and if it fails, retry the fault after waiting 111 * for the buffer to become unreserved. 112 */ 113 ret = ttm_bo_reserve(bo, true, true, false, NULL); 114 if (unlikely(ret != 0)) { 115 if (ret != -EBUSY) 116 return VM_FAULT_NOPAGE; 117 118 if (vmf->flags & FAULT_FLAG_ALLOW_RETRY) { 119 if (!(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) { 120 up_read(&vma->vm_mm->mmap_sem); 121 (void) ttm_bo_wait_unreserved(bo); 122 } 123 124 return VM_FAULT_RETRY; 125 } 126 127 /* 128 * If we'd want to change locking order to 129 * mmap_sem -> bo::reserve, we'd use a blocking reserve here 130 * instead of retrying the fault... 131 */ 132 return VM_FAULT_NOPAGE; 133 } 134 135 if (bdev->driver->fault_reserve_notify) { 136 ret = bdev->driver->fault_reserve_notify(bo); 137 switch (ret) { 138 case 0: 139 break; 140 case -EBUSY: 141 case -ERESTARTSYS: 142 retval = VM_FAULT_NOPAGE; 143 goto out_unlock; 144 default: 145 retval = VM_FAULT_SIGBUS; 146 goto out_unlock; 147 } 148 } 149 150 /* 151 * Wait for buffer data in transit, due to a pipelined 152 * move. 153 */ 154 ret = ttm_bo_vm_fault_idle(bo, vma, vmf); 155 if (unlikely(ret != 0)) { 156 retval = ret; 157 goto out_unlock; 158 } 159 160 ret = ttm_mem_io_lock(man, true); 161 if (unlikely(ret != 0)) { 162 retval = VM_FAULT_NOPAGE; 163 goto out_unlock; 164 } 165 ret = ttm_mem_io_reserve_vm(bo); 166 if (unlikely(ret != 0)) { 167 retval = VM_FAULT_SIGBUS; 168 goto out_io_unlock; 169 } 170 171 page_offset = ((address - vma->vm_start) >> PAGE_SHIFT) + 172 drm_vma_node_start(&bo->vma_node) - vma->vm_pgoff; 173 page_last = vma_pages(vma) + 174 drm_vma_node_start(&bo->vma_node) - vma->vm_pgoff; 175 176 if (unlikely(page_offset >= bo->num_pages)) { 177 retval = VM_FAULT_SIGBUS; 178 goto out_io_unlock; 179 } 180 181 /* 182 * Make a local vma copy to modify the page_prot member 183 * and vm_flags if necessary. The vma parameter is protected 184 * by mmap_sem in write mode. 185 */ 186 cvma = *vma; 187 cvma.vm_page_prot = vm_get_page_prot(cvma.vm_flags); 188 189 if (bo->mem.bus.is_iomem) { 190 cvma.vm_page_prot = ttm_io_prot(bo->mem.placement, 191 cvma.vm_page_prot); 192 } else { 193 ttm = bo->ttm; 194 if (!(bo->mem.placement & TTM_PL_FLAG_CACHED)) 195 cvma.vm_page_prot = ttm_io_prot(bo->mem.placement, 196 cvma.vm_page_prot); 197 198 /* Allocate all page at once, most common usage */ 199 if (ttm->bdev->driver->ttm_tt_populate(ttm)) { 200 retval = VM_FAULT_OOM; 201 goto out_io_unlock; 202 } 203 } 204 205 /* 206 * Speculatively prefault a number of pages. Only error on 207 * first page. 208 */ 209 for (i = 0; i < TTM_BO_VM_NUM_PREFAULT; ++i) { 210 if (bo->mem.bus.is_iomem) 211 pfn = ((bo->mem.bus.base + bo->mem.bus.offset) >> PAGE_SHIFT) + page_offset; 212 else { 213 page = ttm->pages[page_offset]; 214 if (unlikely(!page && i == 0)) { 215 retval = VM_FAULT_OOM; 216 goto out_io_unlock; 217 } else if (unlikely(!page)) { 218 break; 219 } 220 pfn = page_to_pfn(page); 221 } 222 223 ret = vm_insert_mixed(&cvma, address, pfn); 224 /* 225 * Somebody beat us to this PTE or prefaulting to 226 * an already populated PTE, or prefaulting error. 227 */ 228 229 if (unlikely((ret == -EBUSY) || (ret != 0 && i > 0))) 230 break; 231 else if (unlikely(ret != 0)) { 232 retval = 233 (ret == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS; 234 goto out_io_unlock; 235 } 236 237 address += PAGE_SIZE; 238 if (unlikely(++page_offset >= page_last)) 239 break; 240 } 241 out_io_unlock: 242 ttm_mem_io_unlock(man); 243 out_unlock: 244 ttm_bo_unreserve(bo); 245 return retval; 246 } 247 248 static void ttm_bo_vm_open(struct vm_area_struct *vma) 249 { 250 struct ttm_buffer_object *bo = 251 (struct ttm_buffer_object *)vma->vm_private_data; 252 253 (void)ttm_bo_reference(bo); 254 } 255 256 static void ttm_bo_vm_close(struct vm_area_struct *vma) 257 { 258 struct ttm_buffer_object *bo = (struct ttm_buffer_object *)vma->vm_private_data; 259 260 ttm_bo_unref(&bo); 261 vma->vm_private_data = NULL; 262 } 263 264 static const struct vm_operations_struct ttm_bo_vm_ops = { 265 .fault = ttm_bo_vm_fault, 266 .open = ttm_bo_vm_open, 267 .close = ttm_bo_vm_close 268 }; 269 270 static struct ttm_buffer_object *ttm_bo_vm_lookup(struct ttm_bo_device *bdev, 271 unsigned long offset, 272 unsigned long pages) 273 { 274 struct drm_vma_offset_node *node; 275 struct ttm_buffer_object *bo = NULL; 276 277 drm_vma_offset_lock_lookup(&bdev->vma_manager); 278 279 node = drm_vma_offset_lookup_locked(&bdev->vma_manager, offset, pages); 280 if (likely(node)) { 281 bo = container_of(node, struct ttm_buffer_object, vma_node); 282 if (!kref_get_unless_zero(&bo->kref)) 283 bo = NULL; 284 } 285 286 drm_vma_offset_unlock_lookup(&bdev->vma_manager); 287 288 if (!bo) 289 pr_err("Could not find buffer object to map\n"); 290 291 return bo; 292 } 293 294 int ttm_bo_mmap(struct file *filp, struct vm_area_struct *vma, 295 struct ttm_bo_device *bdev) 296 { 297 struct ttm_bo_driver *driver; 298 struct ttm_buffer_object *bo; 299 int ret; 300 301 bo = ttm_bo_vm_lookup(bdev, vma->vm_pgoff, vma_pages(vma)); 302 if (unlikely(!bo)) 303 return -EINVAL; 304 305 driver = bo->bdev->driver; 306 if (unlikely(!driver->verify_access)) { 307 ret = -EPERM; 308 goto out_unref; 309 } 310 ret = driver->verify_access(bo, filp); 311 if (unlikely(ret != 0)) 312 goto out_unref; 313 314 vma->vm_ops = &ttm_bo_vm_ops; 315 316 /* 317 * Note: We're transferring the bo reference to 318 * vma->vm_private_data here. 319 */ 320 321 vma->vm_private_data = bo; 322 vma->vm_flags |= VM_IO | VM_MIXEDMAP | VM_DONTEXPAND | VM_DONTDUMP; 323 return 0; 324 out_unref: 325 ttm_bo_unref(&bo); 326 return ret; 327 } 328 EXPORT_SYMBOL(ttm_bo_mmap); 329 330 int ttm_fbdev_mmap(struct vm_area_struct *vma, struct ttm_buffer_object *bo) 331 { 332 if (vma->vm_pgoff != 0) 333 return -EACCES; 334 335 vma->vm_ops = &ttm_bo_vm_ops; 336 vma->vm_private_data = ttm_bo_reference(bo); 337 vma->vm_flags |= VM_IO | VM_MIXEDMAP | VM_DONTEXPAND; 338 return 0; 339 } 340 EXPORT_SYMBOL(ttm_fbdev_mmap); 341