1 /* 2 * Copyright (C) 2012 Russell King 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License version 2 as 6 * published by the Free Software Foundation. 7 */ 8 #include <linux/dma-buf.h> 9 #include <linux/dma-mapping.h> 10 #include <linux/shmem_fs.h> 11 #include <drm/drmP.h> 12 #include "armada_drm.h" 13 #include "armada_gem.h" 14 #include <drm/armada_drm.h> 15 #include "armada_ioctlP.h" 16 17 static int armada_gem_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 18 { 19 struct armada_gem_object *obj = drm_to_armada_gem(vma->vm_private_data); 20 unsigned long addr = (unsigned long)vmf->virtual_address; 21 unsigned long pfn = obj->phys_addr >> PAGE_SHIFT; 22 int ret; 23 24 pfn += (addr - vma->vm_start) >> PAGE_SHIFT; 25 ret = vm_insert_pfn(vma, addr, pfn); 26 27 switch (ret) { 28 case 0: 29 case -EBUSY: 30 return VM_FAULT_NOPAGE; 31 case -ENOMEM: 32 return VM_FAULT_OOM; 33 default: 34 return VM_FAULT_SIGBUS; 35 } 36 } 37 38 const struct vm_operations_struct armada_gem_vm_ops = { 39 .fault = armada_gem_vm_fault, 40 .open = drm_gem_vm_open, 41 .close = drm_gem_vm_close, 42 }; 43 44 static size_t roundup_gem_size(size_t size) 45 { 46 return roundup(size, PAGE_SIZE); 47 } 48 49 /* dev->struct_mutex is held here */ 50 void armada_gem_free_object(struct drm_gem_object *obj) 51 { 52 struct armada_gem_object *dobj = drm_to_armada_gem(obj); 53 54 DRM_DEBUG_DRIVER("release obj %p\n", dobj); 55 56 drm_gem_free_mmap_offset(&dobj->obj); 57 58 if (dobj->page) { 59 /* page backed memory */ 60 unsigned int order = get_order(dobj->obj.size); 61 __free_pages(dobj->page, order); 62 } else if (dobj->linear) { 63 /* linear backed memory */ 64 drm_mm_remove_node(dobj->linear); 65 kfree(dobj->linear); 66 if (dobj->addr) 67 iounmap(dobj->addr); 68 } 69 70 if (dobj->obj.import_attach) { 71 /* We only ever display imported data */ 72 dma_buf_unmap_attachment(dobj->obj.import_attach, dobj->sgt, 73 DMA_TO_DEVICE); 74 drm_prime_gem_destroy(&dobj->obj, NULL); 75 } 76 77 drm_gem_object_release(&dobj->obj); 78 79 kfree(dobj); 80 } 81 82 int 83 armada_gem_linear_back(struct drm_device *dev, struct armada_gem_object *obj) 84 { 85 struct armada_private *priv = dev->dev_private; 86 size_t size = obj->obj.size; 87 88 if (obj->page || obj->linear) 89 return 0; 90 91 /* 92 * If it is a small allocation (typically cursor, which will 93 * be 32x64 or 64x32 ARGB pixels) try to get it from the system. 94 * Framebuffers will never be this small (our minimum size for 95 * framebuffers is larger than this anyway.) Such objects are 96 * only accessed by the CPU so we don't need any special handing 97 * here. 98 */ 99 if (size <= 8192) { 100 unsigned int order = get_order(size); 101 struct page *p = alloc_pages(GFP_KERNEL, order); 102 103 if (p) { 104 obj->addr = page_address(p); 105 obj->phys_addr = page_to_phys(p); 106 obj->page = p; 107 108 memset(obj->addr, 0, PAGE_ALIGN(size)); 109 } 110 } 111 112 /* 113 * We could grab something from CMA if it's enabled, but that 114 * involves building in a problem: 115 * 116 * CMA's interface uses dma_alloc_coherent(), which provides us 117 * with an CPU virtual address and a device address. 118 * 119 * The CPU virtual address may be either an address in the kernel 120 * direct mapped region (for example, as it would be on x86) or 121 * it may be remapped into another part of kernel memory space 122 * (eg, as it would be on ARM.) This means virt_to_phys() on the 123 * returned virtual address is invalid depending on the architecture 124 * implementation. 125 * 126 * The device address may also not be a physical address; it may 127 * be that there is some kind of remapping between the device and 128 * system RAM, which makes the use of the device address also 129 * unsafe to re-use as a physical address. 130 * 131 * This makes DRM usage of dma_alloc_coherent() in a generic way 132 * at best very questionable and unsafe. 133 */ 134 135 /* Otherwise, grab it from our linear allocation */ 136 if (!obj->page) { 137 struct drm_mm_node *node; 138 unsigned align = min_t(unsigned, size, SZ_2M); 139 void __iomem *ptr; 140 int ret; 141 142 node = kzalloc(sizeof(*node), GFP_KERNEL); 143 if (!node) 144 return -ENOSPC; 145 146 mutex_lock(&dev->struct_mutex); 147 ret = drm_mm_insert_node(&priv->linear, node, size, align, 148 DRM_MM_SEARCH_DEFAULT); 149 mutex_unlock(&dev->struct_mutex); 150 if (ret) { 151 kfree(node); 152 return ret; 153 } 154 155 obj->linear = node; 156 157 /* Ensure that the memory we're returning is cleared. */ 158 ptr = ioremap_wc(obj->linear->start, size); 159 if (!ptr) { 160 mutex_lock(&dev->struct_mutex); 161 drm_mm_remove_node(obj->linear); 162 mutex_unlock(&dev->struct_mutex); 163 kfree(obj->linear); 164 obj->linear = NULL; 165 return -ENOMEM; 166 } 167 168 memset_io(ptr, 0, size); 169 iounmap(ptr); 170 171 obj->phys_addr = obj->linear->start; 172 obj->dev_addr = obj->linear->start; 173 } 174 175 DRM_DEBUG_DRIVER("obj %p phys %#x dev %#x\n", 176 obj, obj->phys_addr, obj->dev_addr); 177 178 return 0; 179 } 180 181 void * 182 armada_gem_map_object(struct drm_device *dev, struct armada_gem_object *dobj) 183 { 184 /* only linear objects need to be ioremap'd */ 185 if (!dobj->addr && dobj->linear) 186 dobj->addr = ioremap_wc(dobj->phys_addr, dobj->obj.size); 187 return dobj->addr; 188 } 189 190 struct armada_gem_object * 191 armada_gem_alloc_private_object(struct drm_device *dev, size_t size) 192 { 193 struct armada_gem_object *obj; 194 195 size = roundup_gem_size(size); 196 197 obj = kzalloc(sizeof(*obj), GFP_KERNEL); 198 if (!obj) 199 return NULL; 200 201 drm_gem_private_object_init(dev, &obj->obj, size); 202 obj->dev_addr = DMA_ERROR_CODE; 203 204 DRM_DEBUG_DRIVER("alloc private obj %p size %zu\n", obj, size); 205 206 return obj; 207 } 208 209 struct armada_gem_object *armada_gem_alloc_object(struct drm_device *dev, 210 size_t size) 211 { 212 struct armada_gem_object *obj; 213 struct address_space *mapping; 214 215 size = roundup_gem_size(size); 216 217 obj = kzalloc(sizeof(*obj), GFP_KERNEL); 218 if (!obj) 219 return NULL; 220 221 if (drm_gem_object_init(dev, &obj->obj, size)) { 222 kfree(obj); 223 return NULL; 224 } 225 226 obj->dev_addr = DMA_ERROR_CODE; 227 228 mapping = obj->obj.filp->f_path.dentry->d_inode->i_mapping; 229 mapping_set_gfp_mask(mapping, GFP_HIGHUSER | __GFP_RECLAIMABLE); 230 231 DRM_DEBUG_DRIVER("alloc obj %p size %zu\n", obj, size); 232 233 return obj; 234 } 235 236 /* Dumb alloc support */ 237 int armada_gem_dumb_create(struct drm_file *file, struct drm_device *dev, 238 struct drm_mode_create_dumb *args) 239 { 240 struct armada_gem_object *dobj; 241 u32 handle; 242 size_t size; 243 int ret; 244 245 args->pitch = armada_pitch(args->width, args->bpp); 246 args->size = size = args->pitch * args->height; 247 248 dobj = armada_gem_alloc_private_object(dev, size); 249 if (dobj == NULL) 250 return -ENOMEM; 251 252 ret = armada_gem_linear_back(dev, dobj); 253 if (ret) 254 goto err; 255 256 ret = drm_gem_handle_create(file, &dobj->obj, &handle); 257 if (ret) 258 goto err; 259 260 args->handle = handle; 261 262 /* drop reference from allocate - handle holds it now */ 263 DRM_DEBUG_DRIVER("obj %p size %zu handle %#x\n", dobj, size, handle); 264 err: 265 drm_gem_object_unreference_unlocked(&dobj->obj); 266 return ret; 267 } 268 269 int armada_gem_dumb_map_offset(struct drm_file *file, struct drm_device *dev, 270 uint32_t handle, uint64_t *offset) 271 { 272 struct armada_gem_object *obj; 273 int ret = 0; 274 275 mutex_lock(&dev->struct_mutex); 276 obj = armada_gem_object_lookup(dev, file, handle); 277 if (!obj) { 278 DRM_ERROR("failed to lookup gem object\n"); 279 ret = -EINVAL; 280 goto err_unlock; 281 } 282 283 /* Don't allow imported objects to be mapped */ 284 if (obj->obj.import_attach) { 285 ret = -EINVAL; 286 goto err_unlock; 287 } 288 289 ret = drm_gem_create_mmap_offset(&obj->obj); 290 if (ret == 0) { 291 *offset = drm_vma_node_offset_addr(&obj->obj.vma_node); 292 DRM_DEBUG_DRIVER("handle %#x offset %llx\n", handle, *offset); 293 } 294 295 drm_gem_object_unreference(&obj->obj); 296 err_unlock: 297 mutex_unlock(&dev->struct_mutex); 298 299 return ret; 300 } 301 302 int armada_gem_dumb_destroy(struct drm_file *file, struct drm_device *dev, 303 uint32_t handle) 304 { 305 return drm_gem_handle_delete(file, handle); 306 } 307 308 /* Private driver gem ioctls */ 309 int armada_gem_create_ioctl(struct drm_device *dev, void *data, 310 struct drm_file *file) 311 { 312 struct drm_armada_gem_create *args = data; 313 struct armada_gem_object *dobj; 314 size_t size; 315 u32 handle; 316 int ret; 317 318 if (args->size == 0) 319 return -ENOMEM; 320 321 size = args->size; 322 323 dobj = armada_gem_alloc_object(dev, size); 324 if (dobj == NULL) 325 return -ENOMEM; 326 327 ret = drm_gem_handle_create(file, &dobj->obj, &handle); 328 if (ret) 329 goto err; 330 331 args->handle = handle; 332 333 /* drop reference from allocate - handle holds it now */ 334 DRM_DEBUG_DRIVER("obj %p size %zu handle %#x\n", dobj, size, handle); 335 err: 336 drm_gem_object_unreference_unlocked(&dobj->obj); 337 return ret; 338 } 339 340 /* Map a shmem-backed object into process memory space */ 341 int armada_gem_mmap_ioctl(struct drm_device *dev, void *data, 342 struct drm_file *file) 343 { 344 struct drm_armada_gem_mmap *args = data; 345 struct armada_gem_object *dobj; 346 unsigned long addr; 347 348 dobj = armada_gem_object_lookup(dev, file, args->handle); 349 if (dobj == NULL) 350 return -ENOENT; 351 352 if (!dobj->obj.filp) { 353 drm_gem_object_unreference(&dobj->obj); 354 return -EINVAL; 355 } 356 357 addr = vm_mmap(dobj->obj.filp, 0, args->size, PROT_READ | PROT_WRITE, 358 MAP_SHARED, args->offset); 359 drm_gem_object_unreference(&dobj->obj); 360 if (IS_ERR_VALUE(addr)) 361 return addr; 362 363 args->addr = addr; 364 365 return 0; 366 } 367 368 int armada_gem_pwrite_ioctl(struct drm_device *dev, void *data, 369 struct drm_file *file) 370 { 371 struct drm_armada_gem_pwrite *args = data; 372 struct armada_gem_object *dobj; 373 char __user *ptr; 374 int ret; 375 376 DRM_DEBUG_DRIVER("handle %u off %u size %u ptr 0x%llx\n", 377 args->handle, args->offset, args->size, args->ptr); 378 379 if (args->size == 0) 380 return 0; 381 382 ptr = (char __user *)(uintptr_t)args->ptr; 383 384 if (!access_ok(VERIFY_READ, ptr, args->size)) 385 return -EFAULT; 386 387 ret = fault_in_multipages_readable(ptr, args->size); 388 if (ret) 389 return ret; 390 391 dobj = armada_gem_object_lookup(dev, file, args->handle); 392 if (dobj == NULL) 393 return -ENOENT; 394 395 /* Must be a kernel-mapped object */ 396 if (!dobj->addr) 397 return -EINVAL; 398 399 if (args->offset > dobj->obj.size || 400 args->size > dobj->obj.size - args->offset) { 401 DRM_ERROR("invalid size: object size %u\n", dobj->obj.size); 402 ret = -EINVAL; 403 goto unref; 404 } 405 406 if (copy_from_user(dobj->addr + args->offset, ptr, args->size)) { 407 ret = -EFAULT; 408 } else if (dobj->update) { 409 dobj->update(dobj->update_data); 410 ret = 0; 411 } 412 413 unref: 414 drm_gem_object_unreference_unlocked(&dobj->obj); 415 return ret; 416 } 417 418 /* Prime support */ 419 struct sg_table * 420 armada_gem_prime_map_dma_buf(struct dma_buf_attachment *attach, 421 enum dma_data_direction dir) 422 { 423 struct drm_gem_object *obj = attach->dmabuf->priv; 424 struct armada_gem_object *dobj = drm_to_armada_gem(obj); 425 struct scatterlist *sg; 426 struct sg_table *sgt; 427 int i, num; 428 429 sgt = kmalloc(sizeof(*sgt), GFP_KERNEL); 430 if (!sgt) 431 return NULL; 432 433 if (dobj->obj.filp) { 434 struct address_space *mapping; 435 gfp_t gfp; 436 int count; 437 438 count = dobj->obj.size / PAGE_SIZE; 439 if (sg_alloc_table(sgt, count, GFP_KERNEL)) 440 goto free_sgt; 441 442 mapping = file_inode(dobj->obj.filp)->i_mapping; 443 gfp = mapping_gfp_mask(mapping); 444 445 for_each_sg(sgt->sgl, sg, count, i) { 446 struct page *page; 447 448 page = shmem_read_mapping_page_gfp(mapping, i, gfp); 449 if (IS_ERR(page)) { 450 num = i; 451 goto release; 452 } 453 454 sg_set_page(sg, page, PAGE_SIZE, 0); 455 } 456 457 if (dma_map_sg(attach->dev, sgt->sgl, sgt->nents, dir) == 0) { 458 num = sgt->nents; 459 goto release; 460 } 461 } else if (dobj->page) { 462 /* Single contiguous page */ 463 if (sg_alloc_table(sgt, 1, GFP_KERNEL)) 464 goto free_sgt; 465 466 sg_set_page(sgt->sgl, dobj->page, dobj->obj.size, 0); 467 468 if (dma_map_sg(attach->dev, sgt->sgl, sgt->nents, dir) == 0) 469 goto free_table; 470 } else if (dobj->linear) { 471 /* Single contiguous physical region - no struct page */ 472 if (sg_alloc_table(sgt, 1, GFP_KERNEL)) 473 goto free_sgt; 474 sg_dma_address(sgt->sgl) = dobj->dev_addr; 475 sg_dma_len(sgt->sgl) = dobj->obj.size; 476 } else { 477 goto free_sgt; 478 } 479 return sgt; 480 481 release: 482 for_each_sg(sgt->sgl, sg, num, i) 483 page_cache_release(sg_page(sg)); 484 free_table: 485 sg_free_table(sgt); 486 free_sgt: 487 kfree(sgt); 488 return NULL; 489 } 490 491 static void armada_gem_prime_unmap_dma_buf(struct dma_buf_attachment *attach, 492 struct sg_table *sgt, enum dma_data_direction dir) 493 { 494 struct drm_gem_object *obj = attach->dmabuf->priv; 495 struct armada_gem_object *dobj = drm_to_armada_gem(obj); 496 int i; 497 498 if (!dobj->linear) 499 dma_unmap_sg(attach->dev, sgt->sgl, sgt->nents, dir); 500 501 if (dobj->obj.filp) { 502 struct scatterlist *sg; 503 for_each_sg(sgt->sgl, sg, sgt->nents, i) 504 page_cache_release(sg_page(sg)); 505 } 506 507 sg_free_table(sgt); 508 kfree(sgt); 509 } 510 511 static void *armada_gem_dmabuf_no_kmap(struct dma_buf *buf, unsigned long n) 512 { 513 return NULL; 514 } 515 516 static void 517 armada_gem_dmabuf_no_kunmap(struct dma_buf *buf, unsigned long n, void *addr) 518 { 519 } 520 521 static int 522 armada_gem_dmabuf_mmap(struct dma_buf *buf, struct vm_area_struct *vma) 523 { 524 return -EINVAL; 525 } 526 527 static const struct dma_buf_ops armada_gem_prime_dmabuf_ops = { 528 .map_dma_buf = armada_gem_prime_map_dma_buf, 529 .unmap_dma_buf = armada_gem_prime_unmap_dma_buf, 530 .release = drm_gem_dmabuf_release, 531 .kmap_atomic = armada_gem_dmabuf_no_kmap, 532 .kunmap_atomic = armada_gem_dmabuf_no_kunmap, 533 .kmap = armada_gem_dmabuf_no_kmap, 534 .kunmap = armada_gem_dmabuf_no_kunmap, 535 .mmap = armada_gem_dmabuf_mmap, 536 }; 537 538 struct dma_buf * 539 armada_gem_prime_export(struct drm_device *dev, struct drm_gem_object *obj, 540 int flags) 541 { 542 return dma_buf_export(obj, &armada_gem_prime_dmabuf_ops, obj->size, 543 O_RDWR); 544 } 545 546 struct drm_gem_object * 547 armada_gem_prime_import(struct drm_device *dev, struct dma_buf *buf) 548 { 549 struct dma_buf_attachment *attach; 550 struct armada_gem_object *dobj; 551 552 if (buf->ops == &armada_gem_prime_dmabuf_ops) { 553 struct drm_gem_object *obj = buf->priv; 554 if (obj->dev == dev) { 555 /* 556 * Importing our own dmabuf(s) increases the 557 * refcount on the gem object itself. 558 */ 559 drm_gem_object_reference(obj); 560 dma_buf_put(buf); 561 return obj; 562 } 563 } 564 565 attach = dma_buf_attach(buf, dev->dev); 566 if (IS_ERR(attach)) 567 return ERR_CAST(attach); 568 569 dobj = armada_gem_alloc_private_object(dev, buf->size); 570 if (!dobj) { 571 dma_buf_detach(buf, attach); 572 return ERR_PTR(-ENOMEM); 573 } 574 575 dobj->obj.import_attach = attach; 576 577 /* 578 * Don't call dma_buf_map_attachment() here - it maps the 579 * scatterlist immediately for DMA, and this is not always 580 * an appropriate thing to do. 581 */ 582 return &dobj->obj; 583 } 584 585 int armada_gem_map_import(struct armada_gem_object *dobj) 586 { 587 int ret; 588 589 dobj->sgt = dma_buf_map_attachment(dobj->obj.import_attach, 590 DMA_TO_DEVICE); 591 if (!dobj->sgt) { 592 DRM_ERROR("dma_buf_map_attachment() returned NULL\n"); 593 return -EINVAL; 594 } 595 if (IS_ERR(dobj->sgt)) { 596 ret = PTR_ERR(dobj->sgt); 597 dobj->sgt = NULL; 598 DRM_ERROR("dma_buf_map_attachment() error: %d\n", ret); 599 return ret; 600 } 601 if (dobj->sgt->nents > 1) { 602 DRM_ERROR("dma_buf_map_attachment() returned an (unsupported) scattered list\n"); 603 return -EINVAL; 604 } 605 if (sg_dma_len(dobj->sgt->sgl) < dobj->obj.size) { 606 DRM_ERROR("dma_buf_map_attachment() returned a small buffer\n"); 607 return -EINVAL; 608 } 609 dobj->dev_addr = sg_dma_address(dobj->sgt->sgl); 610 return 0; 611 } 612