1 /* 2 * Copyright (C) 2012 Russell King 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License version 2 as 6 * published by the Free Software Foundation. 7 */ 8 #include <linux/dma-buf.h> 9 #include <linux/dma-mapping.h> 10 #include <linux/shmem_fs.h> 11 #include <drm/drmP.h> 12 #include "armada_drm.h" 13 #include "armada_gem.h" 14 #include <drm/armada_drm.h> 15 #include "armada_ioctlP.h" 16 17 static int armada_gem_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 18 { 19 struct armada_gem_object *obj = drm_to_armada_gem(vma->vm_private_data); 20 unsigned long addr = (unsigned long)vmf->virtual_address; 21 unsigned long pfn = obj->phys_addr >> PAGE_SHIFT; 22 int ret; 23 24 pfn += (addr - vma->vm_start) >> PAGE_SHIFT; 25 ret = vm_insert_pfn(vma, addr, pfn); 26 27 switch (ret) { 28 case 0: 29 case -EBUSY: 30 return VM_FAULT_NOPAGE; 31 case -ENOMEM: 32 return VM_FAULT_OOM; 33 default: 34 return VM_FAULT_SIGBUS; 35 } 36 } 37 38 const struct vm_operations_struct armada_gem_vm_ops = { 39 .fault = armada_gem_vm_fault, 40 .open = drm_gem_vm_open, 41 .close = drm_gem_vm_close, 42 }; 43 44 static size_t roundup_gem_size(size_t size) 45 { 46 return roundup(size, PAGE_SIZE); 47 } 48 49 /* dev->struct_mutex is held here */ 50 void armada_gem_free_object(struct drm_gem_object *obj) 51 { 52 struct armada_gem_object *dobj = drm_to_armada_gem(obj); 53 54 DRM_DEBUG_DRIVER("release obj %p\n", dobj); 55 56 drm_gem_free_mmap_offset(&dobj->obj); 57 58 if (dobj->page) { 59 /* page backed memory */ 60 unsigned int order = get_order(dobj->obj.size); 61 __free_pages(dobj->page, order); 62 } else if (dobj->linear) { 63 /* linear backed memory */ 64 drm_mm_remove_node(dobj->linear); 65 kfree(dobj->linear); 66 if (dobj->addr) 67 iounmap(dobj->addr); 68 } 69 70 if (dobj->obj.import_attach) { 71 /* We only ever display imported data */ 72 if (dobj->sgt) 73 dma_buf_unmap_attachment(dobj->obj.import_attach, 74 dobj->sgt, DMA_TO_DEVICE); 75 drm_prime_gem_destroy(&dobj->obj, NULL); 76 } 77 78 drm_gem_object_release(&dobj->obj); 79 80 kfree(dobj); 81 } 82 83 int 84 armada_gem_linear_back(struct drm_device *dev, struct armada_gem_object *obj) 85 { 86 struct armada_private *priv = dev->dev_private; 87 size_t size = obj->obj.size; 88 89 if (obj->page || obj->linear) 90 return 0; 91 92 /* 93 * If it is a small allocation (typically cursor, which will 94 * be 32x64 or 64x32 ARGB pixels) try to get it from the system. 95 * Framebuffers will never be this small (our minimum size for 96 * framebuffers is larger than this anyway.) Such objects are 97 * only accessed by the CPU so we don't need any special handing 98 * here. 99 */ 100 if (size <= 8192) { 101 unsigned int order = get_order(size); 102 struct page *p = alloc_pages(GFP_KERNEL, order); 103 104 if (p) { 105 obj->addr = page_address(p); 106 obj->phys_addr = page_to_phys(p); 107 obj->page = p; 108 109 memset(obj->addr, 0, PAGE_ALIGN(size)); 110 } 111 } 112 113 /* 114 * We could grab something from CMA if it's enabled, but that 115 * involves building in a problem: 116 * 117 * CMA's interface uses dma_alloc_coherent(), which provides us 118 * with an CPU virtual address and a device address. 119 * 120 * The CPU virtual address may be either an address in the kernel 121 * direct mapped region (for example, as it would be on x86) or 122 * it may be remapped into another part of kernel memory space 123 * (eg, as it would be on ARM.) This means virt_to_phys() on the 124 * returned virtual address is invalid depending on the architecture 125 * implementation. 126 * 127 * The device address may also not be a physical address; it may 128 * be that there is some kind of remapping between the device and 129 * system RAM, which makes the use of the device address also 130 * unsafe to re-use as a physical address. 131 * 132 * This makes DRM usage of dma_alloc_coherent() in a generic way 133 * at best very questionable and unsafe. 134 */ 135 136 /* Otherwise, grab it from our linear allocation */ 137 if (!obj->page) { 138 struct drm_mm_node *node; 139 unsigned align = min_t(unsigned, size, SZ_2M); 140 void __iomem *ptr; 141 int ret; 142 143 node = kzalloc(sizeof(*node), GFP_KERNEL); 144 if (!node) 145 return -ENOSPC; 146 147 mutex_lock(&dev->struct_mutex); 148 ret = drm_mm_insert_node(&priv->linear, node, size, align, 149 DRM_MM_SEARCH_DEFAULT); 150 mutex_unlock(&dev->struct_mutex); 151 if (ret) { 152 kfree(node); 153 return ret; 154 } 155 156 obj->linear = node; 157 158 /* Ensure that the memory we're returning is cleared. */ 159 ptr = ioremap_wc(obj->linear->start, size); 160 if (!ptr) { 161 mutex_lock(&dev->struct_mutex); 162 drm_mm_remove_node(obj->linear); 163 mutex_unlock(&dev->struct_mutex); 164 kfree(obj->linear); 165 obj->linear = NULL; 166 return -ENOMEM; 167 } 168 169 memset_io(ptr, 0, size); 170 iounmap(ptr); 171 172 obj->phys_addr = obj->linear->start; 173 obj->dev_addr = obj->linear->start; 174 } 175 176 DRM_DEBUG_DRIVER("obj %p phys %#llx dev %#llx\n", obj, 177 (unsigned long long)obj->phys_addr, 178 (unsigned long long)obj->dev_addr); 179 180 return 0; 181 } 182 183 void * 184 armada_gem_map_object(struct drm_device *dev, struct armada_gem_object *dobj) 185 { 186 /* only linear objects need to be ioremap'd */ 187 if (!dobj->addr && dobj->linear) 188 dobj->addr = ioremap_wc(dobj->phys_addr, dobj->obj.size); 189 return dobj->addr; 190 } 191 192 struct armada_gem_object * 193 armada_gem_alloc_private_object(struct drm_device *dev, size_t size) 194 { 195 struct armada_gem_object *obj; 196 197 size = roundup_gem_size(size); 198 199 obj = kzalloc(sizeof(*obj), GFP_KERNEL); 200 if (!obj) 201 return NULL; 202 203 drm_gem_private_object_init(dev, &obj->obj, size); 204 obj->dev_addr = DMA_ERROR_CODE; 205 206 DRM_DEBUG_DRIVER("alloc private obj %p size %zu\n", obj, size); 207 208 return obj; 209 } 210 211 struct armada_gem_object *armada_gem_alloc_object(struct drm_device *dev, 212 size_t size) 213 { 214 struct armada_gem_object *obj; 215 struct address_space *mapping; 216 217 size = roundup_gem_size(size); 218 219 obj = kzalloc(sizeof(*obj), GFP_KERNEL); 220 if (!obj) 221 return NULL; 222 223 if (drm_gem_object_init(dev, &obj->obj, size)) { 224 kfree(obj); 225 return NULL; 226 } 227 228 obj->dev_addr = DMA_ERROR_CODE; 229 230 mapping = file_inode(obj->obj.filp)->i_mapping; 231 mapping_set_gfp_mask(mapping, GFP_HIGHUSER | __GFP_RECLAIMABLE); 232 233 DRM_DEBUG_DRIVER("alloc obj %p size %zu\n", obj, size); 234 235 return obj; 236 } 237 238 /* Dumb alloc support */ 239 int armada_gem_dumb_create(struct drm_file *file, struct drm_device *dev, 240 struct drm_mode_create_dumb *args) 241 { 242 struct armada_gem_object *dobj; 243 u32 handle; 244 size_t size; 245 int ret; 246 247 args->pitch = armada_pitch(args->width, args->bpp); 248 args->size = size = args->pitch * args->height; 249 250 dobj = armada_gem_alloc_private_object(dev, size); 251 if (dobj == NULL) 252 return -ENOMEM; 253 254 ret = armada_gem_linear_back(dev, dobj); 255 if (ret) 256 goto err; 257 258 ret = drm_gem_handle_create(file, &dobj->obj, &handle); 259 if (ret) 260 goto err; 261 262 args->handle = handle; 263 264 /* drop reference from allocate - handle holds it now */ 265 DRM_DEBUG_DRIVER("obj %p size %zu handle %#x\n", dobj, size, handle); 266 err: 267 drm_gem_object_unreference_unlocked(&dobj->obj); 268 return ret; 269 } 270 271 int armada_gem_dumb_map_offset(struct drm_file *file, struct drm_device *dev, 272 uint32_t handle, uint64_t *offset) 273 { 274 struct armada_gem_object *obj; 275 int ret = 0; 276 277 mutex_lock(&dev->struct_mutex); 278 obj = armada_gem_object_lookup(dev, file, handle); 279 if (!obj) { 280 DRM_ERROR("failed to lookup gem object\n"); 281 ret = -EINVAL; 282 goto err_unlock; 283 } 284 285 /* Don't allow imported objects to be mapped */ 286 if (obj->obj.import_attach) { 287 ret = -EINVAL; 288 goto err_unlock; 289 } 290 291 ret = drm_gem_create_mmap_offset(&obj->obj); 292 if (ret == 0) { 293 *offset = drm_vma_node_offset_addr(&obj->obj.vma_node); 294 DRM_DEBUG_DRIVER("handle %#x offset %llx\n", handle, *offset); 295 } 296 297 drm_gem_object_unreference(&obj->obj); 298 err_unlock: 299 mutex_unlock(&dev->struct_mutex); 300 301 return ret; 302 } 303 304 int armada_gem_dumb_destroy(struct drm_file *file, struct drm_device *dev, 305 uint32_t handle) 306 { 307 return drm_gem_handle_delete(file, handle); 308 } 309 310 /* Private driver gem ioctls */ 311 int armada_gem_create_ioctl(struct drm_device *dev, void *data, 312 struct drm_file *file) 313 { 314 struct drm_armada_gem_create *args = data; 315 struct armada_gem_object *dobj; 316 size_t size; 317 u32 handle; 318 int ret; 319 320 if (args->size == 0) 321 return -ENOMEM; 322 323 size = args->size; 324 325 dobj = armada_gem_alloc_object(dev, size); 326 if (dobj == NULL) 327 return -ENOMEM; 328 329 ret = drm_gem_handle_create(file, &dobj->obj, &handle); 330 if (ret) 331 goto err; 332 333 args->handle = handle; 334 335 /* drop reference from allocate - handle holds it now */ 336 DRM_DEBUG_DRIVER("obj %p size %zu handle %#x\n", dobj, size, handle); 337 err: 338 drm_gem_object_unreference_unlocked(&dobj->obj); 339 return ret; 340 } 341 342 /* Map a shmem-backed object into process memory space */ 343 int armada_gem_mmap_ioctl(struct drm_device *dev, void *data, 344 struct drm_file *file) 345 { 346 struct drm_armada_gem_mmap *args = data; 347 struct armada_gem_object *dobj; 348 unsigned long addr; 349 350 dobj = armada_gem_object_lookup(dev, file, args->handle); 351 if (dobj == NULL) 352 return -ENOENT; 353 354 if (!dobj->obj.filp) { 355 drm_gem_object_unreference(&dobj->obj); 356 return -EINVAL; 357 } 358 359 addr = vm_mmap(dobj->obj.filp, 0, args->size, PROT_READ | PROT_WRITE, 360 MAP_SHARED, args->offset); 361 drm_gem_object_unreference(&dobj->obj); 362 if (IS_ERR_VALUE(addr)) 363 return addr; 364 365 args->addr = addr; 366 367 return 0; 368 } 369 370 int armada_gem_pwrite_ioctl(struct drm_device *dev, void *data, 371 struct drm_file *file) 372 { 373 struct drm_armada_gem_pwrite *args = data; 374 struct armada_gem_object *dobj; 375 char __user *ptr; 376 int ret; 377 378 DRM_DEBUG_DRIVER("handle %u off %u size %u ptr 0x%llx\n", 379 args->handle, args->offset, args->size, args->ptr); 380 381 if (args->size == 0) 382 return 0; 383 384 ptr = (char __user *)(uintptr_t)args->ptr; 385 386 if (!access_ok(VERIFY_READ, ptr, args->size)) 387 return -EFAULT; 388 389 ret = fault_in_multipages_readable(ptr, args->size); 390 if (ret) 391 return ret; 392 393 dobj = armada_gem_object_lookup(dev, file, args->handle); 394 if (dobj == NULL) 395 return -ENOENT; 396 397 /* Must be a kernel-mapped object */ 398 if (!dobj->addr) 399 return -EINVAL; 400 401 if (args->offset > dobj->obj.size || 402 args->size > dobj->obj.size - args->offset) { 403 DRM_ERROR("invalid size: object size %u\n", dobj->obj.size); 404 ret = -EINVAL; 405 goto unref; 406 } 407 408 if (copy_from_user(dobj->addr + args->offset, ptr, args->size)) { 409 ret = -EFAULT; 410 } else if (dobj->update) { 411 dobj->update(dobj->update_data); 412 ret = 0; 413 } 414 415 unref: 416 drm_gem_object_unreference_unlocked(&dobj->obj); 417 return ret; 418 } 419 420 /* Prime support */ 421 struct sg_table * 422 armada_gem_prime_map_dma_buf(struct dma_buf_attachment *attach, 423 enum dma_data_direction dir) 424 { 425 struct drm_gem_object *obj = attach->dmabuf->priv; 426 struct armada_gem_object *dobj = drm_to_armada_gem(obj); 427 struct scatterlist *sg; 428 struct sg_table *sgt; 429 int i, num; 430 431 sgt = kmalloc(sizeof(*sgt), GFP_KERNEL); 432 if (!sgt) 433 return NULL; 434 435 if (dobj->obj.filp) { 436 struct address_space *mapping; 437 int count; 438 439 count = dobj->obj.size / PAGE_SIZE; 440 if (sg_alloc_table(sgt, count, GFP_KERNEL)) 441 goto free_sgt; 442 443 mapping = file_inode(dobj->obj.filp)->i_mapping; 444 445 for_each_sg(sgt->sgl, sg, count, i) { 446 struct page *page; 447 448 page = shmem_read_mapping_page(mapping, i); 449 if (IS_ERR(page)) { 450 num = i; 451 goto release; 452 } 453 454 sg_set_page(sg, page, PAGE_SIZE, 0); 455 } 456 457 if (dma_map_sg(attach->dev, sgt->sgl, sgt->nents, dir) == 0) { 458 num = sgt->nents; 459 goto release; 460 } 461 } else if (dobj->page) { 462 /* Single contiguous page */ 463 if (sg_alloc_table(sgt, 1, GFP_KERNEL)) 464 goto free_sgt; 465 466 sg_set_page(sgt->sgl, dobj->page, dobj->obj.size, 0); 467 468 if (dma_map_sg(attach->dev, sgt->sgl, sgt->nents, dir) == 0) 469 goto free_table; 470 } else if (dobj->linear) { 471 /* Single contiguous physical region - no struct page */ 472 if (sg_alloc_table(sgt, 1, GFP_KERNEL)) 473 goto free_sgt; 474 sg_dma_address(sgt->sgl) = dobj->dev_addr; 475 sg_dma_len(sgt->sgl) = dobj->obj.size; 476 } else { 477 goto free_sgt; 478 } 479 return sgt; 480 481 release: 482 for_each_sg(sgt->sgl, sg, num, i) 483 page_cache_release(sg_page(sg)); 484 free_table: 485 sg_free_table(sgt); 486 free_sgt: 487 kfree(sgt); 488 return NULL; 489 } 490 491 static void armada_gem_prime_unmap_dma_buf(struct dma_buf_attachment *attach, 492 struct sg_table *sgt, enum dma_data_direction dir) 493 { 494 struct drm_gem_object *obj = attach->dmabuf->priv; 495 struct armada_gem_object *dobj = drm_to_armada_gem(obj); 496 int i; 497 498 if (!dobj->linear) 499 dma_unmap_sg(attach->dev, sgt->sgl, sgt->nents, dir); 500 501 if (dobj->obj.filp) { 502 struct scatterlist *sg; 503 for_each_sg(sgt->sgl, sg, sgt->nents, i) 504 page_cache_release(sg_page(sg)); 505 } 506 507 sg_free_table(sgt); 508 kfree(sgt); 509 } 510 511 static void *armada_gem_dmabuf_no_kmap(struct dma_buf *buf, unsigned long n) 512 { 513 return NULL; 514 } 515 516 static void 517 armada_gem_dmabuf_no_kunmap(struct dma_buf *buf, unsigned long n, void *addr) 518 { 519 } 520 521 static int 522 armada_gem_dmabuf_mmap(struct dma_buf *buf, struct vm_area_struct *vma) 523 { 524 return -EINVAL; 525 } 526 527 static const struct dma_buf_ops armada_gem_prime_dmabuf_ops = { 528 .map_dma_buf = armada_gem_prime_map_dma_buf, 529 .unmap_dma_buf = armada_gem_prime_unmap_dma_buf, 530 .release = drm_gem_dmabuf_release, 531 .kmap_atomic = armada_gem_dmabuf_no_kmap, 532 .kunmap_atomic = armada_gem_dmabuf_no_kunmap, 533 .kmap = armada_gem_dmabuf_no_kmap, 534 .kunmap = armada_gem_dmabuf_no_kunmap, 535 .mmap = armada_gem_dmabuf_mmap, 536 }; 537 538 struct dma_buf * 539 armada_gem_prime_export(struct drm_device *dev, struct drm_gem_object *obj, 540 int flags) 541 { 542 DEFINE_DMA_BUF_EXPORT_INFO(exp_info); 543 544 exp_info.ops = &armada_gem_prime_dmabuf_ops; 545 exp_info.size = obj->size; 546 exp_info.flags = O_RDWR; 547 exp_info.priv = obj; 548 549 return dma_buf_export(&exp_info); 550 } 551 552 struct drm_gem_object * 553 armada_gem_prime_import(struct drm_device *dev, struct dma_buf *buf) 554 { 555 struct dma_buf_attachment *attach; 556 struct armada_gem_object *dobj; 557 558 if (buf->ops == &armada_gem_prime_dmabuf_ops) { 559 struct drm_gem_object *obj = buf->priv; 560 if (obj->dev == dev) { 561 /* 562 * Importing our own dmabuf(s) increases the 563 * refcount on the gem object itself. 564 */ 565 drm_gem_object_reference(obj); 566 return obj; 567 } 568 } 569 570 attach = dma_buf_attach(buf, dev->dev); 571 if (IS_ERR(attach)) 572 return ERR_CAST(attach); 573 574 dobj = armada_gem_alloc_private_object(dev, buf->size); 575 if (!dobj) { 576 dma_buf_detach(buf, attach); 577 return ERR_PTR(-ENOMEM); 578 } 579 580 dobj->obj.import_attach = attach; 581 get_dma_buf(buf); 582 583 /* 584 * Don't call dma_buf_map_attachment() here - it maps the 585 * scatterlist immediately for DMA, and this is not always 586 * an appropriate thing to do. 587 */ 588 return &dobj->obj; 589 } 590 591 int armada_gem_map_import(struct armada_gem_object *dobj) 592 { 593 int ret; 594 595 dobj->sgt = dma_buf_map_attachment(dobj->obj.import_attach, 596 DMA_TO_DEVICE); 597 if (!dobj->sgt) { 598 DRM_ERROR("dma_buf_map_attachment() returned NULL\n"); 599 return -EINVAL; 600 } 601 if (IS_ERR(dobj->sgt)) { 602 ret = PTR_ERR(dobj->sgt); 603 dobj->sgt = NULL; 604 DRM_ERROR("dma_buf_map_attachment() error: %d\n", ret); 605 return ret; 606 } 607 if (dobj->sgt->nents > 1) { 608 DRM_ERROR("dma_buf_map_attachment() returned an (unsupported) scattered list\n"); 609 return -EINVAL; 610 } 611 if (sg_dma_len(dobj->sgt->sgl) < dobj->obj.size) { 612 DRM_ERROR("dma_buf_map_attachment() returned a small buffer\n"); 613 return -EINVAL; 614 } 615 dobj->dev_addr = sg_dma_address(dobj->sgt->sgl); 616 return 0; 617 } 618