1 /* 2 * Copyright (C) Fuzhou Rockchip Electronics Co.Ltd 3 * Author:Mark Yao <mark.yao@rock-chips.com> 4 * 5 * This software is licensed under the terms of the GNU General Public 6 * License version 2, as published by the Free Software Foundation, and 7 * may be copied, distributed, and modified under those terms. 8 * 9 * This program is distributed in the hope that it will be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 */ 14 15 #include <drm/drm.h> 16 #include <drm/drmP.h> 17 #include <drm/drm_gem.h> 18 #include <drm/drm_vma_manager.h> 19 20 #include <linux/dma-buf.h> 21 #include <linux/iommu.h> 22 23 #include "rockchip_drm_drv.h" 24 #include "rockchip_drm_gem.h" 25 26 static int rockchip_gem_iommu_map(struct rockchip_gem_object *rk_obj) 27 { 28 struct drm_device *drm = rk_obj->base.dev; 29 struct rockchip_drm_private *private = drm->dev_private; 30 int prot = IOMMU_READ | IOMMU_WRITE; 31 ssize_t ret; 32 33 mutex_lock(&private->mm_lock); 34 ret = drm_mm_insert_node_generic(&private->mm, &rk_obj->mm, 35 rk_obj->base.size, PAGE_SIZE, 36 0, 0); 37 mutex_unlock(&private->mm_lock); 38 39 if (ret < 0) { 40 DRM_ERROR("out of I/O virtual memory: %zd\n", ret); 41 return ret; 42 } 43 44 rk_obj->dma_addr = rk_obj->mm.start; 45 46 ret = iommu_map_sg(private->domain, rk_obj->dma_addr, rk_obj->sgt->sgl, 47 rk_obj->sgt->nents, prot); 48 if (ret < rk_obj->base.size) { 49 DRM_ERROR("failed to map buffer: size=%zd request_size=%zd\n", 50 ret, rk_obj->base.size); 51 ret = -ENOMEM; 52 goto err_remove_node; 53 } 54 55 rk_obj->size = ret; 56 57 return 0; 58 59 err_remove_node: 60 mutex_lock(&private->mm_lock); 61 drm_mm_remove_node(&rk_obj->mm); 62 mutex_unlock(&private->mm_lock); 63 64 return ret; 65 } 66 67 static int rockchip_gem_iommu_unmap(struct rockchip_gem_object *rk_obj) 68 { 69 struct drm_device *drm = rk_obj->base.dev; 70 struct rockchip_drm_private *private = drm->dev_private; 71 72 iommu_unmap(private->domain, rk_obj->dma_addr, rk_obj->size); 73 74 mutex_lock(&private->mm_lock); 75 76 drm_mm_remove_node(&rk_obj->mm); 77 78 mutex_unlock(&private->mm_lock); 79 80 return 0; 81 } 82 83 static int rockchip_gem_get_pages(struct rockchip_gem_object *rk_obj) 84 { 85 struct drm_device *drm = rk_obj->base.dev; 86 int ret, i; 87 struct scatterlist *s; 88 89 rk_obj->pages = drm_gem_get_pages(&rk_obj->base); 90 if (IS_ERR(rk_obj->pages)) 91 return PTR_ERR(rk_obj->pages); 92 93 rk_obj->num_pages = rk_obj->base.size >> PAGE_SHIFT; 94 95 rk_obj->sgt = drm_prime_pages_to_sg(rk_obj->pages, rk_obj->num_pages); 96 if (IS_ERR(rk_obj->sgt)) { 97 ret = PTR_ERR(rk_obj->sgt); 98 goto err_put_pages; 99 } 100 101 /* 102 * Fake up the SG table so that dma_sync_sg_for_device() can be used 103 * to flush the pages associated with it. 104 * 105 * TODO: Replace this by drm_clflush_sg() once it can be implemented 106 * without relying on symbols that are not exported. 107 */ 108 for_each_sg(rk_obj->sgt->sgl, s, rk_obj->sgt->nents, i) 109 sg_dma_address(s) = sg_phys(s); 110 111 dma_sync_sg_for_device(drm->dev, rk_obj->sgt->sgl, rk_obj->sgt->nents, 112 DMA_TO_DEVICE); 113 114 return 0; 115 116 err_put_pages: 117 drm_gem_put_pages(&rk_obj->base, rk_obj->pages, false, false); 118 return ret; 119 } 120 121 static void rockchip_gem_put_pages(struct rockchip_gem_object *rk_obj) 122 { 123 sg_free_table(rk_obj->sgt); 124 kfree(rk_obj->sgt); 125 drm_gem_put_pages(&rk_obj->base, rk_obj->pages, true, true); 126 } 127 128 static int rockchip_gem_alloc_iommu(struct rockchip_gem_object *rk_obj, 129 bool alloc_kmap) 130 { 131 int ret; 132 133 ret = rockchip_gem_get_pages(rk_obj); 134 if (ret < 0) 135 return ret; 136 137 ret = rockchip_gem_iommu_map(rk_obj); 138 if (ret < 0) 139 goto err_free; 140 141 if (alloc_kmap) { 142 rk_obj->kvaddr = vmap(rk_obj->pages, rk_obj->num_pages, VM_MAP, 143 pgprot_writecombine(PAGE_KERNEL)); 144 if (!rk_obj->kvaddr) { 145 DRM_ERROR("failed to vmap() buffer\n"); 146 ret = -ENOMEM; 147 goto err_unmap; 148 } 149 } 150 151 return 0; 152 153 err_unmap: 154 rockchip_gem_iommu_unmap(rk_obj); 155 err_free: 156 rockchip_gem_put_pages(rk_obj); 157 158 return ret; 159 } 160 161 static int rockchip_gem_alloc_dma(struct rockchip_gem_object *rk_obj, 162 bool alloc_kmap) 163 { 164 struct drm_gem_object *obj = &rk_obj->base; 165 struct drm_device *drm = obj->dev; 166 167 rk_obj->dma_attrs = DMA_ATTR_WRITE_COMBINE; 168 169 if (!alloc_kmap) 170 rk_obj->dma_attrs |= DMA_ATTR_NO_KERNEL_MAPPING; 171 172 rk_obj->kvaddr = dma_alloc_attrs(drm->dev, obj->size, 173 &rk_obj->dma_addr, GFP_KERNEL, 174 rk_obj->dma_attrs); 175 if (!rk_obj->kvaddr) { 176 DRM_ERROR("failed to allocate %zu byte dma buffer", obj->size); 177 return -ENOMEM; 178 } 179 180 return 0; 181 } 182 183 static int rockchip_gem_alloc_buf(struct rockchip_gem_object *rk_obj, 184 bool alloc_kmap) 185 { 186 struct drm_gem_object *obj = &rk_obj->base; 187 struct drm_device *drm = obj->dev; 188 struct rockchip_drm_private *private = drm->dev_private; 189 190 if (private->domain) 191 return rockchip_gem_alloc_iommu(rk_obj, alloc_kmap); 192 else 193 return rockchip_gem_alloc_dma(rk_obj, alloc_kmap); 194 } 195 196 static void rockchip_gem_free_iommu(struct rockchip_gem_object *rk_obj) 197 { 198 vunmap(rk_obj->kvaddr); 199 rockchip_gem_iommu_unmap(rk_obj); 200 rockchip_gem_put_pages(rk_obj); 201 } 202 203 static void rockchip_gem_free_dma(struct rockchip_gem_object *rk_obj) 204 { 205 struct drm_gem_object *obj = &rk_obj->base; 206 struct drm_device *drm = obj->dev; 207 208 dma_free_attrs(drm->dev, obj->size, rk_obj->kvaddr, rk_obj->dma_addr, 209 rk_obj->dma_attrs); 210 } 211 212 static void rockchip_gem_free_buf(struct rockchip_gem_object *rk_obj) 213 { 214 if (rk_obj->pages) 215 rockchip_gem_free_iommu(rk_obj); 216 else 217 rockchip_gem_free_dma(rk_obj); 218 } 219 220 static int rockchip_drm_gem_object_mmap_iommu(struct drm_gem_object *obj, 221 struct vm_area_struct *vma) 222 { 223 struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj); 224 unsigned int count = obj->size >> PAGE_SHIFT; 225 unsigned long user_count = vma_pages(vma); 226 227 if (user_count == 0) 228 return -ENXIO; 229 230 return vm_map_pages(vma, rk_obj->pages, count); 231 } 232 233 static int rockchip_drm_gem_object_mmap_dma(struct drm_gem_object *obj, 234 struct vm_area_struct *vma) 235 { 236 struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj); 237 struct drm_device *drm = obj->dev; 238 239 return dma_mmap_attrs(drm->dev, vma, rk_obj->kvaddr, rk_obj->dma_addr, 240 obj->size, rk_obj->dma_attrs); 241 } 242 243 static int rockchip_drm_gem_object_mmap(struct drm_gem_object *obj, 244 struct vm_area_struct *vma) 245 { 246 int ret; 247 struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj); 248 249 /* 250 * We allocated a struct page table for rk_obj, so clear 251 * VM_PFNMAP flag that was set by drm_gem_mmap_obj()/drm_gem_mmap(). 252 */ 253 vma->vm_flags &= ~VM_PFNMAP; 254 255 if (rk_obj->pages) 256 ret = rockchip_drm_gem_object_mmap_iommu(obj, vma); 257 else 258 ret = rockchip_drm_gem_object_mmap_dma(obj, vma); 259 260 if (ret) 261 drm_gem_vm_close(vma); 262 263 return ret; 264 } 265 266 int rockchip_gem_mmap_buf(struct drm_gem_object *obj, 267 struct vm_area_struct *vma) 268 { 269 int ret; 270 271 ret = drm_gem_mmap_obj(obj, obj->size, vma); 272 if (ret) 273 return ret; 274 275 return rockchip_drm_gem_object_mmap(obj, vma); 276 } 277 278 /* drm driver mmap file operations */ 279 int rockchip_gem_mmap(struct file *filp, struct vm_area_struct *vma) 280 { 281 struct drm_gem_object *obj; 282 int ret; 283 284 ret = drm_gem_mmap(filp, vma); 285 if (ret) 286 return ret; 287 288 /* 289 * Set vm_pgoff (used as a fake buffer offset by DRM) to 0 and map the 290 * whole buffer from the start. 291 */ 292 vma->vm_pgoff = 0; 293 294 obj = vma->vm_private_data; 295 296 return rockchip_drm_gem_object_mmap(obj, vma); 297 } 298 299 static void rockchip_gem_release_object(struct rockchip_gem_object *rk_obj) 300 { 301 drm_gem_object_release(&rk_obj->base); 302 kfree(rk_obj); 303 } 304 305 struct rockchip_gem_object * 306 rockchip_gem_alloc_object(struct drm_device *drm, unsigned int size) 307 { 308 struct rockchip_gem_object *rk_obj; 309 struct drm_gem_object *obj; 310 311 size = round_up(size, PAGE_SIZE); 312 313 rk_obj = kzalloc(sizeof(*rk_obj), GFP_KERNEL); 314 if (!rk_obj) 315 return ERR_PTR(-ENOMEM); 316 317 obj = &rk_obj->base; 318 319 drm_gem_object_init(drm, obj, size); 320 321 return rk_obj; 322 } 323 324 struct rockchip_gem_object * 325 rockchip_gem_create_object(struct drm_device *drm, unsigned int size, 326 bool alloc_kmap) 327 { 328 struct rockchip_gem_object *rk_obj; 329 int ret; 330 331 rk_obj = rockchip_gem_alloc_object(drm, size); 332 if (IS_ERR(rk_obj)) 333 return rk_obj; 334 335 ret = rockchip_gem_alloc_buf(rk_obj, alloc_kmap); 336 if (ret) 337 goto err_free_rk_obj; 338 339 return rk_obj; 340 341 err_free_rk_obj: 342 rockchip_gem_release_object(rk_obj); 343 return ERR_PTR(ret); 344 } 345 346 /* 347 * rockchip_gem_free_object - (struct drm_driver)->gem_free_object_unlocked 348 * callback function 349 */ 350 void rockchip_gem_free_object(struct drm_gem_object *obj) 351 { 352 struct drm_device *drm = obj->dev; 353 struct rockchip_drm_private *private = drm->dev_private; 354 struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj); 355 356 if (obj->import_attach) { 357 if (private->domain) { 358 rockchip_gem_iommu_unmap(rk_obj); 359 } else { 360 dma_unmap_sg(drm->dev, rk_obj->sgt->sgl, 361 rk_obj->sgt->nents, DMA_BIDIRECTIONAL); 362 } 363 drm_prime_gem_destroy(obj, rk_obj->sgt); 364 } else { 365 rockchip_gem_free_buf(rk_obj); 366 } 367 368 rockchip_gem_release_object(rk_obj); 369 } 370 371 /* 372 * rockchip_gem_create_with_handle - allocate an object with the given 373 * size and create a gem handle on it 374 * 375 * returns a struct rockchip_gem_object* on success or ERR_PTR values 376 * on failure. 377 */ 378 static struct rockchip_gem_object * 379 rockchip_gem_create_with_handle(struct drm_file *file_priv, 380 struct drm_device *drm, unsigned int size, 381 unsigned int *handle) 382 { 383 struct rockchip_gem_object *rk_obj; 384 struct drm_gem_object *obj; 385 int ret; 386 387 rk_obj = rockchip_gem_create_object(drm, size, false); 388 if (IS_ERR(rk_obj)) 389 return ERR_CAST(rk_obj); 390 391 obj = &rk_obj->base; 392 393 /* 394 * allocate a id of idr table where the obj is registered 395 * and handle has the id what user can see. 396 */ 397 ret = drm_gem_handle_create(file_priv, obj, handle); 398 if (ret) 399 goto err_handle_create; 400 401 /* drop reference from allocate - handle holds it now. */ 402 drm_gem_object_put_unlocked(obj); 403 404 return rk_obj; 405 406 err_handle_create: 407 rockchip_gem_free_object(obj); 408 409 return ERR_PTR(ret); 410 } 411 412 /* 413 * rockchip_gem_dumb_create - (struct drm_driver)->dumb_create callback 414 * function 415 * 416 * This aligns the pitch and size arguments to the minimum required. wrap 417 * this into your own function if you need bigger alignment. 418 */ 419 int rockchip_gem_dumb_create(struct drm_file *file_priv, 420 struct drm_device *dev, 421 struct drm_mode_create_dumb *args) 422 { 423 struct rockchip_gem_object *rk_obj; 424 int min_pitch = DIV_ROUND_UP(args->width * args->bpp, 8); 425 426 /* 427 * align to 64 bytes since Mali requires it. 428 */ 429 args->pitch = ALIGN(min_pitch, 64); 430 args->size = args->pitch * args->height; 431 432 rk_obj = rockchip_gem_create_with_handle(file_priv, dev, args->size, 433 &args->handle); 434 435 return PTR_ERR_OR_ZERO(rk_obj); 436 } 437 438 /* 439 * Allocate a sg_table for this GEM object. 440 * Note: Both the table's contents, and the sg_table itself must be freed by 441 * the caller. 442 * Returns a pointer to the newly allocated sg_table, or an ERR_PTR() error. 443 */ 444 struct sg_table *rockchip_gem_prime_get_sg_table(struct drm_gem_object *obj) 445 { 446 struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj); 447 struct drm_device *drm = obj->dev; 448 struct sg_table *sgt; 449 int ret; 450 451 if (rk_obj->pages) 452 return drm_prime_pages_to_sg(rk_obj->pages, rk_obj->num_pages); 453 454 sgt = kzalloc(sizeof(*sgt), GFP_KERNEL); 455 if (!sgt) 456 return ERR_PTR(-ENOMEM); 457 458 ret = dma_get_sgtable_attrs(drm->dev, sgt, rk_obj->kvaddr, 459 rk_obj->dma_addr, obj->size, 460 rk_obj->dma_attrs); 461 if (ret) { 462 DRM_ERROR("failed to allocate sgt, %d\n", ret); 463 kfree(sgt); 464 return ERR_PTR(ret); 465 } 466 467 return sgt; 468 } 469 470 static unsigned long rockchip_sg_get_contiguous_size(struct sg_table *sgt, 471 int count) 472 { 473 struct scatterlist *s; 474 dma_addr_t expected = sg_dma_address(sgt->sgl); 475 unsigned int i; 476 unsigned long size = 0; 477 478 for_each_sg(sgt->sgl, s, count, i) { 479 if (sg_dma_address(s) != expected) 480 break; 481 expected = sg_dma_address(s) + sg_dma_len(s); 482 size += sg_dma_len(s); 483 } 484 return size; 485 } 486 487 static int 488 rockchip_gem_iommu_map_sg(struct drm_device *drm, 489 struct dma_buf_attachment *attach, 490 struct sg_table *sg, 491 struct rockchip_gem_object *rk_obj) 492 { 493 rk_obj->sgt = sg; 494 return rockchip_gem_iommu_map(rk_obj); 495 } 496 497 static int 498 rockchip_gem_dma_map_sg(struct drm_device *drm, 499 struct dma_buf_attachment *attach, 500 struct sg_table *sg, 501 struct rockchip_gem_object *rk_obj) 502 { 503 int count = dma_map_sg(drm->dev, sg->sgl, sg->nents, 504 DMA_BIDIRECTIONAL); 505 if (!count) 506 return -EINVAL; 507 508 if (rockchip_sg_get_contiguous_size(sg, count) < attach->dmabuf->size) { 509 DRM_ERROR("failed to map sg_table to contiguous linear address.\n"); 510 dma_unmap_sg(drm->dev, sg->sgl, sg->nents, 511 DMA_BIDIRECTIONAL); 512 return -EINVAL; 513 } 514 515 rk_obj->dma_addr = sg_dma_address(sg->sgl); 516 rk_obj->sgt = sg; 517 return 0; 518 } 519 520 struct drm_gem_object * 521 rockchip_gem_prime_import_sg_table(struct drm_device *drm, 522 struct dma_buf_attachment *attach, 523 struct sg_table *sg) 524 { 525 struct rockchip_drm_private *private = drm->dev_private; 526 struct rockchip_gem_object *rk_obj; 527 int ret; 528 529 rk_obj = rockchip_gem_alloc_object(drm, attach->dmabuf->size); 530 if (IS_ERR(rk_obj)) 531 return ERR_CAST(rk_obj); 532 533 if (private->domain) 534 ret = rockchip_gem_iommu_map_sg(drm, attach, sg, rk_obj); 535 else 536 ret = rockchip_gem_dma_map_sg(drm, attach, sg, rk_obj); 537 538 if (ret < 0) { 539 DRM_ERROR("failed to import sg table: %d\n", ret); 540 goto err_free_rk_obj; 541 } 542 543 return &rk_obj->base; 544 545 err_free_rk_obj: 546 rockchip_gem_release_object(rk_obj); 547 return ERR_PTR(ret); 548 } 549 550 void *rockchip_gem_prime_vmap(struct drm_gem_object *obj) 551 { 552 struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj); 553 554 if (rk_obj->pages) 555 return vmap(rk_obj->pages, rk_obj->num_pages, VM_MAP, 556 pgprot_writecombine(PAGE_KERNEL)); 557 558 if (rk_obj->dma_attrs & DMA_ATTR_NO_KERNEL_MAPPING) 559 return NULL; 560 561 return rk_obj->kvaddr; 562 } 563 564 void rockchip_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr) 565 { 566 struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj); 567 568 if (rk_obj->pages) { 569 vunmap(vaddr); 570 return; 571 } 572 573 /* Nothing to do if allocated by DMA mapping API. */ 574 } 575