47 	struct dma_buf *dmabuf;  in dmabuffs_dname()  local
51 	dmabuf = dentry->d_fsdata;  in dmabuffs_dname()
52 	spin_lock(&dmabuf->name_lock);  in dmabuffs_dname()
53 	if (dmabuf->name)  in dmabuffs_dname()
54 		ret = strlcpy(name, dmabuf->name, DMA_BUF_NAME_LEN);  in dmabuffs_dname()
55 	spin_unlock(&dmabuf->name_lock);  in dmabuffs_dname()
63 	struct dma_buf *dmabuf;  in dma_buf_release()  local
65 	dmabuf = dentry->d_fsdata;  in dma_buf_release()
66 	if (unlikely(!dmabuf))  in dma_buf_release()
69 	BUG_ON(dmabuf->vmapping_counter);  in dma_buf_release()
76 	BUG_ON(dmabuf->cb_in.active || dmabuf->cb_out.active);  in dma_buf_release()
78 	dma_buf_stats_teardown(dmabuf);  in dma_buf_release()
79 	dmabuf->ops->release(dmabuf);  in dma_buf_release()
81 	if (dmabuf->resv == (struct dma_resv *)&dmabuf[1])  in dma_buf_release()
82 		dma_resv_fini(dmabuf->resv);  in dma_buf_release()
84 	WARN_ON(!list_empty(&dmabuf->attachments));  in dma_buf_release()
85 	module_put(dmabuf->owner);  in dma_buf_release()
86 	kfree(dmabuf->name);  in dma_buf_release()
87 	kfree(dmabuf);  in dma_buf_release()
92 	struct dma_buf *dmabuf;  in dma_buf_file_release()  local
97 	dmabuf = file->private_data;  in dma_buf_file_release()
98 	if (dmabuf) {  in dma_buf_file_release()
100 		list_del(&dmabuf->list_node);  in dma_buf_file_release()
133 	struct dma_buf *dmabuf;  in dma_buf_mmap_internal()  local
138 	dmabuf = file->private_data;  in dma_buf_mmap_internal()
141 	if (!dmabuf->ops->mmap)  in dma_buf_mmap_internal()
146 	    dmabuf->size >> PAGE_SHIFT)  in dma_buf_mmap_internal()
149 	return dmabuf->ops->mmap(dmabuf, vma);  in dma_buf_mmap_internal()
154 	struct dma_buf *dmabuf;  in dma_buf_llseek()  local
160 	dmabuf = file->private_data;  in dma_buf_llseek()
166 		base = dmabuf->size;  in dma_buf_llseek()
206 	struct dma_buf *dmabuf = container_of(dcb->poll, struct dma_buf, poll);  in dma_buf_poll_cb()  local
215 	fput(dmabuf->file);  in dma_buf_poll_cb()
239 	struct dma_buf *dmabuf;  in dma_buf_poll()  local
243 	dmabuf = file->private_data;  in dma_buf_poll()
244 	if (!dmabuf || !dmabuf->resv)  in dma_buf_poll()
247 	resv = dmabuf->resv;  in dma_buf_poll()
249 	poll_wait(file, &dmabuf->poll, poll);  in dma_buf_poll()
258 		struct dma_buf_poll_cb_t *dcb = &dmabuf->cb_out;  in dma_buf_poll()
261 		spin_lock_irq(&dmabuf->poll.lock);  in dma_buf_poll()
266 		spin_unlock_irq(&dmabuf->poll.lock);  in dma_buf_poll()
270 			get_file(dmabuf->file);  in dma_buf_poll()
281 		struct dma_buf_poll_cb_t *dcb = &dmabuf->cb_in;  in dma_buf_poll()
284 		spin_lock_irq(&dmabuf->poll.lock);  in dma_buf_poll()
289 		spin_unlock_irq(&dmabuf->poll.lock);  in dma_buf_poll()
293 			get_file(dmabuf->file);  in dma_buf_poll()
320 static long dma_buf_set_name(struct dma_buf *dmabuf, const char __user *buf)  in dma_buf_set_name()  argument
327 	spin_lock(&dmabuf->name_lock);  in dma_buf_set_name()
328 	kfree(dmabuf->name);  in dma_buf_set_name()
329 	dmabuf->name = name;  in dma_buf_set_name()
330 	spin_unlock(&dmabuf->name_lock);  in dma_buf_set_name()
336 static long dma_buf_export_sync_file(struct dma_buf *dmabuf,  in dma_buf_export_sync_file()  argument
359 	ret = dma_resv_get_singleton(dmabuf->resv, usage, &fence);  in dma_buf_export_sync_file()
392 static long dma_buf_import_sync_file(struct dma_buf *dmabuf,  in dma_buf_import_sync_file()  argument
423 		dma_resv_lock(dmabuf->resv, NULL);  in dma_buf_import_sync_file()
425 		ret = dma_resv_reserve_fences(dmabuf->resv, num_fences);  in dma_buf_import_sync_file()
428 				dma_resv_add_fence(dmabuf->resv, f, usage);  in dma_buf_import_sync_file()
431 		dma_resv_unlock(dmabuf->resv);  in dma_buf_import_sync_file()
443 	struct dma_buf *dmabuf;  in dma_buf_ioctl()  local
448 	dmabuf = file->private_data;  in dma_buf_ioctl()
473 			ret = dma_buf_end_cpu_access(dmabuf, direction);  in dma_buf_ioctl()
475 			ret = dma_buf_begin_cpu_access(dmabuf, direction);  in dma_buf_ioctl()
481 		return dma_buf_set_name(dmabuf, (const char __user *)arg);  in dma_buf_ioctl()
485 		return dma_buf_export_sync_file(dmabuf, (void __user *)arg);  in dma_buf_ioctl()
487 		return dma_buf_import_sync_file(dmabuf, (const void __user *)arg);  in dma_buf_ioctl()
497 	struct dma_buf *dmabuf = file->private_data;  in dma_buf_show_fdinfo()  local
499 	seq_printf(m, "size:\t%zu\n", dmabuf->size);  in dma_buf_show_fdinfo()
501 	seq_printf(m, "count:\t%ld\n", file_count(dmabuf->file) - 1);  in dma_buf_show_fdinfo()
502 	seq_printf(m, "exp_name:\t%s\n", dmabuf->exp_name);  in dma_buf_show_fdinfo()
503 	spin_lock(&dmabuf->name_lock);  in dma_buf_show_fdinfo()
504 	if (dmabuf->name)  in dma_buf_show_fdinfo()
505 		seq_printf(m, "name:\t%s\n", dmabuf->name);  in dma_buf_show_fdinfo()
506 	spin_unlock(&dmabuf->name_lock);  in dma_buf_show_fdinfo()
610 	struct dma_buf *dmabuf;  in dma_buf_export()  local
643 	dmabuf = kzalloc(alloc_size, GFP_KERNEL);  in dma_buf_export()
644 	if (!dmabuf) {  in dma_buf_export()
649 	dmabuf->priv = exp_info->priv;  in dma_buf_export()
650 	dmabuf->ops = exp_info->ops;  in dma_buf_export()
651 	dmabuf->size = exp_info->size;  in dma_buf_export()
652 	dmabuf->exp_name = exp_info->exp_name;  in dma_buf_export()
653 	dmabuf->owner = exp_info->owner;  in dma_buf_export()
654 	spin_lock_init(&dmabuf->name_lock);  in dma_buf_export()
655 	init_waitqueue_head(&dmabuf->poll);  in dma_buf_export()
656 	dmabuf->cb_in.poll = dmabuf->cb_out.poll = &dmabuf->poll;  in dma_buf_export()
657 	dmabuf->cb_in.active = dmabuf->cb_out.active = 0;  in dma_buf_export()
658 	INIT_LIST_HEAD(&dmabuf->attachments);  in dma_buf_export()
661 		dmabuf->resv = (struct dma_resv *)&dmabuf[1];  in dma_buf_export()
662 		dma_resv_init(dmabuf->resv);  in dma_buf_export()
664 		dmabuf->resv = resv;  in dma_buf_export()
667 	ret = dma_buf_stats_setup(dmabuf, file);  in dma_buf_export()
671 	file->private_data = dmabuf;  in dma_buf_export()
672 	file->f_path.dentry->d_fsdata = dmabuf;  in dma_buf_export()
673 	dmabuf->file = file;  in dma_buf_export()
676 	list_add(&dmabuf->list_node, &db_list.head);  in dma_buf_export()
679 	return dmabuf;  in dma_buf_export()
683 		dma_resv_fini(dmabuf->resv);  in dma_buf_export()
684 	kfree(dmabuf);  in dma_buf_export()
700 int dma_buf_fd(struct dma_buf *dmabuf, int flags)  in dma_buf_fd()  argument
704 	if (!dmabuf || !dmabuf->file)  in dma_buf_fd()
711 	fd_install(fd, dmabuf->file);  in dma_buf_fd()
753 void dma_buf_put(struct dma_buf *dmabuf)  in dma_buf_put()  argument
755 	if (WARN_ON(!dmabuf || !dmabuf->file))  in dma_buf_put()
758 	fput(dmabuf->file);  in dma_buf_put()
783 	sg_table = attach->dmabuf->ops->map_dma_buf(attach, direction);  in __map_dma_buf()
788 		ret = dma_resv_wait_timeout(attach->dmabuf->resv,  in __map_dma_buf()
792 			attach->dmabuf->ops->unmap_dma_buf(attach, sg_table,  in __map_dma_buf()
889 dma_buf_dynamic_attach(struct dma_buf *dmabuf, struct device *dev,  in dma_buf_dynamic_attach()  argument
896 	if (WARN_ON(!dmabuf || !dev))  in dma_buf_dynamic_attach()
907 	attach->dmabuf = dmabuf;  in dma_buf_dynamic_attach()
913 	if (dmabuf->ops->attach) {  in dma_buf_dynamic_attach()
914 		ret = dmabuf->ops->attach(dmabuf, attach);  in dma_buf_dynamic_attach()
918 	dma_resv_lock(dmabuf->resv, NULL);  in dma_buf_dynamic_attach()
919 	list_add(&attach->node, &dmabuf->attachments);  in dma_buf_dynamic_attach()
920 	dma_resv_unlock(dmabuf->resv);  in dma_buf_dynamic_attach()
927 	    dma_buf_is_dynamic(dmabuf)) {  in dma_buf_dynamic_attach()
930 		dma_resv_lock(attach->dmabuf->resv, NULL);  in dma_buf_dynamic_attach()
931 		if (dma_buf_is_dynamic(attach->dmabuf)) {  in dma_buf_dynamic_attach()
932 			ret = dmabuf->ops->pin(attach);  in dma_buf_dynamic_attach()
944 		dma_resv_unlock(attach->dmabuf->resv);  in dma_buf_dynamic_attach()
956 	if (dma_buf_is_dynamic(attach->dmabuf))  in dma_buf_dynamic_attach()
957 		dmabuf->ops->unpin(attach);  in dma_buf_dynamic_attach()
960 	dma_resv_unlock(attach->dmabuf->resv);  in dma_buf_dynamic_attach()
962 	dma_buf_detach(dmabuf, attach);  in dma_buf_dynamic_attach()
975 struct dma_buf_attachment *dma_buf_attach(struct dma_buf *dmabuf,  in dma_buf_attach()  argument
978 	return dma_buf_dynamic_attach(dmabuf, dev, NULL, NULL);  in dma_buf_attach()
989 	attach->dmabuf->ops->unmap_dma_buf(attach, sg_table, direction);  in __unmap_dma_buf()
1001 void dma_buf_detach(struct dma_buf *dmabuf, struct dma_buf_attachment *attach)  in dma_buf_detach()  argument
1003 	if (WARN_ON(!dmabuf || !attach || dmabuf != attach->dmabuf))  in dma_buf_detach()
1006 	dma_resv_lock(dmabuf->resv, NULL);  in dma_buf_detach()
1012 		if (dma_buf_is_dynamic(attach->dmabuf))  in dma_buf_detach()
1013 			dmabuf->ops->unpin(attach);  in dma_buf_detach()
1017 	dma_resv_unlock(dmabuf->resv);  in dma_buf_detach()
1019 	if (dmabuf->ops->detach)  in dma_buf_detach()
1020 		dmabuf->ops->detach(dmabuf, attach);  in dma_buf_detach()
1042 	struct dma_buf *dmabuf = attach->dmabuf;  in dma_buf_pin()  local
1047 	dma_resv_assert_held(dmabuf->resv);  in dma_buf_pin()
1049 	if (dmabuf->ops->pin)  in dma_buf_pin()
1050 		ret = dmabuf->ops->pin(attach);  in dma_buf_pin()
1066 	struct dma_buf *dmabuf = attach->dmabuf;  in dma_buf_unpin()  local
1070 	dma_resv_assert_held(dmabuf->resv);  in dma_buf_unpin()
1072 	if (dmabuf->ops->unpin)  in dma_buf_unpin()
1073 		dmabuf->ops->unpin(attach);  in dma_buf_unpin()
1106 	if (WARN_ON(!attach || !attach->dmabuf))  in dma_buf_map_attachment()
1109 	dma_resv_assert_held(attach->dmabuf->resv);  in dma_buf_map_attachment()
1123 	if (dma_buf_is_dynamic(attach->dmabuf)) {  in dma_buf_map_attachment()
1125 			r = attach->dmabuf->ops->pin(attach);  in dma_buf_map_attachment()
1135 	if (IS_ERR(sg_table) && dma_buf_is_dynamic(attach->dmabuf) &&  in dma_buf_map_attachment()
1137 		attach->dmabuf->ops->unpin(attach);  in dma_buf_map_attachment()
1139 	if (!IS_ERR(sg_table) && attach->dmabuf->ops->cache_sgt_mapping) {  in dma_buf_map_attachment()
1182 	if (WARN_ON(!attach || !attach->dmabuf))  in dma_buf_map_attachment_unlocked()
1185 	dma_resv_lock(attach->dmabuf->resv, NULL);  in dma_buf_map_attachment_unlocked()
1187 	dma_resv_unlock(attach->dmabuf->resv);  in dma_buf_map_attachment_unlocked()
1209 	if (WARN_ON(!attach || !attach->dmabuf || !sg_table))  in dma_buf_unmap_attachment()
1212 	dma_resv_assert_held(attach->dmabuf->resv);  in dma_buf_unmap_attachment()
1219 	if (dma_buf_is_dynamic(attach->dmabuf) &&  in dma_buf_unmap_attachment()
1241 	if (WARN_ON(!attach || !attach->dmabuf || !sg_table))  in dma_buf_unmap_attachment_unlocked()
1244 	dma_resv_lock(attach->dmabuf->resv, NULL);  in dma_buf_unmap_attachment_unlocked()
1246 	dma_resv_unlock(attach->dmabuf->resv);  in dma_buf_unmap_attachment_unlocked()
1258 void dma_buf_move_notify(struct dma_buf *dmabuf)  in dma_buf_move_notify()  argument
1262 	dma_resv_assert_held(dmabuf->resv);  in dma_buf_move_notify()
1264 	list_for_each_entry(attach, &dmabuf->attachments, node)  in dma_buf_move_notify()
1355 static int __dma_buf_begin_cpu_access(struct dma_buf *dmabuf,  in __dma_buf_begin_cpu_access()  argument
1360 	struct dma_resv *resv = dmabuf->resv;  in __dma_buf_begin_cpu_access()
1391 int dma_buf_begin_cpu_access(struct dma_buf *dmabuf,  in dma_buf_begin_cpu_access()  argument
1396 	if (WARN_ON(!dmabuf))  in dma_buf_begin_cpu_access()
1399 	might_lock(&dmabuf->resv->lock.base);  in dma_buf_begin_cpu_access()
1401 	if (dmabuf->ops->begin_cpu_access)  in dma_buf_begin_cpu_access()
1402 		ret = dmabuf->ops->begin_cpu_access(dmabuf, direction);  in dma_buf_begin_cpu_access()
1409 		ret = __dma_buf_begin_cpu_access(dmabuf, direction);  in dma_buf_begin_cpu_access()
1427 int dma_buf_end_cpu_access(struct dma_buf *dmabuf,  in dma_buf_end_cpu_access()  argument
1432 	WARN_ON(!dmabuf);  in dma_buf_end_cpu_access()
1434 	might_lock(&dmabuf->resv->lock.base);  in dma_buf_end_cpu_access()
1436 	if (dmabuf->ops->end_cpu_access)  in dma_buf_end_cpu_access()
1437 		ret = dmabuf->ops->end_cpu_access(dmabuf, direction);  in dma_buf_end_cpu_access()
1458 int dma_buf_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma,  in dma_buf_mmap()  argument
1461 	if (WARN_ON(!dmabuf || !vma))  in dma_buf_mmap()
1465 	if (!dmabuf->ops->mmap)  in dma_buf_mmap()
1474 	    dmabuf->size >> PAGE_SHIFT)  in dma_buf_mmap()
1478 	vma_set_file(vma, dmabuf->file);  in dma_buf_mmap()
1481 	return dmabuf->ops->mmap(dmabuf, vma);  in dma_buf_mmap()
1501 int dma_buf_vmap(struct dma_buf *dmabuf, struct iosys_map *map)  in dma_buf_vmap()  argument
1508 	if (WARN_ON(!dmabuf))  in dma_buf_vmap()
1511 	dma_resv_assert_held(dmabuf->resv);  in dma_buf_vmap()
1513 	if (!dmabuf->ops->vmap)  in dma_buf_vmap()
1516 	if (dmabuf->vmapping_counter) {  in dma_buf_vmap()
1517 		dmabuf->vmapping_counter++;  in dma_buf_vmap()
1518 		BUG_ON(iosys_map_is_null(&dmabuf->vmap_ptr));  in dma_buf_vmap()
1519 		*map = dmabuf->vmap_ptr;  in dma_buf_vmap()
1523 	BUG_ON(iosys_map_is_set(&dmabuf->vmap_ptr));  in dma_buf_vmap()
1525 	ret = dmabuf->ops->vmap(dmabuf, &ptr);  in dma_buf_vmap()
1529 	dmabuf->vmap_ptr = ptr;  in dma_buf_vmap()
1530 	dmabuf->vmapping_counter = 1;  in dma_buf_vmap()
1532 	*map = dmabuf->vmap_ptr;  in dma_buf_vmap()
1548 int dma_buf_vmap_unlocked(struct dma_buf *dmabuf, struct iosys_map *map)  in dma_buf_vmap_unlocked()  argument
1554 	if (WARN_ON(!dmabuf))  in dma_buf_vmap_unlocked()
1557 	dma_resv_lock(dmabuf->resv, NULL);  in dma_buf_vmap_unlocked()
1558 	ret = dma_buf_vmap(dmabuf, map);  in dma_buf_vmap_unlocked()
1559 	dma_resv_unlock(dmabuf->resv);  in dma_buf_vmap_unlocked()
1570 void dma_buf_vunmap(struct dma_buf *dmabuf, struct iosys_map *map)  in dma_buf_vunmap()  argument
1572 	if (WARN_ON(!dmabuf))  in dma_buf_vunmap()
1575 	dma_resv_assert_held(dmabuf->resv);  in dma_buf_vunmap()
1577 	BUG_ON(iosys_map_is_null(&dmabuf->vmap_ptr));  in dma_buf_vunmap()
1578 	BUG_ON(dmabuf->vmapping_counter == 0);  in dma_buf_vunmap()
1579 	BUG_ON(!iosys_map_is_equal(&dmabuf->vmap_ptr, map));  in dma_buf_vunmap()
1581 	if (--dmabuf->vmapping_counter == 0) {  in dma_buf_vunmap()
1582 		if (dmabuf->ops->vunmap)  in dma_buf_vunmap()
1583 			dmabuf->ops->vunmap(dmabuf, map);  in dma_buf_vunmap()
1584 		iosys_map_clear(&dmabuf->vmap_ptr);  in dma_buf_vunmap()
1594 void dma_buf_vunmap_unlocked(struct dma_buf *dmabuf, struct iosys_map *map)  in dma_buf_vunmap_unlocked()  argument
1596 	if (WARN_ON(!dmabuf))  in dma_buf_vunmap_unlocked()
1599 	dma_resv_lock(dmabuf->resv, NULL);  in dma_buf_vunmap_unlocked()
1600 	dma_buf_vunmap(dmabuf, map);  in dma_buf_vunmap_unlocked()
1601 	dma_resv_unlock(dmabuf->resv);  in dma_buf_vunmap_unlocked()