10b8762e9SThomas Hellstrom /* SPDX-License-Identifier: GPL-2.0 OR MIT */
20b8762e9SThomas Hellstrom /**************************************************************************
30b8762e9SThomas Hellstrom *
4931e09d8SMaaz Mombasawala * Copyright (c) 2009-2022 VMware, Inc., Palo Alto, CA., USA
50b8762e9SThomas Hellstrom * All Rights Reserved.
60b8762e9SThomas Hellstrom *
70b8762e9SThomas Hellstrom * Permission is hereby granted, free of charge, to any person obtaining a
80b8762e9SThomas Hellstrom * copy of this software and associated documentation files (the
90b8762e9SThomas Hellstrom * "Software"), to deal in the Software without restriction, including
100b8762e9SThomas Hellstrom * without limitation the rights to use, copy, modify, merge, publish,
110b8762e9SThomas Hellstrom * distribute, sub license, and/or sell copies of the Software, and to
120b8762e9SThomas Hellstrom * permit persons to whom the Software is furnished to do so, subject to
130b8762e9SThomas Hellstrom * the following conditions:
140b8762e9SThomas Hellstrom *
150b8762e9SThomas Hellstrom * The above copyright notice and this permission notice (including the
160b8762e9SThomas Hellstrom * next paragraph) shall be included in all copies or substantial portions
170b8762e9SThomas Hellstrom * of the Software.
180b8762e9SThomas Hellstrom *
190b8762e9SThomas Hellstrom * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
200b8762e9SThomas Hellstrom * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
210b8762e9SThomas Hellstrom * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
220b8762e9SThomas Hellstrom * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
230b8762e9SThomas Hellstrom * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
240b8762e9SThomas Hellstrom * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
250b8762e9SThomas Hellstrom * USE OR OTHER DEALINGS IN THE SOFTWARE.
260b8762e9SThomas Hellstrom *
270b8762e9SThomas Hellstrom **************************************************************************/
280b8762e9SThomas Hellstrom /*
290b8762e9SThomas Hellstrom * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
300b8762e9SThomas Hellstrom *
310b8762e9SThomas Hellstrom * While no substantial code is shared, the prime code is inspired by
320b8762e9SThomas Hellstrom * drm_prime.c, with
330b8762e9SThomas Hellstrom * Authors:
340b8762e9SThomas Hellstrom * Dave Airlie <airlied@redhat.com>
350b8762e9SThomas Hellstrom * Rob Clark <rob.clark@linaro.org>
360b8762e9SThomas Hellstrom */
370b8762e9SThomas Hellstrom /** @file ttm_ref_object.c
380b8762e9SThomas Hellstrom *
390b8762e9SThomas Hellstrom * Base- and reference object implementation for the various
400b8762e9SThomas Hellstrom * ttm objects. Implements reference counting, minimal security checks
410b8762e9SThomas Hellstrom * and release on file close.
420b8762e9SThomas Hellstrom */
430b8762e9SThomas Hellstrom
440b8762e9SThomas Hellstrom
45d1441394SLee Jones #define pr_fmt(fmt) "[TTM] " fmt
46d1441394SLee Jones
47931e09d8SMaaz Mombasawala #include "ttm_object.h"
48931e09d8SMaaz Mombasawala #include "vmwgfx_drv.h"
49931e09d8SMaaz Mombasawala
50d1441394SLee Jones #include <linux/list.h>
51d1441394SLee Jones #include <linux/spinlock.h>
52d1441394SLee Jones #include <linux/slab.h>
53d1441394SLee Jones #include <linux/atomic.h>
5416b0314aSGreg Kroah-Hartman #include <linux/module.h>
5576a9e07fSMaaz Mombasawala #include <linux/hashtable.h>
56d1441394SLee Jones
5716b0314aSGreg Kroah-Hartman MODULE_IMPORT_NS(DMA_BUF);
5816b0314aSGreg Kroah-Hartman
5976a9e07fSMaaz Mombasawala #define VMW_TTM_OBJECT_REF_HT_ORDER 10
6076a9e07fSMaaz Mombasawala
610b8762e9SThomas Hellstrom /**
620b8762e9SThomas Hellstrom * struct ttm_object_file
630b8762e9SThomas Hellstrom *
640b8762e9SThomas Hellstrom * @tdev: Pointer to the ttm_object_device.
650b8762e9SThomas Hellstrom *
660b8762e9SThomas Hellstrom * @lock: Lock that protects the ref_list list and the
670b8762e9SThomas Hellstrom * ref_hash hash tables.
680b8762e9SThomas Hellstrom *
690b8762e9SThomas Hellstrom * @ref_list: List of ttm_ref_objects to be destroyed at
700b8762e9SThomas Hellstrom * file release.
710b8762e9SThomas Hellstrom *
720b8762e9SThomas Hellstrom * @ref_hash: Hash tables of ref objects, one per ttm_ref_type,
730b8762e9SThomas Hellstrom * for fast lookup of ref objects given a base object.
74d1441394SLee Jones *
75d1441394SLee Jones * @refcount: reference/usage count
760b8762e9SThomas Hellstrom */
770b8762e9SThomas Hellstrom struct ttm_object_file {
780b8762e9SThomas Hellstrom struct ttm_object_device *tdev;
790b8762e9SThomas Hellstrom spinlock_t lock;
800b8762e9SThomas Hellstrom struct list_head ref_list;
8176a9e07fSMaaz Mombasawala DECLARE_HASHTABLE(ref_hash, VMW_TTM_OBJECT_REF_HT_ORDER);
820b8762e9SThomas Hellstrom struct kref refcount;
830b8762e9SThomas Hellstrom };
840b8762e9SThomas Hellstrom
85f322f32aSLee Jones /*
860b8762e9SThomas Hellstrom * struct ttm_object_device
870b8762e9SThomas Hellstrom *
88931e09d8SMaaz Mombasawala * @object_lock: lock that protects idr.
890b8762e9SThomas Hellstrom *
900b8762e9SThomas Hellstrom * @object_count: Per device object count.
910b8762e9SThomas Hellstrom *
920b8762e9SThomas Hellstrom * This is the per-device data structure needed for ttm object management.
930b8762e9SThomas Hellstrom */
940b8762e9SThomas Hellstrom
950b8762e9SThomas Hellstrom struct ttm_object_device {
960b8762e9SThomas Hellstrom spinlock_t object_lock;
970b8762e9SThomas Hellstrom atomic_t object_count;
980b8762e9SThomas Hellstrom struct dma_buf_ops ops;
990b8762e9SThomas Hellstrom void (*dmabuf_release)(struct dma_buf *dma_buf);
100c7eae626SThomas Hellstrom struct idr idr;
1010b8762e9SThomas Hellstrom };
1020b8762e9SThomas Hellstrom
103f322f32aSLee Jones /*
1040b8762e9SThomas Hellstrom * struct ttm_ref_object
1050b8762e9SThomas Hellstrom *
1060b8762e9SThomas Hellstrom * @hash: Hash entry for the per-file object reference hash.
1070b8762e9SThomas Hellstrom *
1080b8762e9SThomas Hellstrom * @head: List entry for the per-file list of ref-objects.
1090b8762e9SThomas Hellstrom *
1100b8762e9SThomas Hellstrom * @kref: Ref count.
1110b8762e9SThomas Hellstrom *
1120b8762e9SThomas Hellstrom * @obj: Base object this ref object is referencing.
1130b8762e9SThomas Hellstrom *
1140b8762e9SThomas Hellstrom * @ref_type: Type of ref object.
1150b8762e9SThomas Hellstrom *
1160b8762e9SThomas Hellstrom * This is similar to an idr object, but it also has a hash table entry
1170b8762e9SThomas Hellstrom * that allows lookup with a pointer to the referenced object as a key. In
1180b8762e9SThomas Hellstrom * that way, one can easily detect whether a base object is referenced by
1190b8762e9SThomas Hellstrom * a particular ttm_object_file. It also carries a ref count to avoid creating
1200b8762e9SThomas Hellstrom * multiple ref objects if a ttm_object_file references the same base
1210b8762e9SThomas Hellstrom * object more than once.
1220b8762e9SThomas Hellstrom */
1230b8762e9SThomas Hellstrom
1240b8762e9SThomas Hellstrom struct ttm_ref_object {
1250b8762e9SThomas Hellstrom struct rcu_head rcu_head;
1262985c964SThomas Zimmermann struct vmwgfx_hash_item hash;
1270b8762e9SThomas Hellstrom struct list_head head;
1280b8762e9SThomas Hellstrom struct kref kref;
1290b8762e9SThomas Hellstrom struct ttm_base_object *obj;
1300b8762e9SThomas Hellstrom struct ttm_object_file *tfile;
1310b8762e9SThomas Hellstrom };
1320b8762e9SThomas Hellstrom
1330b8762e9SThomas Hellstrom static void ttm_prime_dmabuf_release(struct dma_buf *dma_buf);
1340b8762e9SThomas Hellstrom
1350b8762e9SThomas Hellstrom static inline struct ttm_object_file *
ttm_object_file_ref(struct ttm_object_file * tfile)1360b8762e9SThomas Hellstrom ttm_object_file_ref(struct ttm_object_file *tfile)
1370b8762e9SThomas Hellstrom {
1380b8762e9SThomas Hellstrom kref_get(&tfile->refcount);
1390b8762e9SThomas Hellstrom return tfile;
1400b8762e9SThomas Hellstrom }
1410b8762e9SThomas Hellstrom
ttm_tfile_find_ref_rcu(struct ttm_object_file * tfile,uint64_t key,struct vmwgfx_hash_item ** p_hash)14276a9e07fSMaaz Mombasawala static int ttm_tfile_find_ref_rcu(struct ttm_object_file *tfile,
14376a9e07fSMaaz Mombasawala uint64_t key,
14476a9e07fSMaaz Mombasawala struct vmwgfx_hash_item **p_hash)
14576a9e07fSMaaz Mombasawala {
14676a9e07fSMaaz Mombasawala struct vmwgfx_hash_item *hash;
14776a9e07fSMaaz Mombasawala
14876a9e07fSMaaz Mombasawala hash_for_each_possible_rcu(tfile->ref_hash, hash, head, key) {
14976a9e07fSMaaz Mombasawala if (hash->key == key) {
15076a9e07fSMaaz Mombasawala *p_hash = hash;
15176a9e07fSMaaz Mombasawala return 0;
15276a9e07fSMaaz Mombasawala }
15376a9e07fSMaaz Mombasawala }
15476a9e07fSMaaz Mombasawala return -EINVAL;
15576a9e07fSMaaz Mombasawala }
15676a9e07fSMaaz Mombasawala
ttm_tfile_find_ref(struct ttm_object_file * tfile,uint64_t key,struct vmwgfx_hash_item ** p_hash)15776a9e07fSMaaz Mombasawala static int ttm_tfile_find_ref(struct ttm_object_file *tfile,
15876a9e07fSMaaz Mombasawala uint64_t key,
15976a9e07fSMaaz Mombasawala struct vmwgfx_hash_item **p_hash)
16076a9e07fSMaaz Mombasawala {
16176a9e07fSMaaz Mombasawala struct vmwgfx_hash_item *hash;
16276a9e07fSMaaz Mombasawala
16376a9e07fSMaaz Mombasawala hash_for_each_possible(tfile->ref_hash, hash, head, key) {
16476a9e07fSMaaz Mombasawala if (hash->key == key) {
16576a9e07fSMaaz Mombasawala *p_hash = hash;
16676a9e07fSMaaz Mombasawala return 0;
16776a9e07fSMaaz Mombasawala }
16876a9e07fSMaaz Mombasawala }
16976a9e07fSMaaz Mombasawala return -EINVAL;
17076a9e07fSMaaz Mombasawala }
17176a9e07fSMaaz Mombasawala
ttm_object_file_destroy(struct kref * kref)1720b8762e9SThomas Hellstrom static void ttm_object_file_destroy(struct kref *kref)
1730b8762e9SThomas Hellstrom {
1740b8762e9SThomas Hellstrom struct ttm_object_file *tfile =
1750b8762e9SThomas Hellstrom container_of(kref, struct ttm_object_file, refcount);
1760b8762e9SThomas Hellstrom
1770b8762e9SThomas Hellstrom kfree(tfile);
1780b8762e9SThomas Hellstrom }
1790b8762e9SThomas Hellstrom
1800b8762e9SThomas Hellstrom
ttm_object_file_unref(struct ttm_object_file ** p_tfile)1810b8762e9SThomas Hellstrom static inline void ttm_object_file_unref(struct ttm_object_file **p_tfile)
1820b8762e9SThomas Hellstrom {
1830b8762e9SThomas Hellstrom struct ttm_object_file *tfile = *p_tfile;
1840b8762e9SThomas Hellstrom
1850b8762e9SThomas Hellstrom *p_tfile = NULL;
1860b8762e9SThomas Hellstrom kref_put(&tfile->refcount, ttm_object_file_destroy);
1870b8762e9SThomas Hellstrom }
1880b8762e9SThomas Hellstrom
1890b8762e9SThomas Hellstrom
ttm_base_object_init(struct ttm_object_file * tfile,struct ttm_base_object * base,bool shareable,enum ttm_object_type object_type,void (* refcount_release)(struct ttm_base_object **))1900b8762e9SThomas Hellstrom int ttm_base_object_init(struct ttm_object_file *tfile,
1910b8762e9SThomas Hellstrom struct ttm_base_object *base,
1920b8762e9SThomas Hellstrom bool shareable,
1930b8762e9SThomas Hellstrom enum ttm_object_type object_type,
1948afa13a0SZack Rusin void (*refcount_release) (struct ttm_base_object **))
1950b8762e9SThomas Hellstrom {
1960b8762e9SThomas Hellstrom struct ttm_object_device *tdev = tfile->tdev;
1970b8762e9SThomas Hellstrom int ret;
1980b8762e9SThomas Hellstrom
1990b8762e9SThomas Hellstrom base->shareable = shareable;
2000b8762e9SThomas Hellstrom base->tfile = ttm_object_file_ref(tfile);
2010b8762e9SThomas Hellstrom base->refcount_release = refcount_release;
2020b8762e9SThomas Hellstrom base->object_type = object_type;
2030b8762e9SThomas Hellstrom kref_init(&base->refcount);
204c7eae626SThomas Hellstrom idr_preload(GFP_KERNEL);
2050b8762e9SThomas Hellstrom spin_lock(&tdev->object_lock);
2068407f8a1SThomas Hellstrom ret = idr_alloc(&tdev->idr, base, 1, 0, GFP_NOWAIT);
2070b8762e9SThomas Hellstrom spin_unlock(&tdev->object_lock);
208c7eae626SThomas Hellstrom idr_preload_end();
209c7eae626SThomas Hellstrom if (ret < 0)
210c7eae626SThomas Hellstrom return ret;
2110b8762e9SThomas Hellstrom
212c7eae626SThomas Hellstrom base->handle = ret;
2138afa13a0SZack Rusin ret = ttm_ref_object_add(tfile, base, NULL, false);
2140b8762e9SThomas Hellstrom if (unlikely(ret != 0))
2150b8762e9SThomas Hellstrom goto out_err1;
2160b8762e9SThomas Hellstrom
2170b8762e9SThomas Hellstrom ttm_base_object_unref(&base);
2180b8762e9SThomas Hellstrom
2190b8762e9SThomas Hellstrom return 0;
2200b8762e9SThomas Hellstrom out_err1:
2210b8762e9SThomas Hellstrom spin_lock(&tdev->object_lock);
222c7eae626SThomas Hellstrom idr_remove(&tdev->idr, base->handle);
2230b8762e9SThomas Hellstrom spin_unlock(&tdev->object_lock);
2240b8762e9SThomas Hellstrom return ret;
2250b8762e9SThomas Hellstrom }
2260b8762e9SThomas Hellstrom
ttm_release_base(struct kref * kref)2270b8762e9SThomas Hellstrom static void ttm_release_base(struct kref *kref)
2280b8762e9SThomas Hellstrom {
2290b8762e9SThomas Hellstrom struct ttm_base_object *base =
2300b8762e9SThomas Hellstrom container_of(kref, struct ttm_base_object, refcount);
2310b8762e9SThomas Hellstrom struct ttm_object_device *tdev = base->tfile->tdev;
2320b8762e9SThomas Hellstrom
2330b8762e9SThomas Hellstrom spin_lock(&tdev->object_lock);
234c7eae626SThomas Hellstrom idr_remove(&tdev->idr, base->handle);
2350b8762e9SThomas Hellstrom spin_unlock(&tdev->object_lock);
2360b8762e9SThomas Hellstrom
2370b8762e9SThomas Hellstrom /*
2380b8762e9SThomas Hellstrom * Note: We don't use synchronize_rcu() here because it's far
2390b8762e9SThomas Hellstrom * too slow. It's up to the user to free the object using
2400b8762e9SThomas Hellstrom * call_rcu() or ttm_base_object_kfree().
2410b8762e9SThomas Hellstrom */
2420b8762e9SThomas Hellstrom
2430b8762e9SThomas Hellstrom ttm_object_file_unref(&base->tfile);
2440b8762e9SThomas Hellstrom if (base->refcount_release)
2450b8762e9SThomas Hellstrom base->refcount_release(&base);
2460b8762e9SThomas Hellstrom }
2470b8762e9SThomas Hellstrom
ttm_base_object_unref(struct ttm_base_object ** p_base)2480b8762e9SThomas Hellstrom void ttm_base_object_unref(struct ttm_base_object **p_base)
2490b8762e9SThomas Hellstrom {
2500b8762e9SThomas Hellstrom struct ttm_base_object *base = *p_base;
2510b8762e9SThomas Hellstrom
2520b8762e9SThomas Hellstrom *p_base = NULL;
2530b8762e9SThomas Hellstrom
2540b8762e9SThomas Hellstrom kref_put(&base->refcount, ttm_release_base);
2550b8762e9SThomas Hellstrom }
2560b8762e9SThomas Hellstrom
ttm_base_object_lookup(struct ttm_object_file * tfile,uint64_t key)2570b8762e9SThomas Hellstrom struct ttm_base_object *ttm_base_object_lookup(struct ttm_object_file *tfile,
25876a9e07fSMaaz Mombasawala uint64_t key)
2590b8762e9SThomas Hellstrom {
2600b8762e9SThomas Hellstrom struct ttm_base_object *base = NULL;
2612985c964SThomas Zimmermann struct vmwgfx_hash_item *hash;
2620b8762e9SThomas Hellstrom int ret;
2630b8762e9SThomas Hellstrom
264*a309c719SZack Rusin spin_lock(&tfile->lock);
265*a309c719SZack Rusin ret = ttm_tfile_find_ref(tfile, key, &hash);
2660b8762e9SThomas Hellstrom
2670b8762e9SThomas Hellstrom if (likely(ret == 0)) {
2689da30cddSMaaz Mombasawala base = hlist_entry(hash, struct ttm_ref_object, hash)->obj;
2690b8762e9SThomas Hellstrom if (!kref_get_unless_zero(&base->refcount))
2700b8762e9SThomas Hellstrom base = NULL;
2710b8762e9SThomas Hellstrom }
272*a309c719SZack Rusin spin_unlock(&tfile->lock);
273*a309c719SZack Rusin
2740b8762e9SThomas Hellstrom
2750b8762e9SThomas Hellstrom return base;
2760b8762e9SThomas Hellstrom }
2770b8762e9SThomas Hellstrom
2780b8762e9SThomas Hellstrom struct ttm_base_object *
ttm_base_object_lookup_for_ref(struct ttm_object_device * tdev,uint64_t key)27976a9e07fSMaaz Mombasawala ttm_base_object_lookup_for_ref(struct ttm_object_device *tdev, uint64_t key)
2800b8762e9SThomas Hellstrom {
281c7eae626SThomas Hellstrom struct ttm_base_object *base;
2820b8762e9SThomas Hellstrom
2830b8762e9SThomas Hellstrom rcu_read_lock();
284c7eae626SThomas Hellstrom base = idr_find(&tdev->idr, key);
2850b8762e9SThomas Hellstrom
286c7eae626SThomas Hellstrom if (base && !kref_get_unless_zero(&base->refcount))
2870b8762e9SThomas Hellstrom base = NULL;
2880b8762e9SThomas Hellstrom rcu_read_unlock();
2890b8762e9SThomas Hellstrom
2900b8762e9SThomas Hellstrom return base;
2910b8762e9SThomas Hellstrom }
2920b8762e9SThomas Hellstrom
ttm_ref_object_add(struct ttm_object_file * tfile,struct ttm_base_object * base,bool * existed,bool require_existed)2930b8762e9SThomas Hellstrom int ttm_ref_object_add(struct ttm_object_file *tfile,
2940b8762e9SThomas Hellstrom struct ttm_base_object *base,
2958afa13a0SZack Rusin bool *existed,
2960b8762e9SThomas Hellstrom bool require_existed)
2970b8762e9SThomas Hellstrom {
2980b8762e9SThomas Hellstrom struct ttm_ref_object *ref;
2992985c964SThomas Zimmermann struct vmwgfx_hash_item *hash;
3000b8762e9SThomas Hellstrom int ret = -EINVAL;
3010b8762e9SThomas Hellstrom
3020b8762e9SThomas Hellstrom if (base->tfile != tfile && !base->shareable)
3030b8762e9SThomas Hellstrom return -EPERM;
3040b8762e9SThomas Hellstrom
3050b8762e9SThomas Hellstrom if (existed != NULL)
3060b8762e9SThomas Hellstrom *existed = true;
3070b8762e9SThomas Hellstrom
3080b8762e9SThomas Hellstrom while (ret == -EINVAL) {
3090b8762e9SThomas Hellstrom rcu_read_lock();
31076a9e07fSMaaz Mombasawala ret = ttm_tfile_find_ref_rcu(tfile, base->handle, &hash);
3110b8762e9SThomas Hellstrom
3120b8762e9SThomas Hellstrom if (ret == 0) {
3139da30cddSMaaz Mombasawala ref = hlist_entry(hash, struct ttm_ref_object, hash);
3140b8762e9SThomas Hellstrom if (kref_get_unless_zero(&ref->kref)) {
3150b8762e9SThomas Hellstrom rcu_read_unlock();
3160b8762e9SThomas Hellstrom break;
3170b8762e9SThomas Hellstrom }
3180b8762e9SThomas Hellstrom }
3190b8762e9SThomas Hellstrom
3200b8762e9SThomas Hellstrom rcu_read_unlock();
3210b8762e9SThomas Hellstrom if (require_existed)
3220b8762e9SThomas Hellstrom return -EPERM;
3230b8762e9SThomas Hellstrom
3240b8762e9SThomas Hellstrom ref = kmalloc(sizeof(*ref), GFP_KERNEL);
3250b8762e9SThomas Hellstrom if (unlikely(ref == NULL)) {
3260b8762e9SThomas Hellstrom return -ENOMEM;
3270b8762e9SThomas Hellstrom }
3280b8762e9SThomas Hellstrom
329c7eae626SThomas Hellstrom ref->hash.key = base->handle;
3300b8762e9SThomas Hellstrom ref->obj = base;
3310b8762e9SThomas Hellstrom ref->tfile = tfile;
3320b8762e9SThomas Hellstrom kref_init(&ref->kref);
3330b8762e9SThomas Hellstrom
3340b8762e9SThomas Hellstrom spin_lock(&tfile->lock);
33576a9e07fSMaaz Mombasawala hash_add_rcu(tfile->ref_hash, &ref->hash.head, ref->hash.key);
33676a9e07fSMaaz Mombasawala ret = 0;
3370b8762e9SThomas Hellstrom
3380b8762e9SThomas Hellstrom list_add_tail(&ref->head, &tfile->ref_list);
3390b8762e9SThomas Hellstrom kref_get(&base->refcount);
3400b8762e9SThomas Hellstrom spin_unlock(&tfile->lock);
3410b8762e9SThomas Hellstrom if (existed != NULL)
3420b8762e9SThomas Hellstrom *existed = false;
3430b8762e9SThomas Hellstrom }
3440b8762e9SThomas Hellstrom
3450b8762e9SThomas Hellstrom return ret;
3460b8762e9SThomas Hellstrom }
3470b8762e9SThomas Hellstrom
3480b8762e9SThomas Hellstrom static void __releases(tfile->lock) __acquires(tfile->lock)
ttm_ref_object_release(struct kref * kref)3490b8762e9SThomas Hellstrom ttm_ref_object_release(struct kref *kref)
3500b8762e9SThomas Hellstrom {
3510b8762e9SThomas Hellstrom struct ttm_ref_object *ref =
3520b8762e9SThomas Hellstrom container_of(kref, struct ttm_ref_object, kref);
3530b8762e9SThomas Hellstrom struct ttm_object_file *tfile = ref->tfile;
3540b8762e9SThomas Hellstrom
35576a9e07fSMaaz Mombasawala hash_del_rcu(&ref->hash.head);
3560b8762e9SThomas Hellstrom list_del(&ref->head);
3570b8762e9SThomas Hellstrom spin_unlock(&tfile->lock);
3580b8762e9SThomas Hellstrom
3590b8762e9SThomas Hellstrom ttm_base_object_unref(&ref->obj);
3600b8762e9SThomas Hellstrom kfree_rcu(ref, rcu_head);
3610b8762e9SThomas Hellstrom spin_lock(&tfile->lock);
3620b8762e9SThomas Hellstrom }
3630b8762e9SThomas Hellstrom
ttm_ref_object_base_unref(struct ttm_object_file * tfile,unsigned long key)3640b8762e9SThomas Hellstrom int ttm_ref_object_base_unref(struct ttm_object_file *tfile,
3658afa13a0SZack Rusin unsigned long key)
3660b8762e9SThomas Hellstrom {
3670b8762e9SThomas Hellstrom struct ttm_ref_object *ref;
3682985c964SThomas Zimmermann struct vmwgfx_hash_item *hash;
3690b8762e9SThomas Hellstrom int ret;
3700b8762e9SThomas Hellstrom
3710b8762e9SThomas Hellstrom spin_lock(&tfile->lock);
37276a9e07fSMaaz Mombasawala ret = ttm_tfile_find_ref(tfile, key, &hash);
3730b8762e9SThomas Hellstrom if (unlikely(ret != 0)) {
3740b8762e9SThomas Hellstrom spin_unlock(&tfile->lock);
3750b8762e9SThomas Hellstrom return -EINVAL;
3760b8762e9SThomas Hellstrom }
3779da30cddSMaaz Mombasawala ref = hlist_entry(hash, struct ttm_ref_object, hash);
3780b8762e9SThomas Hellstrom kref_put(&ref->kref, ttm_ref_object_release);
3790b8762e9SThomas Hellstrom spin_unlock(&tfile->lock);
3800b8762e9SThomas Hellstrom return 0;
3810b8762e9SThomas Hellstrom }
3820b8762e9SThomas Hellstrom
ttm_object_file_release(struct ttm_object_file ** p_tfile)3830b8762e9SThomas Hellstrom void ttm_object_file_release(struct ttm_object_file **p_tfile)
3840b8762e9SThomas Hellstrom {
3850b8762e9SThomas Hellstrom struct ttm_ref_object *ref;
3860b8762e9SThomas Hellstrom struct list_head *list;
3870b8762e9SThomas Hellstrom struct ttm_object_file *tfile = *p_tfile;
3880b8762e9SThomas Hellstrom
3890b8762e9SThomas Hellstrom *p_tfile = NULL;
3900b8762e9SThomas Hellstrom spin_lock(&tfile->lock);
3910b8762e9SThomas Hellstrom
3920b8762e9SThomas Hellstrom /*
3930b8762e9SThomas Hellstrom * Since we release the lock within the loop, we have to
3940b8762e9SThomas Hellstrom * restart it from the beginning each time.
3950b8762e9SThomas Hellstrom */
3960b8762e9SThomas Hellstrom
3970b8762e9SThomas Hellstrom while (!list_empty(&tfile->ref_list)) {
3980b8762e9SThomas Hellstrom list = tfile->ref_list.next;
3990b8762e9SThomas Hellstrom ref = list_entry(list, struct ttm_ref_object, head);
4000b8762e9SThomas Hellstrom ttm_ref_object_release(&ref->kref);
4010b8762e9SThomas Hellstrom }
4020b8762e9SThomas Hellstrom
4030b8762e9SThomas Hellstrom spin_unlock(&tfile->lock);
4040b8762e9SThomas Hellstrom
4050b8762e9SThomas Hellstrom ttm_object_file_unref(&tfile);
4060b8762e9SThomas Hellstrom }
4070b8762e9SThomas Hellstrom
ttm_object_file_init(struct ttm_object_device * tdev)40876a9e07fSMaaz Mombasawala struct ttm_object_file *ttm_object_file_init(struct ttm_object_device *tdev)
4090b8762e9SThomas Hellstrom {
4100b8762e9SThomas Hellstrom struct ttm_object_file *tfile = kmalloc(sizeof(*tfile), GFP_KERNEL);
4110b8762e9SThomas Hellstrom
4120b8762e9SThomas Hellstrom if (unlikely(tfile == NULL))
4130b8762e9SThomas Hellstrom return NULL;
4140b8762e9SThomas Hellstrom
4150b8762e9SThomas Hellstrom spin_lock_init(&tfile->lock);
4160b8762e9SThomas Hellstrom tfile->tdev = tdev;
4170b8762e9SThomas Hellstrom kref_init(&tfile->refcount);
4180b8762e9SThomas Hellstrom INIT_LIST_HEAD(&tfile->ref_list);
4190b8762e9SThomas Hellstrom
42076a9e07fSMaaz Mombasawala hash_init(tfile->ref_hash);
4210b8762e9SThomas Hellstrom
4220b8762e9SThomas Hellstrom return tfile;
4230b8762e9SThomas Hellstrom }
4240b8762e9SThomas Hellstrom
4250b8762e9SThomas Hellstrom struct ttm_object_device *
ttm_object_device_init(const struct dma_buf_ops * ops)426931e09d8SMaaz Mombasawala ttm_object_device_init(const struct dma_buf_ops *ops)
4270b8762e9SThomas Hellstrom {
4280b8762e9SThomas Hellstrom struct ttm_object_device *tdev = kmalloc(sizeof(*tdev), GFP_KERNEL);
4290b8762e9SThomas Hellstrom
4300b8762e9SThomas Hellstrom if (unlikely(tdev == NULL))
4310b8762e9SThomas Hellstrom return NULL;
4320b8762e9SThomas Hellstrom
4330b8762e9SThomas Hellstrom spin_lock_init(&tdev->object_lock);
4340b8762e9SThomas Hellstrom atomic_set(&tdev->object_count, 0);
4350b8762e9SThomas Hellstrom
4368afa13a0SZack Rusin /*
4378afa13a0SZack Rusin * Our base is at VMWGFX_NUM_MOB + 1 because we want to create
4388afa13a0SZack Rusin * a seperate namespace for GEM handles (which are
4398afa13a0SZack Rusin * 1..VMWGFX_NUM_MOB) and the surface handles. Some ioctl's
4408afa13a0SZack Rusin * can take either handle as an argument so we want to
4418afa13a0SZack Rusin * easily be able to tell whether the handle refers to a
4428afa13a0SZack Rusin * GEM buffer or a surface.
4438afa13a0SZack Rusin */
4448afa13a0SZack Rusin idr_init_base(&tdev->idr, VMWGFX_NUM_MOB + 1);
4450b8762e9SThomas Hellstrom tdev->ops = *ops;
4460b8762e9SThomas Hellstrom tdev->dmabuf_release = tdev->ops.release;
4470b8762e9SThomas Hellstrom tdev->ops.release = ttm_prime_dmabuf_release;
4480b8762e9SThomas Hellstrom return tdev;
4490b8762e9SThomas Hellstrom }
4500b8762e9SThomas Hellstrom
ttm_object_device_release(struct ttm_object_device ** p_tdev)4510b8762e9SThomas Hellstrom void ttm_object_device_release(struct ttm_object_device **p_tdev)
4520b8762e9SThomas Hellstrom {
4530b8762e9SThomas Hellstrom struct ttm_object_device *tdev = *p_tdev;
4540b8762e9SThomas Hellstrom
4550b8762e9SThomas Hellstrom *p_tdev = NULL;
4560b8762e9SThomas Hellstrom
457c7eae626SThomas Hellstrom WARN_ON_ONCE(!idr_is_empty(&tdev->idr));
458c7eae626SThomas Hellstrom idr_destroy(&tdev->idr);
4590b8762e9SThomas Hellstrom
4600b8762e9SThomas Hellstrom kfree(tdev);
4610b8762e9SThomas Hellstrom }
4620b8762e9SThomas Hellstrom
4630b8762e9SThomas Hellstrom /**
4640b8762e9SThomas Hellstrom * get_dma_buf_unless_doomed - get a dma_buf reference if possible.
4650b8762e9SThomas Hellstrom *
466f322f32aSLee Jones * @dmabuf: Non-refcounted pointer to a struct dma-buf.
4670b8762e9SThomas Hellstrom *
4680b8762e9SThomas Hellstrom * Obtain a file reference from a lookup structure that doesn't refcount
4690b8762e9SThomas Hellstrom * the file, but synchronizes with its release method to make sure it has
4700b8762e9SThomas Hellstrom * not been freed yet. See for example kref_get_unless_zero documentation.
4710b8762e9SThomas Hellstrom * Returns true if refcounting succeeds, false otherwise.
4720b8762e9SThomas Hellstrom *
4730b8762e9SThomas Hellstrom * Nobody really wants this as a public API yet, so let it mature here
4740b8762e9SThomas Hellstrom * for some time...
4750b8762e9SThomas Hellstrom */
get_dma_buf_unless_doomed(struct dma_buf * dmabuf)4760b8762e9SThomas Hellstrom static bool __must_check get_dma_buf_unless_doomed(struct dma_buf *dmabuf)
4770b8762e9SThomas Hellstrom {
4780b8762e9SThomas Hellstrom return atomic_long_inc_not_zero(&dmabuf->file->f_count) != 0L;
4790b8762e9SThomas Hellstrom }
4800b8762e9SThomas Hellstrom
4810b8762e9SThomas Hellstrom /**
4820b8762e9SThomas Hellstrom * ttm_prime_refcount_release - refcount release method for a prime object.
4830b8762e9SThomas Hellstrom *
4840b8762e9SThomas Hellstrom * @p_base: Pointer to ttm_base_object pointer.
4850b8762e9SThomas Hellstrom *
4860b8762e9SThomas Hellstrom * This is a wrapper that calls the refcount_release founction of the
4870b8762e9SThomas Hellstrom * underlying object. At the same time it cleans up the prime object.
4880b8762e9SThomas Hellstrom * This function is called when all references to the base object we
4890b8762e9SThomas Hellstrom * derive from are gone.
4900b8762e9SThomas Hellstrom */
ttm_prime_refcount_release(struct ttm_base_object ** p_base)4910b8762e9SThomas Hellstrom static void ttm_prime_refcount_release(struct ttm_base_object **p_base)
4920b8762e9SThomas Hellstrom {
4930b8762e9SThomas Hellstrom struct ttm_base_object *base = *p_base;
4940b8762e9SThomas Hellstrom struct ttm_prime_object *prime;
4950b8762e9SThomas Hellstrom
4960b8762e9SThomas Hellstrom *p_base = NULL;
4970b8762e9SThomas Hellstrom prime = container_of(base, struct ttm_prime_object, base);
4980b8762e9SThomas Hellstrom BUG_ON(prime->dma_buf != NULL);
4990b8762e9SThomas Hellstrom mutex_destroy(&prime->mutex);
5000b8762e9SThomas Hellstrom if (prime->refcount_release)
5010b8762e9SThomas Hellstrom prime->refcount_release(&base);
5020b8762e9SThomas Hellstrom }
5030b8762e9SThomas Hellstrom
5040b8762e9SThomas Hellstrom /**
5050b8762e9SThomas Hellstrom * ttm_prime_dmabuf_release - Release method for the dma-bufs we export
5060b8762e9SThomas Hellstrom *
5070b8762e9SThomas Hellstrom * @dma_buf:
5080b8762e9SThomas Hellstrom *
5090b8762e9SThomas Hellstrom * This function first calls the dma_buf release method the driver
5100b8762e9SThomas Hellstrom * provides. Then it cleans up our dma_buf pointer used for lookup,
5110b8762e9SThomas Hellstrom * and finally releases the reference the dma_buf has on our base
5120b8762e9SThomas Hellstrom * object.
5130b8762e9SThomas Hellstrom */
ttm_prime_dmabuf_release(struct dma_buf * dma_buf)5140b8762e9SThomas Hellstrom static void ttm_prime_dmabuf_release(struct dma_buf *dma_buf)
5150b8762e9SThomas Hellstrom {
5160b8762e9SThomas Hellstrom struct ttm_prime_object *prime =
5170b8762e9SThomas Hellstrom (struct ttm_prime_object *) dma_buf->priv;
5180b8762e9SThomas Hellstrom struct ttm_base_object *base = &prime->base;
5190b8762e9SThomas Hellstrom struct ttm_object_device *tdev = base->tfile->tdev;
5200b8762e9SThomas Hellstrom
5210b8762e9SThomas Hellstrom if (tdev->dmabuf_release)
5220b8762e9SThomas Hellstrom tdev->dmabuf_release(dma_buf);
5230b8762e9SThomas Hellstrom mutex_lock(&prime->mutex);
5240b8762e9SThomas Hellstrom if (prime->dma_buf == dma_buf)
5250b8762e9SThomas Hellstrom prime->dma_buf = NULL;
5260b8762e9SThomas Hellstrom mutex_unlock(&prime->mutex);
5270b8762e9SThomas Hellstrom ttm_base_object_unref(&base);
5280b8762e9SThomas Hellstrom }
5290b8762e9SThomas Hellstrom
5300b8762e9SThomas Hellstrom /**
5310b8762e9SThomas Hellstrom * ttm_prime_fd_to_handle - Get a base object handle from a prime fd
5320b8762e9SThomas Hellstrom *
5330b8762e9SThomas Hellstrom * @tfile: A struct ttm_object_file identifying the caller.
5340b8762e9SThomas Hellstrom * @fd: The prime / dmabuf fd.
5350b8762e9SThomas Hellstrom * @handle: The returned handle.
5360b8762e9SThomas Hellstrom *
5370b8762e9SThomas Hellstrom * This function returns a handle to an object that previously exported
5380b8762e9SThomas Hellstrom * a dma-buf. Note that we don't handle imports yet, because we simply
5390b8762e9SThomas Hellstrom * have no consumers of that implementation.
5400b8762e9SThomas Hellstrom */
ttm_prime_fd_to_handle(struct ttm_object_file * tfile,int fd,u32 * handle)5410b8762e9SThomas Hellstrom int ttm_prime_fd_to_handle(struct ttm_object_file *tfile,
5420b8762e9SThomas Hellstrom int fd, u32 *handle)
5430b8762e9SThomas Hellstrom {
5440b8762e9SThomas Hellstrom struct ttm_object_device *tdev = tfile->tdev;
5450b8762e9SThomas Hellstrom struct dma_buf *dma_buf;
5460b8762e9SThomas Hellstrom struct ttm_prime_object *prime;
5470b8762e9SThomas Hellstrom struct ttm_base_object *base;
5480b8762e9SThomas Hellstrom int ret;
5490b8762e9SThomas Hellstrom
5500b8762e9SThomas Hellstrom dma_buf = dma_buf_get(fd);
5510b8762e9SThomas Hellstrom if (IS_ERR(dma_buf))
5520b8762e9SThomas Hellstrom return PTR_ERR(dma_buf);
5530b8762e9SThomas Hellstrom
5540b8762e9SThomas Hellstrom if (dma_buf->ops != &tdev->ops)
5550b8762e9SThomas Hellstrom return -ENOSYS;
5560b8762e9SThomas Hellstrom
5570b8762e9SThomas Hellstrom prime = (struct ttm_prime_object *) dma_buf->priv;
5580b8762e9SThomas Hellstrom base = &prime->base;
559c7eae626SThomas Hellstrom *handle = base->handle;
5608afa13a0SZack Rusin ret = ttm_ref_object_add(tfile, base, NULL, false);
5610b8762e9SThomas Hellstrom
5620b8762e9SThomas Hellstrom dma_buf_put(dma_buf);
5630b8762e9SThomas Hellstrom
5640b8762e9SThomas Hellstrom return ret;
5650b8762e9SThomas Hellstrom }
5660b8762e9SThomas Hellstrom
5670b8762e9SThomas Hellstrom /**
5680b8762e9SThomas Hellstrom * ttm_prime_handle_to_fd - Return a dma_buf fd from a ttm prime object
5690b8762e9SThomas Hellstrom *
5700b8762e9SThomas Hellstrom * @tfile: Struct ttm_object_file identifying the caller.
5710b8762e9SThomas Hellstrom * @handle: Handle to the object we're exporting from.
5720b8762e9SThomas Hellstrom * @flags: flags for dma-buf creation. We just pass them on.
5730b8762e9SThomas Hellstrom * @prime_fd: The returned file descriptor.
5740b8762e9SThomas Hellstrom *
5750b8762e9SThomas Hellstrom */
ttm_prime_handle_to_fd(struct ttm_object_file * tfile,uint32_t handle,uint32_t flags,int * prime_fd)5760b8762e9SThomas Hellstrom int ttm_prime_handle_to_fd(struct ttm_object_file *tfile,
5770b8762e9SThomas Hellstrom uint32_t handle, uint32_t flags,
5780b8762e9SThomas Hellstrom int *prime_fd)
5790b8762e9SThomas Hellstrom {
5800b8762e9SThomas Hellstrom struct ttm_object_device *tdev = tfile->tdev;
5810b8762e9SThomas Hellstrom struct ttm_base_object *base;
5820b8762e9SThomas Hellstrom struct dma_buf *dma_buf;
5830b8762e9SThomas Hellstrom struct ttm_prime_object *prime;
5840b8762e9SThomas Hellstrom int ret;
5850b8762e9SThomas Hellstrom
5860b8762e9SThomas Hellstrom base = ttm_base_object_lookup(tfile, handle);
5870b8762e9SThomas Hellstrom if (unlikely(base == NULL ||
5880b8762e9SThomas Hellstrom base->object_type != ttm_prime_type)) {
5890b8762e9SThomas Hellstrom ret = -ENOENT;
5900b8762e9SThomas Hellstrom goto out_unref;
5910b8762e9SThomas Hellstrom }
5920b8762e9SThomas Hellstrom
5930b8762e9SThomas Hellstrom prime = container_of(base, struct ttm_prime_object, base);
5940b8762e9SThomas Hellstrom if (unlikely(!base->shareable)) {
5950b8762e9SThomas Hellstrom ret = -EPERM;
5960b8762e9SThomas Hellstrom goto out_unref;
5970b8762e9SThomas Hellstrom }
5980b8762e9SThomas Hellstrom
5990b8762e9SThomas Hellstrom ret = mutex_lock_interruptible(&prime->mutex);
6000b8762e9SThomas Hellstrom if (unlikely(ret != 0)) {
6010b8762e9SThomas Hellstrom ret = -ERESTARTSYS;
6020b8762e9SThomas Hellstrom goto out_unref;
6030b8762e9SThomas Hellstrom }
6040b8762e9SThomas Hellstrom
6050b8762e9SThomas Hellstrom dma_buf = prime->dma_buf;
6060b8762e9SThomas Hellstrom if (!dma_buf || !get_dma_buf_unless_doomed(dma_buf)) {
6070b8762e9SThomas Hellstrom DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
6080b8762e9SThomas Hellstrom exp_info.ops = &tdev->ops;
6090b8762e9SThomas Hellstrom exp_info.size = prime->size;
6100b8762e9SThomas Hellstrom exp_info.flags = flags;
6110b8762e9SThomas Hellstrom exp_info.priv = prime;
6120b8762e9SThomas Hellstrom
6130b8762e9SThomas Hellstrom /*
6148aadeb8aSZack Rusin * Need to create a new dma_buf
6150b8762e9SThomas Hellstrom */
6160b8762e9SThomas Hellstrom
6170b8762e9SThomas Hellstrom dma_buf = dma_buf_export(&exp_info);
6180b8762e9SThomas Hellstrom if (IS_ERR(dma_buf)) {
6190b8762e9SThomas Hellstrom ret = PTR_ERR(dma_buf);
6200b8762e9SThomas Hellstrom mutex_unlock(&prime->mutex);
6210b8762e9SThomas Hellstrom goto out_unref;
6220b8762e9SThomas Hellstrom }
6230b8762e9SThomas Hellstrom
6240b8762e9SThomas Hellstrom /*
6250b8762e9SThomas Hellstrom * dma_buf has taken the base object reference
6260b8762e9SThomas Hellstrom */
6270b8762e9SThomas Hellstrom base = NULL;
6280b8762e9SThomas Hellstrom prime->dma_buf = dma_buf;
6290b8762e9SThomas Hellstrom }
6300b8762e9SThomas Hellstrom mutex_unlock(&prime->mutex);
6310b8762e9SThomas Hellstrom
6320b8762e9SThomas Hellstrom ret = dma_buf_fd(dma_buf, flags);
6330b8762e9SThomas Hellstrom if (ret >= 0) {
6340b8762e9SThomas Hellstrom *prime_fd = ret;
6350b8762e9SThomas Hellstrom ret = 0;
6360b8762e9SThomas Hellstrom } else
6370b8762e9SThomas Hellstrom dma_buf_put(dma_buf);
6380b8762e9SThomas Hellstrom
6390b8762e9SThomas Hellstrom out_unref:
6400b8762e9SThomas Hellstrom if (base)
6410b8762e9SThomas Hellstrom ttm_base_object_unref(&base);
6420b8762e9SThomas Hellstrom return ret;
6430b8762e9SThomas Hellstrom }
6440b8762e9SThomas Hellstrom
6450b8762e9SThomas Hellstrom /**
6460b8762e9SThomas Hellstrom * ttm_prime_object_init - Initialize a ttm_prime_object
6470b8762e9SThomas Hellstrom *
6480b8762e9SThomas Hellstrom * @tfile: struct ttm_object_file identifying the caller
6490b8762e9SThomas Hellstrom * @size: The size of the dma_bufs we export.
6500b8762e9SThomas Hellstrom * @prime: The object to be initialized.
6510b8762e9SThomas Hellstrom * @shareable: See ttm_base_object_init
6520b8762e9SThomas Hellstrom * @type: See ttm_base_object_init
6530b8762e9SThomas Hellstrom * @refcount_release: See ttm_base_object_init
6540b8762e9SThomas Hellstrom *
6550b8762e9SThomas Hellstrom * Initializes an object which is compatible with the drm_prime model
6560b8762e9SThomas Hellstrom * for data sharing between processes and devices.
6570b8762e9SThomas Hellstrom */
ttm_prime_object_init(struct ttm_object_file * tfile,size_t size,struct ttm_prime_object * prime,bool shareable,enum ttm_object_type type,void (* refcount_release)(struct ttm_base_object **))6580b8762e9SThomas Hellstrom int ttm_prime_object_init(struct ttm_object_file *tfile, size_t size,
6590b8762e9SThomas Hellstrom struct ttm_prime_object *prime, bool shareable,
6600b8762e9SThomas Hellstrom enum ttm_object_type type,
6618afa13a0SZack Rusin void (*refcount_release) (struct ttm_base_object **))
6620b8762e9SThomas Hellstrom {
6630b8762e9SThomas Hellstrom mutex_init(&prime->mutex);
6640b8762e9SThomas Hellstrom prime->size = PAGE_ALIGN(size);
6650b8762e9SThomas Hellstrom prime->real_type = type;
6660b8762e9SThomas Hellstrom prime->dma_buf = NULL;
6670b8762e9SThomas Hellstrom prime->refcount_release = refcount_release;
6680b8762e9SThomas Hellstrom return ttm_base_object_init(tfile, &prime->base, shareable,
6690b8762e9SThomas Hellstrom ttm_prime_type,
6708afa13a0SZack Rusin ttm_prime_refcount_release);
6710b8762e9SThomas Hellstrom }
672