1d38ceaf9SAlex Deucher /* 2d38ceaf9SAlex Deucher * Copyright 2015 Advanced Micro Devices, Inc. 3d38ceaf9SAlex Deucher * All Rights Reserved. 4d38ceaf9SAlex Deucher * 5d38ceaf9SAlex Deucher * Permission is hereby granted, free of charge, to any person obtaining a 6d38ceaf9SAlex Deucher * copy of this software and associated documentation files (the 7d38ceaf9SAlex Deucher * "Software"), to deal in the Software without restriction, including 8d38ceaf9SAlex Deucher * without limitation the rights to use, copy, modify, merge, publish, 9d38ceaf9SAlex Deucher * distribute, sub license, and/or sell copies of the Software, and to 10d38ceaf9SAlex Deucher * permit persons to whom the Software is furnished to do so, subject to 11d38ceaf9SAlex Deucher * the following conditions: 12d38ceaf9SAlex Deucher * 13d38ceaf9SAlex Deucher * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14d38ceaf9SAlex Deucher * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15d38ceaf9SAlex Deucher * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 16d38ceaf9SAlex Deucher * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 17d38ceaf9SAlex Deucher * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 18d38ceaf9SAlex Deucher * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 19d38ceaf9SAlex Deucher * USE OR OTHER DEALINGS IN THE SOFTWARE. 20d38ceaf9SAlex Deucher * 21d38ceaf9SAlex Deucher * The above copyright notice and this permission notice (including the 22d38ceaf9SAlex Deucher * next paragraph) shall be included in all copies or substantial portions 23d38ceaf9SAlex Deucher * of the Software. 24d38ceaf9SAlex Deucher * 25d38ceaf9SAlex Deucher */ 26d38ceaf9SAlex Deucher /* 27d38ceaf9SAlex Deucher * Authors: 28d38ceaf9SAlex Deucher * Christian König <deathsimple@vodafone.de> 29d38ceaf9SAlex Deucher */ 30d38ceaf9SAlex Deucher 31d38ceaf9SAlex Deucher #include <drm/drmP.h> 32d38ceaf9SAlex Deucher #include "amdgpu.h" 33ec74407aSChristian König #include "amdgpu_trace.h" 34d38ceaf9SAlex Deucher 35636ce25cSChristian König #define AMDGPU_BO_LIST_MAX_PRIORITY 32u 36636ce25cSChristian König #define AMDGPU_BO_LIST_NUM_BUCKETS (AMDGPU_BO_LIST_MAX_PRIORITY + 1) 37636ce25cSChristian König 38d38ceaf9SAlex Deucher static int amdgpu_bo_list_create(struct amdgpu_fpriv *fpriv, 39d38ceaf9SAlex Deucher struct amdgpu_bo_list **result, 40d38ceaf9SAlex Deucher int *id) 41d38ceaf9SAlex Deucher { 42d38ceaf9SAlex Deucher int r; 43d38ceaf9SAlex Deucher 44d38ceaf9SAlex Deucher *result = kzalloc(sizeof(struct amdgpu_bo_list), GFP_KERNEL); 45d38ceaf9SAlex Deucher if (!*result) 46d38ceaf9SAlex Deucher return -ENOMEM; 47d38ceaf9SAlex Deucher 48d38ceaf9SAlex Deucher mutex_lock(&fpriv->bo_list_lock); 49d38ceaf9SAlex Deucher r = idr_alloc(&fpriv->bo_list_handles, *result, 50decee87aSmonk.liu 1, 0, GFP_KERNEL); 51d38ceaf9SAlex Deucher if (r < 0) { 52d38ceaf9SAlex Deucher mutex_unlock(&fpriv->bo_list_lock); 53d38ceaf9SAlex Deucher kfree(*result); 54d38ceaf9SAlex Deucher return r; 55d38ceaf9SAlex Deucher } 56d38ceaf9SAlex Deucher *id = r; 57d38ceaf9SAlex Deucher 58d38ceaf9SAlex Deucher mutex_init(&(*result)->lock); 59d38ceaf9SAlex Deucher (*result)->num_entries = 0; 60d38ceaf9SAlex Deucher (*result)->array = NULL; 61d38ceaf9SAlex Deucher 62d38ceaf9SAlex Deucher mutex_lock(&(*result)->lock); 63d38ceaf9SAlex Deucher mutex_unlock(&fpriv->bo_list_lock); 64d38ceaf9SAlex Deucher 65d38ceaf9SAlex Deucher return 0; 66d38ceaf9SAlex Deucher } 67d38ceaf9SAlex Deucher 68d38ceaf9SAlex Deucher static void amdgpu_bo_list_destroy(struct amdgpu_fpriv *fpriv, int id) 69d38ceaf9SAlex Deucher { 70d38ceaf9SAlex Deucher struct amdgpu_bo_list *list; 71d38ceaf9SAlex Deucher 72d38ceaf9SAlex Deucher mutex_lock(&fpriv->bo_list_lock); 73d38ceaf9SAlex Deucher list = idr_find(&fpriv->bo_list_handles, id); 74d38ceaf9SAlex Deucher if (list) { 75d38ceaf9SAlex Deucher mutex_lock(&list->lock); 76d38ceaf9SAlex Deucher idr_remove(&fpriv->bo_list_handles, id); 77d38ceaf9SAlex Deucher mutex_unlock(&list->lock); 78d38ceaf9SAlex Deucher amdgpu_bo_list_free(list); 79d38ceaf9SAlex Deucher } 80d38ceaf9SAlex Deucher mutex_unlock(&fpriv->bo_list_lock); 81d38ceaf9SAlex Deucher } 82d38ceaf9SAlex Deucher 83d38ceaf9SAlex Deucher static int amdgpu_bo_list_set(struct amdgpu_device *adev, 84d38ceaf9SAlex Deucher struct drm_file *filp, 85d38ceaf9SAlex Deucher struct amdgpu_bo_list *list, 86d38ceaf9SAlex Deucher struct drm_amdgpu_bo_list_entry *info, 87d38ceaf9SAlex Deucher unsigned num_entries) 88d38ceaf9SAlex Deucher { 89d38ceaf9SAlex Deucher struct amdgpu_bo_list_entry *array; 90d38ceaf9SAlex Deucher struct amdgpu_bo *gds_obj = adev->gds.gds_gfx_bo; 91d38ceaf9SAlex Deucher struct amdgpu_bo *gws_obj = adev->gds.gws_gfx_bo; 92d38ceaf9SAlex Deucher struct amdgpu_bo *oa_obj = adev->gds.oa_gfx_bo; 93d38ceaf9SAlex Deucher 94211dff55SChristian König unsigned last_entry = 0, first_userptr = num_entries; 95d38ceaf9SAlex Deucher unsigned i; 96cc325d19SChristian König int r; 97d38ceaf9SAlex Deucher 98d38ceaf9SAlex Deucher array = drm_malloc_ab(num_entries, sizeof(struct amdgpu_bo_list_entry)); 99d38ceaf9SAlex Deucher if (!array) 100d38ceaf9SAlex Deucher return -ENOMEM; 101d38ceaf9SAlex Deucher memset(array, 0, num_entries * sizeof(struct amdgpu_bo_list_entry)); 102d38ceaf9SAlex Deucher 103d38ceaf9SAlex Deucher for (i = 0; i < num_entries; ++i) { 104211dff55SChristian König struct amdgpu_bo_list_entry *entry; 105d38ceaf9SAlex Deucher struct drm_gem_object *gobj; 106211dff55SChristian König struct amdgpu_bo *bo; 107cc325d19SChristian König struct mm_struct *usermm; 108d38ceaf9SAlex Deucher 109d38ceaf9SAlex Deucher gobj = drm_gem_object_lookup(adev->ddev, filp, info[i].bo_handle); 110cc325d19SChristian König if (!gobj) { 111cc325d19SChristian König r = -ENOENT; 112d38ceaf9SAlex Deucher goto error_free; 113cc325d19SChristian König } 114d38ceaf9SAlex Deucher 115211dff55SChristian König bo = amdgpu_bo_ref(gem_to_amdgpu_bo(gobj)); 116d38ceaf9SAlex Deucher drm_gem_object_unreference_unlocked(gobj); 117211dff55SChristian König 118211dff55SChristian König usermm = amdgpu_ttm_tt_get_usermm(bo->tbo.ttm); 119cc325d19SChristian König if (usermm) { 120cc325d19SChristian König if (usermm != current->mm) { 121211dff55SChristian König amdgpu_bo_unref(&bo); 122cc325d19SChristian König r = -EPERM; 123cc325d19SChristian König goto error_free; 124cc325d19SChristian König } 125211dff55SChristian König entry = &array[--first_userptr]; 126211dff55SChristian König } else { 127211dff55SChristian König entry = &array[last_entry++]; 128cc325d19SChristian König } 129211dff55SChristian König 130211dff55SChristian König entry->robj = bo; 131211dff55SChristian König entry->priority = min(info[i].bo_priority, 132211dff55SChristian König AMDGPU_BO_LIST_MAX_PRIORITY); 133d38ceaf9SAlex Deucher entry->tv.bo = &entry->robj->tbo; 134d38ceaf9SAlex Deucher entry->tv.shared = true; 135d38ceaf9SAlex Deucher 1361ea863fdSChristian König if (entry->robj->prefered_domains == AMDGPU_GEM_DOMAIN_GDS) 137d38ceaf9SAlex Deucher gds_obj = entry->robj; 1381ea863fdSChristian König if (entry->robj->prefered_domains == AMDGPU_GEM_DOMAIN_GWS) 139d38ceaf9SAlex Deucher gws_obj = entry->robj; 1401ea863fdSChristian König if (entry->robj->prefered_domains == AMDGPU_GEM_DOMAIN_OA) 141d38ceaf9SAlex Deucher oa_obj = entry->robj; 142ec74407aSChristian König 143ec74407aSChristian König trace_amdgpu_bo_list_set(list, entry->robj); 144d38ceaf9SAlex Deucher } 145d38ceaf9SAlex Deucher 146d38ceaf9SAlex Deucher for (i = 0; i < list->num_entries; ++i) 147d38ceaf9SAlex Deucher amdgpu_bo_unref(&list->array[i].robj); 148d38ceaf9SAlex Deucher 149d38ceaf9SAlex Deucher drm_free_large(list->array); 150d38ceaf9SAlex Deucher 151d38ceaf9SAlex Deucher list->gds_obj = gds_obj; 152d38ceaf9SAlex Deucher list->gws_obj = gws_obj; 153d38ceaf9SAlex Deucher list->oa_obj = oa_obj; 154211dff55SChristian König list->first_userptr = first_userptr; 155d38ceaf9SAlex Deucher list->array = array; 156d38ceaf9SAlex Deucher list->num_entries = num_entries; 157d38ceaf9SAlex Deucher 158d38ceaf9SAlex Deucher return 0; 159d38ceaf9SAlex Deucher 160d38ceaf9SAlex Deucher error_free: 16170eacc72SChristian König while (i--) 16270eacc72SChristian König amdgpu_bo_unref(&array[i].robj); 163d38ceaf9SAlex Deucher drm_free_large(array); 164cc325d19SChristian König return r; 165d38ceaf9SAlex Deucher } 166d38ceaf9SAlex Deucher 167d38ceaf9SAlex Deucher struct amdgpu_bo_list * 168d38ceaf9SAlex Deucher amdgpu_bo_list_get(struct amdgpu_fpriv *fpriv, int id) 169d38ceaf9SAlex Deucher { 170d38ceaf9SAlex Deucher struct amdgpu_bo_list *result; 171d38ceaf9SAlex Deucher 172d38ceaf9SAlex Deucher mutex_lock(&fpriv->bo_list_lock); 173d38ceaf9SAlex Deucher result = idr_find(&fpriv->bo_list_handles, id); 174d38ceaf9SAlex Deucher if (result) 175d38ceaf9SAlex Deucher mutex_lock(&result->lock); 176d38ceaf9SAlex Deucher mutex_unlock(&fpriv->bo_list_lock); 177d38ceaf9SAlex Deucher return result; 178d38ceaf9SAlex Deucher } 179d38ceaf9SAlex Deucher 180636ce25cSChristian König void amdgpu_bo_list_get_list(struct amdgpu_bo_list *list, 181636ce25cSChristian König struct list_head *validated) 182636ce25cSChristian König { 183636ce25cSChristian König /* This is based on the bucket sort with O(n) time complexity. 184636ce25cSChristian König * An item with priority "i" is added to bucket[i]. The lists are then 185636ce25cSChristian König * concatenated in descending order. 186636ce25cSChristian König */ 187636ce25cSChristian König struct list_head bucket[AMDGPU_BO_LIST_NUM_BUCKETS]; 188636ce25cSChristian König unsigned i; 189636ce25cSChristian König 190636ce25cSChristian König for (i = 0; i < AMDGPU_BO_LIST_NUM_BUCKETS; i++) 191636ce25cSChristian König INIT_LIST_HEAD(&bucket[i]); 192636ce25cSChristian König 193636ce25cSChristian König /* Since buffers which appear sooner in the relocation list are 194636ce25cSChristian König * likely to be used more often than buffers which appear later 195636ce25cSChristian König * in the list, the sort mustn't change the ordering of buffers 196636ce25cSChristian König * with the same priority, i.e. it must be stable. 197636ce25cSChristian König */ 198636ce25cSChristian König for (i = 0; i < list->num_entries; i++) { 199636ce25cSChristian König unsigned priority = list->array[i].priority; 200636ce25cSChristian König 201636ce25cSChristian König list_add_tail(&list->array[i].tv.head, 202636ce25cSChristian König &bucket[priority]); 203636ce25cSChristian König } 204636ce25cSChristian König 205636ce25cSChristian König /* Connect the sorted buckets in the output list. */ 206636ce25cSChristian König for (i = 0; i < AMDGPU_BO_LIST_NUM_BUCKETS; i++) 207636ce25cSChristian König list_splice(&bucket[i], validated); 208636ce25cSChristian König } 209636ce25cSChristian König 210d38ceaf9SAlex Deucher void amdgpu_bo_list_put(struct amdgpu_bo_list *list) 211d38ceaf9SAlex Deucher { 212d38ceaf9SAlex Deucher mutex_unlock(&list->lock); 213d38ceaf9SAlex Deucher } 214d38ceaf9SAlex Deucher 215d38ceaf9SAlex Deucher void amdgpu_bo_list_free(struct amdgpu_bo_list *list) 216d38ceaf9SAlex Deucher { 217d38ceaf9SAlex Deucher unsigned i; 218d38ceaf9SAlex Deucher 219d38ceaf9SAlex Deucher for (i = 0; i < list->num_entries; ++i) 220d38ceaf9SAlex Deucher amdgpu_bo_unref(&list->array[i].robj); 221d38ceaf9SAlex Deucher 222d38ceaf9SAlex Deucher mutex_destroy(&list->lock); 223d38ceaf9SAlex Deucher drm_free_large(list->array); 224d38ceaf9SAlex Deucher kfree(list); 225d38ceaf9SAlex Deucher } 226d38ceaf9SAlex Deucher 227d38ceaf9SAlex Deucher int amdgpu_bo_list_ioctl(struct drm_device *dev, void *data, 228d38ceaf9SAlex Deucher struct drm_file *filp) 229d38ceaf9SAlex Deucher { 230d38ceaf9SAlex Deucher const uint32_t info_size = sizeof(struct drm_amdgpu_bo_list_entry); 231d38ceaf9SAlex Deucher 232d38ceaf9SAlex Deucher struct amdgpu_device *adev = dev->dev_private; 233d38ceaf9SAlex Deucher struct amdgpu_fpriv *fpriv = filp->driver_priv; 234d38ceaf9SAlex Deucher union drm_amdgpu_bo_list *args = data; 235d38ceaf9SAlex Deucher uint32_t handle = args->in.list_handle; 236d38ceaf9SAlex Deucher const void __user *uptr = (const void*)(long)args->in.bo_info_ptr; 237d38ceaf9SAlex Deucher 238d38ceaf9SAlex Deucher struct drm_amdgpu_bo_list_entry *info; 239d38ceaf9SAlex Deucher struct amdgpu_bo_list *list; 240d38ceaf9SAlex Deucher 241d38ceaf9SAlex Deucher int r; 242d38ceaf9SAlex Deucher 243d38ceaf9SAlex Deucher info = drm_malloc_ab(args->in.bo_number, 244d38ceaf9SAlex Deucher sizeof(struct drm_amdgpu_bo_list_entry)); 245d38ceaf9SAlex Deucher if (!info) 246d38ceaf9SAlex Deucher return -ENOMEM; 247d38ceaf9SAlex Deucher 248d38ceaf9SAlex Deucher /* copy the handle array from userspace to a kernel buffer */ 249d38ceaf9SAlex Deucher r = -EFAULT; 250d38ceaf9SAlex Deucher if (likely(info_size == args->in.bo_info_size)) { 251d38ceaf9SAlex Deucher unsigned long bytes = args->in.bo_number * 252d38ceaf9SAlex Deucher args->in.bo_info_size; 253d38ceaf9SAlex Deucher 254d38ceaf9SAlex Deucher if (copy_from_user(info, uptr, bytes)) 255d38ceaf9SAlex Deucher goto error_free; 256d38ceaf9SAlex Deucher 257d38ceaf9SAlex Deucher } else { 258d38ceaf9SAlex Deucher unsigned long bytes = min(args->in.bo_info_size, info_size); 259d38ceaf9SAlex Deucher unsigned i; 260d38ceaf9SAlex Deucher 261d38ceaf9SAlex Deucher memset(info, 0, args->in.bo_number * info_size); 262d38ceaf9SAlex Deucher for (i = 0; i < args->in.bo_number; ++i) { 263d38ceaf9SAlex Deucher if (copy_from_user(&info[i], uptr, bytes)) 264d38ceaf9SAlex Deucher goto error_free; 265d38ceaf9SAlex Deucher 266d38ceaf9SAlex Deucher uptr += args->in.bo_info_size; 267d38ceaf9SAlex Deucher } 268d38ceaf9SAlex Deucher } 269d38ceaf9SAlex Deucher 270d38ceaf9SAlex Deucher switch (args->in.operation) { 271d38ceaf9SAlex Deucher case AMDGPU_BO_LIST_OP_CREATE: 272d38ceaf9SAlex Deucher r = amdgpu_bo_list_create(fpriv, &list, &handle); 273d38ceaf9SAlex Deucher if (r) 274d38ceaf9SAlex Deucher goto error_free; 275d38ceaf9SAlex Deucher 276d38ceaf9SAlex Deucher r = amdgpu_bo_list_set(adev, filp, list, info, 277d38ceaf9SAlex Deucher args->in.bo_number); 278d38ceaf9SAlex Deucher amdgpu_bo_list_put(list); 279d38ceaf9SAlex Deucher if (r) 280d38ceaf9SAlex Deucher goto error_free; 281d38ceaf9SAlex Deucher 282d38ceaf9SAlex Deucher break; 283d38ceaf9SAlex Deucher 284d38ceaf9SAlex Deucher case AMDGPU_BO_LIST_OP_DESTROY: 285d38ceaf9SAlex Deucher amdgpu_bo_list_destroy(fpriv, handle); 286d38ceaf9SAlex Deucher handle = 0; 287d38ceaf9SAlex Deucher break; 288d38ceaf9SAlex Deucher 289d38ceaf9SAlex Deucher case AMDGPU_BO_LIST_OP_UPDATE: 290d38ceaf9SAlex Deucher r = -ENOENT; 291d38ceaf9SAlex Deucher list = amdgpu_bo_list_get(fpriv, handle); 292d38ceaf9SAlex Deucher if (!list) 293d38ceaf9SAlex Deucher goto error_free; 294d38ceaf9SAlex Deucher 295d38ceaf9SAlex Deucher r = amdgpu_bo_list_set(adev, filp, list, info, 296d38ceaf9SAlex Deucher args->in.bo_number); 297d38ceaf9SAlex Deucher amdgpu_bo_list_put(list); 298d38ceaf9SAlex Deucher if (r) 299d38ceaf9SAlex Deucher goto error_free; 300d38ceaf9SAlex Deucher 301d38ceaf9SAlex Deucher break; 302d38ceaf9SAlex Deucher 303d38ceaf9SAlex Deucher default: 304d38ceaf9SAlex Deucher r = -EINVAL; 305d38ceaf9SAlex Deucher goto error_free; 306d38ceaf9SAlex Deucher } 307d38ceaf9SAlex Deucher 308d38ceaf9SAlex Deucher memset(args, 0, sizeof(*args)); 309d38ceaf9SAlex Deucher args->out.list_handle = handle; 310d38ceaf9SAlex Deucher drm_free_large(info); 311d38ceaf9SAlex Deucher 312d38ceaf9SAlex Deucher return 0; 313d38ceaf9SAlex Deucher 314d38ceaf9SAlex Deucher error_free: 315d38ceaf9SAlex Deucher drm_free_large(info); 316d38ceaf9SAlex Deucher return r; 317d38ceaf9SAlex Deucher } 318