14a488a7aSOded Gabbay /* 24a488a7aSOded Gabbay * Copyright 2014 Advanced Micro Devices, Inc. 34a488a7aSOded Gabbay * 44a488a7aSOded Gabbay * Permission is hereby granted, free of charge, to any person obtaining a 54a488a7aSOded Gabbay * copy of this software and associated documentation files (the "Software"), 64a488a7aSOded Gabbay * to deal in the Software without restriction, including without limitation 74a488a7aSOded Gabbay * the rights to use, copy, modify, merge, publish, distribute, sublicense, 84a488a7aSOded Gabbay * and/or sell copies of the Software, and to permit persons to whom the 94a488a7aSOded Gabbay * Software is furnished to do so, subject to the following conditions: 104a488a7aSOded Gabbay * 114a488a7aSOded Gabbay * The above copyright notice and this permission notice shall be included in 124a488a7aSOded Gabbay * all copies or substantial portions of the Software. 134a488a7aSOded Gabbay * 144a488a7aSOded Gabbay * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 154a488a7aSOded Gabbay * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 164a488a7aSOded Gabbay * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 174a488a7aSOded Gabbay * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 184a488a7aSOded Gabbay * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 194a488a7aSOded Gabbay * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 204a488a7aSOded Gabbay * OTHER DEALINGS IN THE SOFTWARE. 214a488a7aSOded Gabbay */ 224a488a7aSOded Gabbay 234a488a7aSOded Gabbay #ifndef KFD_PRIV_H_INCLUDED 244a488a7aSOded Gabbay #define KFD_PRIV_H_INCLUDED 254a488a7aSOded Gabbay 264a488a7aSOded Gabbay #include <linux/hashtable.h> 274a488a7aSOded Gabbay #include <linux/mmu_notifier.h> 284a488a7aSOded Gabbay #include <linux/mutex.h> 294a488a7aSOded Gabbay #include <linux/types.h> 304a488a7aSOded Gabbay #include <linux/atomic.h> 314a488a7aSOded Gabbay #include <linux/workqueue.h> 324a488a7aSOded Gabbay #include <linux/spinlock.h> 3319f6d2a6SOded Gabbay #include <linux/kfd_ioctl.h> 344a488a7aSOded Gabbay #include <kgd_kfd_interface.h> 354a488a7aSOded Gabbay 365b5c4e40SEvgeny Pinchuk #define KFD_SYSFS_FILE_MODE 0444 375b5c4e40SEvgeny Pinchuk 385b5c4e40SEvgeny Pinchuk /* GPU ID hash width in bits */ 395b5c4e40SEvgeny Pinchuk #define KFD_GPU_ID_HASH_WIDTH 16 405b5c4e40SEvgeny Pinchuk 415b5c4e40SEvgeny Pinchuk /* Macro for allocating structures */ 425b5c4e40SEvgeny Pinchuk #define kfd_alloc_struct(ptr_to_struct) \ 435b5c4e40SEvgeny Pinchuk ((typeof(ptr_to_struct)) kzalloc(sizeof(*ptr_to_struct), GFP_KERNEL)) 445b5c4e40SEvgeny Pinchuk 4519f6d2a6SOded Gabbay /* Kernel module parameter to specify maximum number of supported processes */ 4619f6d2a6SOded Gabbay extern int max_num_of_processes; 4719f6d2a6SOded Gabbay 4819f6d2a6SOded Gabbay #define KFD_MAX_NUM_OF_PROCESSES_DEFAULT 32 4919f6d2a6SOded Gabbay #define KFD_MAX_NUM_OF_PROCESSES 512 5019f6d2a6SOded Gabbay 5119f6d2a6SOded Gabbay /* 5219f6d2a6SOded Gabbay * Kernel module parameter to specify maximum number of supported queues 5319f6d2a6SOded Gabbay * per process 5419f6d2a6SOded Gabbay */ 5519f6d2a6SOded Gabbay extern int max_num_of_queues_per_process; 5619f6d2a6SOded Gabbay 5719f6d2a6SOded Gabbay #define KFD_MAX_NUM_OF_QUEUES_PER_PROCESS_DEFAULT 128 5819f6d2a6SOded Gabbay #define KFD_MAX_NUM_OF_QUEUES_PER_PROCESS 1024 5919f6d2a6SOded Gabbay 6019f6d2a6SOded Gabbay 614a488a7aSOded Gabbay struct kfd_device_info { 624a488a7aSOded Gabbay unsigned int max_pasid_bits; 634a488a7aSOded Gabbay size_t ih_ring_entry_size; 6419f6d2a6SOded Gabbay uint16_t mqd_size_aligned; 654a488a7aSOded Gabbay }; 664a488a7aSOded Gabbay 674a488a7aSOded Gabbay struct kfd_dev { 684a488a7aSOded Gabbay struct kgd_dev *kgd; 694a488a7aSOded Gabbay 704a488a7aSOded Gabbay const struct kfd_device_info *device_info; 714a488a7aSOded Gabbay struct pci_dev *pdev; 724a488a7aSOded Gabbay 734a488a7aSOded Gabbay unsigned int id; /* topology stub index */ 744a488a7aSOded Gabbay 7519f6d2a6SOded Gabbay phys_addr_t doorbell_base; /* Start of actual doorbells used by 7619f6d2a6SOded Gabbay * KFD. It is aligned for mapping 7719f6d2a6SOded Gabbay * into user mode 7819f6d2a6SOded Gabbay */ 7919f6d2a6SOded Gabbay size_t doorbell_id_offset; /* Doorbell offset (from KFD doorbell 8019f6d2a6SOded Gabbay * to HW doorbell, GFX reserved some 8119f6d2a6SOded Gabbay * at the start) 8219f6d2a6SOded Gabbay */ 8319f6d2a6SOded Gabbay size_t doorbell_process_limit; /* Number of processes we have doorbell 8419f6d2a6SOded Gabbay * space for. 8519f6d2a6SOded Gabbay */ 8619f6d2a6SOded Gabbay u32 __iomem *doorbell_kernel_ptr; /* This is a pointer for a doorbells 8719f6d2a6SOded Gabbay * page used by kernel queue 8819f6d2a6SOded Gabbay */ 8919f6d2a6SOded Gabbay 904a488a7aSOded Gabbay struct kgd2kfd_shared_resources shared_resources; 914a488a7aSOded Gabbay 924a488a7aSOded Gabbay bool init_complete; 934a488a7aSOded Gabbay 944a488a7aSOded Gabbay }; 954a488a7aSOded Gabbay 964a488a7aSOded Gabbay /* KGD2KFD callbacks */ 974a488a7aSOded Gabbay void kgd2kfd_exit(void); 984a488a7aSOded Gabbay struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd, struct pci_dev *pdev); 994a488a7aSOded Gabbay bool kgd2kfd_device_init(struct kfd_dev *kfd, 1004a488a7aSOded Gabbay const struct kgd2kfd_shared_resources *gpu_resources); 1014a488a7aSOded Gabbay void kgd2kfd_device_exit(struct kfd_dev *kfd); 1024a488a7aSOded Gabbay 1034a488a7aSOded Gabbay extern const struct kfd2kgd_calls *kfd2kgd; 1044a488a7aSOded Gabbay 10519f6d2a6SOded Gabbay struct kfd_mem_obj { 10619f6d2a6SOded Gabbay void *bo; 10719f6d2a6SOded Gabbay uint64_t gpu_addr; 10819f6d2a6SOded Gabbay uint32_t *cpu_ptr; 10919f6d2a6SOded Gabbay }; 11019f6d2a6SOded Gabbay 11119f6d2a6SOded Gabbay enum kfd_mempool { 11219f6d2a6SOded Gabbay KFD_MEMPOOL_SYSTEM_CACHEABLE = 1, 11319f6d2a6SOded Gabbay KFD_MEMPOOL_SYSTEM_WRITECOMBINE = 2, 11419f6d2a6SOded Gabbay KFD_MEMPOOL_FRAMEBUFFER = 3, 11519f6d2a6SOded Gabbay }; 11619f6d2a6SOded Gabbay 1174a488a7aSOded Gabbay /* Character device interface */ 1184a488a7aSOded Gabbay int kfd_chardev_init(void); 1194a488a7aSOded Gabbay void kfd_chardev_exit(void); 1204a488a7aSOded Gabbay struct device *kfd_chardev(void); 1214a488a7aSOded Gabbay 12219f6d2a6SOded Gabbay 12319f6d2a6SOded Gabbay /* Data that is per-process-per device. */ 12419f6d2a6SOded Gabbay struct kfd_process_device { 12519f6d2a6SOded Gabbay /* 12619f6d2a6SOded Gabbay * List of all per-device data for a process. 12719f6d2a6SOded Gabbay * Starts from kfd_process.per_device_data. 12819f6d2a6SOded Gabbay */ 12919f6d2a6SOded Gabbay struct list_head per_device_list; 13019f6d2a6SOded Gabbay 13119f6d2a6SOded Gabbay /* The device that owns this data. */ 13219f6d2a6SOded Gabbay struct kfd_dev *dev; 13319f6d2a6SOded Gabbay 13419f6d2a6SOded Gabbay 13519f6d2a6SOded Gabbay /*Apertures*/ 13619f6d2a6SOded Gabbay uint64_t lds_base; 13719f6d2a6SOded Gabbay uint64_t lds_limit; 13819f6d2a6SOded Gabbay uint64_t gpuvm_base; 13919f6d2a6SOded Gabbay uint64_t gpuvm_limit; 14019f6d2a6SOded Gabbay uint64_t scratch_base; 14119f6d2a6SOded Gabbay uint64_t scratch_limit; 14219f6d2a6SOded Gabbay 14319f6d2a6SOded Gabbay /* Is this process/pasid bound to this device? (amd_iommu_bind_pasid) */ 14419f6d2a6SOded Gabbay bool bound; 14519f6d2a6SOded Gabbay }; 14619f6d2a6SOded Gabbay 1474a488a7aSOded Gabbay /* Process data */ 1484a488a7aSOded Gabbay struct kfd_process { 14919f6d2a6SOded Gabbay /* 15019f6d2a6SOded Gabbay * kfd_process are stored in an mm_struct*->kfd_process* 15119f6d2a6SOded Gabbay * hash table (kfd_processes in kfd_process.c) 15219f6d2a6SOded Gabbay */ 15319f6d2a6SOded Gabbay struct hlist_node kfd_processes; 15419f6d2a6SOded Gabbay 15519f6d2a6SOded Gabbay struct mm_struct *mm; 15619f6d2a6SOded Gabbay 15719f6d2a6SOded Gabbay struct mutex mutex; 15819f6d2a6SOded Gabbay 15919f6d2a6SOded Gabbay /* 16019f6d2a6SOded Gabbay * In any process, the thread that started main() is the lead 16119f6d2a6SOded Gabbay * thread and outlives the rest. 16219f6d2a6SOded Gabbay * It is here because amd_iommu_bind_pasid wants a task_struct. 16319f6d2a6SOded Gabbay */ 16419f6d2a6SOded Gabbay struct task_struct *lead_thread; 16519f6d2a6SOded Gabbay 16619f6d2a6SOded Gabbay /* We want to receive a notification when the mm_struct is destroyed */ 16719f6d2a6SOded Gabbay struct mmu_notifier mmu_notifier; 16819f6d2a6SOded Gabbay 16919f6d2a6SOded Gabbay /* Use for delayed freeing of kfd_process structure */ 17019f6d2a6SOded Gabbay struct rcu_head rcu; 17119f6d2a6SOded Gabbay 17219f6d2a6SOded Gabbay unsigned int pasid; 17319f6d2a6SOded Gabbay 17419f6d2a6SOded Gabbay /* 17519f6d2a6SOded Gabbay * List of kfd_process_device structures, 17619f6d2a6SOded Gabbay * one for each device the process is using. 17719f6d2a6SOded Gabbay */ 17819f6d2a6SOded Gabbay struct list_head per_device_data; 17919f6d2a6SOded Gabbay 18019f6d2a6SOded Gabbay /* The process's queues. */ 18119f6d2a6SOded Gabbay size_t queue_array_size; 18219f6d2a6SOded Gabbay 18319f6d2a6SOded Gabbay /* Size is queue_array_size, up to MAX_PROCESS_QUEUES. */ 18419f6d2a6SOded Gabbay struct kfd_queue **queues; 18519f6d2a6SOded Gabbay 18619f6d2a6SOded Gabbay unsigned long allocated_queue_bitmap[DIV_ROUND_UP(KFD_MAX_NUM_OF_QUEUES_PER_PROCESS, BITS_PER_LONG)]; 18719f6d2a6SOded Gabbay 18819f6d2a6SOded Gabbay /*Is the user space process 32 bit?*/ 18919f6d2a6SOded Gabbay bool is_32bit_user_mode; 1904a488a7aSOded Gabbay }; 1914a488a7aSOded Gabbay 19219f6d2a6SOded Gabbay void kfd_process_create_wq(void); 19319f6d2a6SOded Gabbay void kfd_process_destroy_wq(void); 19419f6d2a6SOded Gabbay struct kfd_process *kfd_create_process(const struct task_struct *); 19519f6d2a6SOded Gabbay struct kfd_process *kfd_get_process(const struct task_struct *); 19619f6d2a6SOded Gabbay 197b17f068aSOded Gabbay void kfd_unbind_process_from_device(struct kfd_dev *dev, unsigned int pasid); 19819f6d2a6SOded Gabbay struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev, 19919f6d2a6SOded Gabbay struct kfd_process *p, 20019f6d2a6SOded Gabbay int create_pdd); 20119f6d2a6SOded Gabbay 20219f6d2a6SOded Gabbay /* PASIDs */ 20319f6d2a6SOded Gabbay int kfd_pasid_init(void); 20419f6d2a6SOded Gabbay void kfd_pasid_exit(void); 20519f6d2a6SOded Gabbay bool kfd_set_pasid_limit(unsigned int new_limit); 20619f6d2a6SOded Gabbay unsigned int kfd_get_pasid_limit(void); 20719f6d2a6SOded Gabbay unsigned int kfd_pasid_alloc(void); 20819f6d2a6SOded Gabbay void kfd_pasid_free(unsigned int pasid); 20919f6d2a6SOded Gabbay 21019f6d2a6SOded Gabbay /* Doorbells */ 21119f6d2a6SOded Gabbay void kfd_doorbell_init(struct kfd_dev *kfd); 21219f6d2a6SOded Gabbay int kfd_doorbell_mmap(struct kfd_process *process, struct vm_area_struct *vma); 21319f6d2a6SOded Gabbay u32 __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd, 21419f6d2a6SOded Gabbay unsigned int *doorbell_off); 21519f6d2a6SOded Gabbay void kfd_release_kernel_doorbell(struct kfd_dev *kfd, u32 __iomem *db_addr); 21619f6d2a6SOded Gabbay u32 read_kernel_doorbell(u32 __iomem *db); 21719f6d2a6SOded Gabbay void write_kernel_doorbell(u32 __iomem *db, u32 value); 21819f6d2a6SOded Gabbay unsigned int kfd_queue_id_to_doorbell(struct kfd_dev *kfd, 21919f6d2a6SOded Gabbay struct kfd_process *process, 22019f6d2a6SOded Gabbay unsigned int queue_id); 22119f6d2a6SOded Gabbay 2224a488a7aSOded Gabbay extern struct device *kfd_device; 2234a488a7aSOded Gabbay 2245b5c4e40SEvgeny Pinchuk /* Topology */ 2255b5c4e40SEvgeny Pinchuk int kfd_topology_init(void); 2265b5c4e40SEvgeny Pinchuk void kfd_topology_shutdown(void); 2275b5c4e40SEvgeny Pinchuk int kfd_topology_add_device(struct kfd_dev *gpu); 2285b5c4e40SEvgeny Pinchuk int kfd_topology_remove_device(struct kfd_dev *gpu); 2295b5c4e40SEvgeny Pinchuk struct kfd_dev *kfd_device_by_id(uint32_t gpu_id); 2305b5c4e40SEvgeny Pinchuk struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev); 2315b5c4e40SEvgeny Pinchuk struct kfd_dev *kfd_topology_enum_kfd_devices(uint8_t idx); 2325b5c4e40SEvgeny Pinchuk 2334a488a7aSOded Gabbay /* Interrupts */ 2344a488a7aSOded Gabbay void kgd2kfd_interrupt(struct kfd_dev *dev, const void *ih_ring_entry); 2354a488a7aSOded Gabbay 2364a488a7aSOded Gabbay /* Power Management */ 2374a488a7aSOded Gabbay void kgd2kfd_suspend(struct kfd_dev *dev); 2384a488a7aSOded Gabbay int kgd2kfd_resume(struct kfd_dev *dev); 2394a488a7aSOded Gabbay 24019f6d2a6SOded Gabbay /* amdkfd Apertures */ 24119f6d2a6SOded Gabbay int kfd_init_apertures(struct kfd_process *process); 24219f6d2a6SOded Gabbay 24319f6d2a6SOded Gabbay uint64_t kfd_get_number_elems(struct kfd_dev *kfd); 24419f6d2a6SOded Gabbay phys_addr_t kfd_get_process_doorbells(struct kfd_dev *dev, 24519f6d2a6SOded Gabbay struct kfd_process *process); 24619f6d2a6SOded Gabbay 2474a488a7aSOded Gabbay #endif 248