1a538bbe7SJack Xiao /* 2a538bbe7SJack Xiao * Copyright 2019 Advanced Micro Devices, Inc. 3a538bbe7SJack Xiao * 4a538bbe7SJack Xiao * Permission is hereby granted, free of charge, to any person obtaining a 5a538bbe7SJack Xiao * copy of this software and associated documentation files (the "Software"), 6a538bbe7SJack Xiao * to deal in the Software without restriction, including without limitation 7a538bbe7SJack Xiao * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8a538bbe7SJack Xiao * and/or sell copies of the Software, and to permit persons to whom the 9a538bbe7SJack Xiao * Software is furnished to do so, subject to the following conditions: 10a538bbe7SJack Xiao * 11a538bbe7SJack Xiao * The above copyright notice and this permission notice shall be included in 12a538bbe7SJack Xiao * all copies or substantial portions of the Software. 13a538bbe7SJack Xiao * 14a538bbe7SJack Xiao * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15a538bbe7SJack Xiao * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16a538bbe7SJack Xiao * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17a538bbe7SJack Xiao * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18a538bbe7SJack Xiao * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19a538bbe7SJack Xiao * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20a538bbe7SJack Xiao * OTHER DEALINGS IN THE SOFTWARE. 21a538bbe7SJack Xiao * 22a538bbe7SJack Xiao */ 23a538bbe7SJack Xiao 24a538bbe7SJack Xiao #ifndef __AMDGPU_MES_H__ 25a538bbe7SJack Xiao #define __AMDGPU_MES_H__ 26a538bbe7SJack Xiao 273a42c7f3SJack Xiao #include "amdgpu_irq.h" 283a42c7f3SJack Xiao #include "kgd_kfd_interface.h" 293a42c7f3SJack Xiao #include "amdgpu_gfx.h" 303a42c7f3SJack Xiao #include <linux/sched/mm.h> 313a42c7f3SJack Xiao 323bbd31e0SJack Xiao #define AMDGPU_MES_MAX_COMPUTE_PIPES 8 333bbd31e0SJack Xiao #define AMDGPU_MES_MAX_GFX_PIPES 2 343bbd31e0SJack Xiao #define AMDGPU_MES_MAX_SDMA_PIPES 2 353bbd31e0SJack Xiao 363bbd31e0SJack Xiao enum amdgpu_mes_priority_level { 373bbd31e0SJack Xiao AMDGPU_MES_PRIORITY_LEVEL_LOW = 0, 383bbd31e0SJack Xiao AMDGPU_MES_PRIORITY_LEVEL_NORMAL = 1, 393bbd31e0SJack Xiao AMDGPU_MES_PRIORITY_LEVEL_MEDIUM = 2, 403bbd31e0SJack Xiao AMDGPU_MES_PRIORITY_LEVEL_HIGH = 3, 413bbd31e0SJack Xiao AMDGPU_MES_PRIORITY_LEVEL_REALTIME = 4, 423bbd31e0SJack Xiao AMDGPU_MES_PRIORITY_NUM_LEVELS 433bbd31e0SJack Xiao }; 443bbd31e0SJack Xiao 453a42c7f3SJack Xiao #define AMDGPU_MES_PROC_CTX_SIZE 0x1000 /* one page area */ 463a42c7f3SJack Xiao #define AMDGPU_MES_GANG_CTX_SIZE 0x1000 /* one page area */ 473a42c7f3SJack Xiao 487bbc3676SJack Xiao struct amdgpu_mes_funcs; 49a538bbe7SJack Xiao 50207e8bbeSJack Xiao enum admgpu_mes_pipe { 51207e8bbeSJack Xiao AMDGPU_MES_SCHED_PIPE = 0, 52207e8bbeSJack Xiao AMDGPU_MES_KIQ_PIPE, 53207e8bbeSJack Xiao AMDGPU_MAX_MES_PIPES = 2, 54207e8bbeSJack Xiao }; 55207e8bbeSJack Xiao 567bbc3676SJack Xiao struct amdgpu_mes { 573bbd31e0SJack Xiao struct amdgpu_device *adev; 583bbd31e0SJack Xiao 593a42c7f3SJack Xiao struct mutex mutex; 603a42c7f3SJack Xiao 613a42c7f3SJack Xiao struct idr pasid_idr; 623a42c7f3SJack Xiao struct idr gang_id_idr; 633a42c7f3SJack Xiao struct idr queue_id_idr; 643a42c7f3SJack Xiao struct ida doorbell_ida; 653a42c7f3SJack Xiao 663a42c7f3SJack Xiao spinlock_t queue_id_lock; 673a42c7f3SJack Xiao 683bbd31e0SJack Xiao uint32_t total_max_queue; 693bbd31e0SJack Xiao uint32_t doorbell_id_offset; 703bbd31e0SJack Xiao uint32_t max_doorbell_slices; 713bbd31e0SJack Xiao 723bbd31e0SJack Xiao uint64_t default_process_quantum; 733bbd31e0SJack Xiao uint64_t default_gang_quantum; 743bbd31e0SJack Xiao 753bbd31e0SJack Xiao struct amdgpu_ring ring; 767bbc3676SJack Xiao 77207e8bbeSJack Xiao const struct firmware *fw[AMDGPU_MAX_MES_PIPES]; 785aa91248SJack Xiao 795aa91248SJack Xiao /* mes ucode */ 80207e8bbeSJack Xiao struct amdgpu_bo *ucode_fw_obj[AMDGPU_MAX_MES_PIPES]; 81207e8bbeSJack Xiao uint64_t ucode_fw_gpu_addr[AMDGPU_MAX_MES_PIPES]; 82207e8bbeSJack Xiao uint32_t *ucode_fw_ptr[AMDGPU_MAX_MES_PIPES]; 83207e8bbeSJack Xiao uint32_t ucode_fw_version[AMDGPU_MAX_MES_PIPES]; 84207e8bbeSJack Xiao uint64_t uc_start_addr[AMDGPU_MAX_MES_PIPES]; 855aa91248SJack Xiao 865aa91248SJack Xiao /* mes ucode data */ 87207e8bbeSJack Xiao struct amdgpu_bo *data_fw_obj[AMDGPU_MAX_MES_PIPES]; 88207e8bbeSJack Xiao uint64_t data_fw_gpu_addr[AMDGPU_MAX_MES_PIPES]; 89207e8bbeSJack Xiao uint32_t *data_fw_ptr[AMDGPU_MAX_MES_PIPES]; 90207e8bbeSJack Xiao uint32_t data_fw_version[AMDGPU_MAX_MES_PIPES]; 91207e8bbeSJack Xiao uint64_t data_start_addr[AMDGPU_MAX_MES_PIPES]; 925aa91248SJack Xiao 933bbd31e0SJack Xiao /* eop gpu obj */ 94207e8bbeSJack Xiao struct amdgpu_bo *eop_gpu_obj[AMDGPU_MAX_MES_PIPES]; 95207e8bbeSJack Xiao uint64_t eop_gpu_addr[AMDGPU_MAX_MES_PIPES]; 963bbd31e0SJack Xiao 97207e8bbeSJack Xiao void *mqd_backup[AMDGPU_MAX_MES_PIPES]; 98207e8bbeSJack Xiao struct amdgpu_irq_src irq[AMDGPU_MAX_MES_PIPES]; 993bbd31e0SJack Xiao 1003bbd31e0SJack Xiao uint32_t vmid_mask_gfxhub; 1013bbd31e0SJack Xiao uint32_t vmid_mask_mmhub; 1023bbd31e0SJack Xiao uint32_t compute_hqd_mask[AMDGPU_MES_MAX_COMPUTE_PIPES]; 1033bbd31e0SJack Xiao uint32_t gfx_hqd_mask[AMDGPU_MES_MAX_GFX_PIPES]; 1043bbd31e0SJack Xiao uint32_t sdma_hqd_mask[AMDGPU_MES_MAX_SDMA_PIPES]; 1053bbd31e0SJack Xiao uint32_t agreegated_doorbells[AMDGPU_MES_PRIORITY_NUM_LEVELS]; 1063bbd31e0SJack Xiao uint32_t sch_ctx_offs; 1073bbd31e0SJack Xiao uint64_t sch_ctx_gpu_addr; 1083bbd31e0SJack Xiao uint64_t *sch_ctx_ptr; 109ae4e3b62SLe Ma uint32_t query_status_fence_offs; 110ae4e3b62SLe Ma uint64_t query_status_fence_gpu_addr; 111ae4e3b62SLe Ma uint64_t *query_status_fence_ptr; 1123bbd31e0SJack Xiao 113cf064b45SJack Xiao /* initialize kiq pipe */ 114cf064b45SJack Xiao int (*kiq_hw_init)(struct amdgpu_device *adev); 115cf064b45SJack Xiao 1167bbc3676SJack Xiao /* ip specific functions */ 1173bbd31e0SJack Xiao const struct amdgpu_mes_funcs *funcs; 1187bbc3676SJack Xiao }; 1197bbc3676SJack Xiao 1203a42c7f3SJack Xiao struct amdgpu_mes_process { 1213a42c7f3SJack Xiao int pasid; 1223a42c7f3SJack Xiao struct amdgpu_vm *vm; 1233a42c7f3SJack Xiao uint64_t pd_gpu_addr; 1243a42c7f3SJack Xiao struct amdgpu_bo *proc_ctx_bo; 1253a42c7f3SJack Xiao uint64_t proc_ctx_gpu_addr; 1263a42c7f3SJack Xiao void *proc_ctx_cpu_ptr; 1273a42c7f3SJack Xiao uint64_t process_quantum; 1283a42c7f3SJack Xiao struct list_head gang_list; 1293a42c7f3SJack Xiao uint32_t doorbell_index; 1303a42c7f3SJack Xiao unsigned long *doorbell_bitmap; 1313a42c7f3SJack Xiao struct mutex doorbell_lock; 1323a42c7f3SJack Xiao }; 1333a42c7f3SJack Xiao 1343a42c7f3SJack Xiao struct amdgpu_mes_gang { 1353a42c7f3SJack Xiao int gang_id; 1363a42c7f3SJack Xiao int priority; 1373a42c7f3SJack Xiao int inprocess_gang_priority; 1383a42c7f3SJack Xiao int global_priority_level; 1393a42c7f3SJack Xiao struct list_head list; 1403a42c7f3SJack Xiao struct amdgpu_mes_process *process; 1413a42c7f3SJack Xiao struct amdgpu_bo *gang_ctx_bo; 1423a42c7f3SJack Xiao uint64_t gang_ctx_gpu_addr; 1433a42c7f3SJack Xiao void *gang_ctx_cpu_ptr; 1443a42c7f3SJack Xiao uint64_t gang_quantum; 1453a42c7f3SJack Xiao struct list_head queue_list; 1463a42c7f3SJack Xiao }; 1473a42c7f3SJack Xiao 1483a42c7f3SJack Xiao struct amdgpu_mes_queue { 1493a42c7f3SJack Xiao struct list_head list; 1503a42c7f3SJack Xiao struct amdgpu_mes_gang *gang; 1513a42c7f3SJack Xiao int queue_id; 1523a42c7f3SJack Xiao uint64_t doorbell_off; 1533a42c7f3SJack Xiao struct amdgpu_bo *mqd_obj; 1543a42c7f3SJack Xiao void *mqd_cpu_ptr; 1553a42c7f3SJack Xiao uint64_t mqd_gpu_addr; 1563a42c7f3SJack Xiao uint64_t wptr_gpu_addr; 1573a42c7f3SJack Xiao int queue_type; 1583a42c7f3SJack Xiao int paging; 1593a42c7f3SJack Xiao struct amdgpu_ring *ring; 1603a42c7f3SJack Xiao }; 1613a42c7f3SJack Xiao 162*5d0f619fSJack Xiao struct amdgpu_mes_gang_properties { 163*5d0f619fSJack Xiao uint32_t priority; 164*5d0f619fSJack Xiao uint32_t gang_quantum; 165*5d0f619fSJack Xiao uint32_t inprocess_gang_priority; 166*5d0f619fSJack Xiao uint32_t priority_level; 167*5d0f619fSJack Xiao int global_priority_level; 168*5d0f619fSJack Xiao }; 169*5d0f619fSJack Xiao 1707bbc3676SJack Xiao struct mes_add_queue_input { 1717bbc3676SJack Xiao uint32_t process_id; 1727bbc3676SJack Xiao uint64_t page_table_base_addr; 1737bbc3676SJack Xiao uint64_t process_va_start; 1747bbc3676SJack Xiao uint64_t process_va_end; 1757bbc3676SJack Xiao uint64_t process_quantum; 1767bbc3676SJack Xiao uint64_t process_context_addr; 1777bbc3676SJack Xiao uint64_t gang_quantum; 1787bbc3676SJack Xiao uint64_t gang_context_addr; 1797bbc3676SJack Xiao uint32_t inprocess_gang_priority; 1807bbc3676SJack Xiao uint32_t gang_global_priority_level; 1817bbc3676SJack Xiao uint32_t doorbell_offset; 1827bbc3676SJack Xiao uint64_t mqd_addr; 1837bbc3676SJack Xiao uint64_t wptr_addr; 1847bbc3676SJack Xiao uint32_t queue_type; 1857bbc3676SJack Xiao uint32_t paging; 1867bbc3676SJack Xiao }; 1877bbc3676SJack Xiao 1887bbc3676SJack Xiao struct mes_remove_queue_input { 1897bbc3676SJack Xiao uint32_t doorbell_offset; 1907bbc3676SJack Xiao uint64_t gang_context_addr; 1917bbc3676SJack Xiao }; 1927bbc3676SJack Xiao 1937bbc3676SJack Xiao struct mes_suspend_gang_input { 1947bbc3676SJack Xiao bool suspend_all_gangs; 1957bbc3676SJack Xiao uint64_t gang_context_addr; 1967bbc3676SJack Xiao uint64_t suspend_fence_addr; 1977bbc3676SJack Xiao uint32_t suspend_fence_value; 1987bbc3676SJack Xiao }; 1997bbc3676SJack Xiao 2007bbc3676SJack Xiao struct mes_resume_gang_input { 2017bbc3676SJack Xiao bool resume_all_gangs; 2027bbc3676SJack Xiao uint64_t gang_context_addr; 2037bbc3676SJack Xiao }; 2047bbc3676SJack Xiao 2057bbc3676SJack Xiao struct amdgpu_mes_funcs { 2067bbc3676SJack Xiao int (*add_hw_queue)(struct amdgpu_mes *mes, 2077bbc3676SJack Xiao struct mes_add_queue_input *input); 2087bbc3676SJack Xiao 2097bbc3676SJack Xiao int (*remove_hw_queue)(struct amdgpu_mes *mes, 2107bbc3676SJack Xiao struct mes_remove_queue_input *input); 2117bbc3676SJack Xiao 2127bbc3676SJack Xiao int (*suspend_gang)(struct amdgpu_mes *mes, 2137bbc3676SJack Xiao struct mes_suspend_gang_input *input); 2147bbc3676SJack Xiao 2157bbc3676SJack Xiao int (*resume_gang)(struct amdgpu_mes *mes, 2167bbc3676SJack Xiao struct mes_resume_gang_input *input); 217a538bbe7SJack Xiao }; 218a538bbe7SJack Xiao 219b04c1d64SJack Xiao 220cf064b45SJack Xiao #define amdgpu_mes_kiq_hw_init(adev) (adev)->mes.kiq_hw_init((adev)) 221cf064b45SJack Xiao 222b04c1d64SJack Xiao int amdgpu_mes_init(struct amdgpu_device *adev); 223b04c1d64SJack Xiao void amdgpu_mes_fini(struct amdgpu_device *adev); 224b04c1d64SJack Xiao 22548dcd2b7SJack Xiao int amdgpu_mes_create_process(struct amdgpu_device *adev, int pasid, 22648dcd2b7SJack Xiao struct amdgpu_vm *vm); 227063a38d6SJack Xiao void amdgpu_mes_destroy_process(struct amdgpu_device *adev, int pasid); 22848dcd2b7SJack Xiao 229*5d0f619fSJack Xiao int amdgpu_mes_add_gang(struct amdgpu_device *adev, int pasid, 230*5d0f619fSJack Xiao struct amdgpu_mes_gang_properties *gprops, 231*5d0f619fSJack Xiao int *gang_id); 232*5d0f619fSJack Xiao 233a538bbe7SJack Xiao #endif /* __AMDGPU_MES_H__ */ 234