1a538bbe7SJack Xiao /* 2a538bbe7SJack Xiao * Copyright 2019 Advanced Micro Devices, Inc. 3a538bbe7SJack Xiao * 4a538bbe7SJack Xiao * Permission is hereby granted, free of charge, to any person obtaining a 5a538bbe7SJack Xiao * copy of this software and associated documentation files (the "Software"), 6a538bbe7SJack Xiao * to deal in the Software without restriction, including without limitation 7a538bbe7SJack Xiao * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8a538bbe7SJack Xiao * and/or sell copies of the Software, and to permit persons to whom the 9a538bbe7SJack Xiao * Software is furnished to do so, subject to the following conditions: 10a538bbe7SJack Xiao * 11a538bbe7SJack Xiao * The above copyright notice and this permission notice shall be included in 12a538bbe7SJack Xiao * all copies or substantial portions of the Software. 13a538bbe7SJack Xiao * 14a538bbe7SJack Xiao * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15a538bbe7SJack Xiao * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16a538bbe7SJack Xiao * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17a538bbe7SJack Xiao * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18a538bbe7SJack Xiao * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19a538bbe7SJack Xiao * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20a538bbe7SJack Xiao * OTHER DEALINGS IN THE SOFTWARE. 21a538bbe7SJack Xiao * 22a538bbe7SJack Xiao */ 23a538bbe7SJack Xiao 24a538bbe7SJack Xiao #ifndef __AMDGPU_MES_H__ 25a538bbe7SJack Xiao #define __AMDGPU_MES_H__ 26a538bbe7SJack Xiao 273a42c7f3SJack Xiao #include "amdgpu_irq.h" 283a42c7f3SJack Xiao #include "kgd_kfd_interface.h" 293a42c7f3SJack Xiao #include "amdgpu_gfx.h" 303a42c7f3SJack Xiao #include <linux/sched/mm.h> 313a42c7f3SJack Xiao 323bbd31e0SJack Xiao #define AMDGPU_MES_MAX_COMPUTE_PIPES 8 333bbd31e0SJack Xiao #define AMDGPU_MES_MAX_GFX_PIPES 2 343bbd31e0SJack Xiao #define AMDGPU_MES_MAX_SDMA_PIPES 2 353bbd31e0SJack Xiao 363bbd31e0SJack Xiao enum amdgpu_mes_priority_level { 373bbd31e0SJack Xiao AMDGPU_MES_PRIORITY_LEVEL_LOW = 0, 383bbd31e0SJack Xiao AMDGPU_MES_PRIORITY_LEVEL_NORMAL = 1, 393bbd31e0SJack Xiao AMDGPU_MES_PRIORITY_LEVEL_MEDIUM = 2, 403bbd31e0SJack Xiao AMDGPU_MES_PRIORITY_LEVEL_HIGH = 3, 413bbd31e0SJack Xiao AMDGPU_MES_PRIORITY_LEVEL_REALTIME = 4, 423bbd31e0SJack Xiao AMDGPU_MES_PRIORITY_NUM_LEVELS 433bbd31e0SJack Xiao }; 443bbd31e0SJack Xiao 453a42c7f3SJack Xiao #define AMDGPU_MES_PROC_CTX_SIZE 0x1000 /* one page area */ 463a42c7f3SJack Xiao #define AMDGPU_MES_GANG_CTX_SIZE 0x1000 /* one page area */ 473a42c7f3SJack Xiao 487bbc3676SJack Xiao struct amdgpu_mes_funcs; 49a538bbe7SJack Xiao 50207e8bbeSJack Xiao enum admgpu_mes_pipe { 51207e8bbeSJack Xiao AMDGPU_MES_SCHED_PIPE = 0, 52207e8bbeSJack Xiao AMDGPU_MES_KIQ_PIPE, 53207e8bbeSJack Xiao AMDGPU_MAX_MES_PIPES = 2, 54207e8bbeSJack Xiao }; 55207e8bbeSJack Xiao 567bbc3676SJack Xiao struct amdgpu_mes { 573bbd31e0SJack Xiao struct amdgpu_device *adev; 583bbd31e0SJack Xiao 593a42c7f3SJack Xiao struct mutex mutex; 603a42c7f3SJack Xiao 613a42c7f3SJack Xiao struct idr pasid_idr; 623a42c7f3SJack Xiao struct idr gang_id_idr; 633a42c7f3SJack Xiao struct idr queue_id_idr; 643a42c7f3SJack Xiao struct ida doorbell_ida; 653a42c7f3SJack Xiao 663a42c7f3SJack Xiao spinlock_t queue_id_lock; 673a42c7f3SJack Xiao 683bbd31e0SJack Xiao uint32_t total_max_queue; 693bbd31e0SJack Xiao uint32_t doorbell_id_offset; 703bbd31e0SJack Xiao uint32_t max_doorbell_slices; 713bbd31e0SJack Xiao 723bbd31e0SJack Xiao uint64_t default_process_quantum; 733bbd31e0SJack Xiao uint64_t default_gang_quantum; 743bbd31e0SJack Xiao 753bbd31e0SJack Xiao struct amdgpu_ring ring; 767bbc3676SJack Xiao 77207e8bbeSJack Xiao const struct firmware *fw[AMDGPU_MAX_MES_PIPES]; 785aa91248SJack Xiao 795aa91248SJack Xiao /* mes ucode */ 80207e8bbeSJack Xiao struct amdgpu_bo *ucode_fw_obj[AMDGPU_MAX_MES_PIPES]; 81207e8bbeSJack Xiao uint64_t ucode_fw_gpu_addr[AMDGPU_MAX_MES_PIPES]; 82207e8bbeSJack Xiao uint32_t *ucode_fw_ptr[AMDGPU_MAX_MES_PIPES]; 83207e8bbeSJack Xiao uint32_t ucode_fw_version[AMDGPU_MAX_MES_PIPES]; 84207e8bbeSJack Xiao uint64_t uc_start_addr[AMDGPU_MAX_MES_PIPES]; 855aa91248SJack Xiao 865aa91248SJack Xiao /* mes ucode data */ 87207e8bbeSJack Xiao struct amdgpu_bo *data_fw_obj[AMDGPU_MAX_MES_PIPES]; 88207e8bbeSJack Xiao uint64_t data_fw_gpu_addr[AMDGPU_MAX_MES_PIPES]; 89207e8bbeSJack Xiao uint32_t *data_fw_ptr[AMDGPU_MAX_MES_PIPES]; 90207e8bbeSJack Xiao uint32_t data_fw_version[AMDGPU_MAX_MES_PIPES]; 91207e8bbeSJack Xiao uint64_t data_start_addr[AMDGPU_MAX_MES_PIPES]; 925aa91248SJack Xiao 933bbd31e0SJack Xiao /* eop gpu obj */ 94207e8bbeSJack Xiao struct amdgpu_bo *eop_gpu_obj[AMDGPU_MAX_MES_PIPES]; 95207e8bbeSJack Xiao uint64_t eop_gpu_addr[AMDGPU_MAX_MES_PIPES]; 963bbd31e0SJack Xiao 97207e8bbeSJack Xiao void *mqd_backup[AMDGPU_MAX_MES_PIPES]; 98207e8bbeSJack Xiao struct amdgpu_irq_src irq[AMDGPU_MAX_MES_PIPES]; 993bbd31e0SJack Xiao 1003bbd31e0SJack Xiao uint32_t vmid_mask_gfxhub; 1013bbd31e0SJack Xiao uint32_t vmid_mask_mmhub; 1023bbd31e0SJack Xiao uint32_t compute_hqd_mask[AMDGPU_MES_MAX_COMPUTE_PIPES]; 1033bbd31e0SJack Xiao uint32_t gfx_hqd_mask[AMDGPU_MES_MAX_GFX_PIPES]; 1043bbd31e0SJack Xiao uint32_t sdma_hqd_mask[AMDGPU_MES_MAX_SDMA_PIPES]; 1053bbd31e0SJack Xiao uint32_t agreegated_doorbells[AMDGPU_MES_PRIORITY_NUM_LEVELS]; 1063bbd31e0SJack Xiao uint32_t sch_ctx_offs; 1073bbd31e0SJack Xiao uint64_t sch_ctx_gpu_addr; 1083bbd31e0SJack Xiao uint64_t *sch_ctx_ptr; 109ae4e3b62SLe Ma uint32_t query_status_fence_offs; 110ae4e3b62SLe Ma uint64_t query_status_fence_gpu_addr; 111ae4e3b62SLe Ma uint64_t *query_status_fence_ptr; 1123bbd31e0SJack Xiao 113cf064b45SJack Xiao /* initialize kiq pipe */ 114cf064b45SJack Xiao int (*kiq_hw_init)(struct amdgpu_device *adev); 115cf064b45SJack Xiao 1167bbc3676SJack Xiao /* ip specific functions */ 1173bbd31e0SJack Xiao const struct amdgpu_mes_funcs *funcs; 1187bbc3676SJack Xiao }; 1197bbc3676SJack Xiao 1203a42c7f3SJack Xiao struct amdgpu_mes_process { 1213a42c7f3SJack Xiao int pasid; 1223a42c7f3SJack Xiao struct amdgpu_vm *vm; 1233a42c7f3SJack Xiao uint64_t pd_gpu_addr; 1243a42c7f3SJack Xiao struct amdgpu_bo *proc_ctx_bo; 1253a42c7f3SJack Xiao uint64_t proc_ctx_gpu_addr; 1263a42c7f3SJack Xiao void *proc_ctx_cpu_ptr; 1273a42c7f3SJack Xiao uint64_t process_quantum; 1283a42c7f3SJack Xiao struct list_head gang_list; 1293a42c7f3SJack Xiao uint32_t doorbell_index; 1303a42c7f3SJack Xiao unsigned long *doorbell_bitmap; 1313a42c7f3SJack Xiao struct mutex doorbell_lock; 1323a42c7f3SJack Xiao }; 1333a42c7f3SJack Xiao 1343a42c7f3SJack Xiao struct amdgpu_mes_gang { 1353a42c7f3SJack Xiao int gang_id; 1363a42c7f3SJack Xiao int priority; 1373a42c7f3SJack Xiao int inprocess_gang_priority; 1383a42c7f3SJack Xiao int global_priority_level; 1393a42c7f3SJack Xiao struct list_head list; 1403a42c7f3SJack Xiao struct amdgpu_mes_process *process; 1413a42c7f3SJack Xiao struct amdgpu_bo *gang_ctx_bo; 1423a42c7f3SJack Xiao uint64_t gang_ctx_gpu_addr; 1433a42c7f3SJack Xiao void *gang_ctx_cpu_ptr; 1443a42c7f3SJack Xiao uint64_t gang_quantum; 1453a42c7f3SJack Xiao struct list_head queue_list; 1463a42c7f3SJack Xiao }; 1473a42c7f3SJack Xiao 1483a42c7f3SJack Xiao struct amdgpu_mes_queue { 1493a42c7f3SJack Xiao struct list_head list; 1503a42c7f3SJack Xiao struct amdgpu_mes_gang *gang; 1513a42c7f3SJack Xiao int queue_id; 1523a42c7f3SJack Xiao uint64_t doorbell_off; 1533a42c7f3SJack Xiao struct amdgpu_bo *mqd_obj; 1543a42c7f3SJack Xiao void *mqd_cpu_ptr; 1553a42c7f3SJack Xiao uint64_t mqd_gpu_addr; 1563a42c7f3SJack Xiao uint64_t wptr_gpu_addr; 1573a42c7f3SJack Xiao int queue_type; 1583a42c7f3SJack Xiao int paging; 1593a42c7f3SJack Xiao struct amdgpu_ring *ring; 1603a42c7f3SJack Xiao }; 1613a42c7f3SJack Xiao 162be5609deSJack Xiao struct amdgpu_mes_queue_properties { 163be5609deSJack Xiao int queue_type; 164be5609deSJack Xiao uint64_t hqd_base_gpu_addr; 165be5609deSJack Xiao uint64_t rptr_gpu_addr; 166be5609deSJack Xiao uint64_t wptr_gpu_addr; 167be5609deSJack Xiao uint32_t queue_size; 168be5609deSJack Xiao uint64_t eop_gpu_addr; 169be5609deSJack Xiao uint32_t hqd_pipe_priority; 170be5609deSJack Xiao uint32_t hqd_queue_priority; 171be5609deSJack Xiao bool paging; 172be5609deSJack Xiao struct amdgpu_ring *ring; 173be5609deSJack Xiao /* out */ 174be5609deSJack Xiao uint64_t doorbell_off; 175be5609deSJack Xiao }; 176be5609deSJack Xiao 1775d0f619fSJack Xiao struct amdgpu_mes_gang_properties { 1785d0f619fSJack Xiao uint32_t priority; 1795d0f619fSJack Xiao uint32_t gang_quantum; 1805d0f619fSJack Xiao uint32_t inprocess_gang_priority; 1815d0f619fSJack Xiao uint32_t priority_level; 1825d0f619fSJack Xiao int global_priority_level; 1835d0f619fSJack Xiao }; 1845d0f619fSJack Xiao 1857bbc3676SJack Xiao struct mes_add_queue_input { 1867bbc3676SJack Xiao uint32_t process_id; 1877bbc3676SJack Xiao uint64_t page_table_base_addr; 1887bbc3676SJack Xiao uint64_t process_va_start; 1897bbc3676SJack Xiao uint64_t process_va_end; 1907bbc3676SJack Xiao uint64_t process_quantum; 1917bbc3676SJack Xiao uint64_t process_context_addr; 1927bbc3676SJack Xiao uint64_t gang_quantum; 1937bbc3676SJack Xiao uint64_t gang_context_addr; 1947bbc3676SJack Xiao uint32_t inprocess_gang_priority; 1957bbc3676SJack Xiao uint32_t gang_global_priority_level; 1967bbc3676SJack Xiao uint32_t doorbell_offset; 1977bbc3676SJack Xiao uint64_t mqd_addr; 1987bbc3676SJack Xiao uint64_t wptr_addr; 1997bbc3676SJack Xiao uint32_t queue_type; 2007bbc3676SJack Xiao uint32_t paging; 2017bbc3676SJack Xiao }; 2027bbc3676SJack Xiao 2037bbc3676SJack Xiao struct mes_remove_queue_input { 2047bbc3676SJack Xiao uint32_t doorbell_offset; 2057bbc3676SJack Xiao uint64_t gang_context_addr; 2067bbc3676SJack Xiao }; 2077bbc3676SJack Xiao 2087bbc3676SJack Xiao struct mes_suspend_gang_input { 2097bbc3676SJack Xiao bool suspend_all_gangs; 2107bbc3676SJack Xiao uint64_t gang_context_addr; 2117bbc3676SJack Xiao uint64_t suspend_fence_addr; 2127bbc3676SJack Xiao uint32_t suspend_fence_value; 2137bbc3676SJack Xiao }; 2147bbc3676SJack Xiao 2157bbc3676SJack Xiao struct mes_resume_gang_input { 2167bbc3676SJack Xiao bool resume_all_gangs; 2177bbc3676SJack Xiao uint64_t gang_context_addr; 2187bbc3676SJack Xiao }; 2197bbc3676SJack Xiao 2207bbc3676SJack Xiao struct amdgpu_mes_funcs { 2217bbc3676SJack Xiao int (*add_hw_queue)(struct amdgpu_mes *mes, 2227bbc3676SJack Xiao struct mes_add_queue_input *input); 2237bbc3676SJack Xiao 2247bbc3676SJack Xiao int (*remove_hw_queue)(struct amdgpu_mes *mes, 2257bbc3676SJack Xiao struct mes_remove_queue_input *input); 2267bbc3676SJack Xiao 2277bbc3676SJack Xiao int (*suspend_gang)(struct amdgpu_mes *mes, 2287bbc3676SJack Xiao struct mes_suspend_gang_input *input); 2297bbc3676SJack Xiao 2307bbc3676SJack Xiao int (*resume_gang)(struct amdgpu_mes *mes, 2317bbc3676SJack Xiao struct mes_resume_gang_input *input); 232a538bbe7SJack Xiao }; 233a538bbe7SJack Xiao 234cf064b45SJack Xiao #define amdgpu_mes_kiq_hw_init(adev) (adev)->mes.kiq_hw_init((adev)) 235cf064b45SJack Xiao 23611ec5b36SJack Xiao int amdgpu_mes_ctx_get_offs(struct amdgpu_ring *ring, unsigned int id_offs); 23711ec5b36SJack Xiao 238b04c1d64SJack Xiao int amdgpu_mes_init(struct amdgpu_device *adev); 239b04c1d64SJack Xiao void amdgpu_mes_fini(struct amdgpu_device *adev); 240b04c1d64SJack Xiao 24148dcd2b7SJack Xiao int amdgpu_mes_create_process(struct amdgpu_device *adev, int pasid, 24248dcd2b7SJack Xiao struct amdgpu_vm *vm); 243063a38d6SJack Xiao void amdgpu_mes_destroy_process(struct amdgpu_device *adev, int pasid); 24448dcd2b7SJack Xiao 2455d0f619fSJack Xiao int amdgpu_mes_add_gang(struct amdgpu_device *adev, int pasid, 2465d0f619fSJack Xiao struct amdgpu_mes_gang_properties *gprops, 2475d0f619fSJack Xiao int *gang_id); 248b0306e58SJack Xiao int amdgpu_mes_remove_gang(struct amdgpu_device *adev, int gang_id); 2495d0f619fSJack Xiao 250c8bb1057SJack Xiao int amdgpu_mes_suspend(struct amdgpu_device *adev); 251ea756bd5SJack Xiao int amdgpu_mes_resume(struct amdgpu_device *adev); 252c8bb1057SJack Xiao 253be5609deSJack Xiao int amdgpu_mes_add_hw_queue(struct amdgpu_device *adev, int gang_id, 254be5609deSJack Xiao struct amdgpu_mes_queue_properties *qprops, 255be5609deSJack Xiao int *queue_id); 256bcc4e1e1SJack Xiao int amdgpu_mes_remove_hw_queue(struct amdgpu_device *adev, int queue_id); 257be5609deSJack Xiao 258*d0c423b6SJack Xiao int amdgpu_mes_add_ring(struct amdgpu_device *adev, int gang_id, 259*d0c423b6SJack Xiao int queue_type, int idx, 260*d0c423b6SJack Xiao struct amdgpu_mes_ctx_data *ctx_data, 261*d0c423b6SJack Xiao struct amdgpu_ring **out); 262*d0c423b6SJack Xiao 263a538bbe7SJack Xiao #endif /* __AMDGPU_MES_H__ */ 264