14a488a7aSOded Gabbay /* 24a488a7aSOded Gabbay * Copyright 2014 Advanced Micro Devices, Inc. 34a488a7aSOded Gabbay * 44a488a7aSOded Gabbay * Permission is hereby granted, free of charge, to any person obtaining a 54a488a7aSOded Gabbay * copy of this software and associated documentation files (the "Software"), 64a488a7aSOded Gabbay * to deal in the Software without restriction, including without limitation 74a488a7aSOded Gabbay * the rights to use, copy, modify, merge, publish, distribute, sublicense, 84a488a7aSOded Gabbay * and/or sell copies of the Software, and to permit persons to whom the 94a488a7aSOded Gabbay * Software is furnished to do so, subject to the following conditions: 104a488a7aSOded Gabbay * 114a488a7aSOded Gabbay * The above copyright notice and this permission notice shall be included in 124a488a7aSOded Gabbay * all copies or substantial portions of the Software. 134a488a7aSOded Gabbay * 144a488a7aSOded Gabbay * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 154a488a7aSOded Gabbay * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 164a488a7aSOded Gabbay * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 174a488a7aSOded Gabbay * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 184a488a7aSOded Gabbay * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 194a488a7aSOded Gabbay * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 204a488a7aSOded Gabbay * OTHER DEALINGS IN THE SOFTWARE. 214a488a7aSOded Gabbay */ 224a488a7aSOded Gabbay 234a488a7aSOded Gabbay #include <linux/bsearch.h> 244a488a7aSOded Gabbay #include <linux/pci.h> 254a488a7aSOded Gabbay #include <linux/slab.h> 264a488a7aSOded Gabbay #include "kfd_priv.h" 2764c7f8cfSBen Goz #include "kfd_device_queue_manager.h" 28507968ddSFelix Kuehling #include "kfd_pm4_headers_vi.h" 290db54b24SYong Zhao #include "cwsr_trap_handler.h" 3064d1c3a4SFelix Kuehling #include "kfd_iommu.h" 315b87245fSAmber Lin #include "amdgpu_amdkfd.h" 324a488a7aSOded Gabbay 3319f6d2a6SOded Gabbay #define MQD_SIZE_ALIGNED 768 34e42051d2SShaoyun Liu 35e42051d2SShaoyun Liu /* 36e42051d2SShaoyun Liu * kfd_locked is used to lock the kfd driver during suspend or reset 37e42051d2SShaoyun Liu * once locked, kfd driver will stop any further GPU execution. 38e42051d2SShaoyun Liu * create process (open) will return -EAGAIN. 39e42051d2SShaoyun Liu */ 40e42051d2SShaoyun Liu static atomic_t kfd_locked = ATOMIC_INIT(0); 4119f6d2a6SOded Gabbay 4264d1c3a4SFelix Kuehling #ifdef KFD_SUPPORT_IOMMU_V2 434a488a7aSOded Gabbay static const struct kfd_device_info kaveri_device_info = { 440da7558cSBen Goz .asic_family = CHIP_KAVERI, 45c181159aSYong Zhao .asic_name = "kaveri", 460da7558cSBen Goz .max_pasid_bits = 16, 47992839adSYair Shachar /* max num of queues for KV.TODO should be a dynamic value */ 48992839adSYair Shachar .max_no_of_hqd = 24, 49ada2b29cSFelix Kuehling .doorbell_size = 4, 500da7558cSBen Goz .ih_ring_entry_size = 4 * sizeof(uint32_t), 51f3a39818SAndrew Lewycky .event_interrupt_class = &event_interrupt_class_cik, 52fbeb661bSYair Shachar .num_of_watch_points = 4, 53373d7080SFelix Kuehling .mqd_size_aligned = MQD_SIZE_ALIGNED, 54373d7080SFelix Kuehling .supports_cwsr = false, 5564d1c3a4SFelix Kuehling .needs_iommu_device = true, 563ee2d00cSFelix Kuehling .needs_pci_atomics = false, 5798bb9222SYong Zhao .num_sdma_engines = 2, 581b4670f6SOak Zeng .num_xgmi_sdma_engines = 0, 59d5094189SShaoyun Liu .num_sdma_queues_per_engine = 2, 600da7558cSBen Goz }; 610da7558cSBen Goz 620da7558cSBen Goz static const struct kfd_device_info carrizo_device_info = { 630da7558cSBen Goz .asic_family = CHIP_CARRIZO, 64c181159aSYong Zhao .asic_name = "carrizo", 654a488a7aSOded Gabbay .max_pasid_bits = 16, 66eaccd6e7SOded Gabbay /* max num of queues for CZ.TODO should be a dynamic value */ 67eaccd6e7SOded Gabbay .max_no_of_hqd = 24, 68ada2b29cSFelix Kuehling .doorbell_size = 4, 69b3f5e6b4SAndrew Lewycky .ih_ring_entry_size = 4 * sizeof(uint32_t), 70eaccd6e7SOded Gabbay .event_interrupt_class = &event_interrupt_class_cik, 71f7c826adSAlexey Skidanov .num_of_watch_points = 4, 72373d7080SFelix Kuehling .mqd_size_aligned = MQD_SIZE_ALIGNED, 73373d7080SFelix Kuehling .supports_cwsr = true, 7464d1c3a4SFelix Kuehling .needs_iommu_device = true, 753ee2d00cSFelix Kuehling .needs_pci_atomics = false, 7698bb9222SYong Zhao .num_sdma_engines = 2, 771b4670f6SOak Zeng .num_xgmi_sdma_engines = 0, 78d5094189SShaoyun Liu .num_sdma_queues_per_engine = 2, 794a488a7aSOded Gabbay }; 804d663df6SYong Zhao 814d663df6SYong Zhao static const struct kfd_device_info raven_device_info = { 824d663df6SYong Zhao .asic_family = CHIP_RAVEN, 83c181159aSYong Zhao .asic_name = "raven", 844d663df6SYong Zhao .max_pasid_bits = 16, 854d663df6SYong Zhao .max_no_of_hqd = 24, 864d663df6SYong Zhao .doorbell_size = 8, 874d663df6SYong Zhao .ih_ring_entry_size = 8 * sizeof(uint32_t), 884d663df6SYong Zhao .event_interrupt_class = &event_interrupt_class_v9, 894d663df6SYong Zhao .num_of_watch_points = 4, 904d663df6SYong Zhao .mqd_size_aligned = MQD_SIZE_ALIGNED, 914d663df6SYong Zhao .supports_cwsr = true, 924d663df6SYong Zhao .needs_iommu_device = true, 934d663df6SYong Zhao .needs_pci_atomics = true, 944d663df6SYong Zhao .num_sdma_engines = 1, 951b4670f6SOak Zeng .num_xgmi_sdma_engines = 0, 96d5094189SShaoyun Liu .num_sdma_queues_per_engine = 2, 974d663df6SYong Zhao }; 9864d1c3a4SFelix Kuehling #endif 994a488a7aSOded Gabbay 100a3084e6cSFelix Kuehling static const struct kfd_device_info hawaii_device_info = { 101a3084e6cSFelix Kuehling .asic_family = CHIP_HAWAII, 102c181159aSYong Zhao .asic_name = "hawaii", 103a3084e6cSFelix Kuehling .max_pasid_bits = 16, 104a3084e6cSFelix Kuehling /* max num of queues for KV.TODO should be a dynamic value */ 105a3084e6cSFelix Kuehling .max_no_of_hqd = 24, 106ada2b29cSFelix Kuehling .doorbell_size = 4, 107a3084e6cSFelix Kuehling .ih_ring_entry_size = 4 * sizeof(uint32_t), 108a3084e6cSFelix Kuehling .event_interrupt_class = &event_interrupt_class_cik, 109a3084e6cSFelix Kuehling .num_of_watch_points = 4, 110a3084e6cSFelix Kuehling .mqd_size_aligned = MQD_SIZE_ALIGNED, 111a3084e6cSFelix Kuehling .supports_cwsr = false, 11264d1c3a4SFelix Kuehling .needs_iommu_device = false, 113a3084e6cSFelix Kuehling .needs_pci_atomics = false, 11498bb9222SYong Zhao .num_sdma_engines = 2, 1151b4670f6SOak Zeng .num_xgmi_sdma_engines = 0, 116d5094189SShaoyun Liu .num_sdma_queues_per_engine = 2, 117a3084e6cSFelix Kuehling }; 118a3084e6cSFelix Kuehling 119a3084e6cSFelix Kuehling static const struct kfd_device_info tonga_device_info = { 120a3084e6cSFelix Kuehling .asic_family = CHIP_TONGA, 121c181159aSYong Zhao .asic_name = "tonga", 122a3084e6cSFelix Kuehling .max_pasid_bits = 16, 123a3084e6cSFelix Kuehling .max_no_of_hqd = 24, 124ada2b29cSFelix Kuehling .doorbell_size = 4, 125a3084e6cSFelix Kuehling .ih_ring_entry_size = 4 * sizeof(uint32_t), 126a3084e6cSFelix Kuehling .event_interrupt_class = &event_interrupt_class_cik, 127a3084e6cSFelix Kuehling .num_of_watch_points = 4, 128a3084e6cSFelix Kuehling .mqd_size_aligned = MQD_SIZE_ALIGNED, 129a3084e6cSFelix Kuehling .supports_cwsr = false, 13064d1c3a4SFelix Kuehling .needs_iommu_device = false, 131a3084e6cSFelix Kuehling .needs_pci_atomics = true, 13298bb9222SYong Zhao .num_sdma_engines = 2, 1331b4670f6SOak Zeng .num_xgmi_sdma_engines = 0, 134d5094189SShaoyun Liu .num_sdma_queues_per_engine = 2, 135a3084e6cSFelix Kuehling }; 136a3084e6cSFelix Kuehling 137a3084e6cSFelix Kuehling static const struct kfd_device_info fiji_device_info = { 138a3084e6cSFelix Kuehling .asic_family = CHIP_FIJI, 139c181159aSYong Zhao .asic_name = "fiji", 140a3084e6cSFelix Kuehling .max_pasid_bits = 16, 141a3084e6cSFelix Kuehling .max_no_of_hqd = 24, 142ada2b29cSFelix Kuehling .doorbell_size = 4, 143a3084e6cSFelix Kuehling .ih_ring_entry_size = 4 * sizeof(uint32_t), 144a3084e6cSFelix Kuehling .event_interrupt_class = &event_interrupt_class_cik, 145a3084e6cSFelix Kuehling .num_of_watch_points = 4, 146a3084e6cSFelix Kuehling .mqd_size_aligned = MQD_SIZE_ALIGNED, 147a3084e6cSFelix Kuehling .supports_cwsr = true, 14864d1c3a4SFelix Kuehling .needs_iommu_device = false, 149a3084e6cSFelix Kuehling .needs_pci_atomics = true, 15098bb9222SYong Zhao .num_sdma_engines = 2, 1511b4670f6SOak Zeng .num_xgmi_sdma_engines = 0, 152d5094189SShaoyun Liu .num_sdma_queues_per_engine = 2, 153a3084e6cSFelix Kuehling }; 154a3084e6cSFelix Kuehling 155a3084e6cSFelix Kuehling static const struct kfd_device_info fiji_vf_device_info = { 156a3084e6cSFelix Kuehling .asic_family = CHIP_FIJI, 157c181159aSYong Zhao .asic_name = "fiji", 158a3084e6cSFelix Kuehling .max_pasid_bits = 16, 159a3084e6cSFelix Kuehling .max_no_of_hqd = 24, 160ada2b29cSFelix Kuehling .doorbell_size = 4, 161a3084e6cSFelix Kuehling .ih_ring_entry_size = 4 * sizeof(uint32_t), 162a3084e6cSFelix Kuehling .event_interrupt_class = &event_interrupt_class_cik, 163a3084e6cSFelix Kuehling .num_of_watch_points = 4, 164a3084e6cSFelix Kuehling .mqd_size_aligned = MQD_SIZE_ALIGNED, 165a3084e6cSFelix Kuehling .supports_cwsr = true, 16664d1c3a4SFelix Kuehling .needs_iommu_device = false, 167a3084e6cSFelix Kuehling .needs_pci_atomics = false, 16898bb9222SYong Zhao .num_sdma_engines = 2, 1691b4670f6SOak Zeng .num_xgmi_sdma_engines = 0, 170d5094189SShaoyun Liu .num_sdma_queues_per_engine = 2, 171a3084e6cSFelix Kuehling }; 172a3084e6cSFelix Kuehling 173a3084e6cSFelix Kuehling 174a3084e6cSFelix Kuehling static const struct kfd_device_info polaris10_device_info = { 175a3084e6cSFelix Kuehling .asic_family = CHIP_POLARIS10, 176c181159aSYong Zhao .asic_name = "polaris10", 177a3084e6cSFelix Kuehling .max_pasid_bits = 16, 178a3084e6cSFelix Kuehling .max_no_of_hqd = 24, 179ada2b29cSFelix Kuehling .doorbell_size = 4, 180a3084e6cSFelix Kuehling .ih_ring_entry_size = 4 * sizeof(uint32_t), 181a3084e6cSFelix Kuehling .event_interrupt_class = &event_interrupt_class_cik, 182a3084e6cSFelix Kuehling .num_of_watch_points = 4, 183a3084e6cSFelix Kuehling .mqd_size_aligned = MQD_SIZE_ALIGNED, 184a3084e6cSFelix Kuehling .supports_cwsr = true, 18564d1c3a4SFelix Kuehling .needs_iommu_device = false, 186a3084e6cSFelix Kuehling .needs_pci_atomics = true, 18798bb9222SYong Zhao .num_sdma_engines = 2, 1881b4670f6SOak Zeng .num_xgmi_sdma_engines = 0, 189d5094189SShaoyun Liu .num_sdma_queues_per_engine = 2, 190a3084e6cSFelix Kuehling }; 191a3084e6cSFelix Kuehling 192a3084e6cSFelix Kuehling static const struct kfd_device_info polaris10_vf_device_info = { 193a3084e6cSFelix Kuehling .asic_family = CHIP_POLARIS10, 194c181159aSYong Zhao .asic_name = "polaris10", 195a3084e6cSFelix Kuehling .max_pasid_bits = 16, 196a3084e6cSFelix Kuehling .max_no_of_hqd = 24, 197ada2b29cSFelix Kuehling .doorbell_size = 4, 198a3084e6cSFelix Kuehling .ih_ring_entry_size = 4 * sizeof(uint32_t), 199a3084e6cSFelix Kuehling .event_interrupt_class = &event_interrupt_class_cik, 200a3084e6cSFelix Kuehling .num_of_watch_points = 4, 201a3084e6cSFelix Kuehling .mqd_size_aligned = MQD_SIZE_ALIGNED, 202a3084e6cSFelix Kuehling .supports_cwsr = true, 20364d1c3a4SFelix Kuehling .needs_iommu_device = false, 204a3084e6cSFelix Kuehling .needs_pci_atomics = false, 20598bb9222SYong Zhao .num_sdma_engines = 2, 2061b4670f6SOak Zeng .num_xgmi_sdma_engines = 0, 207d5094189SShaoyun Liu .num_sdma_queues_per_engine = 2, 208a3084e6cSFelix Kuehling }; 209a3084e6cSFelix Kuehling 210a3084e6cSFelix Kuehling static const struct kfd_device_info polaris11_device_info = { 211a3084e6cSFelix Kuehling .asic_family = CHIP_POLARIS11, 212c181159aSYong Zhao .asic_name = "polaris11", 213a3084e6cSFelix Kuehling .max_pasid_bits = 16, 214a3084e6cSFelix Kuehling .max_no_of_hqd = 24, 215ada2b29cSFelix Kuehling .doorbell_size = 4, 216a3084e6cSFelix Kuehling .ih_ring_entry_size = 4 * sizeof(uint32_t), 217a3084e6cSFelix Kuehling .event_interrupt_class = &event_interrupt_class_cik, 218a3084e6cSFelix Kuehling .num_of_watch_points = 4, 219a3084e6cSFelix Kuehling .mqd_size_aligned = MQD_SIZE_ALIGNED, 220a3084e6cSFelix Kuehling .supports_cwsr = true, 22164d1c3a4SFelix Kuehling .needs_iommu_device = false, 222a3084e6cSFelix Kuehling .needs_pci_atomics = true, 22398bb9222SYong Zhao .num_sdma_engines = 2, 2241b4670f6SOak Zeng .num_xgmi_sdma_engines = 0, 225d5094189SShaoyun Liu .num_sdma_queues_per_engine = 2, 226a3084e6cSFelix Kuehling }; 227a3084e6cSFelix Kuehling 228846a44d7SGang Ba static const struct kfd_device_info polaris12_device_info = { 229846a44d7SGang Ba .asic_family = CHIP_POLARIS12, 230c181159aSYong Zhao .asic_name = "polaris12", 231846a44d7SGang Ba .max_pasid_bits = 16, 232846a44d7SGang Ba .max_no_of_hqd = 24, 233846a44d7SGang Ba .doorbell_size = 4, 234846a44d7SGang Ba .ih_ring_entry_size = 4 * sizeof(uint32_t), 235846a44d7SGang Ba .event_interrupt_class = &event_interrupt_class_cik, 236846a44d7SGang Ba .num_of_watch_points = 4, 237846a44d7SGang Ba .mqd_size_aligned = MQD_SIZE_ALIGNED, 238846a44d7SGang Ba .supports_cwsr = true, 239846a44d7SGang Ba .needs_iommu_device = false, 240846a44d7SGang Ba .needs_pci_atomics = true, 241846a44d7SGang Ba .num_sdma_engines = 2, 2421b4670f6SOak Zeng .num_xgmi_sdma_engines = 0, 243846a44d7SGang Ba .num_sdma_queues_per_engine = 2, 244846a44d7SGang Ba }; 245846a44d7SGang Ba 246ed81cd6eSKent Russell static const struct kfd_device_info vegam_device_info = { 247ed81cd6eSKent Russell .asic_family = CHIP_VEGAM, 248c181159aSYong Zhao .asic_name = "vegam", 249ed81cd6eSKent Russell .max_pasid_bits = 16, 250ed81cd6eSKent Russell .max_no_of_hqd = 24, 251ed81cd6eSKent Russell .doorbell_size = 4, 252ed81cd6eSKent Russell .ih_ring_entry_size = 4 * sizeof(uint32_t), 253ed81cd6eSKent Russell .event_interrupt_class = &event_interrupt_class_cik, 254ed81cd6eSKent Russell .num_of_watch_points = 4, 255ed81cd6eSKent Russell .mqd_size_aligned = MQD_SIZE_ALIGNED, 256ed81cd6eSKent Russell .supports_cwsr = true, 257ed81cd6eSKent Russell .needs_iommu_device = false, 258ed81cd6eSKent Russell .needs_pci_atomics = true, 259ed81cd6eSKent Russell .num_sdma_engines = 2, 260ed81cd6eSKent Russell .num_xgmi_sdma_engines = 0, 261a3084e6cSFelix Kuehling .num_sdma_queues_per_engine = 2, 262a3084e6cSFelix Kuehling }; 263a3084e6cSFelix Kuehling 264389056e5SFelix Kuehling static const struct kfd_device_info vega10_device_info = { 265389056e5SFelix Kuehling .asic_family = CHIP_VEGA10, 266c181159aSYong Zhao .asic_name = "vega10", 267389056e5SFelix Kuehling .max_pasid_bits = 16, 268389056e5SFelix Kuehling .max_no_of_hqd = 24, 269389056e5SFelix Kuehling .doorbell_size = 8, 270389056e5SFelix Kuehling .ih_ring_entry_size = 8 * sizeof(uint32_t), 271389056e5SFelix Kuehling .event_interrupt_class = &event_interrupt_class_v9, 272389056e5SFelix Kuehling .num_of_watch_points = 4, 273389056e5SFelix Kuehling .mqd_size_aligned = MQD_SIZE_ALIGNED, 274389056e5SFelix Kuehling .supports_cwsr = true, 275389056e5SFelix Kuehling .needs_iommu_device = false, 276389056e5SFelix Kuehling .needs_pci_atomics = false, 27798bb9222SYong Zhao .num_sdma_engines = 2, 2781b4670f6SOak Zeng .num_xgmi_sdma_engines = 0, 279d5094189SShaoyun Liu .num_sdma_queues_per_engine = 2, 280389056e5SFelix Kuehling }; 281389056e5SFelix Kuehling 282389056e5SFelix Kuehling static const struct kfd_device_info vega10_vf_device_info = { 283389056e5SFelix Kuehling .asic_family = CHIP_VEGA10, 284c181159aSYong Zhao .asic_name = "vega10", 285389056e5SFelix Kuehling .max_pasid_bits = 16, 286389056e5SFelix Kuehling .max_no_of_hqd = 24, 287389056e5SFelix Kuehling .doorbell_size = 8, 288389056e5SFelix Kuehling .ih_ring_entry_size = 8 * sizeof(uint32_t), 289389056e5SFelix Kuehling .event_interrupt_class = &event_interrupt_class_v9, 290389056e5SFelix Kuehling .num_of_watch_points = 4, 291389056e5SFelix Kuehling .mqd_size_aligned = MQD_SIZE_ALIGNED, 292389056e5SFelix Kuehling .supports_cwsr = true, 293389056e5SFelix Kuehling .needs_iommu_device = false, 294389056e5SFelix Kuehling .needs_pci_atomics = false, 29598bb9222SYong Zhao .num_sdma_engines = 2, 2961b4670f6SOak Zeng .num_xgmi_sdma_engines = 0, 297d5094189SShaoyun Liu .num_sdma_queues_per_engine = 2, 298389056e5SFelix Kuehling }; 299389056e5SFelix Kuehling 300846a44d7SGang Ba static const struct kfd_device_info vega12_device_info = { 301846a44d7SGang Ba .asic_family = CHIP_VEGA12, 302c181159aSYong Zhao .asic_name = "vega12", 303846a44d7SGang Ba .max_pasid_bits = 16, 304846a44d7SGang Ba .max_no_of_hqd = 24, 305846a44d7SGang Ba .doorbell_size = 8, 306846a44d7SGang Ba .ih_ring_entry_size = 8 * sizeof(uint32_t), 307846a44d7SGang Ba .event_interrupt_class = &event_interrupt_class_v9, 308846a44d7SGang Ba .num_of_watch_points = 4, 309846a44d7SGang Ba .mqd_size_aligned = MQD_SIZE_ALIGNED, 310846a44d7SGang Ba .supports_cwsr = true, 311846a44d7SGang Ba .needs_iommu_device = false, 312846a44d7SGang Ba .needs_pci_atomics = false, 313846a44d7SGang Ba .num_sdma_engines = 2, 3141b4670f6SOak Zeng .num_xgmi_sdma_engines = 0, 315846a44d7SGang Ba .num_sdma_queues_per_engine = 2, 316846a44d7SGang Ba }; 317846a44d7SGang Ba 31822a3a294SShaoyun Liu static const struct kfd_device_info vega20_device_info = { 31922a3a294SShaoyun Liu .asic_family = CHIP_VEGA20, 320c181159aSYong Zhao .asic_name = "vega20", 32122a3a294SShaoyun Liu .max_pasid_bits = 16, 32222a3a294SShaoyun Liu .max_no_of_hqd = 24, 32322a3a294SShaoyun Liu .doorbell_size = 8, 32422a3a294SShaoyun Liu .ih_ring_entry_size = 8 * sizeof(uint32_t), 32522a3a294SShaoyun Liu .event_interrupt_class = &event_interrupt_class_v9, 32622a3a294SShaoyun Liu .num_of_watch_points = 4, 32722a3a294SShaoyun Liu .mqd_size_aligned = MQD_SIZE_ALIGNED, 32822a3a294SShaoyun Liu .supports_cwsr = true, 32922a3a294SShaoyun Liu .needs_iommu_device = false, 330006a0b3dSShaoyun Liu .needs_pci_atomics = false, 33122a3a294SShaoyun Liu .num_sdma_engines = 2, 3321b4670f6SOak Zeng .num_xgmi_sdma_engines = 0, 33322a3a294SShaoyun Liu .num_sdma_queues_per_engine = 8, 33422a3a294SShaoyun Liu }; 33522a3a294SShaoyun Liu 33649adcf8aSYong Zhao static const struct kfd_device_info arcturus_device_info = { 33749adcf8aSYong Zhao .asic_family = CHIP_ARCTURUS, 338c181159aSYong Zhao .asic_name = "arcturus", 33949adcf8aSYong Zhao .max_pasid_bits = 16, 34049adcf8aSYong Zhao .max_no_of_hqd = 24, 34149adcf8aSYong Zhao .doorbell_size = 8, 34249adcf8aSYong Zhao .ih_ring_entry_size = 8 * sizeof(uint32_t), 34349adcf8aSYong Zhao .event_interrupt_class = &event_interrupt_class_v9, 34449adcf8aSYong Zhao .num_of_watch_points = 4, 34549adcf8aSYong Zhao .mqd_size_aligned = MQD_SIZE_ALIGNED, 34649adcf8aSYong Zhao .supports_cwsr = true, 34749adcf8aSYong Zhao .needs_iommu_device = false, 34849adcf8aSYong Zhao .needs_pci_atomics = false, 349b6689cf7SOak Zeng .num_sdma_engines = 2, 350b6689cf7SOak Zeng .num_xgmi_sdma_engines = 6, 35149adcf8aSYong Zhao .num_sdma_queues_per_engine = 8, 35249adcf8aSYong Zhao }; 35349adcf8aSYong Zhao 35414328aa5SPhilip Cox static const struct kfd_device_info navi10_device_info = { 35514328aa5SPhilip Cox .asic_family = CHIP_NAVI10, 356c181159aSYong Zhao .asic_name = "navi10", 35714328aa5SPhilip Cox .max_pasid_bits = 16, 35814328aa5SPhilip Cox .max_no_of_hqd = 24, 35914328aa5SPhilip Cox .doorbell_size = 8, 36014328aa5SPhilip Cox .ih_ring_entry_size = 8 * sizeof(uint32_t), 36114328aa5SPhilip Cox .event_interrupt_class = &event_interrupt_class_v9, 36214328aa5SPhilip Cox .num_of_watch_points = 4, 36314328aa5SPhilip Cox .mqd_size_aligned = MQD_SIZE_ALIGNED, 36414328aa5SPhilip Cox .needs_iommu_device = false, 36514328aa5SPhilip Cox .supports_cwsr = true, 36614328aa5SPhilip Cox .needs_pci_atomics = false, 36714328aa5SPhilip Cox .num_sdma_engines = 2, 36814328aa5SPhilip Cox .num_xgmi_sdma_engines = 0, 36914328aa5SPhilip Cox .num_sdma_queues_per_engine = 8, 37014328aa5SPhilip Cox }; 37114328aa5SPhilip Cox 372*8099ae40SYong Zhao static const struct kfd_device_info navi14_device_info = { 373*8099ae40SYong Zhao .asic_family = CHIP_NAVI14, 374*8099ae40SYong Zhao .asic_name = "navi14", 375*8099ae40SYong Zhao .max_pasid_bits = 16, 376*8099ae40SYong Zhao .max_no_of_hqd = 24, 377*8099ae40SYong Zhao .doorbell_size = 8, 378*8099ae40SYong Zhao .ih_ring_entry_size = 8 * sizeof(uint32_t), 379*8099ae40SYong Zhao .event_interrupt_class = &event_interrupt_class_v9, 380*8099ae40SYong Zhao .num_of_watch_points = 4, 381*8099ae40SYong Zhao .mqd_size_aligned = MQD_SIZE_ALIGNED, 382*8099ae40SYong Zhao .needs_iommu_device = false, 383*8099ae40SYong Zhao .supports_cwsr = true, 384*8099ae40SYong Zhao .needs_pci_atomics = false, 385*8099ae40SYong Zhao .num_sdma_engines = 2, 386*8099ae40SYong Zhao .num_xgmi_sdma_engines = 0, 387*8099ae40SYong Zhao .num_sdma_queues_per_engine = 8, 388*8099ae40SYong Zhao }; 389*8099ae40SYong Zhao 390050091abSYong Zhao /* For each entry, [0] is regular and [1] is virtualisation device. */ 391050091abSYong Zhao static const struct kfd_device_info *kfd_supported_devices[][2] = { 39295a5bd1bSYong Zhao #ifdef KFD_SUPPORT_IOMMU_V2 393050091abSYong Zhao [CHIP_KAVERI] = {&kaveri_device_info, NULL}, 39495a5bd1bSYong Zhao [CHIP_CARRIZO] = {&carrizo_device_info, NULL}, 39595a5bd1bSYong Zhao [CHIP_RAVEN] = {&raven_device_info, NULL}, 39695a5bd1bSYong Zhao #endif 397050091abSYong Zhao [CHIP_HAWAII] = {&hawaii_device_info, NULL}, 398050091abSYong Zhao [CHIP_TONGA] = {&tonga_device_info, NULL}, 399050091abSYong Zhao [CHIP_FIJI] = {&fiji_device_info, &fiji_vf_device_info}, 400050091abSYong Zhao [CHIP_POLARIS10] = {&polaris10_device_info, &polaris10_vf_device_info}, 401050091abSYong Zhao [CHIP_POLARIS11] = {&polaris11_device_info, NULL}, 402050091abSYong Zhao [CHIP_POLARIS12] = {&polaris12_device_info, NULL}, 403050091abSYong Zhao [CHIP_VEGAM] = {&vegam_device_info, NULL}, 404050091abSYong Zhao [CHIP_VEGA10] = {&vega10_device_info, &vega10_vf_device_info}, 405050091abSYong Zhao [CHIP_VEGA12] = {&vega12_device_info, NULL}, 406050091abSYong Zhao [CHIP_VEGA20] = {&vega20_device_info, NULL}, 407050091abSYong Zhao [CHIP_ARCTURUS] = {&arcturus_device_info, &arcturus_device_info}, 408050091abSYong Zhao [CHIP_NAVI10] = {&navi10_device_info, NULL}, 409*8099ae40SYong Zhao [CHIP_NAVI14] = {&navi14_device_info, NULL}, 4104a488a7aSOded Gabbay }; 4114a488a7aSOded Gabbay 4126e81090bSOded Gabbay static int kfd_gtt_sa_init(struct kfd_dev *kfd, unsigned int buf_size, 4136e81090bSOded Gabbay unsigned int chunk_size); 4146e81090bSOded Gabbay static void kfd_gtt_sa_fini(struct kfd_dev *kfd); 4156e81090bSOded Gabbay 416b8935a7cSYong Zhao static int kfd_resume(struct kfd_dev *kfd); 417b8935a7cSYong Zhao 418cea405b1SXihan Zhang struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd, 419050091abSYong Zhao struct pci_dev *pdev, const struct kfd2kgd_calls *f2g, 420050091abSYong Zhao unsigned int asic_type, bool vf) 4214a488a7aSOded Gabbay { 4224a488a7aSOded Gabbay struct kfd_dev *kfd; 423050091abSYong Zhao const struct kfd_device_info *device_info; 424050091abSYong Zhao 425050091abSYong Zhao if (asic_type >= sizeof(kfd_supported_devices) / (sizeof(void *) * 2)) { 426050091abSYong Zhao dev_err(kfd_device, "asic_type %d out of range\n", asic_type); 427050091abSYong Zhao return NULL; /* asic_type out of range */ 428050091abSYong Zhao } 429050091abSYong Zhao 430050091abSYong Zhao device_info = kfd_supported_devices[asic_type][vf]; 4314a488a7aSOded Gabbay 4324ebc7182SYong Zhao if (!device_info) { 433050091abSYong Zhao dev_err(kfd_device, "%s %s not supported in kfd\n", 434050091abSYong Zhao amdgpu_asic_name[asic_type], vf ? "VF" : ""); 4354a488a7aSOded Gabbay return NULL; 4364ebc7182SYong Zhao } 4374a488a7aSOded Gabbay 438d35f00d8SEric Huang kfd = kzalloc(sizeof(*kfd), GFP_KERNEL); 439d35f00d8SEric Huang if (!kfd) 440d35f00d8SEric Huang return NULL; 441d35f00d8SEric Huang 4426106dce9Swelu /* Allow BIF to recode atomics to PCIe 3.0 AtomicOps. 4436106dce9Swelu * 32 and 64-bit requests are possible and must be 4446106dce9Swelu * supported. 4453ee2d00cSFelix Kuehling */ 446aabf3a95SJack Xiao kfd->pci_atomic_requested = amdgpu_amdkfd_have_atomics_support(kgd); 447aabf3a95SJack Xiao if (device_info->needs_pci_atomics && 448aabf3a95SJack Xiao !kfd->pci_atomic_requested) { 4493ee2d00cSFelix Kuehling dev_info(kfd_device, 4506106dce9Swelu "skipped device %x:%x, PCI rejects atomics\n", 4513ee2d00cSFelix Kuehling pdev->vendor, pdev->device); 452d35f00d8SEric Huang kfree(kfd); 4533ee2d00cSFelix Kuehling return NULL; 454aabf3a95SJack Xiao } 4554a488a7aSOded Gabbay 4564a488a7aSOded Gabbay kfd->kgd = kgd; 4574a488a7aSOded Gabbay kfd->device_info = device_info; 4584a488a7aSOded Gabbay kfd->pdev = pdev; 45919f6d2a6SOded Gabbay kfd->init_complete = false; 460cea405b1SXihan Zhang kfd->kfd2kgd = f2g; 46143d8107fSHarish Kasiviswanathan atomic_set(&kfd->compute_profile, 0); 462cea405b1SXihan Zhang 463cea405b1SXihan Zhang mutex_init(&kfd->doorbell_mutex); 464cea405b1SXihan Zhang memset(&kfd->doorbell_available_index, 0, 465cea405b1SXihan Zhang sizeof(kfd->doorbell_available_index)); 4664a488a7aSOded Gabbay 4679b54d201SEric Huang atomic_set(&kfd->sram_ecc_flag, 0); 4689b54d201SEric Huang 4694a488a7aSOded Gabbay return kfd; 4704a488a7aSOded Gabbay } 4714a488a7aSOded Gabbay 472373d7080SFelix Kuehling static void kfd_cwsr_init(struct kfd_dev *kfd) 473373d7080SFelix Kuehling { 474373d7080SFelix Kuehling if (cwsr_enable && kfd->device_info->supports_cwsr) { 4753e76c239SFelix Kuehling if (kfd->device_info->asic_family < CHIP_VEGA10) { 476373d7080SFelix Kuehling BUILD_BUG_ON(sizeof(cwsr_trap_gfx8_hex) > PAGE_SIZE); 477373d7080SFelix Kuehling kfd->cwsr_isa = cwsr_trap_gfx8_hex; 478373d7080SFelix Kuehling kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx8_hex); 4793baa24f0SOak Zeng } else if (kfd->device_info->asic_family == CHIP_ARCTURUS) { 4803baa24f0SOak Zeng BUILD_BUG_ON(sizeof(cwsr_trap_arcturus_hex) > PAGE_SIZE); 4813baa24f0SOak Zeng kfd->cwsr_isa = cwsr_trap_arcturus_hex; 4823baa24f0SOak Zeng kfd->cwsr_isa_size = sizeof(cwsr_trap_arcturus_hex); 48314328aa5SPhilip Cox } else if (kfd->device_info->asic_family < CHIP_NAVI10) { 4843e76c239SFelix Kuehling BUILD_BUG_ON(sizeof(cwsr_trap_gfx9_hex) > PAGE_SIZE); 4853e76c239SFelix Kuehling kfd->cwsr_isa = cwsr_trap_gfx9_hex; 4863e76c239SFelix Kuehling kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx9_hex); 48714328aa5SPhilip Cox } else { 48814328aa5SPhilip Cox BUILD_BUG_ON(sizeof(cwsr_trap_gfx10_hex) > PAGE_SIZE); 48914328aa5SPhilip Cox kfd->cwsr_isa = cwsr_trap_gfx10_hex; 49014328aa5SPhilip Cox kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx10_hex); 4913e76c239SFelix Kuehling } 4923e76c239SFelix Kuehling 493373d7080SFelix Kuehling kfd->cwsr_enabled = true; 494373d7080SFelix Kuehling } 495373d7080SFelix Kuehling } 496373d7080SFelix Kuehling 4974a488a7aSOded Gabbay bool kgd2kfd_device_init(struct kfd_dev *kfd, 4984a488a7aSOded Gabbay const struct kgd2kfd_shared_resources *gpu_resources) 4994a488a7aSOded Gabbay { 50019f6d2a6SOded Gabbay unsigned int size; 50119f6d2a6SOded Gabbay 5020da8b10eSAmber Lin kfd->mec_fw_version = amdgpu_amdkfd_get_fw_version(kfd->kgd, 5035ade6c9cSFelix Kuehling KGD_ENGINE_MEC1); 5040da8b10eSAmber Lin kfd->sdma_fw_version = amdgpu_amdkfd_get_fw_version(kfd->kgd, 5055ade6c9cSFelix Kuehling KGD_ENGINE_SDMA1); 5064a488a7aSOded Gabbay kfd->shared_resources = *gpu_resources; 5074a488a7aSOded Gabbay 50844008d7aSYong Zhao kfd->vm_info.first_vmid_kfd = ffs(gpu_resources->compute_vmid_bitmap)-1; 50944008d7aSYong Zhao kfd->vm_info.last_vmid_kfd = fls(gpu_resources->compute_vmid_bitmap)-1; 51044008d7aSYong Zhao kfd->vm_info.vmid_num_kfd = kfd->vm_info.last_vmid_kfd 51144008d7aSYong Zhao - kfd->vm_info.first_vmid_kfd + 1; 51244008d7aSYong Zhao 513a99c6d4fSFelix Kuehling /* Verify module parameters regarding mapped process number*/ 514a99c6d4fSFelix Kuehling if ((hws_max_conc_proc < 0) 515a99c6d4fSFelix Kuehling || (hws_max_conc_proc > kfd->vm_info.vmid_num_kfd)) { 516a99c6d4fSFelix Kuehling dev_err(kfd_device, 517a99c6d4fSFelix Kuehling "hws_max_conc_proc %d must be between 0 and %d, use %d instead\n", 518a99c6d4fSFelix Kuehling hws_max_conc_proc, kfd->vm_info.vmid_num_kfd, 519a99c6d4fSFelix Kuehling kfd->vm_info.vmid_num_kfd); 520a99c6d4fSFelix Kuehling kfd->max_proc_per_quantum = kfd->vm_info.vmid_num_kfd; 521a99c6d4fSFelix Kuehling } else 522a99c6d4fSFelix Kuehling kfd->max_proc_per_quantum = hws_max_conc_proc; 523a99c6d4fSFelix Kuehling 524e09d4fc8SOak Zeng /* Allocate global GWS that is shared by all KFD processes */ 525e09d4fc8SOak Zeng if (hws_gws_support && amdgpu_amdkfd_alloc_gws(kfd->kgd, 526e09d4fc8SOak Zeng amdgpu_amdkfd_get_num_gws(kfd->kgd), &kfd->gws)) { 527e09d4fc8SOak Zeng dev_err(kfd_device, "Could not allocate %d gws\n", 528e09d4fc8SOak Zeng amdgpu_amdkfd_get_num_gws(kfd->kgd)); 529e09d4fc8SOak Zeng goto out; 530e09d4fc8SOak Zeng } 53119f6d2a6SOded Gabbay /* calculate max size of mqds needed for queues */ 532b8cbab04SOded Gabbay size = max_num_of_queues_per_device * 53319f6d2a6SOded Gabbay kfd->device_info->mqd_size_aligned; 53419f6d2a6SOded Gabbay 535e18e794eSOded Gabbay /* 536e18e794eSOded Gabbay * calculate max size of runlist packet. 537e18e794eSOded Gabbay * There can be only 2 packets at once 538e18e794eSOded Gabbay */ 539507968ddSFelix Kuehling size += (KFD_MAX_NUM_OF_PROCESSES * sizeof(struct pm4_mes_map_process) + 540507968ddSFelix Kuehling max_num_of_queues_per_device * sizeof(struct pm4_mes_map_queues) 541507968ddSFelix Kuehling + sizeof(struct pm4_mes_runlist)) * 2; 542e18e794eSOded Gabbay 543e18e794eSOded Gabbay /* Add size of HIQ & DIQ */ 544e18e794eSOded Gabbay size += KFD_KERNEL_QUEUE_SIZE * 2; 545e18e794eSOded Gabbay 546e18e794eSOded Gabbay /* add another 512KB for all other allocations on gart (HPD, fences) */ 54719f6d2a6SOded Gabbay size += 512 * 1024; 54819f6d2a6SOded Gabbay 5497cd52c91SAmber Lin if (amdgpu_amdkfd_alloc_gtt_mem( 550cea405b1SXihan Zhang kfd->kgd, size, &kfd->gtt_mem, 55115426dbbSYong Zhao &kfd->gtt_start_gpu_addr, &kfd->gtt_start_cpu_ptr, 55215426dbbSYong Zhao false)) { 55379775b62SKent Russell dev_err(kfd_device, "Could not allocate %d bytes\n", size); 554e09d4fc8SOak Zeng goto alloc_gtt_mem_failure; 55519f6d2a6SOded Gabbay } 55619f6d2a6SOded Gabbay 55779775b62SKent Russell dev_info(kfd_device, "Allocated %d bytes on gart\n", size); 558e18e794eSOded Gabbay 55973a1da0bSOded Gabbay /* Initialize GTT sa with 512 byte chunk size */ 56073a1da0bSOded Gabbay if (kfd_gtt_sa_init(kfd, size, 512) != 0) { 56179775b62SKent Russell dev_err(kfd_device, "Error initializing gtt sub-allocator\n"); 56273a1da0bSOded Gabbay goto kfd_gtt_sa_init_error; 56373a1da0bSOded Gabbay } 56473a1da0bSOded Gabbay 565735df2baSFelix Kuehling if (kfd_doorbell_init(kfd)) { 566735df2baSFelix Kuehling dev_err(kfd_device, 567735df2baSFelix Kuehling "Error initializing doorbell aperture\n"); 568735df2baSFelix Kuehling goto kfd_doorbell_error; 569735df2baSFelix Kuehling } 57019f6d2a6SOded Gabbay 5710c1690e3SShaoyun Liu if (kfd->kfd2kgd->get_hive_id) 5720c1690e3SShaoyun Liu kfd->hive_id = kfd->kfd2kgd->get_hive_id(kfd->kgd); 5730c1690e3SShaoyun Liu 5742249d558SAndrew Lewycky if (kfd_interrupt_init(kfd)) { 57579775b62SKent Russell dev_err(kfd_device, "Error initializing interrupts\n"); 5762249d558SAndrew Lewycky goto kfd_interrupt_error; 5772249d558SAndrew Lewycky } 5782249d558SAndrew Lewycky 57964c7f8cfSBen Goz kfd->dqm = device_queue_manager_init(kfd); 58064c7f8cfSBen Goz if (!kfd->dqm) { 58179775b62SKent Russell dev_err(kfd_device, "Error initializing queue manager\n"); 58264c7f8cfSBen Goz goto device_queue_manager_error; 58364c7f8cfSBen Goz } 58464c7f8cfSBen Goz 58564d1c3a4SFelix Kuehling if (kfd_iommu_device_init(kfd)) { 58664d1c3a4SFelix Kuehling dev_err(kfd_device, "Error initializing iommuv2\n"); 58764d1c3a4SFelix Kuehling goto device_iommu_error; 58864c7f8cfSBen Goz } 58964c7f8cfSBen Goz 590373d7080SFelix Kuehling kfd_cwsr_init(kfd); 591373d7080SFelix Kuehling 592b8935a7cSYong Zhao if (kfd_resume(kfd)) 593b8935a7cSYong Zhao goto kfd_resume_error; 594b8935a7cSYong Zhao 595fbeb661bSYair Shachar kfd->dbgmgr = NULL; 596fbeb661bSYair Shachar 597465ab9e0SOak Zeng if (kfd_topology_add_device(kfd)) { 598465ab9e0SOak Zeng dev_err(kfd_device, "Error adding device to topology\n"); 599465ab9e0SOak Zeng goto kfd_topology_add_device_error; 600465ab9e0SOak Zeng } 601465ab9e0SOak Zeng 6024a488a7aSOded Gabbay kfd->init_complete = true; 60379775b62SKent Russell dev_info(kfd_device, "added device %x:%x\n", kfd->pdev->vendor, 6044a488a7aSOded Gabbay kfd->pdev->device); 6054a488a7aSOded Gabbay 60679775b62SKent Russell pr_debug("Starting kfd with the following scheduling policy %d\n", 607d146c5a7SFelix Kuehling kfd->dqm->sched_policy); 60864c7f8cfSBen Goz 60919f6d2a6SOded Gabbay goto out; 61019f6d2a6SOded Gabbay 611465ab9e0SOak Zeng kfd_topology_add_device_error: 612b8935a7cSYong Zhao kfd_resume_error: 61364d1c3a4SFelix Kuehling device_iommu_error: 61464c7f8cfSBen Goz device_queue_manager_uninit(kfd->dqm); 61564c7f8cfSBen Goz device_queue_manager_error: 6162249d558SAndrew Lewycky kfd_interrupt_exit(kfd); 6172249d558SAndrew Lewycky kfd_interrupt_error: 618735df2baSFelix Kuehling kfd_doorbell_fini(kfd); 619735df2baSFelix Kuehling kfd_doorbell_error: 62073a1da0bSOded Gabbay kfd_gtt_sa_fini(kfd); 62173a1da0bSOded Gabbay kfd_gtt_sa_init_error: 6227cd52c91SAmber Lin amdgpu_amdkfd_free_gtt_mem(kfd->kgd, kfd->gtt_mem); 623e09d4fc8SOak Zeng alloc_gtt_mem_failure: 624e09d4fc8SOak Zeng if (hws_gws_support) 625e09d4fc8SOak Zeng amdgpu_amdkfd_free_gws(kfd->kgd, kfd->gws); 62619f6d2a6SOded Gabbay dev_err(kfd_device, 62779775b62SKent Russell "device %x:%x NOT added due to errors\n", 62819f6d2a6SOded Gabbay kfd->pdev->vendor, kfd->pdev->device); 62919f6d2a6SOded Gabbay out: 63019f6d2a6SOded Gabbay return kfd->init_complete; 6314a488a7aSOded Gabbay } 6324a488a7aSOded Gabbay 6334a488a7aSOded Gabbay void kgd2kfd_device_exit(struct kfd_dev *kfd) 6344a488a7aSOded Gabbay { 635b17f068aSOded Gabbay if (kfd->init_complete) { 636b8935a7cSYong Zhao kgd2kfd_suspend(kfd); 63764c7f8cfSBen Goz device_queue_manager_uninit(kfd->dqm); 6382249d558SAndrew Lewycky kfd_interrupt_exit(kfd); 63919f6d2a6SOded Gabbay kfd_topology_remove_device(kfd); 640735df2baSFelix Kuehling kfd_doorbell_fini(kfd); 64173a1da0bSOded Gabbay kfd_gtt_sa_fini(kfd); 6427cd52c91SAmber Lin amdgpu_amdkfd_free_gtt_mem(kfd->kgd, kfd->gtt_mem); 643e09d4fc8SOak Zeng if (hws_gws_support) 644e09d4fc8SOak Zeng amdgpu_amdkfd_free_gws(kfd->kgd, kfd->gws); 645b17f068aSOded Gabbay } 6465b5c4e40SEvgeny Pinchuk 6474a488a7aSOded Gabbay kfree(kfd); 6484a488a7aSOded Gabbay } 6494a488a7aSOded Gabbay 650e3b7a967SShaoyun Liu int kgd2kfd_pre_reset(struct kfd_dev *kfd) 651e3b7a967SShaoyun Liu { 652e42051d2SShaoyun Liu if (!kfd->init_complete) 653e42051d2SShaoyun Liu return 0; 654e42051d2SShaoyun Liu kgd2kfd_suspend(kfd); 655e42051d2SShaoyun Liu 656e42051d2SShaoyun Liu /* hold dqm->lock to prevent further execution*/ 657e42051d2SShaoyun Liu dqm_lock(kfd->dqm); 658e42051d2SShaoyun Liu 659e42051d2SShaoyun Liu kfd_signal_reset_event(kfd); 660e3b7a967SShaoyun Liu return 0; 661e3b7a967SShaoyun Liu } 662e3b7a967SShaoyun Liu 663e42051d2SShaoyun Liu /* 664e42051d2SShaoyun Liu * Fix me. KFD won't be able to resume existing process for now. 665e42051d2SShaoyun Liu * We will keep all existing process in a evicted state and 666e42051d2SShaoyun Liu * wait the process to be terminated. 667e42051d2SShaoyun Liu */ 668e42051d2SShaoyun Liu 669e3b7a967SShaoyun Liu int kgd2kfd_post_reset(struct kfd_dev *kfd) 670e3b7a967SShaoyun Liu { 671e42051d2SShaoyun Liu int ret, count; 672e42051d2SShaoyun Liu 673e42051d2SShaoyun Liu if (!kfd->init_complete) 674e3b7a967SShaoyun Liu return 0; 675e42051d2SShaoyun Liu 676e42051d2SShaoyun Liu dqm_unlock(kfd->dqm); 677e42051d2SShaoyun Liu 678e42051d2SShaoyun Liu ret = kfd_resume(kfd); 679e42051d2SShaoyun Liu if (ret) 680e42051d2SShaoyun Liu return ret; 681e42051d2SShaoyun Liu count = atomic_dec_return(&kfd_locked); 6829b54d201SEric Huang 6839b54d201SEric Huang atomic_set(&kfd->sram_ecc_flag, 0); 6849b54d201SEric Huang 685e42051d2SShaoyun Liu return 0; 686e42051d2SShaoyun Liu } 687e42051d2SShaoyun Liu 688e42051d2SShaoyun Liu bool kfd_is_locked(void) 689e42051d2SShaoyun Liu { 690e42051d2SShaoyun Liu return (atomic_read(&kfd_locked) > 0); 691e3b7a967SShaoyun Liu } 692e3b7a967SShaoyun Liu 6934a488a7aSOded Gabbay void kgd2kfd_suspend(struct kfd_dev *kfd) 6944a488a7aSOded Gabbay { 695733fa1f7SYong Zhao if (!kfd->init_complete) 696733fa1f7SYong Zhao return; 697733fa1f7SYong Zhao 69826103436SFelix Kuehling /* For first KFD device suspend all the KFD processes */ 699e42051d2SShaoyun Liu if (atomic_inc_return(&kfd_locked) == 1) 70026103436SFelix Kuehling kfd_suspend_all_processes(); 70126103436SFelix Kuehling 70245c9a5e4SOded Gabbay kfd->dqm->ops.stop(kfd->dqm); 703733fa1f7SYong Zhao 70464d1c3a4SFelix Kuehling kfd_iommu_suspend(kfd); 7054a488a7aSOded Gabbay } 7064a488a7aSOded Gabbay 7074a488a7aSOded Gabbay int kgd2kfd_resume(struct kfd_dev *kfd) 7084a488a7aSOded Gabbay { 70926103436SFelix Kuehling int ret, count; 71026103436SFelix Kuehling 711b8935a7cSYong Zhao if (!kfd->init_complete) 712b8935a7cSYong Zhao return 0; 713b17f068aSOded Gabbay 71426103436SFelix Kuehling ret = kfd_resume(kfd); 71526103436SFelix Kuehling if (ret) 71626103436SFelix Kuehling return ret; 717b17f068aSOded Gabbay 718e42051d2SShaoyun Liu count = atomic_dec_return(&kfd_locked); 71926103436SFelix Kuehling WARN_ONCE(count < 0, "KFD suspend / resume ref. error"); 72026103436SFelix Kuehling if (count == 0) 72126103436SFelix Kuehling ret = kfd_resume_all_processes(); 72226103436SFelix Kuehling 72326103436SFelix Kuehling return ret; 7244ebc7182SYong Zhao } 7254ebc7182SYong Zhao 726b8935a7cSYong Zhao static int kfd_resume(struct kfd_dev *kfd) 727b8935a7cSYong Zhao { 728b8935a7cSYong Zhao int err = 0; 729b8935a7cSYong Zhao 73064d1c3a4SFelix Kuehling err = kfd_iommu_resume(kfd); 73164d1c3a4SFelix Kuehling if (err) { 73264d1c3a4SFelix Kuehling dev_err(kfd_device, 73364d1c3a4SFelix Kuehling "Failed to resume IOMMU for device %x:%x\n", 73464d1c3a4SFelix Kuehling kfd->pdev->vendor, kfd->pdev->device); 73564d1c3a4SFelix Kuehling return err; 73664d1c3a4SFelix Kuehling } 737733fa1f7SYong Zhao 738b8935a7cSYong Zhao err = kfd->dqm->ops.start(kfd->dqm); 739b8935a7cSYong Zhao if (err) { 740b8935a7cSYong Zhao dev_err(kfd_device, 741b8935a7cSYong Zhao "Error starting queue manager for device %x:%x\n", 742b8935a7cSYong Zhao kfd->pdev->vendor, kfd->pdev->device); 743b8935a7cSYong Zhao goto dqm_start_error; 744b17f068aSOded Gabbay } 745b17f068aSOded Gabbay 746b8935a7cSYong Zhao return err; 747b8935a7cSYong Zhao 748b8935a7cSYong Zhao dqm_start_error: 74964d1c3a4SFelix Kuehling kfd_iommu_suspend(kfd); 750b8935a7cSYong Zhao return err; 7514a488a7aSOded Gabbay } 7524a488a7aSOded Gabbay 753b3f5e6b4SAndrew Lewycky /* This is called directly from KGD at ISR. */ 754b3f5e6b4SAndrew Lewycky void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry) 7554a488a7aSOded Gabbay { 75658e69886SLan Xiao uint32_t patched_ihre[KFD_MAX_RING_ENTRY_SIZE]; 75758e69886SLan Xiao bool is_patched = false; 7582383a767SChristian König unsigned long flags; 75958e69886SLan Xiao 7602249d558SAndrew Lewycky if (!kfd->init_complete) 7612249d558SAndrew Lewycky return; 7622249d558SAndrew Lewycky 76358e69886SLan Xiao if (kfd->device_info->ih_ring_entry_size > sizeof(patched_ihre)) { 76458e69886SLan Xiao dev_err_once(kfd_device, "Ring entry too small\n"); 76558e69886SLan Xiao return; 76658e69886SLan Xiao } 76758e69886SLan Xiao 7682383a767SChristian König spin_lock_irqsave(&kfd->interrupt_lock, flags); 7692249d558SAndrew Lewycky 7702249d558SAndrew Lewycky if (kfd->interrupts_active 77158e69886SLan Xiao && interrupt_is_wanted(kfd, ih_ring_entry, 77258e69886SLan Xiao patched_ihre, &is_patched) 77358e69886SLan Xiao && enqueue_ih_ring_entry(kfd, 77458e69886SLan Xiao is_patched ? patched_ihre : ih_ring_entry)) 77548e876a2SAndres Rodriguez queue_work(kfd->ih_wq, &kfd->interrupt_work); 7762249d558SAndrew Lewycky 7772383a767SChristian König spin_unlock_irqrestore(&kfd->interrupt_lock, flags); 7784a488a7aSOded Gabbay } 7796e81090bSOded Gabbay 7806b95e797SFelix Kuehling int kgd2kfd_quiesce_mm(struct mm_struct *mm) 7816b95e797SFelix Kuehling { 7826b95e797SFelix Kuehling struct kfd_process *p; 7836b95e797SFelix Kuehling int r; 7846b95e797SFelix Kuehling 7856b95e797SFelix Kuehling /* Because we are called from arbitrary context (workqueue) as opposed 7866b95e797SFelix Kuehling * to process context, kfd_process could attempt to exit while we are 7876b95e797SFelix Kuehling * running so the lookup function increments the process ref count. 7886b95e797SFelix Kuehling */ 7896b95e797SFelix Kuehling p = kfd_lookup_process_by_mm(mm); 7906b95e797SFelix Kuehling if (!p) 7916b95e797SFelix Kuehling return -ESRCH; 7926b95e797SFelix Kuehling 7936b95e797SFelix Kuehling r = kfd_process_evict_queues(p); 7946b95e797SFelix Kuehling 7956b95e797SFelix Kuehling kfd_unref_process(p); 7966b95e797SFelix Kuehling return r; 7976b95e797SFelix Kuehling } 7986b95e797SFelix Kuehling 7996b95e797SFelix Kuehling int kgd2kfd_resume_mm(struct mm_struct *mm) 8006b95e797SFelix Kuehling { 8016b95e797SFelix Kuehling struct kfd_process *p; 8026b95e797SFelix Kuehling int r; 8036b95e797SFelix Kuehling 8046b95e797SFelix Kuehling /* Because we are called from arbitrary context (workqueue) as opposed 8056b95e797SFelix Kuehling * to process context, kfd_process could attempt to exit while we are 8066b95e797SFelix Kuehling * running so the lookup function increments the process ref count. 8076b95e797SFelix Kuehling */ 8086b95e797SFelix Kuehling p = kfd_lookup_process_by_mm(mm); 8096b95e797SFelix Kuehling if (!p) 8106b95e797SFelix Kuehling return -ESRCH; 8116b95e797SFelix Kuehling 8126b95e797SFelix Kuehling r = kfd_process_restore_queues(p); 8136b95e797SFelix Kuehling 8146b95e797SFelix Kuehling kfd_unref_process(p); 8156b95e797SFelix Kuehling return r; 8166b95e797SFelix Kuehling } 8176b95e797SFelix Kuehling 81826103436SFelix Kuehling /** kgd2kfd_schedule_evict_and_restore_process - Schedules work queue that will 81926103436SFelix Kuehling * prepare for safe eviction of KFD BOs that belong to the specified 82026103436SFelix Kuehling * process. 82126103436SFelix Kuehling * 82226103436SFelix Kuehling * @mm: mm_struct that identifies the specified KFD process 82326103436SFelix Kuehling * @fence: eviction fence attached to KFD process BOs 82426103436SFelix Kuehling * 82526103436SFelix Kuehling */ 82626103436SFelix Kuehling int kgd2kfd_schedule_evict_and_restore_process(struct mm_struct *mm, 82726103436SFelix Kuehling struct dma_fence *fence) 82826103436SFelix Kuehling { 82926103436SFelix Kuehling struct kfd_process *p; 83026103436SFelix Kuehling unsigned long active_time; 83126103436SFelix Kuehling unsigned long delay_jiffies = msecs_to_jiffies(PROCESS_ACTIVE_TIME_MS); 83226103436SFelix Kuehling 83326103436SFelix Kuehling if (!fence) 83426103436SFelix Kuehling return -EINVAL; 83526103436SFelix Kuehling 83626103436SFelix Kuehling if (dma_fence_is_signaled(fence)) 83726103436SFelix Kuehling return 0; 83826103436SFelix Kuehling 83926103436SFelix Kuehling p = kfd_lookup_process_by_mm(mm); 84026103436SFelix Kuehling if (!p) 84126103436SFelix Kuehling return -ENODEV; 84226103436SFelix Kuehling 84326103436SFelix Kuehling if (fence->seqno == p->last_eviction_seqno) 84426103436SFelix Kuehling goto out; 84526103436SFelix Kuehling 84626103436SFelix Kuehling p->last_eviction_seqno = fence->seqno; 84726103436SFelix Kuehling 84826103436SFelix Kuehling /* Avoid KFD process starvation. Wait for at least 84926103436SFelix Kuehling * PROCESS_ACTIVE_TIME_MS before evicting the process again 85026103436SFelix Kuehling */ 85126103436SFelix Kuehling active_time = get_jiffies_64() - p->last_restore_timestamp; 85226103436SFelix Kuehling if (delay_jiffies > active_time) 85326103436SFelix Kuehling delay_jiffies -= active_time; 85426103436SFelix Kuehling else 85526103436SFelix Kuehling delay_jiffies = 0; 85626103436SFelix Kuehling 85726103436SFelix Kuehling /* During process initialization eviction_work.dwork is initialized 85826103436SFelix Kuehling * to kfd_evict_bo_worker 85926103436SFelix Kuehling */ 86026103436SFelix Kuehling schedule_delayed_work(&p->eviction_work, delay_jiffies); 86126103436SFelix Kuehling out: 86226103436SFelix Kuehling kfd_unref_process(p); 86326103436SFelix Kuehling return 0; 86426103436SFelix Kuehling } 86526103436SFelix Kuehling 8666e81090bSOded Gabbay static int kfd_gtt_sa_init(struct kfd_dev *kfd, unsigned int buf_size, 8676e81090bSOded Gabbay unsigned int chunk_size) 8686e81090bSOded Gabbay { 8698625ff9cSFelix Kuehling unsigned int num_of_longs; 8706e81090bSOded Gabbay 87132fa8219SFelix Kuehling if (WARN_ON(buf_size < chunk_size)) 87232fa8219SFelix Kuehling return -EINVAL; 87332fa8219SFelix Kuehling if (WARN_ON(buf_size == 0)) 87432fa8219SFelix Kuehling return -EINVAL; 87532fa8219SFelix Kuehling if (WARN_ON(chunk_size == 0)) 87632fa8219SFelix Kuehling return -EINVAL; 8776e81090bSOded Gabbay 8786e81090bSOded Gabbay kfd->gtt_sa_chunk_size = chunk_size; 8796e81090bSOded Gabbay kfd->gtt_sa_num_of_chunks = buf_size / chunk_size; 8806e81090bSOded Gabbay 8818625ff9cSFelix Kuehling num_of_longs = (kfd->gtt_sa_num_of_chunks + BITS_PER_LONG - 1) / 8828625ff9cSFelix Kuehling BITS_PER_LONG; 8836e81090bSOded Gabbay 8848625ff9cSFelix Kuehling kfd->gtt_sa_bitmap = kcalloc(num_of_longs, sizeof(long), GFP_KERNEL); 8856e81090bSOded Gabbay 8866e81090bSOded Gabbay if (!kfd->gtt_sa_bitmap) 8876e81090bSOded Gabbay return -ENOMEM; 8886e81090bSOded Gabbay 88979775b62SKent Russell pr_debug("gtt_sa_num_of_chunks = %d, gtt_sa_bitmap = %p\n", 8906e81090bSOded Gabbay kfd->gtt_sa_num_of_chunks, kfd->gtt_sa_bitmap); 8916e81090bSOded Gabbay 8926e81090bSOded Gabbay mutex_init(&kfd->gtt_sa_lock); 8936e81090bSOded Gabbay 8946e81090bSOded Gabbay return 0; 8956e81090bSOded Gabbay 8966e81090bSOded Gabbay } 8976e81090bSOded Gabbay 8986e81090bSOded Gabbay static void kfd_gtt_sa_fini(struct kfd_dev *kfd) 8996e81090bSOded Gabbay { 9006e81090bSOded Gabbay mutex_destroy(&kfd->gtt_sa_lock); 9016e81090bSOded Gabbay kfree(kfd->gtt_sa_bitmap); 9026e81090bSOded Gabbay } 9036e81090bSOded Gabbay 9046e81090bSOded Gabbay static inline uint64_t kfd_gtt_sa_calc_gpu_addr(uint64_t start_addr, 9056e81090bSOded Gabbay unsigned int bit_num, 9066e81090bSOded Gabbay unsigned int chunk_size) 9076e81090bSOded Gabbay { 9086e81090bSOded Gabbay return start_addr + bit_num * chunk_size; 9096e81090bSOded Gabbay } 9106e81090bSOded Gabbay 9116e81090bSOded Gabbay static inline uint32_t *kfd_gtt_sa_calc_cpu_addr(void *start_addr, 9126e81090bSOded Gabbay unsigned int bit_num, 9136e81090bSOded Gabbay unsigned int chunk_size) 9146e81090bSOded Gabbay { 9156e81090bSOded Gabbay return (uint32_t *) ((uint64_t) start_addr + bit_num * chunk_size); 9166e81090bSOded Gabbay } 9176e81090bSOded Gabbay 9186e81090bSOded Gabbay int kfd_gtt_sa_allocate(struct kfd_dev *kfd, unsigned int size, 9196e81090bSOded Gabbay struct kfd_mem_obj **mem_obj) 9206e81090bSOded Gabbay { 9216e81090bSOded Gabbay unsigned int found, start_search, cur_size; 9226e81090bSOded Gabbay 9236e81090bSOded Gabbay if (size == 0) 9246e81090bSOded Gabbay return -EINVAL; 9256e81090bSOded Gabbay 9266e81090bSOded Gabbay if (size > kfd->gtt_sa_num_of_chunks * kfd->gtt_sa_chunk_size) 9276e81090bSOded Gabbay return -ENOMEM; 9286e81090bSOded Gabbay 9291cd106ecSFelix Kuehling *mem_obj = kzalloc(sizeof(struct kfd_mem_obj), GFP_KERNEL); 9301cd106ecSFelix Kuehling if (!(*mem_obj)) 9316e81090bSOded Gabbay return -ENOMEM; 9326e81090bSOded Gabbay 93379775b62SKent Russell pr_debug("Allocated mem_obj = %p for size = %d\n", *mem_obj, size); 9346e81090bSOded Gabbay 9356e81090bSOded Gabbay start_search = 0; 9366e81090bSOded Gabbay 9376e81090bSOded Gabbay mutex_lock(&kfd->gtt_sa_lock); 9386e81090bSOded Gabbay 9396e81090bSOded Gabbay kfd_gtt_restart_search: 9406e81090bSOded Gabbay /* Find the first chunk that is free */ 9416e81090bSOded Gabbay found = find_next_zero_bit(kfd->gtt_sa_bitmap, 9426e81090bSOded Gabbay kfd->gtt_sa_num_of_chunks, 9436e81090bSOded Gabbay start_search); 9446e81090bSOded Gabbay 94579775b62SKent Russell pr_debug("Found = %d\n", found); 9466e81090bSOded Gabbay 9476e81090bSOded Gabbay /* If there wasn't any free chunk, bail out */ 9486e81090bSOded Gabbay if (found == kfd->gtt_sa_num_of_chunks) 9496e81090bSOded Gabbay goto kfd_gtt_no_free_chunk; 9506e81090bSOded Gabbay 9516e81090bSOded Gabbay /* Update fields of mem_obj */ 9526e81090bSOded Gabbay (*mem_obj)->range_start = found; 9536e81090bSOded Gabbay (*mem_obj)->range_end = found; 9546e81090bSOded Gabbay (*mem_obj)->gpu_addr = kfd_gtt_sa_calc_gpu_addr( 9556e81090bSOded Gabbay kfd->gtt_start_gpu_addr, 9566e81090bSOded Gabbay found, 9576e81090bSOded Gabbay kfd->gtt_sa_chunk_size); 9586e81090bSOded Gabbay (*mem_obj)->cpu_ptr = kfd_gtt_sa_calc_cpu_addr( 9596e81090bSOded Gabbay kfd->gtt_start_cpu_ptr, 9606e81090bSOded Gabbay found, 9616e81090bSOded Gabbay kfd->gtt_sa_chunk_size); 9626e81090bSOded Gabbay 96379775b62SKent Russell pr_debug("gpu_addr = %p, cpu_addr = %p\n", 9646e81090bSOded Gabbay (uint64_t *) (*mem_obj)->gpu_addr, (*mem_obj)->cpu_ptr); 9656e81090bSOded Gabbay 9666e81090bSOded Gabbay /* If we need only one chunk, mark it as allocated and get out */ 9676e81090bSOded Gabbay if (size <= kfd->gtt_sa_chunk_size) { 96879775b62SKent Russell pr_debug("Single bit\n"); 9696e81090bSOded Gabbay set_bit(found, kfd->gtt_sa_bitmap); 9706e81090bSOded Gabbay goto kfd_gtt_out; 9716e81090bSOded Gabbay } 9726e81090bSOded Gabbay 9736e81090bSOded Gabbay /* Otherwise, try to see if we have enough contiguous chunks */ 9746e81090bSOded Gabbay cur_size = size - kfd->gtt_sa_chunk_size; 9756e81090bSOded Gabbay do { 9766e81090bSOded Gabbay (*mem_obj)->range_end = 9776e81090bSOded Gabbay find_next_zero_bit(kfd->gtt_sa_bitmap, 9786e81090bSOded Gabbay kfd->gtt_sa_num_of_chunks, ++found); 9796e81090bSOded Gabbay /* 9806e81090bSOded Gabbay * If next free chunk is not contiguous than we need to 9816e81090bSOded Gabbay * restart our search from the last free chunk we found (which 9826e81090bSOded Gabbay * wasn't contiguous to the previous ones 9836e81090bSOded Gabbay */ 9846e81090bSOded Gabbay if ((*mem_obj)->range_end != found) { 9856e81090bSOded Gabbay start_search = found; 9866e81090bSOded Gabbay goto kfd_gtt_restart_search; 9876e81090bSOded Gabbay } 9886e81090bSOded Gabbay 9896e81090bSOded Gabbay /* 9906e81090bSOded Gabbay * If we reached end of buffer, bail out with error 9916e81090bSOded Gabbay */ 9926e81090bSOded Gabbay if (found == kfd->gtt_sa_num_of_chunks) 9936e81090bSOded Gabbay goto kfd_gtt_no_free_chunk; 9946e81090bSOded Gabbay 9956e81090bSOded Gabbay /* Check if we don't need another chunk */ 9966e81090bSOded Gabbay if (cur_size <= kfd->gtt_sa_chunk_size) 9976e81090bSOded Gabbay cur_size = 0; 9986e81090bSOded Gabbay else 9996e81090bSOded Gabbay cur_size -= kfd->gtt_sa_chunk_size; 10006e81090bSOded Gabbay 10016e81090bSOded Gabbay } while (cur_size > 0); 10026e81090bSOded Gabbay 100379775b62SKent Russell pr_debug("range_start = %d, range_end = %d\n", 10046e81090bSOded Gabbay (*mem_obj)->range_start, (*mem_obj)->range_end); 10056e81090bSOded Gabbay 10066e81090bSOded Gabbay /* Mark the chunks as allocated */ 10076e81090bSOded Gabbay for (found = (*mem_obj)->range_start; 10086e81090bSOded Gabbay found <= (*mem_obj)->range_end; 10096e81090bSOded Gabbay found++) 10106e81090bSOded Gabbay set_bit(found, kfd->gtt_sa_bitmap); 10116e81090bSOded Gabbay 10126e81090bSOded Gabbay kfd_gtt_out: 10136e81090bSOded Gabbay mutex_unlock(&kfd->gtt_sa_lock); 10146e81090bSOded Gabbay return 0; 10156e81090bSOded Gabbay 10166e81090bSOded Gabbay kfd_gtt_no_free_chunk: 101779775b62SKent Russell pr_debug("Allocation failed with mem_obj = %p\n", mem_obj); 10186e81090bSOded Gabbay mutex_unlock(&kfd->gtt_sa_lock); 10196e81090bSOded Gabbay kfree(mem_obj); 10206e81090bSOded Gabbay return -ENOMEM; 10216e81090bSOded Gabbay } 10226e81090bSOded Gabbay 10236e81090bSOded Gabbay int kfd_gtt_sa_free(struct kfd_dev *kfd, struct kfd_mem_obj *mem_obj) 10246e81090bSOded Gabbay { 10256e81090bSOded Gabbay unsigned int bit; 10266e81090bSOded Gabbay 10279216ed29SOded Gabbay /* Act like kfree when trying to free a NULL object */ 10289216ed29SOded Gabbay if (!mem_obj) 10299216ed29SOded Gabbay return 0; 10306e81090bSOded Gabbay 103179775b62SKent Russell pr_debug("Free mem_obj = %p, range_start = %d, range_end = %d\n", 10326e81090bSOded Gabbay mem_obj, mem_obj->range_start, mem_obj->range_end); 10336e81090bSOded Gabbay 10346e81090bSOded Gabbay mutex_lock(&kfd->gtt_sa_lock); 10356e81090bSOded Gabbay 10366e81090bSOded Gabbay /* Mark the chunks as free */ 10376e81090bSOded Gabbay for (bit = mem_obj->range_start; 10386e81090bSOded Gabbay bit <= mem_obj->range_end; 10396e81090bSOded Gabbay bit++) 10406e81090bSOded Gabbay clear_bit(bit, kfd->gtt_sa_bitmap); 10416e81090bSOded Gabbay 10426e81090bSOded Gabbay mutex_unlock(&kfd->gtt_sa_lock); 10436e81090bSOded Gabbay 10446e81090bSOded Gabbay kfree(mem_obj); 10456e81090bSOded Gabbay return 0; 10466e81090bSOded Gabbay } 1047a29ec470SShaoyun Liu 10489b54d201SEric Huang void kgd2kfd_set_sram_ecc_flag(struct kfd_dev *kfd) 10499b54d201SEric Huang { 10509b54d201SEric Huang if (kfd) 10519b54d201SEric Huang atomic_inc(&kfd->sram_ecc_flag); 10529b54d201SEric Huang } 10539b54d201SEric Huang 105443d8107fSHarish Kasiviswanathan void kfd_inc_compute_active(struct kfd_dev *kfd) 105543d8107fSHarish Kasiviswanathan { 105643d8107fSHarish Kasiviswanathan if (atomic_inc_return(&kfd->compute_profile) == 1) 105743d8107fSHarish Kasiviswanathan amdgpu_amdkfd_set_compute_idle(kfd->kgd, false); 105843d8107fSHarish Kasiviswanathan } 105943d8107fSHarish Kasiviswanathan 106043d8107fSHarish Kasiviswanathan void kfd_dec_compute_active(struct kfd_dev *kfd) 106143d8107fSHarish Kasiviswanathan { 106243d8107fSHarish Kasiviswanathan int count = atomic_dec_return(&kfd->compute_profile); 106343d8107fSHarish Kasiviswanathan 106443d8107fSHarish Kasiviswanathan if (count == 0) 106543d8107fSHarish Kasiviswanathan amdgpu_amdkfd_set_compute_idle(kfd->kgd, true); 106643d8107fSHarish Kasiviswanathan WARN_ONCE(count < 0, "Compute profile ref. count error"); 106743d8107fSHarish Kasiviswanathan } 106843d8107fSHarish Kasiviswanathan 1069a29ec470SShaoyun Liu #if defined(CONFIG_DEBUG_FS) 1070a29ec470SShaoyun Liu 1071a29ec470SShaoyun Liu /* This function will send a package to HIQ to hang the HWS 1072a29ec470SShaoyun Liu * which will trigger a GPU reset and bring the HWS back to normal state 1073a29ec470SShaoyun Liu */ 1074a29ec470SShaoyun Liu int kfd_debugfs_hang_hws(struct kfd_dev *dev) 1075a29ec470SShaoyun Liu { 1076a29ec470SShaoyun Liu int r = 0; 1077a29ec470SShaoyun Liu 1078a29ec470SShaoyun Liu if (dev->dqm->sched_policy != KFD_SCHED_POLICY_HWS) { 1079a29ec470SShaoyun Liu pr_err("HWS is not enabled"); 1080a29ec470SShaoyun Liu return -EINVAL; 1081a29ec470SShaoyun Liu } 1082a29ec470SShaoyun Liu 1083a29ec470SShaoyun Liu r = pm_debugfs_hang_hws(&dev->dqm->packets); 1084a29ec470SShaoyun Liu if (!r) 1085a29ec470SShaoyun Liu r = dqm_debugfs_execute_queues(dev->dqm); 1086a29ec470SShaoyun Liu 1087a29ec470SShaoyun Liu return r; 1088a29ec470SShaoyun Liu } 1089a29ec470SShaoyun Liu 1090a29ec470SShaoyun Liu #endif 1091