14a488a7aSOded Gabbay /* 24a488a7aSOded Gabbay * Copyright 2014 Advanced Micro Devices, Inc. 34a488a7aSOded Gabbay * 44a488a7aSOded Gabbay * Permission is hereby granted, free of charge, to any person obtaining a 54a488a7aSOded Gabbay * copy of this software and associated documentation files (the "Software"), 64a488a7aSOded Gabbay * to deal in the Software without restriction, including without limitation 74a488a7aSOded Gabbay * the rights to use, copy, modify, merge, publish, distribute, sublicense, 84a488a7aSOded Gabbay * and/or sell copies of the Software, and to permit persons to whom the 94a488a7aSOded Gabbay * Software is furnished to do so, subject to the following conditions: 104a488a7aSOded Gabbay * 114a488a7aSOded Gabbay * The above copyright notice and this permission notice shall be included in 124a488a7aSOded Gabbay * all copies or substantial portions of the Software. 134a488a7aSOded Gabbay * 144a488a7aSOded Gabbay * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 154a488a7aSOded Gabbay * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 164a488a7aSOded Gabbay * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 174a488a7aSOded Gabbay * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 184a488a7aSOded Gabbay * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 194a488a7aSOded Gabbay * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 204a488a7aSOded Gabbay * OTHER DEALINGS IN THE SOFTWARE. 214a488a7aSOded Gabbay */ 224a488a7aSOded Gabbay 234a488a7aSOded Gabbay #include <linux/bsearch.h> 244a488a7aSOded Gabbay #include <linux/pci.h> 254a488a7aSOded Gabbay #include <linux/slab.h> 264a488a7aSOded Gabbay #include "kfd_priv.h" 2764c7f8cfSBen Goz #include "kfd_device_queue_manager.h" 28507968ddSFelix Kuehling #include "kfd_pm4_headers_vi.h" 290db54b24SYong Zhao #include "cwsr_trap_handler.h" 3064d1c3a4SFelix Kuehling #include "kfd_iommu.h" 315b87245fSAmber Lin #include "amdgpu_amdkfd.h" 324a488a7aSOded Gabbay 3319f6d2a6SOded Gabbay #define MQD_SIZE_ALIGNED 768 34e42051d2SShaoyun Liu 35e42051d2SShaoyun Liu /* 36e42051d2SShaoyun Liu * kfd_locked is used to lock the kfd driver during suspend or reset 37e42051d2SShaoyun Liu * once locked, kfd driver will stop any further GPU execution. 38e42051d2SShaoyun Liu * create process (open) will return -EAGAIN. 39e42051d2SShaoyun Liu */ 40e42051d2SShaoyun Liu static atomic_t kfd_locked = ATOMIC_INIT(0); 4119f6d2a6SOded Gabbay 4264d1c3a4SFelix Kuehling #ifdef KFD_SUPPORT_IOMMU_V2 434a488a7aSOded Gabbay static const struct kfd_device_info kaveri_device_info = { 440da7558cSBen Goz .asic_family = CHIP_KAVERI, 45c181159aSYong Zhao .asic_name = "kaveri", 460da7558cSBen Goz .max_pasid_bits = 16, 47992839adSYair Shachar /* max num of queues for KV.TODO should be a dynamic value */ 48992839adSYair Shachar .max_no_of_hqd = 24, 49ada2b29cSFelix Kuehling .doorbell_size = 4, 500da7558cSBen Goz .ih_ring_entry_size = 4 * sizeof(uint32_t), 51f3a39818SAndrew Lewycky .event_interrupt_class = &event_interrupt_class_cik, 52fbeb661bSYair Shachar .num_of_watch_points = 4, 53373d7080SFelix Kuehling .mqd_size_aligned = MQD_SIZE_ALIGNED, 54373d7080SFelix Kuehling .supports_cwsr = false, 5564d1c3a4SFelix Kuehling .needs_iommu_device = true, 563ee2d00cSFelix Kuehling .needs_pci_atomics = false, 5798bb9222SYong Zhao .num_sdma_engines = 2, 581b4670f6SOak Zeng .num_xgmi_sdma_engines = 0, 59d5094189SShaoyun Liu .num_sdma_queues_per_engine = 2, 600da7558cSBen Goz }; 610da7558cSBen Goz 620da7558cSBen Goz static const struct kfd_device_info carrizo_device_info = { 630da7558cSBen Goz .asic_family = CHIP_CARRIZO, 64c181159aSYong Zhao .asic_name = "carrizo", 654a488a7aSOded Gabbay .max_pasid_bits = 16, 66eaccd6e7SOded Gabbay /* max num of queues for CZ.TODO should be a dynamic value */ 67eaccd6e7SOded Gabbay .max_no_of_hqd = 24, 68ada2b29cSFelix Kuehling .doorbell_size = 4, 69b3f5e6b4SAndrew Lewycky .ih_ring_entry_size = 4 * sizeof(uint32_t), 70eaccd6e7SOded Gabbay .event_interrupt_class = &event_interrupt_class_cik, 71f7c826adSAlexey Skidanov .num_of_watch_points = 4, 72373d7080SFelix Kuehling .mqd_size_aligned = MQD_SIZE_ALIGNED, 73373d7080SFelix Kuehling .supports_cwsr = true, 7464d1c3a4SFelix Kuehling .needs_iommu_device = true, 753ee2d00cSFelix Kuehling .needs_pci_atomics = false, 7698bb9222SYong Zhao .num_sdma_engines = 2, 771b4670f6SOak Zeng .num_xgmi_sdma_engines = 0, 78d5094189SShaoyun Liu .num_sdma_queues_per_engine = 2, 794a488a7aSOded Gabbay }; 804d663df6SYong Zhao 814d663df6SYong Zhao static const struct kfd_device_info raven_device_info = { 824d663df6SYong Zhao .asic_family = CHIP_RAVEN, 83c181159aSYong Zhao .asic_name = "raven", 844d663df6SYong Zhao .max_pasid_bits = 16, 854d663df6SYong Zhao .max_no_of_hqd = 24, 864d663df6SYong Zhao .doorbell_size = 8, 874d663df6SYong Zhao .ih_ring_entry_size = 8 * sizeof(uint32_t), 884d663df6SYong Zhao .event_interrupt_class = &event_interrupt_class_v9, 894d663df6SYong Zhao .num_of_watch_points = 4, 904d663df6SYong Zhao .mqd_size_aligned = MQD_SIZE_ALIGNED, 914d663df6SYong Zhao .supports_cwsr = true, 924d663df6SYong Zhao .needs_iommu_device = true, 934d663df6SYong Zhao .needs_pci_atomics = true, 944d663df6SYong Zhao .num_sdma_engines = 1, 951b4670f6SOak Zeng .num_xgmi_sdma_engines = 0, 96d5094189SShaoyun Liu .num_sdma_queues_per_engine = 2, 974d663df6SYong Zhao }; 9864d1c3a4SFelix Kuehling #endif 994a488a7aSOded Gabbay 100a3084e6cSFelix Kuehling static const struct kfd_device_info hawaii_device_info = { 101a3084e6cSFelix Kuehling .asic_family = CHIP_HAWAII, 102c181159aSYong Zhao .asic_name = "hawaii", 103a3084e6cSFelix Kuehling .max_pasid_bits = 16, 104a3084e6cSFelix Kuehling /* max num of queues for KV.TODO should be a dynamic value */ 105a3084e6cSFelix Kuehling .max_no_of_hqd = 24, 106ada2b29cSFelix Kuehling .doorbell_size = 4, 107a3084e6cSFelix Kuehling .ih_ring_entry_size = 4 * sizeof(uint32_t), 108a3084e6cSFelix Kuehling .event_interrupt_class = &event_interrupt_class_cik, 109a3084e6cSFelix Kuehling .num_of_watch_points = 4, 110a3084e6cSFelix Kuehling .mqd_size_aligned = MQD_SIZE_ALIGNED, 111a3084e6cSFelix Kuehling .supports_cwsr = false, 11264d1c3a4SFelix Kuehling .needs_iommu_device = false, 113a3084e6cSFelix Kuehling .needs_pci_atomics = false, 11498bb9222SYong Zhao .num_sdma_engines = 2, 1151b4670f6SOak Zeng .num_xgmi_sdma_engines = 0, 116d5094189SShaoyun Liu .num_sdma_queues_per_engine = 2, 117a3084e6cSFelix Kuehling }; 118a3084e6cSFelix Kuehling 119a3084e6cSFelix Kuehling static const struct kfd_device_info tonga_device_info = { 120a3084e6cSFelix Kuehling .asic_family = CHIP_TONGA, 121c181159aSYong Zhao .asic_name = "tonga", 122a3084e6cSFelix Kuehling .max_pasid_bits = 16, 123a3084e6cSFelix Kuehling .max_no_of_hqd = 24, 124ada2b29cSFelix Kuehling .doorbell_size = 4, 125a3084e6cSFelix Kuehling .ih_ring_entry_size = 4 * sizeof(uint32_t), 126a3084e6cSFelix Kuehling .event_interrupt_class = &event_interrupt_class_cik, 127a3084e6cSFelix Kuehling .num_of_watch_points = 4, 128a3084e6cSFelix Kuehling .mqd_size_aligned = MQD_SIZE_ALIGNED, 129a3084e6cSFelix Kuehling .supports_cwsr = false, 13064d1c3a4SFelix Kuehling .needs_iommu_device = false, 131a3084e6cSFelix Kuehling .needs_pci_atomics = true, 13298bb9222SYong Zhao .num_sdma_engines = 2, 1331b4670f6SOak Zeng .num_xgmi_sdma_engines = 0, 134d5094189SShaoyun Liu .num_sdma_queues_per_engine = 2, 135a3084e6cSFelix Kuehling }; 136a3084e6cSFelix Kuehling 137a3084e6cSFelix Kuehling static const struct kfd_device_info fiji_device_info = { 138a3084e6cSFelix Kuehling .asic_family = CHIP_FIJI, 139c181159aSYong Zhao .asic_name = "fiji", 140a3084e6cSFelix Kuehling .max_pasid_bits = 16, 141a3084e6cSFelix Kuehling .max_no_of_hqd = 24, 142ada2b29cSFelix Kuehling .doorbell_size = 4, 143a3084e6cSFelix Kuehling .ih_ring_entry_size = 4 * sizeof(uint32_t), 144a3084e6cSFelix Kuehling .event_interrupt_class = &event_interrupt_class_cik, 145a3084e6cSFelix Kuehling .num_of_watch_points = 4, 146a3084e6cSFelix Kuehling .mqd_size_aligned = MQD_SIZE_ALIGNED, 147a3084e6cSFelix Kuehling .supports_cwsr = true, 14864d1c3a4SFelix Kuehling .needs_iommu_device = false, 149a3084e6cSFelix Kuehling .needs_pci_atomics = true, 15098bb9222SYong Zhao .num_sdma_engines = 2, 1511b4670f6SOak Zeng .num_xgmi_sdma_engines = 0, 152d5094189SShaoyun Liu .num_sdma_queues_per_engine = 2, 153a3084e6cSFelix Kuehling }; 154a3084e6cSFelix Kuehling 155a3084e6cSFelix Kuehling static const struct kfd_device_info fiji_vf_device_info = { 156a3084e6cSFelix Kuehling .asic_family = CHIP_FIJI, 157c181159aSYong Zhao .asic_name = "fiji", 158a3084e6cSFelix Kuehling .max_pasid_bits = 16, 159a3084e6cSFelix Kuehling .max_no_of_hqd = 24, 160ada2b29cSFelix Kuehling .doorbell_size = 4, 161a3084e6cSFelix Kuehling .ih_ring_entry_size = 4 * sizeof(uint32_t), 162a3084e6cSFelix Kuehling .event_interrupt_class = &event_interrupt_class_cik, 163a3084e6cSFelix Kuehling .num_of_watch_points = 4, 164a3084e6cSFelix Kuehling .mqd_size_aligned = MQD_SIZE_ALIGNED, 165a3084e6cSFelix Kuehling .supports_cwsr = true, 16664d1c3a4SFelix Kuehling .needs_iommu_device = false, 167a3084e6cSFelix Kuehling .needs_pci_atomics = false, 16898bb9222SYong Zhao .num_sdma_engines = 2, 1691b4670f6SOak Zeng .num_xgmi_sdma_engines = 0, 170d5094189SShaoyun Liu .num_sdma_queues_per_engine = 2, 171a3084e6cSFelix Kuehling }; 172a3084e6cSFelix Kuehling 173a3084e6cSFelix Kuehling 174a3084e6cSFelix Kuehling static const struct kfd_device_info polaris10_device_info = { 175a3084e6cSFelix Kuehling .asic_family = CHIP_POLARIS10, 176c181159aSYong Zhao .asic_name = "polaris10", 177a3084e6cSFelix Kuehling .max_pasid_bits = 16, 178a3084e6cSFelix Kuehling .max_no_of_hqd = 24, 179ada2b29cSFelix Kuehling .doorbell_size = 4, 180a3084e6cSFelix Kuehling .ih_ring_entry_size = 4 * sizeof(uint32_t), 181a3084e6cSFelix Kuehling .event_interrupt_class = &event_interrupt_class_cik, 182a3084e6cSFelix Kuehling .num_of_watch_points = 4, 183a3084e6cSFelix Kuehling .mqd_size_aligned = MQD_SIZE_ALIGNED, 184a3084e6cSFelix Kuehling .supports_cwsr = true, 18564d1c3a4SFelix Kuehling .needs_iommu_device = false, 186a3084e6cSFelix Kuehling .needs_pci_atomics = true, 18798bb9222SYong Zhao .num_sdma_engines = 2, 1881b4670f6SOak Zeng .num_xgmi_sdma_engines = 0, 189d5094189SShaoyun Liu .num_sdma_queues_per_engine = 2, 190a3084e6cSFelix Kuehling }; 191a3084e6cSFelix Kuehling 192a3084e6cSFelix Kuehling static const struct kfd_device_info polaris10_vf_device_info = { 193a3084e6cSFelix Kuehling .asic_family = CHIP_POLARIS10, 194c181159aSYong Zhao .asic_name = "polaris10", 195a3084e6cSFelix Kuehling .max_pasid_bits = 16, 196a3084e6cSFelix Kuehling .max_no_of_hqd = 24, 197ada2b29cSFelix Kuehling .doorbell_size = 4, 198a3084e6cSFelix Kuehling .ih_ring_entry_size = 4 * sizeof(uint32_t), 199a3084e6cSFelix Kuehling .event_interrupt_class = &event_interrupt_class_cik, 200a3084e6cSFelix Kuehling .num_of_watch_points = 4, 201a3084e6cSFelix Kuehling .mqd_size_aligned = MQD_SIZE_ALIGNED, 202a3084e6cSFelix Kuehling .supports_cwsr = true, 20364d1c3a4SFelix Kuehling .needs_iommu_device = false, 204a3084e6cSFelix Kuehling .needs_pci_atomics = false, 20598bb9222SYong Zhao .num_sdma_engines = 2, 2061b4670f6SOak Zeng .num_xgmi_sdma_engines = 0, 207d5094189SShaoyun Liu .num_sdma_queues_per_engine = 2, 208a3084e6cSFelix Kuehling }; 209a3084e6cSFelix Kuehling 210a3084e6cSFelix Kuehling static const struct kfd_device_info polaris11_device_info = { 211a3084e6cSFelix Kuehling .asic_family = CHIP_POLARIS11, 212c181159aSYong Zhao .asic_name = "polaris11", 213a3084e6cSFelix Kuehling .max_pasid_bits = 16, 214a3084e6cSFelix Kuehling .max_no_of_hqd = 24, 215ada2b29cSFelix Kuehling .doorbell_size = 4, 216a3084e6cSFelix Kuehling .ih_ring_entry_size = 4 * sizeof(uint32_t), 217a3084e6cSFelix Kuehling .event_interrupt_class = &event_interrupt_class_cik, 218a3084e6cSFelix Kuehling .num_of_watch_points = 4, 219a3084e6cSFelix Kuehling .mqd_size_aligned = MQD_SIZE_ALIGNED, 220a3084e6cSFelix Kuehling .supports_cwsr = true, 22164d1c3a4SFelix Kuehling .needs_iommu_device = false, 222a3084e6cSFelix Kuehling .needs_pci_atomics = true, 22398bb9222SYong Zhao .num_sdma_engines = 2, 2241b4670f6SOak Zeng .num_xgmi_sdma_engines = 0, 225d5094189SShaoyun Liu .num_sdma_queues_per_engine = 2, 226a3084e6cSFelix Kuehling }; 227a3084e6cSFelix Kuehling 228846a44d7SGang Ba static const struct kfd_device_info polaris12_device_info = { 229846a44d7SGang Ba .asic_family = CHIP_POLARIS12, 230c181159aSYong Zhao .asic_name = "polaris12", 231846a44d7SGang Ba .max_pasid_bits = 16, 232846a44d7SGang Ba .max_no_of_hqd = 24, 233846a44d7SGang Ba .doorbell_size = 4, 234846a44d7SGang Ba .ih_ring_entry_size = 4 * sizeof(uint32_t), 235846a44d7SGang Ba .event_interrupt_class = &event_interrupt_class_cik, 236846a44d7SGang Ba .num_of_watch_points = 4, 237846a44d7SGang Ba .mqd_size_aligned = MQD_SIZE_ALIGNED, 238846a44d7SGang Ba .supports_cwsr = true, 239846a44d7SGang Ba .needs_iommu_device = false, 240846a44d7SGang Ba .needs_pci_atomics = true, 241846a44d7SGang Ba .num_sdma_engines = 2, 2421b4670f6SOak Zeng .num_xgmi_sdma_engines = 0, 243846a44d7SGang Ba .num_sdma_queues_per_engine = 2, 244846a44d7SGang Ba }; 245846a44d7SGang Ba 246ed81cd6eSKent Russell static const struct kfd_device_info vegam_device_info = { 247ed81cd6eSKent Russell .asic_family = CHIP_VEGAM, 248c181159aSYong Zhao .asic_name = "vegam", 249ed81cd6eSKent Russell .max_pasid_bits = 16, 250ed81cd6eSKent Russell .max_no_of_hqd = 24, 251ed81cd6eSKent Russell .doorbell_size = 4, 252ed81cd6eSKent Russell .ih_ring_entry_size = 4 * sizeof(uint32_t), 253ed81cd6eSKent Russell .event_interrupt_class = &event_interrupt_class_cik, 254ed81cd6eSKent Russell .num_of_watch_points = 4, 255ed81cd6eSKent Russell .mqd_size_aligned = MQD_SIZE_ALIGNED, 256ed81cd6eSKent Russell .supports_cwsr = true, 257ed81cd6eSKent Russell .needs_iommu_device = false, 258ed81cd6eSKent Russell .needs_pci_atomics = true, 259ed81cd6eSKent Russell .num_sdma_engines = 2, 260ed81cd6eSKent Russell .num_xgmi_sdma_engines = 0, 261a3084e6cSFelix Kuehling .num_sdma_queues_per_engine = 2, 262a3084e6cSFelix Kuehling }; 263a3084e6cSFelix Kuehling 264389056e5SFelix Kuehling static const struct kfd_device_info vega10_device_info = { 265389056e5SFelix Kuehling .asic_family = CHIP_VEGA10, 266c181159aSYong Zhao .asic_name = "vega10", 267389056e5SFelix Kuehling .max_pasid_bits = 16, 268389056e5SFelix Kuehling .max_no_of_hqd = 24, 269389056e5SFelix Kuehling .doorbell_size = 8, 270389056e5SFelix Kuehling .ih_ring_entry_size = 8 * sizeof(uint32_t), 271389056e5SFelix Kuehling .event_interrupt_class = &event_interrupt_class_v9, 272389056e5SFelix Kuehling .num_of_watch_points = 4, 273389056e5SFelix Kuehling .mqd_size_aligned = MQD_SIZE_ALIGNED, 274389056e5SFelix Kuehling .supports_cwsr = true, 275389056e5SFelix Kuehling .needs_iommu_device = false, 276389056e5SFelix Kuehling .needs_pci_atomics = false, 27798bb9222SYong Zhao .num_sdma_engines = 2, 2781b4670f6SOak Zeng .num_xgmi_sdma_engines = 0, 279d5094189SShaoyun Liu .num_sdma_queues_per_engine = 2, 280389056e5SFelix Kuehling }; 281389056e5SFelix Kuehling 282389056e5SFelix Kuehling static const struct kfd_device_info vega10_vf_device_info = { 283389056e5SFelix Kuehling .asic_family = CHIP_VEGA10, 284c181159aSYong Zhao .asic_name = "vega10", 285389056e5SFelix Kuehling .max_pasid_bits = 16, 286389056e5SFelix Kuehling .max_no_of_hqd = 24, 287389056e5SFelix Kuehling .doorbell_size = 8, 288389056e5SFelix Kuehling .ih_ring_entry_size = 8 * sizeof(uint32_t), 289389056e5SFelix Kuehling .event_interrupt_class = &event_interrupt_class_v9, 290389056e5SFelix Kuehling .num_of_watch_points = 4, 291389056e5SFelix Kuehling .mqd_size_aligned = MQD_SIZE_ALIGNED, 292389056e5SFelix Kuehling .supports_cwsr = true, 293389056e5SFelix Kuehling .needs_iommu_device = false, 294389056e5SFelix Kuehling .needs_pci_atomics = false, 29598bb9222SYong Zhao .num_sdma_engines = 2, 2961b4670f6SOak Zeng .num_xgmi_sdma_engines = 0, 297d5094189SShaoyun Liu .num_sdma_queues_per_engine = 2, 298389056e5SFelix Kuehling }; 299389056e5SFelix Kuehling 300846a44d7SGang Ba static const struct kfd_device_info vega12_device_info = { 301846a44d7SGang Ba .asic_family = CHIP_VEGA12, 302c181159aSYong Zhao .asic_name = "vega12", 303846a44d7SGang Ba .max_pasid_bits = 16, 304846a44d7SGang Ba .max_no_of_hqd = 24, 305846a44d7SGang Ba .doorbell_size = 8, 306846a44d7SGang Ba .ih_ring_entry_size = 8 * sizeof(uint32_t), 307846a44d7SGang Ba .event_interrupt_class = &event_interrupt_class_v9, 308846a44d7SGang Ba .num_of_watch_points = 4, 309846a44d7SGang Ba .mqd_size_aligned = MQD_SIZE_ALIGNED, 310846a44d7SGang Ba .supports_cwsr = true, 311846a44d7SGang Ba .needs_iommu_device = false, 312846a44d7SGang Ba .needs_pci_atomics = false, 313846a44d7SGang Ba .num_sdma_engines = 2, 3141b4670f6SOak Zeng .num_xgmi_sdma_engines = 0, 315846a44d7SGang Ba .num_sdma_queues_per_engine = 2, 316846a44d7SGang Ba }; 317846a44d7SGang Ba 31822a3a294SShaoyun Liu static const struct kfd_device_info vega20_device_info = { 31922a3a294SShaoyun Liu .asic_family = CHIP_VEGA20, 320c181159aSYong Zhao .asic_name = "vega20", 32122a3a294SShaoyun Liu .max_pasid_bits = 16, 32222a3a294SShaoyun Liu .max_no_of_hqd = 24, 32322a3a294SShaoyun Liu .doorbell_size = 8, 32422a3a294SShaoyun Liu .ih_ring_entry_size = 8 * sizeof(uint32_t), 32522a3a294SShaoyun Liu .event_interrupt_class = &event_interrupt_class_v9, 32622a3a294SShaoyun Liu .num_of_watch_points = 4, 32722a3a294SShaoyun Liu .mqd_size_aligned = MQD_SIZE_ALIGNED, 32822a3a294SShaoyun Liu .supports_cwsr = true, 32922a3a294SShaoyun Liu .needs_iommu_device = false, 330006a0b3dSShaoyun Liu .needs_pci_atomics = false, 33122a3a294SShaoyun Liu .num_sdma_engines = 2, 3321b4670f6SOak Zeng .num_xgmi_sdma_engines = 0, 33322a3a294SShaoyun Liu .num_sdma_queues_per_engine = 8, 33422a3a294SShaoyun Liu }; 33522a3a294SShaoyun Liu 33649adcf8aSYong Zhao static const struct kfd_device_info arcturus_device_info = { 33749adcf8aSYong Zhao .asic_family = CHIP_ARCTURUS, 338c181159aSYong Zhao .asic_name = "arcturus", 33949adcf8aSYong Zhao .max_pasid_bits = 16, 34049adcf8aSYong Zhao .max_no_of_hqd = 24, 34149adcf8aSYong Zhao .doorbell_size = 8, 34249adcf8aSYong Zhao .ih_ring_entry_size = 8 * sizeof(uint32_t), 34349adcf8aSYong Zhao .event_interrupt_class = &event_interrupt_class_v9, 34449adcf8aSYong Zhao .num_of_watch_points = 4, 34549adcf8aSYong Zhao .mqd_size_aligned = MQD_SIZE_ALIGNED, 34649adcf8aSYong Zhao .supports_cwsr = true, 34749adcf8aSYong Zhao .needs_iommu_device = false, 34849adcf8aSYong Zhao .needs_pci_atomics = false, 349b6689cf7SOak Zeng .num_sdma_engines = 2, 350b6689cf7SOak Zeng .num_xgmi_sdma_engines = 6, 35149adcf8aSYong Zhao .num_sdma_queues_per_engine = 8, 35249adcf8aSYong Zhao }; 35349adcf8aSYong Zhao 35414328aa5SPhilip Cox static const struct kfd_device_info navi10_device_info = { 35514328aa5SPhilip Cox .asic_family = CHIP_NAVI10, 356c181159aSYong Zhao .asic_name = "navi10", 35714328aa5SPhilip Cox .max_pasid_bits = 16, 35814328aa5SPhilip Cox .max_no_of_hqd = 24, 35914328aa5SPhilip Cox .doorbell_size = 8, 36014328aa5SPhilip Cox .ih_ring_entry_size = 8 * sizeof(uint32_t), 36114328aa5SPhilip Cox .event_interrupt_class = &event_interrupt_class_v9, 36214328aa5SPhilip Cox .num_of_watch_points = 4, 36314328aa5SPhilip Cox .mqd_size_aligned = MQD_SIZE_ALIGNED, 36414328aa5SPhilip Cox .needs_iommu_device = false, 36514328aa5SPhilip Cox .supports_cwsr = true, 36614328aa5SPhilip Cox .needs_pci_atomics = false, 36714328aa5SPhilip Cox .num_sdma_engines = 2, 36814328aa5SPhilip Cox .num_xgmi_sdma_engines = 0, 36914328aa5SPhilip Cox .num_sdma_queues_per_engine = 8, 37014328aa5SPhilip Cox }; 37114328aa5SPhilip Cox 372050091abSYong Zhao /* For each entry, [0] is regular and [1] is virtualisation device. */ 373050091abSYong Zhao static const struct kfd_device_info *kfd_supported_devices[][2] = { 374*95a5bd1bSYong Zhao #ifdef KFD_SUPPORT_IOMMU_V2 375050091abSYong Zhao [CHIP_KAVERI] = {&kaveri_device_info, NULL}, 376*95a5bd1bSYong Zhao [CHIP_CARRIZO] = {&carrizo_device_info, NULL}, 377*95a5bd1bSYong Zhao [CHIP_RAVEN] = {&raven_device_info, NULL}, 378*95a5bd1bSYong Zhao #endif 379050091abSYong Zhao [CHIP_HAWAII] = {&hawaii_device_info, NULL}, 380050091abSYong Zhao [CHIP_TONGA] = {&tonga_device_info, NULL}, 381050091abSYong Zhao [CHIP_FIJI] = {&fiji_device_info, &fiji_vf_device_info}, 382050091abSYong Zhao [CHIP_POLARIS10] = {&polaris10_device_info, &polaris10_vf_device_info}, 383050091abSYong Zhao [CHIP_POLARIS11] = {&polaris11_device_info, NULL}, 384050091abSYong Zhao [CHIP_POLARIS12] = {&polaris12_device_info, NULL}, 385050091abSYong Zhao [CHIP_VEGAM] = {&vegam_device_info, NULL}, 386050091abSYong Zhao [CHIP_VEGA10] = {&vega10_device_info, &vega10_vf_device_info}, 387050091abSYong Zhao [CHIP_VEGA12] = {&vega12_device_info, NULL}, 388050091abSYong Zhao [CHIP_VEGA20] = {&vega20_device_info, NULL}, 389050091abSYong Zhao [CHIP_ARCTURUS] = {&arcturus_device_info, &arcturus_device_info}, 390050091abSYong Zhao [CHIP_NAVI10] = {&navi10_device_info, NULL}, 3914a488a7aSOded Gabbay }; 3924a488a7aSOded Gabbay 3936e81090bSOded Gabbay static int kfd_gtt_sa_init(struct kfd_dev *kfd, unsigned int buf_size, 3946e81090bSOded Gabbay unsigned int chunk_size); 3956e81090bSOded Gabbay static void kfd_gtt_sa_fini(struct kfd_dev *kfd); 3966e81090bSOded Gabbay 397b8935a7cSYong Zhao static int kfd_resume(struct kfd_dev *kfd); 398b8935a7cSYong Zhao 399cea405b1SXihan Zhang struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd, 400050091abSYong Zhao struct pci_dev *pdev, const struct kfd2kgd_calls *f2g, 401050091abSYong Zhao unsigned int asic_type, bool vf) 4024a488a7aSOded Gabbay { 4034a488a7aSOded Gabbay struct kfd_dev *kfd; 404050091abSYong Zhao const struct kfd_device_info *device_info; 405050091abSYong Zhao 406050091abSYong Zhao if (asic_type >= sizeof(kfd_supported_devices) / (sizeof(void *) * 2)) { 407050091abSYong Zhao dev_err(kfd_device, "asic_type %d out of range\n", asic_type); 408050091abSYong Zhao return NULL; /* asic_type out of range */ 409050091abSYong Zhao } 410050091abSYong Zhao 411050091abSYong Zhao device_info = kfd_supported_devices[asic_type][vf]; 4124a488a7aSOded Gabbay 4134ebc7182SYong Zhao if (!device_info) { 414050091abSYong Zhao dev_err(kfd_device, "%s %s not supported in kfd\n", 415050091abSYong Zhao amdgpu_asic_name[asic_type], vf ? "VF" : ""); 4164a488a7aSOded Gabbay return NULL; 4174ebc7182SYong Zhao } 4184a488a7aSOded Gabbay 419d35f00d8SEric Huang kfd = kzalloc(sizeof(*kfd), GFP_KERNEL); 420d35f00d8SEric Huang if (!kfd) 421d35f00d8SEric Huang return NULL; 422d35f00d8SEric Huang 4236106dce9Swelu /* Allow BIF to recode atomics to PCIe 3.0 AtomicOps. 4246106dce9Swelu * 32 and 64-bit requests are possible and must be 4256106dce9Swelu * supported. 4263ee2d00cSFelix Kuehling */ 427aabf3a95SJack Xiao kfd->pci_atomic_requested = amdgpu_amdkfd_have_atomics_support(kgd); 428aabf3a95SJack Xiao if (device_info->needs_pci_atomics && 429aabf3a95SJack Xiao !kfd->pci_atomic_requested) { 4303ee2d00cSFelix Kuehling dev_info(kfd_device, 4316106dce9Swelu "skipped device %x:%x, PCI rejects atomics\n", 4323ee2d00cSFelix Kuehling pdev->vendor, pdev->device); 433d35f00d8SEric Huang kfree(kfd); 4343ee2d00cSFelix Kuehling return NULL; 435aabf3a95SJack Xiao } 4364a488a7aSOded Gabbay 4374a488a7aSOded Gabbay kfd->kgd = kgd; 4384a488a7aSOded Gabbay kfd->device_info = device_info; 4394a488a7aSOded Gabbay kfd->pdev = pdev; 44019f6d2a6SOded Gabbay kfd->init_complete = false; 441cea405b1SXihan Zhang kfd->kfd2kgd = f2g; 44243d8107fSHarish Kasiviswanathan atomic_set(&kfd->compute_profile, 0); 443cea405b1SXihan Zhang 444cea405b1SXihan Zhang mutex_init(&kfd->doorbell_mutex); 445cea405b1SXihan Zhang memset(&kfd->doorbell_available_index, 0, 446cea405b1SXihan Zhang sizeof(kfd->doorbell_available_index)); 4474a488a7aSOded Gabbay 4489b54d201SEric Huang atomic_set(&kfd->sram_ecc_flag, 0); 4499b54d201SEric Huang 4504a488a7aSOded Gabbay return kfd; 4514a488a7aSOded Gabbay } 4524a488a7aSOded Gabbay 453373d7080SFelix Kuehling static void kfd_cwsr_init(struct kfd_dev *kfd) 454373d7080SFelix Kuehling { 455373d7080SFelix Kuehling if (cwsr_enable && kfd->device_info->supports_cwsr) { 4563e76c239SFelix Kuehling if (kfd->device_info->asic_family < CHIP_VEGA10) { 457373d7080SFelix Kuehling BUILD_BUG_ON(sizeof(cwsr_trap_gfx8_hex) > PAGE_SIZE); 458373d7080SFelix Kuehling kfd->cwsr_isa = cwsr_trap_gfx8_hex; 459373d7080SFelix Kuehling kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx8_hex); 4603baa24f0SOak Zeng } else if (kfd->device_info->asic_family == CHIP_ARCTURUS) { 4613baa24f0SOak Zeng BUILD_BUG_ON(sizeof(cwsr_trap_arcturus_hex) > PAGE_SIZE); 4623baa24f0SOak Zeng kfd->cwsr_isa = cwsr_trap_arcturus_hex; 4633baa24f0SOak Zeng kfd->cwsr_isa_size = sizeof(cwsr_trap_arcturus_hex); 46414328aa5SPhilip Cox } else if (kfd->device_info->asic_family < CHIP_NAVI10) { 4653e76c239SFelix Kuehling BUILD_BUG_ON(sizeof(cwsr_trap_gfx9_hex) > PAGE_SIZE); 4663e76c239SFelix Kuehling kfd->cwsr_isa = cwsr_trap_gfx9_hex; 4673e76c239SFelix Kuehling kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx9_hex); 46814328aa5SPhilip Cox } else { 46914328aa5SPhilip Cox BUILD_BUG_ON(sizeof(cwsr_trap_gfx10_hex) > PAGE_SIZE); 47014328aa5SPhilip Cox kfd->cwsr_isa = cwsr_trap_gfx10_hex; 47114328aa5SPhilip Cox kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx10_hex); 4723e76c239SFelix Kuehling } 4733e76c239SFelix Kuehling 474373d7080SFelix Kuehling kfd->cwsr_enabled = true; 475373d7080SFelix Kuehling } 476373d7080SFelix Kuehling } 477373d7080SFelix Kuehling 4784a488a7aSOded Gabbay bool kgd2kfd_device_init(struct kfd_dev *kfd, 4794a488a7aSOded Gabbay const struct kgd2kfd_shared_resources *gpu_resources) 4804a488a7aSOded Gabbay { 48119f6d2a6SOded Gabbay unsigned int size; 48219f6d2a6SOded Gabbay 4830da8b10eSAmber Lin kfd->mec_fw_version = amdgpu_amdkfd_get_fw_version(kfd->kgd, 4845ade6c9cSFelix Kuehling KGD_ENGINE_MEC1); 4850da8b10eSAmber Lin kfd->sdma_fw_version = amdgpu_amdkfd_get_fw_version(kfd->kgd, 4865ade6c9cSFelix Kuehling KGD_ENGINE_SDMA1); 4874a488a7aSOded Gabbay kfd->shared_resources = *gpu_resources; 4884a488a7aSOded Gabbay 48944008d7aSYong Zhao kfd->vm_info.first_vmid_kfd = ffs(gpu_resources->compute_vmid_bitmap)-1; 49044008d7aSYong Zhao kfd->vm_info.last_vmid_kfd = fls(gpu_resources->compute_vmid_bitmap)-1; 49144008d7aSYong Zhao kfd->vm_info.vmid_num_kfd = kfd->vm_info.last_vmid_kfd 49244008d7aSYong Zhao - kfd->vm_info.first_vmid_kfd + 1; 49344008d7aSYong Zhao 494a99c6d4fSFelix Kuehling /* Verify module parameters regarding mapped process number*/ 495a99c6d4fSFelix Kuehling if ((hws_max_conc_proc < 0) 496a99c6d4fSFelix Kuehling || (hws_max_conc_proc > kfd->vm_info.vmid_num_kfd)) { 497a99c6d4fSFelix Kuehling dev_err(kfd_device, 498a99c6d4fSFelix Kuehling "hws_max_conc_proc %d must be between 0 and %d, use %d instead\n", 499a99c6d4fSFelix Kuehling hws_max_conc_proc, kfd->vm_info.vmid_num_kfd, 500a99c6d4fSFelix Kuehling kfd->vm_info.vmid_num_kfd); 501a99c6d4fSFelix Kuehling kfd->max_proc_per_quantum = kfd->vm_info.vmid_num_kfd; 502a99c6d4fSFelix Kuehling } else 503a99c6d4fSFelix Kuehling kfd->max_proc_per_quantum = hws_max_conc_proc; 504a99c6d4fSFelix Kuehling 505e09d4fc8SOak Zeng /* Allocate global GWS that is shared by all KFD processes */ 506e09d4fc8SOak Zeng if (hws_gws_support && amdgpu_amdkfd_alloc_gws(kfd->kgd, 507e09d4fc8SOak Zeng amdgpu_amdkfd_get_num_gws(kfd->kgd), &kfd->gws)) { 508e09d4fc8SOak Zeng dev_err(kfd_device, "Could not allocate %d gws\n", 509e09d4fc8SOak Zeng amdgpu_amdkfd_get_num_gws(kfd->kgd)); 510e09d4fc8SOak Zeng goto out; 511e09d4fc8SOak Zeng } 51219f6d2a6SOded Gabbay /* calculate max size of mqds needed for queues */ 513b8cbab04SOded Gabbay size = max_num_of_queues_per_device * 51419f6d2a6SOded Gabbay kfd->device_info->mqd_size_aligned; 51519f6d2a6SOded Gabbay 516e18e794eSOded Gabbay /* 517e18e794eSOded Gabbay * calculate max size of runlist packet. 518e18e794eSOded Gabbay * There can be only 2 packets at once 519e18e794eSOded Gabbay */ 520507968ddSFelix Kuehling size += (KFD_MAX_NUM_OF_PROCESSES * sizeof(struct pm4_mes_map_process) + 521507968ddSFelix Kuehling max_num_of_queues_per_device * sizeof(struct pm4_mes_map_queues) 522507968ddSFelix Kuehling + sizeof(struct pm4_mes_runlist)) * 2; 523e18e794eSOded Gabbay 524e18e794eSOded Gabbay /* Add size of HIQ & DIQ */ 525e18e794eSOded Gabbay size += KFD_KERNEL_QUEUE_SIZE * 2; 526e18e794eSOded Gabbay 527e18e794eSOded Gabbay /* add another 512KB for all other allocations on gart (HPD, fences) */ 52819f6d2a6SOded Gabbay size += 512 * 1024; 52919f6d2a6SOded Gabbay 5307cd52c91SAmber Lin if (amdgpu_amdkfd_alloc_gtt_mem( 531cea405b1SXihan Zhang kfd->kgd, size, &kfd->gtt_mem, 53215426dbbSYong Zhao &kfd->gtt_start_gpu_addr, &kfd->gtt_start_cpu_ptr, 53315426dbbSYong Zhao false)) { 53479775b62SKent Russell dev_err(kfd_device, "Could not allocate %d bytes\n", size); 535e09d4fc8SOak Zeng goto alloc_gtt_mem_failure; 53619f6d2a6SOded Gabbay } 53719f6d2a6SOded Gabbay 53879775b62SKent Russell dev_info(kfd_device, "Allocated %d bytes on gart\n", size); 539e18e794eSOded Gabbay 54073a1da0bSOded Gabbay /* Initialize GTT sa with 512 byte chunk size */ 54173a1da0bSOded Gabbay if (kfd_gtt_sa_init(kfd, size, 512) != 0) { 54279775b62SKent Russell dev_err(kfd_device, "Error initializing gtt sub-allocator\n"); 54373a1da0bSOded Gabbay goto kfd_gtt_sa_init_error; 54473a1da0bSOded Gabbay } 54573a1da0bSOded Gabbay 546735df2baSFelix Kuehling if (kfd_doorbell_init(kfd)) { 547735df2baSFelix Kuehling dev_err(kfd_device, 548735df2baSFelix Kuehling "Error initializing doorbell aperture\n"); 549735df2baSFelix Kuehling goto kfd_doorbell_error; 550735df2baSFelix Kuehling } 55119f6d2a6SOded Gabbay 5520c1690e3SShaoyun Liu if (kfd->kfd2kgd->get_hive_id) 5530c1690e3SShaoyun Liu kfd->hive_id = kfd->kfd2kgd->get_hive_id(kfd->kgd); 5540c1690e3SShaoyun Liu 5552249d558SAndrew Lewycky if (kfd_interrupt_init(kfd)) { 55679775b62SKent Russell dev_err(kfd_device, "Error initializing interrupts\n"); 5572249d558SAndrew Lewycky goto kfd_interrupt_error; 5582249d558SAndrew Lewycky } 5592249d558SAndrew Lewycky 56064c7f8cfSBen Goz kfd->dqm = device_queue_manager_init(kfd); 56164c7f8cfSBen Goz if (!kfd->dqm) { 56279775b62SKent Russell dev_err(kfd_device, "Error initializing queue manager\n"); 56364c7f8cfSBen Goz goto device_queue_manager_error; 56464c7f8cfSBen Goz } 56564c7f8cfSBen Goz 56664d1c3a4SFelix Kuehling if (kfd_iommu_device_init(kfd)) { 56764d1c3a4SFelix Kuehling dev_err(kfd_device, "Error initializing iommuv2\n"); 56864d1c3a4SFelix Kuehling goto device_iommu_error; 56964c7f8cfSBen Goz } 57064c7f8cfSBen Goz 571373d7080SFelix Kuehling kfd_cwsr_init(kfd); 572373d7080SFelix Kuehling 573b8935a7cSYong Zhao if (kfd_resume(kfd)) 574b8935a7cSYong Zhao goto kfd_resume_error; 575b8935a7cSYong Zhao 576fbeb661bSYair Shachar kfd->dbgmgr = NULL; 577fbeb661bSYair Shachar 578465ab9e0SOak Zeng if (kfd_topology_add_device(kfd)) { 579465ab9e0SOak Zeng dev_err(kfd_device, "Error adding device to topology\n"); 580465ab9e0SOak Zeng goto kfd_topology_add_device_error; 581465ab9e0SOak Zeng } 582465ab9e0SOak Zeng 5834a488a7aSOded Gabbay kfd->init_complete = true; 58479775b62SKent Russell dev_info(kfd_device, "added device %x:%x\n", kfd->pdev->vendor, 5854a488a7aSOded Gabbay kfd->pdev->device); 5864a488a7aSOded Gabbay 58779775b62SKent Russell pr_debug("Starting kfd with the following scheduling policy %d\n", 588d146c5a7SFelix Kuehling kfd->dqm->sched_policy); 58964c7f8cfSBen Goz 59019f6d2a6SOded Gabbay goto out; 59119f6d2a6SOded Gabbay 592465ab9e0SOak Zeng kfd_topology_add_device_error: 593b8935a7cSYong Zhao kfd_resume_error: 59464d1c3a4SFelix Kuehling device_iommu_error: 59564c7f8cfSBen Goz device_queue_manager_uninit(kfd->dqm); 59664c7f8cfSBen Goz device_queue_manager_error: 5972249d558SAndrew Lewycky kfd_interrupt_exit(kfd); 5982249d558SAndrew Lewycky kfd_interrupt_error: 599735df2baSFelix Kuehling kfd_doorbell_fini(kfd); 600735df2baSFelix Kuehling kfd_doorbell_error: 60173a1da0bSOded Gabbay kfd_gtt_sa_fini(kfd); 60273a1da0bSOded Gabbay kfd_gtt_sa_init_error: 6037cd52c91SAmber Lin amdgpu_amdkfd_free_gtt_mem(kfd->kgd, kfd->gtt_mem); 604e09d4fc8SOak Zeng alloc_gtt_mem_failure: 605e09d4fc8SOak Zeng if (hws_gws_support) 606e09d4fc8SOak Zeng amdgpu_amdkfd_free_gws(kfd->kgd, kfd->gws); 60719f6d2a6SOded Gabbay dev_err(kfd_device, 60879775b62SKent Russell "device %x:%x NOT added due to errors\n", 60919f6d2a6SOded Gabbay kfd->pdev->vendor, kfd->pdev->device); 61019f6d2a6SOded Gabbay out: 61119f6d2a6SOded Gabbay return kfd->init_complete; 6124a488a7aSOded Gabbay } 6134a488a7aSOded Gabbay 6144a488a7aSOded Gabbay void kgd2kfd_device_exit(struct kfd_dev *kfd) 6154a488a7aSOded Gabbay { 616b17f068aSOded Gabbay if (kfd->init_complete) { 617b8935a7cSYong Zhao kgd2kfd_suspend(kfd); 61864c7f8cfSBen Goz device_queue_manager_uninit(kfd->dqm); 6192249d558SAndrew Lewycky kfd_interrupt_exit(kfd); 62019f6d2a6SOded Gabbay kfd_topology_remove_device(kfd); 621735df2baSFelix Kuehling kfd_doorbell_fini(kfd); 62273a1da0bSOded Gabbay kfd_gtt_sa_fini(kfd); 6237cd52c91SAmber Lin amdgpu_amdkfd_free_gtt_mem(kfd->kgd, kfd->gtt_mem); 624e09d4fc8SOak Zeng if (hws_gws_support) 625e09d4fc8SOak Zeng amdgpu_amdkfd_free_gws(kfd->kgd, kfd->gws); 626b17f068aSOded Gabbay } 6275b5c4e40SEvgeny Pinchuk 6284a488a7aSOded Gabbay kfree(kfd); 6294a488a7aSOded Gabbay } 6304a488a7aSOded Gabbay 631e3b7a967SShaoyun Liu int kgd2kfd_pre_reset(struct kfd_dev *kfd) 632e3b7a967SShaoyun Liu { 633e42051d2SShaoyun Liu if (!kfd->init_complete) 634e42051d2SShaoyun Liu return 0; 635e42051d2SShaoyun Liu kgd2kfd_suspend(kfd); 636e42051d2SShaoyun Liu 637e42051d2SShaoyun Liu /* hold dqm->lock to prevent further execution*/ 638e42051d2SShaoyun Liu dqm_lock(kfd->dqm); 639e42051d2SShaoyun Liu 640e42051d2SShaoyun Liu kfd_signal_reset_event(kfd); 641e3b7a967SShaoyun Liu return 0; 642e3b7a967SShaoyun Liu } 643e3b7a967SShaoyun Liu 644e42051d2SShaoyun Liu /* 645e42051d2SShaoyun Liu * Fix me. KFD won't be able to resume existing process for now. 646e42051d2SShaoyun Liu * We will keep all existing process in a evicted state and 647e42051d2SShaoyun Liu * wait the process to be terminated. 648e42051d2SShaoyun Liu */ 649e42051d2SShaoyun Liu 650e3b7a967SShaoyun Liu int kgd2kfd_post_reset(struct kfd_dev *kfd) 651e3b7a967SShaoyun Liu { 652e42051d2SShaoyun Liu int ret, count; 653e42051d2SShaoyun Liu 654e42051d2SShaoyun Liu if (!kfd->init_complete) 655e3b7a967SShaoyun Liu return 0; 656e42051d2SShaoyun Liu 657e42051d2SShaoyun Liu dqm_unlock(kfd->dqm); 658e42051d2SShaoyun Liu 659e42051d2SShaoyun Liu ret = kfd_resume(kfd); 660e42051d2SShaoyun Liu if (ret) 661e42051d2SShaoyun Liu return ret; 662e42051d2SShaoyun Liu count = atomic_dec_return(&kfd_locked); 6639b54d201SEric Huang 6649b54d201SEric Huang atomic_set(&kfd->sram_ecc_flag, 0); 6659b54d201SEric Huang 666e42051d2SShaoyun Liu return 0; 667e42051d2SShaoyun Liu } 668e42051d2SShaoyun Liu 669e42051d2SShaoyun Liu bool kfd_is_locked(void) 670e42051d2SShaoyun Liu { 671e42051d2SShaoyun Liu return (atomic_read(&kfd_locked) > 0); 672e3b7a967SShaoyun Liu } 673e3b7a967SShaoyun Liu 6744a488a7aSOded Gabbay void kgd2kfd_suspend(struct kfd_dev *kfd) 6754a488a7aSOded Gabbay { 676733fa1f7SYong Zhao if (!kfd->init_complete) 677733fa1f7SYong Zhao return; 678733fa1f7SYong Zhao 67926103436SFelix Kuehling /* For first KFD device suspend all the KFD processes */ 680e42051d2SShaoyun Liu if (atomic_inc_return(&kfd_locked) == 1) 68126103436SFelix Kuehling kfd_suspend_all_processes(); 68226103436SFelix Kuehling 68345c9a5e4SOded Gabbay kfd->dqm->ops.stop(kfd->dqm); 684733fa1f7SYong Zhao 68564d1c3a4SFelix Kuehling kfd_iommu_suspend(kfd); 6864a488a7aSOded Gabbay } 6874a488a7aSOded Gabbay 6884a488a7aSOded Gabbay int kgd2kfd_resume(struct kfd_dev *kfd) 6894a488a7aSOded Gabbay { 69026103436SFelix Kuehling int ret, count; 69126103436SFelix Kuehling 692b8935a7cSYong Zhao if (!kfd->init_complete) 693b8935a7cSYong Zhao return 0; 694b17f068aSOded Gabbay 69526103436SFelix Kuehling ret = kfd_resume(kfd); 69626103436SFelix Kuehling if (ret) 69726103436SFelix Kuehling return ret; 698b17f068aSOded Gabbay 699e42051d2SShaoyun Liu count = atomic_dec_return(&kfd_locked); 70026103436SFelix Kuehling WARN_ONCE(count < 0, "KFD suspend / resume ref. error"); 70126103436SFelix Kuehling if (count == 0) 70226103436SFelix Kuehling ret = kfd_resume_all_processes(); 70326103436SFelix Kuehling 70426103436SFelix Kuehling return ret; 7054ebc7182SYong Zhao } 7064ebc7182SYong Zhao 707b8935a7cSYong Zhao static int kfd_resume(struct kfd_dev *kfd) 708b8935a7cSYong Zhao { 709b8935a7cSYong Zhao int err = 0; 710b8935a7cSYong Zhao 71164d1c3a4SFelix Kuehling err = kfd_iommu_resume(kfd); 71264d1c3a4SFelix Kuehling if (err) { 71364d1c3a4SFelix Kuehling dev_err(kfd_device, 71464d1c3a4SFelix Kuehling "Failed to resume IOMMU for device %x:%x\n", 71564d1c3a4SFelix Kuehling kfd->pdev->vendor, kfd->pdev->device); 71664d1c3a4SFelix Kuehling return err; 71764d1c3a4SFelix Kuehling } 718733fa1f7SYong Zhao 719b8935a7cSYong Zhao err = kfd->dqm->ops.start(kfd->dqm); 720b8935a7cSYong Zhao if (err) { 721b8935a7cSYong Zhao dev_err(kfd_device, 722b8935a7cSYong Zhao "Error starting queue manager for device %x:%x\n", 723b8935a7cSYong Zhao kfd->pdev->vendor, kfd->pdev->device); 724b8935a7cSYong Zhao goto dqm_start_error; 725b17f068aSOded Gabbay } 726b17f068aSOded Gabbay 727b8935a7cSYong Zhao return err; 728b8935a7cSYong Zhao 729b8935a7cSYong Zhao dqm_start_error: 73064d1c3a4SFelix Kuehling kfd_iommu_suspend(kfd); 731b8935a7cSYong Zhao return err; 7324a488a7aSOded Gabbay } 7334a488a7aSOded Gabbay 734b3f5e6b4SAndrew Lewycky /* This is called directly from KGD at ISR. */ 735b3f5e6b4SAndrew Lewycky void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry) 7364a488a7aSOded Gabbay { 73758e69886SLan Xiao uint32_t patched_ihre[KFD_MAX_RING_ENTRY_SIZE]; 73858e69886SLan Xiao bool is_patched = false; 7392383a767SChristian König unsigned long flags; 74058e69886SLan Xiao 7412249d558SAndrew Lewycky if (!kfd->init_complete) 7422249d558SAndrew Lewycky return; 7432249d558SAndrew Lewycky 74458e69886SLan Xiao if (kfd->device_info->ih_ring_entry_size > sizeof(patched_ihre)) { 74558e69886SLan Xiao dev_err_once(kfd_device, "Ring entry too small\n"); 74658e69886SLan Xiao return; 74758e69886SLan Xiao } 74858e69886SLan Xiao 7492383a767SChristian König spin_lock_irqsave(&kfd->interrupt_lock, flags); 7502249d558SAndrew Lewycky 7512249d558SAndrew Lewycky if (kfd->interrupts_active 75258e69886SLan Xiao && interrupt_is_wanted(kfd, ih_ring_entry, 75358e69886SLan Xiao patched_ihre, &is_patched) 75458e69886SLan Xiao && enqueue_ih_ring_entry(kfd, 75558e69886SLan Xiao is_patched ? patched_ihre : ih_ring_entry)) 75648e876a2SAndres Rodriguez queue_work(kfd->ih_wq, &kfd->interrupt_work); 7572249d558SAndrew Lewycky 7582383a767SChristian König spin_unlock_irqrestore(&kfd->interrupt_lock, flags); 7594a488a7aSOded Gabbay } 7606e81090bSOded Gabbay 7616b95e797SFelix Kuehling int kgd2kfd_quiesce_mm(struct mm_struct *mm) 7626b95e797SFelix Kuehling { 7636b95e797SFelix Kuehling struct kfd_process *p; 7646b95e797SFelix Kuehling int r; 7656b95e797SFelix Kuehling 7666b95e797SFelix Kuehling /* Because we are called from arbitrary context (workqueue) as opposed 7676b95e797SFelix Kuehling * to process context, kfd_process could attempt to exit while we are 7686b95e797SFelix Kuehling * running so the lookup function increments the process ref count. 7696b95e797SFelix Kuehling */ 7706b95e797SFelix Kuehling p = kfd_lookup_process_by_mm(mm); 7716b95e797SFelix Kuehling if (!p) 7726b95e797SFelix Kuehling return -ESRCH; 7736b95e797SFelix Kuehling 7746b95e797SFelix Kuehling r = kfd_process_evict_queues(p); 7756b95e797SFelix Kuehling 7766b95e797SFelix Kuehling kfd_unref_process(p); 7776b95e797SFelix Kuehling return r; 7786b95e797SFelix Kuehling } 7796b95e797SFelix Kuehling 7806b95e797SFelix Kuehling int kgd2kfd_resume_mm(struct mm_struct *mm) 7816b95e797SFelix Kuehling { 7826b95e797SFelix Kuehling struct kfd_process *p; 7836b95e797SFelix Kuehling int r; 7846b95e797SFelix Kuehling 7856b95e797SFelix Kuehling /* Because we are called from arbitrary context (workqueue) as opposed 7866b95e797SFelix Kuehling * to process context, kfd_process could attempt to exit while we are 7876b95e797SFelix Kuehling * running so the lookup function increments the process ref count. 7886b95e797SFelix Kuehling */ 7896b95e797SFelix Kuehling p = kfd_lookup_process_by_mm(mm); 7906b95e797SFelix Kuehling if (!p) 7916b95e797SFelix Kuehling return -ESRCH; 7926b95e797SFelix Kuehling 7936b95e797SFelix Kuehling r = kfd_process_restore_queues(p); 7946b95e797SFelix Kuehling 7956b95e797SFelix Kuehling kfd_unref_process(p); 7966b95e797SFelix Kuehling return r; 7976b95e797SFelix Kuehling } 7986b95e797SFelix Kuehling 79926103436SFelix Kuehling /** kgd2kfd_schedule_evict_and_restore_process - Schedules work queue that will 80026103436SFelix Kuehling * prepare for safe eviction of KFD BOs that belong to the specified 80126103436SFelix Kuehling * process. 80226103436SFelix Kuehling * 80326103436SFelix Kuehling * @mm: mm_struct that identifies the specified KFD process 80426103436SFelix Kuehling * @fence: eviction fence attached to KFD process BOs 80526103436SFelix Kuehling * 80626103436SFelix Kuehling */ 80726103436SFelix Kuehling int kgd2kfd_schedule_evict_and_restore_process(struct mm_struct *mm, 80826103436SFelix Kuehling struct dma_fence *fence) 80926103436SFelix Kuehling { 81026103436SFelix Kuehling struct kfd_process *p; 81126103436SFelix Kuehling unsigned long active_time; 81226103436SFelix Kuehling unsigned long delay_jiffies = msecs_to_jiffies(PROCESS_ACTIVE_TIME_MS); 81326103436SFelix Kuehling 81426103436SFelix Kuehling if (!fence) 81526103436SFelix Kuehling return -EINVAL; 81626103436SFelix Kuehling 81726103436SFelix Kuehling if (dma_fence_is_signaled(fence)) 81826103436SFelix Kuehling return 0; 81926103436SFelix Kuehling 82026103436SFelix Kuehling p = kfd_lookup_process_by_mm(mm); 82126103436SFelix Kuehling if (!p) 82226103436SFelix Kuehling return -ENODEV; 82326103436SFelix Kuehling 82426103436SFelix Kuehling if (fence->seqno == p->last_eviction_seqno) 82526103436SFelix Kuehling goto out; 82626103436SFelix Kuehling 82726103436SFelix Kuehling p->last_eviction_seqno = fence->seqno; 82826103436SFelix Kuehling 82926103436SFelix Kuehling /* Avoid KFD process starvation. Wait for at least 83026103436SFelix Kuehling * PROCESS_ACTIVE_TIME_MS before evicting the process again 83126103436SFelix Kuehling */ 83226103436SFelix Kuehling active_time = get_jiffies_64() - p->last_restore_timestamp; 83326103436SFelix Kuehling if (delay_jiffies > active_time) 83426103436SFelix Kuehling delay_jiffies -= active_time; 83526103436SFelix Kuehling else 83626103436SFelix Kuehling delay_jiffies = 0; 83726103436SFelix Kuehling 83826103436SFelix Kuehling /* During process initialization eviction_work.dwork is initialized 83926103436SFelix Kuehling * to kfd_evict_bo_worker 84026103436SFelix Kuehling */ 84126103436SFelix Kuehling schedule_delayed_work(&p->eviction_work, delay_jiffies); 84226103436SFelix Kuehling out: 84326103436SFelix Kuehling kfd_unref_process(p); 84426103436SFelix Kuehling return 0; 84526103436SFelix Kuehling } 84626103436SFelix Kuehling 8476e81090bSOded Gabbay static int kfd_gtt_sa_init(struct kfd_dev *kfd, unsigned int buf_size, 8486e81090bSOded Gabbay unsigned int chunk_size) 8496e81090bSOded Gabbay { 8508625ff9cSFelix Kuehling unsigned int num_of_longs; 8516e81090bSOded Gabbay 85232fa8219SFelix Kuehling if (WARN_ON(buf_size < chunk_size)) 85332fa8219SFelix Kuehling return -EINVAL; 85432fa8219SFelix Kuehling if (WARN_ON(buf_size == 0)) 85532fa8219SFelix Kuehling return -EINVAL; 85632fa8219SFelix Kuehling if (WARN_ON(chunk_size == 0)) 85732fa8219SFelix Kuehling return -EINVAL; 8586e81090bSOded Gabbay 8596e81090bSOded Gabbay kfd->gtt_sa_chunk_size = chunk_size; 8606e81090bSOded Gabbay kfd->gtt_sa_num_of_chunks = buf_size / chunk_size; 8616e81090bSOded Gabbay 8628625ff9cSFelix Kuehling num_of_longs = (kfd->gtt_sa_num_of_chunks + BITS_PER_LONG - 1) / 8638625ff9cSFelix Kuehling BITS_PER_LONG; 8646e81090bSOded Gabbay 8658625ff9cSFelix Kuehling kfd->gtt_sa_bitmap = kcalloc(num_of_longs, sizeof(long), GFP_KERNEL); 8666e81090bSOded Gabbay 8676e81090bSOded Gabbay if (!kfd->gtt_sa_bitmap) 8686e81090bSOded Gabbay return -ENOMEM; 8696e81090bSOded Gabbay 87079775b62SKent Russell pr_debug("gtt_sa_num_of_chunks = %d, gtt_sa_bitmap = %p\n", 8716e81090bSOded Gabbay kfd->gtt_sa_num_of_chunks, kfd->gtt_sa_bitmap); 8726e81090bSOded Gabbay 8736e81090bSOded Gabbay mutex_init(&kfd->gtt_sa_lock); 8746e81090bSOded Gabbay 8756e81090bSOded Gabbay return 0; 8766e81090bSOded Gabbay 8776e81090bSOded Gabbay } 8786e81090bSOded Gabbay 8796e81090bSOded Gabbay static void kfd_gtt_sa_fini(struct kfd_dev *kfd) 8806e81090bSOded Gabbay { 8816e81090bSOded Gabbay mutex_destroy(&kfd->gtt_sa_lock); 8826e81090bSOded Gabbay kfree(kfd->gtt_sa_bitmap); 8836e81090bSOded Gabbay } 8846e81090bSOded Gabbay 8856e81090bSOded Gabbay static inline uint64_t kfd_gtt_sa_calc_gpu_addr(uint64_t start_addr, 8866e81090bSOded Gabbay unsigned int bit_num, 8876e81090bSOded Gabbay unsigned int chunk_size) 8886e81090bSOded Gabbay { 8896e81090bSOded Gabbay return start_addr + bit_num * chunk_size; 8906e81090bSOded Gabbay } 8916e81090bSOded Gabbay 8926e81090bSOded Gabbay static inline uint32_t *kfd_gtt_sa_calc_cpu_addr(void *start_addr, 8936e81090bSOded Gabbay unsigned int bit_num, 8946e81090bSOded Gabbay unsigned int chunk_size) 8956e81090bSOded Gabbay { 8966e81090bSOded Gabbay return (uint32_t *) ((uint64_t) start_addr + bit_num * chunk_size); 8976e81090bSOded Gabbay } 8986e81090bSOded Gabbay 8996e81090bSOded Gabbay int kfd_gtt_sa_allocate(struct kfd_dev *kfd, unsigned int size, 9006e81090bSOded Gabbay struct kfd_mem_obj **mem_obj) 9016e81090bSOded Gabbay { 9026e81090bSOded Gabbay unsigned int found, start_search, cur_size; 9036e81090bSOded Gabbay 9046e81090bSOded Gabbay if (size == 0) 9056e81090bSOded Gabbay return -EINVAL; 9066e81090bSOded Gabbay 9076e81090bSOded Gabbay if (size > kfd->gtt_sa_num_of_chunks * kfd->gtt_sa_chunk_size) 9086e81090bSOded Gabbay return -ENOMEM; 9096e81090bSOded Gabbay 9101cd106ecSFelix Kuehling *mem_obj = kzalloc(sizeof(struct kfd_mem_obj), GFP_KERNEL); 9111cd106ecSFelix Kuehling if (!(*mem_obj)) 9126e81090bSOded Gabbay return -ENOMEM; 9136e81090bSOded Gabbay 91479775b62SKent Russell pr_debug("Allocated mem_obj = %p for size = %d\n", *mem_obj, size); 9156e81090bSOded Gabbay 9166e81090bSOded Gabbay start_search = 0; 9176e81090bSOded Gabbay 9186e81090bSOded Gabbay mutex_lock(&kfd->gtt_sa_lock); 9196e81090bSOded Gabbay 9206e81090bSOded Gabbay kfd_gtt_restart_search: 9216e81090bSOded Gabbay /* Find the first chunk that is free */ 9226e81090bSOded Gabbay found = find_next_zero_bit(kfd->gtt_sa_bitmap, 9236e81090bSOded Gabbay kfd->gtt_sa_num_of_chunks, 9246e81090bSOded Gabbay start_search); 9256e81090bSOded Gabbay 92679775b62SKent Russell pr_debug("Found = %d\n", found); 9276e81090bSOded Gabbay 9286e81090bSOded Gabbay /* If there wasn't any free chunk, bail out */ 9296e81090bSOded Gabbay if (found == kfd->gtt_sa_num_of_chunks) 9306e81090bSOded Gabbay goto kfd_gtt_no_free_chunk; 9316e81090bSOded Gabbay 9326e81090bSOded Gabbay /* Update fields of mem_obj */ 9336e81090bSOded Gabbay (*mem_obj)->range_start = found; 9346e81090bSOded Gabbay (*mem_obj)->range_end = found; 9356e81090bSOded Gabbay (*mem_obj)->gpu_addr = kfd_gtt_sa_calc_gpu_addr( 9366e81090bSOded Gabbay kfd->gtt_start_gpu_addr, 9376e81090bSOded Gabbay found, 9386e81090bSOded Gabbay kfd->gtt_sa_chunk_size); 9396e81090bSOded Gabbay (*mem_obj)->cpu_ptr = kfd_gtt_sa_calc_cpu_addr( 9406e81090bSOded Gabbay kfd->gtt_start_cpu_ptr, 9416e81090bSOded Gabbay found, 9426e81090bSOded Gabbay kfd->gtt_sa_chunk_size); 9436e81090bSOded Gabbay 94479775b62SKent Russell pr_debug("gpu_addr = %p, cpu_addr = %p\n", 9456e81090bSOded Gabbay (uint64_t *) (*mem_obj)->gpu_addr, (*mem_obj)->cpu_ptr); 9466e81090bSOded Gabbay 9476e81090bSOded Gabbay /* If we need only one chunk, mark it as allocated and get out */ 9486e81090bSOded Gabbay if (size <= kfd->gtt_sa_chunk_size) { 94979775b62SKent Russell pr_debug("Single bit\n"); 9506e81090bSOded Gabbay set_bit(found, kfd->gtt_sa_bitmap); 9516e81090bSOded Gabbay goto kfd_gtt_out; 9526e81090bSOded Gabbay } 9536e81090bSOded Gabbay 9546e81090bSOded Gabbay /* Otherwise, try to see if we have enough contiguous chunks */ 9556e81090bSOded Gabbay cur_size = size - kfd->gtt_sa_chunk_size; 9566e81090bSOded Gabbay do { 9576e81090bSOded Gabbay (*mem_obj)->range_end = 9586e81090bSOded Gabbay find_next_zero_bit(kfd->gtt_sa_bitmap, 9596e81090bSOded Gabbay kfd->gtt_sa_num_of_chunks, ++found); 9606e81090bSOded Gabbay /* 9616e81090bSOded Gabbay * If next free chunk is not contiguous than we need to 9626e81090bSOded Gabbay * restart our search from the last free chunk we found (which 9636e81090bSOded Gabbay * wasn't contiguous to the previous ones 9646e81090bSOded Gabbay */ 9656e81090bSOded Gabbay if ((*mem_obj)->range_end != found) { 9666e81090bSOded Gabbay start_search = found; 9676e81090bSOded Gabbay goto kfd_gtt_restart_search; 9686e81090bSOded Gabbay } 9696e81090bSOded Gabbay 9706e81090bSOded Gabbay /* 9716e81090bSOded Gabbay * If we reached end of buffer, bail out with error 9726e81090bSOded Gabbay */ 9736e81090bSOded Gabbay if (found == kfd->gtt_sa_num_of_chunks) 9746e81090bSOded Gabbay goto kfd_gtt_no_free_chunk; 9756e81090bSOded Gabbay 9766e81090bSOded Gabbay /* Check if we don't need another chunk */ 9776e81090bSOded Gabbay if (cur_size <= kfd->gtt_sa_chunk_size) 9786e81090bSOded Gabbay cur_size = 0; 9796e81090bSOded Gabbay else 9806e81090bSOded Gabbay cur_size -= kfd->gtt_sa_chunk_size; 9816e81090bSOded Gabbay 9826e81090bSOded Gabbay } while (cur_size > 0); 9836e81090bSOded Gabbay 98479775b62SKent Russell pr_debug("range_start = %d, range_end = %d\n", 9856e81090bSOded Gabbay (*mem_obj)->range_start, (*mem_obj)->range_end); 9866e81090bSOded Gabbay 9876e81090bSOded Gabbay /* Mark the chunks as allocated */ 9886e81090bSOded Gabbay for (found = (*mem_obj)->range_start; 9896e81090bSOded Gabbay found <= (*mem_obj)->range_end; 9906e81090bSOded Gabbay found++) 9916e81090bSOded Gabbay set_bit(found, kfd->gtt_sa_bitmap); 9926e81090bSOded Gabbay 9936e81090bSOded Gabbay kfd_gtt_out: 9946e81090bSOded Gabbay mutex_unlock(&kfd->gtt_sa_lock); 9956e81090bSOded Gabbay return 0; 9966e81090bSOded Gabbay 9976e81090bSOded Gabbay kfd_gtt_no_free_chunk: 99879775b62SKent Russell pr_debug("Allocation failed with mem_obj = %p\n", mem_obj); 9996e81090bSOded Gabbay mutex_unlock(&kfd->gtt_sa_lock); 10006e81090bSOded Gabbay kfree(mem_obj); 10016e81090bSOded Gabbay return -ENOMEM; 10026e81090bSOded Gabbay } 10036e81090bSOded Gabbay 10046e81090bSOded Gabbay int kfd_gtt_sa_free(struct kfd_dev *kfd, struct kfd_mem_obj *mem_obj) 10056e81090bSOded Gabbay { 10066e81090bSOded Gabbay unsigned int bit; 10076e81090bSOded Gabbay 10089216ed29SOded Gabbay /* Act like kfree when trying to free a NULL object */ 10099216ed29SOded Gabbay if (!mem_obj) 10109216ed29SOded Gabbay return 0; 10116e81090bSOded Gabbay 101279775b62SKent Russell pr_debug("Free mem_obj = %p, range_start = %d, range_end = %d\n", 10136e81090bSOded Gabbay mem_obj, mem_obj->range_start, mem_obj->range_end); 10146e81090bSOded Gabbay 10156e81090bSOded Gabbay mutex_lock(&kfd->gtt_sa_lock); 10166e81090bSOded Gabbay 10176e81090bSOded Gabbay /* Mark the chunks as free */ 10186e81090bSOded Gabbay for (bit = mem_obj->range_start; 10196e81090bSOded Gabbay bit <= mem_obj->range_end; 10206e81090bSOded Gabbay bit++) 10216e81090bSOded Gabbay clear_bit(bit, kfd->gtt_sa_bitmap); 10226e81090bSOded Gabbay 10236e81090bSOded Gabbay mutex_unlock(&kfd->gtt_sa_lock); 10246e81090bSOded Gabbay 10256e81090bSOded Gabbay kfree(mem_obj); 10266e81090bSOded Gabbay return 0; 10276e81090bSOded Gabbay } 1028a29ec470SShaoyun Liu 10299b54d201SEric Huang void kgd2kfd_set_sram_ecc_flag(struct kfd_dev *kfd) 10309b54d201SEric Huang { 10319b54d201SEric Huang if (kfd) 10329b54d201SEric Huang atomic_inc(&kfd->sram_ecc_flag); 10339b54d201SEric Huang } 10349b54d201SEric Huang 103543d8107fSHarish Kasiviswanathan void kfd_inc_compute_active(struct kfd_dev *kfd) 103643d8107fSHarish Kasiviswanathan { 103743d8107fSHarish Kasiviswanathan if (atomic_inc_return(&kfd->compute_profile) == 1) 103843d8107fSHarish Kasiviswanathan amdgpu_amdkfd_set_compute_idle(kfd->kgd, false); 103943d8107fSHarish Kasiviswanathan } 104043d8107fSHarish Kasiviswanathan 104143d8107fSHarish Kasiviswanathan void kfd_dec_compute_active(struct kfd_dev *kfd) 104243d8107fSHarish Kasiviswanathan { 104343d8107fSHarish Kasiviswanathan int count = atomic_dec_return(&kfd->compute_profile); 104443d8107fSHarish Kasiviswanathan 104543d8107fSHarish Kasiviswanathan if (count == 0) 104643d8107fSHarish Kasiviswanathan amdgpu_amdkfd_set_compute_idle(kfd->kgd, true); 104743d8107fSHarish Kasiviswanathan WARN_ONCE(count < 0, "Compute profile ref. count error"); 104843d8107fSHarish Kasiviswanathan } 104943d8107fSHarish Kasiviswanathan 1050a29ec470SShaoyun Liu #if defined(CONFIG_DEBUG_FS) 1051a29ec470SShaoyun Liu 1052a29ec470SShaoyun Liu /* This function will send a package to HIQ to hang the HWS 1053a29ec470SShaoyun Liu * which will trigger a GPU reset and bring the HWS back to normal state 1054a29ec470SShaoyun Liu */ 1055a29ec470SShaoyun Liu int kfd_debugfs_hang_hws(struct kfd_dev *dev) 1056a29ec470SShaoyun Liu { 1057a29ec470SShaoyun Liu int r = 0; 1058a29ec470SShaoyun Liu 1059a29ec470SShaoyun Liu if (dev->dqm->sched_policy != KFD_SCHED_POLICY_HWS) { 1060a29ec470SShaoyun Liu pr_err("HWS is not enabled"); 1061a29ec470SShaoyun Liu return -EINVAL; 1062a29ec470SShaoyun Liu } 1063a29ec470SShaoyun Liu 1064a29ec470SShaoyun Liu r = pm_debugfs_hang_hws(&dev->dqm->packets); 1065a29ec470SShaoyun Liu if (!r) 1066a29ec470SShaoyun Liu r = dqm_debugfs_execute_queues(dev->dqm); 1067a29ec470SShaoyun Liu 1068a29ec470SShaoyun Liu return r; 1069a29ec470SShaoyun Liu } 1070a29ec470SShaoyun Liu 1071a29ec470SShaoyun Liu #endif 1072