14a488a7aSOded Gabbay /* 24a488a7aSOded Gabbay * Copyright 2014 Advanced Micro Devices, Inc. 34a488a7aSOded Gabbay * 44a488a7aSOded Gabbay * Permission is hereby granted, free of charge, to any person obtaining a 54a488a7aSOded Gabbay * copy of this software and associated documentation files (the "Software"), 64a488a7aSOded Gabbay * to deal in the Software without restriction, including without limitation 74a488a7aSOded Gabbay * the rights to use, copy, modify, merge, publish, distribute, sublicense, 84a488a7aSOded Gabbay * and/or sell copies of the Software, and to permit persons to whom the 94a488a7aSOded Gabbay * Software is furnished to do so, subject to the following conditions: 104a488a7aSOded Gabbay * 114a488a7aSOded Gabbay * The above copyright notice and this permission notice shall be included in 124a488a7aSOded Gabbay * all copies or substantial portions of the Software. 134a488a7aSOded Gabbay * 144a488a7aSOded Gabbay * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 154a488a7aSOded Gabbay * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 164a488a7aSOded Gabbay * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 174a488a7aSOded Gabbay * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 184a488a7aSOded Gabbay * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 194a488a7aSOded Gabbay * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 204a488a7aSOded Gabbay * OTHER DEALINGS IN THE SOFTWARE. 214a488a7aSOded Gabbay */ 224a488a7aSOded Gabbay 234a488a7aSOded Gabbay #include <linux/bsearch.h> 244a488a7aSOded Gabbay #include <linux/pci.h> 254a488a7aSOded Gabbay #include <linux/slab.h> 264a488a7aSOded Gabbay #include "kfd_priv.h" 2764c7f8cfSBen Goz #include "kfd_device_queue_manager.h" 28507968ddSFelix Kuehling #include "kfd_pm4_headers_vi.h" 29fd6a440eSJonathan Kim #include "kfd_pm4_headers_aldebaran.h" 300db54b24SYong Zhao #include "cwsr_trap_handler.h" 3164d1c3a4SFelix Kuehling #include "kfd_iommu.h" 325b87245fSAmber Lin #include "amdgpu_amdkfd.h" 332c2b0d88SMukul Joshi #include "kfd_smi_events.h" 34814ab993SPhilip Yang #include "kfd_migrate.h" 354a488a7aSOded Gabbay 3619f6d2a6SOded Gabbay #define MQD_SIZE_ALIGNED 768 37e42051d2SShaoyun Liu 38e42051d2SShaoyun Liu /* 39e42051d2SShaoyun Liu * kfd_locked is used to lock the kfd driver during suspend or reset 40e42051d2SShaoyun Liu * once locked, kfd driver will stop any further GPU execution. 41e42051d2SShaoyun Liu * create process (open) will return -EAGAIN. 42e42051d2SShaoyun Liu */ 43e42051d2SShaoyun Liu static atomic_t kfd_locked = ATOMIC_INIT(0); 4419f6d2a6SOded Gabbay 45a3e520a2SAlex Deucher #ifdef CONFIG_DRM_AMDGPU_CIK 46e392c887SYong Zhao extern const struct kfd2kgd_calls gfx_v7_kfd2kgd; 47a3e520a2SAlex Deucher #endif 48e392c887SYong Zhao extern const struct kfd2kgd_calls gfx_v8_kfd2kgd; 49e392c887SYong Zhao extern const struct kfd2kgd_calls gfx_v9_kfd2kgd; 50e392c887SYong Zhao extern const struct kfd2kgd_calls arcturus_kfd2kgd; 515073506cSJonathan Kim extern const struct kfd2kgd_calls aldebaran_kfd2kgd; 52e392c887SYong Zhao extern const struct kfd2kgd_calls gfx_v10_kfd2kgd; 533a2f0c81SYong Zhao extern const struct kfd2kgd_calls gfx_v10_3_kfd2kgd; 54e392c887SYong Zhao 55e392c887SYong Zhao static const struct kfd2kgd_calls *kfd2kgd_funcs[] = { 56e392c887SYong Zhao #ifdef KFD_SUPPORT_IOMMU_V2 57a3e520a2SAlex Deucher #ifdef CONFIG_DRM_AMDGPU_CIK 58e392c887SYong Zhao [CHIP_KAVERI] = &gfx_v7_kfd2kgd, 59a3e520a2SAlex Deucher #endif 60e392c887SYong Zhao [CHIP_CARRIZO] = &gfx_v8_kfd2kgd, 61e392c887SYong Zhao [CHIP_RAVEN] = &gfx_v9_kfd2kgd, 62e392c887SYong Zhao #endif 63a3e520a2SAlex Deucher #ifdef CONFIG_DRM_AMDGPU_CIK 64e392c887SYong Zhao [CHIP_HAWAII] = &gfx_v7_kfd2kgd, 65a3e520a2SAlex Deucher #endif 66e392c887SYong Zhao [CHIP_TONGA] = &gfx_v8_kfd2kgd, 67e392c887SYong Zhao [CHIP_FIJI] = &gfx_v8_kfd2kgd, 68e392c887SYong Zhao [CHIP_POLARIS10] = &gfx_v8_kfd2kgd, 69e392c887SYong Zhao [CHIP_POLARIS11] = &gfx_v8_kfd2kgd, 70e392c887SYong Zhao [CHIP_POLARIS12] = &gfx_v8_kfd2kgd, 71e392c887SYong Zhao [CHIP_VEGAM] = &gfx_v8_kfd2kgd, 72e392c887SYong Zhao [CHIP_VEGA10] = &gfx_v9_kfd2kgd, 73e392c887SYong Zhao [CHIP_VEGA12] = &gfx_v9_kfd2kgd, 74e392c887SYong Zhao [CHIP_VEGA20] = &gfx_v9_kfd2kgd, 75e392c887SYong Zhao [CHIP_RENOIR] = &gfx_v9_kfd2kgd, 76e392c887SYong Zhao [CHIP_ARCTURUS] = &arcturus_kfd2kgd, 775073506cSJonathan Kim [CHIP_ALDEBARAN] = &aldebaran_kfd2kgd, 78e392c887SYong Zhao [CHIP_NAVI10] = &gfx_v10_kfd2kgd, 79e392c887SYong Zhao [CHIP_NAVI12] = &gfx_v10_kfd2kgd, 80e392c887SYong Zhao [CHIP_NAVI14] = &gfx_v10_kfd2kgd, 813a2f0c81SYong Zhao [CHIP_SIENNA_CICHLID] = &gfx_v10_3_kfd2kgd, 8209759e13SChengming Gui [CHIP_NAVY_FLOUNDER] = &gfx_v10_3_kfd2kgd, 833a5e715dSHuang Rui [CHIP_VANGOGH] = &gfx_v10_3_kfd2kgd, 848f72ce64SChengming Gui [CHIP_DIMGREY_CAVEFISH] = &gfx_v10_3_kfd2kgd, 85e392c887SYong Zhao }; 86e392c887SYong Zhao 8764d1c3a4SFelix Kuehling #ifdef KFD_SUPPORT_IOMMU_V2 884a488a7aSOded Gabbay static const struct kfd_device_info kaveri_device_info = { 890da7558cSBen Goz .asic_family = CHIP_KAVERI, 90c181159aSYong Zhao .asic_name = "kaveri", 910da7558cSBen Goz .max_pasid_bits = 16, 92992839adSYair Shachar /* max num of queues for KV.TODO should be a dynamic value */ 93992839adSYair Shachar .max_no_of_hqd = 24, 94ada2b29cSFelix Kuehling .doorbell_size = 4, 950da7558cSBen Goz .ih_ring_entry_size = 4 * sizeof(uint32_t), 96f3a39818SAndrew Lewycky .event_interrupt_class = &event_interrupt_class_cik, 97fbeb661bSYair Shachar .num_of_watch_points = 4, 98373d7080SFelix Kuehling .mqd_size_aligned = MQD_SIZE_ALIGNED, 99373d7080SFelix Kuehling .supports_cwsr = false, 10064d1c3a4SFelix Kuehling .needs_iommu_device = true, 1013ee2d00cSFelix Kuehling .needs_pci_atomics = false, 10298bb9222SYong Zhao .num_sdma_engines = 2, 1031b4670f6SOak Zeng .num_xgmi_sdma_engines = 0, 104d5094189SShaoyun Liu .num_sdma_queues_per_engine = 2, 1050da7558cSBen Goz }; 1060da7558cSBen Goz 1070da7558cSBen Goz static const struct kfd_device_info carrizo_device_info = { 1080da7558cSBen Goz .asic_family = CHIP_CARRIZO, 109c181159aSYong Zhao .asic_name = "carrizo", 1104a488a7aSOded Gabbay .max_pasid_bits = 16, 111eaccd6e7SOded Gabbay /* max num of queues for CZ.TODO should be a dynamic value */ 112eaccd6e7SOded Gabbay .max_no_of_hqd = 24, 113ada2b29cSFelix Kuehling .doorbell_size = 4, 114b3f5e6b4SAndrew Lewycky .ih_ring_entry_size = 4 * sizeof(uint32_t), 115eaccd6e7SOded Gabbay .event_interrupt_class = &event_interrupt_class_cik, 116f7c826adSAlexey Skidanov .num_of_watch_points = 4, 117373d7080SFelix Kuehling .mqd_size_aligned = MQD_SIZE_ALIGNED, 118373d7080SFelix Kuehling .supports_cwsr = true, 11964d1c3a4SFelix Kuehling .needs_iommu_device = true, 1203ee2d00cSFelix Kuehling .needs_pci_atomics = false, 12198bb9222SYong Zhao .num_sdma_engines = 2, 1221b4670f6SOak Zeng .num_xgmi_sdma_engines = 0, 123d5094189SShaoyun Liu .num_sdma_queues_per_engine = 2, 1244a488a7aSOded Gabbay }; 1256127896fSHuang Rui #endif 1264d663df6SYong Zhao 1274d663df6SYong Zhao static const struct kfd_device_info raven_device_info = { 1284d663df6SYong Zhao .asic_family = CHIP_RAVEN, 129c181159aSYong Zhao .asic_name = "raven", 1304d663df6SYong Zhao .max_pasid_bits = 16, 1314d663df6SYong Zhao .max_no_of_hqd = 24, 1324d663df6SYong Zhao .doorbell_size = 8, 1334d663df6SYong Zhao .ih_ring_entry_size = 8 * sizeof(uint32_t), 1344d663df6SYong Zhao .event_interrupt_class = &event_interrupt_class_v9, 1354d663df6SYong Zhao .num_of_watch_points = 4, 1364d663df6SYong Zhao .mqd_size_aligned = MQD_SIZE_ALIGNED, 1374d663df6SYong Zhao .supports_cwsr = true, 1384d663df6SYong Zhao .needs_iommu_device = true, 1394d663df6SYong Zhao .needs_pci_atomics = true, 1404d663df6SYong Zhao .num_sdma_engines = 1, 1411b4670f6SOak Zeng .num_xgmi_sdma_engines = 0, 142d5094189SShaoyun Liu .num_sdma_queues_per_engine = 2, 1434d663df6SYong Zhao }; 1444a488a7aSOded Gabbay 145a3084e6cSFelix Kuehling static const struct kfd_device_info hawaii_device_info = { 146a3084e6cSFelix Kuehling .asic_family = CHIP_HAWAII, 147c181159aSYong Zhao .asic_name = "hawaii", 148a3084e6cSFelix Kuehling .max_pasid_bits = 16, 149a3084e6cSFelix Kuehling /* max num of queues for KV.TODO should be a dynamic value */ 150a3084e6cSFelix Kuehling .max_no_of_hqd = 24, 151ada2b29cSFelix Kuehling .doorbell_size = 4, 152a3084e6cSFelix Kuehling .ih_ring_entry_size = 4 * sizeof(uint32_t), 153a3084e6cSFelix Kuehling .event_interrupt_class = &event_interrupt_class_cik, 154a3084e6cSFelix Kuehling .num_of_watch_points = 4, 155a3084e6cSFelix Kuehling .mqd_size_aligned = MQD_SIZE_ALIGNED, 156a3084e6cSFelix Kuehling .supports_cwsr = false, 15764d1c3a4SFelix Kuehling .needs_iommu_device = false, 158a3084e6cSFelix Kuehling .needs_pci_atomics = false, 15998bb9222SYong Zhao .num_sdma_engines = 2, 1601b4670f6SOak Zeng .num_xgmi_sdma_engines = 0, 161d5094189SShaoyun Liu .num_sdma_queues_per_engine = 2, 162a3084e6cSFelix Kuehling }; 163a3084e6cSFelix Kuehling 164a3084e6cSFelix Kuehling static const struct kfd_device_info tonga_device_info = { 165a3084e6cSFelix Kuehling .asic_family = CHIP_TONGA, 166c181159aSYong Zhao .asic_name = "tonga", 167a3084e6cSFelix Kuehling .max_pasid_bits = 16, 168a3084e6cSFelix Kuehling .max_no_of_hqd = 24, 169ada2b29cSFelix Kuehling .doorbell_size = 4, 170a3084e6cSFelix Kuehling .ih_ring_entry_size = 4 * sizeof(uint32_t), 171a3084e6cSFelix Kuehling .event_interrupt_class = &event_interrupt_class_cik, 172a3084e6cSFelix Kuehling .num_of_watch_points = 4, 173a3084e6cSFelix Kuehling .mqd_size_aligned = MQD_SIZE_ALIGNED, 174a3084e6cSFelix Kuehling .supports_cwsr = false, 17564d1c3a4SFelix Kuehling .needs_iommu_device = false, 176a3084e6cSFelix Kuehling .needs_pci_atomics = true, 17798bb9222SYong Zhao .num_sdma_engines = 2, 1781b4670f6SOak Zeng .num_xgmi_sdma_engines = 0, 179d5094189SShaoyun Liu .num_sdma_queues_per_engine = 2, 180a3084e6cSFelix Kuehling }; 181a3084e6cSFelix Kuehling 182a3084e6cSFelix Kuehling static const struct kfd_device_info fiji_device_info = { 183a3084e6cSFelix Kuehling .asic_family = CHIP_FIJI, 184c181159aSYong Zhao .asic_name = "fiji", 185a3084e6cSFelix Kuehling .max_pasid_bits = 16, 186a3084e6cSFelix Kuehling .max_no_of_hqd = 24, 187ada2b29cSFelix Kuehling .doorbell_size = 4, 188a3084e6cSFelix Kuehling .ih_ring_entry_size = 4 * sizeof(uint32_t), 189a3084e6cSFelix Kuehling .event_interrupt_class = &event_interrupt_class_cik, 190a3084e6cSFelix Kuehling .num_of_watch_points = 4, 191a3084e6cSFelix Kuehling .mqd_size_aligned = MQD_SIZE_ALIGNED, 192a3084e6cSFelix Kuehling .supports_cwsr = true, 19364d1c3a4SFelix Kuehling .needs_iommu_device = false, 194a3084e6cSFelix Kuehling .needs_pci_atomics = true, 19598bb9222SYong Zhao .num_sdma_engines = 2, 1961b4670f6SOak Zeng .num_xgmi_sdma_engines = 0, 197d5094189SShaoyun Liu .num_sdma_queues_per_engine = 2, 198a3084e6cSFelix Kuehling }; 199a3084e6cSFelix Kuehling 200a3084e6cSFelix Kuehling static const struct kfd_device_info fiji_vf_device_info = { 201a3084e6cSFelix Kuehling .asic_family = CHIP_FIJI, 202c181159aSYong Zhao .asic_name = "fiji", 203a3084e6cSFelix Kuehling .max_pasid_bits = 16, 204a3084e6cSFelix Kuehling .max_no_of_hqd = 24, 205ada2b29cSFelix Kuehling .doorbell_size = 4, 206a3084e6cSFelix Kuehling .ih_ring_entry_size = 4 * sizeof(uint32_t), 207a3084e6cSFelix Kuehling .event_interrupt_class = &event_interrupt_class_cik, 208a3084e6cSFelix Kuehling .num_of_watch_points = 4, 209a3084e6cSFelix Kuehling .mqd_size_aligned = MQD_SIZE_ALIGNED, 210a3084e6cSFelix Kuehling .supports_cwsr = true, 21164d1c3a4SFelix Kuehling .needs_iommu_device = false, 212a3084e6cSFelix Kuehling .needs_pci_atomics = false, 21398bb9222SYong Zhao .num_sdma_engines = 2, 2141b4670f6SOak Zeng .num_xgmi_sdma_engines = 0, 215d5094189SShaoyun Liu .num_sdma_queues_per_engine = 2, 216a3084e6cSFelix Kuehling }; 217a3084e6cSFelix Kuehling 218a3084e6cSFelix Kuehling 219a3084e6cSFelix Kuehling static const struct kfd_device_info polaris10_device_info = { 220a3084e6cSFelix Kuehling .asic_family = CHIP_POLARIS10, 221c181159aSYong Zhao .asic_name = "polaris10", 222a3084e6cSFelix Kuehling .max_pasid_bits = 16, 223a3084e6cSFelix Kuehling .max_no_of_hqd = 24, 224ada2b29cSFelix Kuehling .doorbell_size = 4, 225a3084e6cSFelix Kuehling .ih_ring_entry_size = 4 * sizeof(uint32_t), 226a3084e6cSFelix Kuehling .event_interrupt_class = &event_interrupt_class_cik, 227a3084e6cSFelix Kuehling .num_of_watch_points = 4, 228a3084e6cSFelix Kuehling .mqd_size_aligned = MQD_SIZE_ALIGNED, 229a3084e6cSFelix Kuehling .supports_cwsr = true, 23064d1c3a4SFelix Kuehling .needs_iommu_device = false, 231a3084e6cSFelix Kuehling .needs_pci_atomics = true, 23298bb9222SYong Zhao .num_sdma_engines = 2, 2331b4670f6SOak Zeng .num_xgmi_sdma_engines = 0, 234d5094189SShaoyun Liu .num_sdma_queues_per_engine = 2, 235a3084e6cSFelix Kuehling }; 236a3084e6cSFelix Kuehling 237a3084e6cSFelix Kuehling static const struct kfd_device_info polaris10_vf_device_info = { 238a3084e6cSFelix Kuehling .asic_family = CHIP_POLARIS10, 239c181159aSYong Zhao .asic_name = "polaris10", 240a3084e6cSFelix Kuehling .max_pasid_bits = 16, 241a3084e6cSFelix Kuehling .max_no_of_hqd = 24, 242ada2b29cSFelix Kuehling .doorbell_size = 4, 243a3084e6cSFelix Kuehling .ih_ring_entry_size = 4 * sizeof(uint32_t), 244a3084e6cSFelix Kuehling .event_interrupt_class = &event_interrupt_class_cik, 245a3084e6cSFelix Kuehling .num_of_watch_points = 4, 246a3084e6cSFelix Kuehling .mqd_size_aligned = MQD_SIZE_ALIGNED, 247a3084e6cSFelix Kuehling .supports_cwsr = true, 24864d1c3a4SFelix Kuehling .needs_iommu_device = false, 249a3084e6cSFelix Kuehling .needs_pci_atomics = false, 25098bb9222SYong Zhao .num_sdma_engines = 2, 2511b4670f6SOak Zeng .num_xgmi_sdma_engines = 0, 252d5094189SShaoyun Liu .num_sdma_queues_per_engine = 2, 253a3084e6cSFelix Kuehling }; 254a3084e6cSFelix Kuehling 255a3084e6cSFelix Kuehling static const struct kfd_device_info polaris11_device_info = { 256a3084e6cSFelix Kuehling .asic_family = CHIP_POLARIS11, 257c181159aSYong Zhao .asic_name = "polaris11", 258a3084e6cSFelix Kuehling .max_pasid_bits = 16, 259a3084e6cSFelix Kuehling .max_no_of_hqd = 24, 260ada2b29cSFelix Kuehling .doorbell_size = 4, 261a3084e6cSFelix Kuehling .ih_ring_entry_size = 4 * sizeof(uint32_t), 262a3084e6cSFelix Kuehling .event_interrupt_class = &event_interrupt_class_cik, 263a3084e6cSFelix Kuehling .num_of_watch_points = 4, 264a3084e6cSFelix Kuehling .mqd_size_aligned = MQD_SIZE_ALIGNED, 265a3084e6cSFelix Kuehling .supports_cwsr = true, 26664d1c3a4SFelix Kuehling .needs_iommu_device = false, 267a3084e6cSFelix Kuehling .needs_pci_atomics = true, 26898bb9222SYong Zhao .num_sdma_engines = 2, 2691b4670f6SOak Zeng .num_xgmi_sdma_engines = 0, 270d5094189SShaoyun Liu .num_sdma_queues_per_engine = 2, 271a3084e6cSFelix Kuehling }; 272a3084e6cSFelix Kuehling 273846a44d7SGang Ba static const struct kfd_device_info polaris12_device_info = { 274846a44d7SGang Ba .asic_family = CHIP_POLARIS12, 275c181159aSYong Zhao .asic_name = "polaris12", 276846a44d7SGang Ba .max_pasid_bits = 16, 277846a44d7SGang Ba .max_no_of_hqd = 24, 278846a44d7SGang Ba .doorbell_size = 4, 279846a44d7SGang Ba .ih_ring_entry_size = 4 * sizeof(uint32_t), 280846a44d7SGang Ba .event_interrupt_class = &event_interrupt_class_cik, 281846a44d7SGang Ba .num_of_watch_points = 4, 282846a44d7SGang Ba .mqd_size_aligned = MQD_SIZE_ALIGNED, 283846a44d7SGang Ba .supports_cwsr = true, 284846a44d7SGang Ba .needs_iommu_device = false, 285846a44d7SGang Ba .needs_pci_atomics = true, 286846a44d7SGang Ba .num_sdma_engines = 2, 2871b4670f6SOak Zeng .num_xgmi_sdma_engines = 0, 288846a44d7SGang Ba .num_sdma_queues_per_engine = 2, 289846a44d7SGang Ba }; 290846a44d7SGang Ba 291ed81cd6eSKent Russell static const struct kfd_device_info vegam_device_info = { 292ed81cd6eSKent Russell .asic_family = CHIP_VEGAM, 293c181159aSYong Zhao .asic_name = "vegam", 294ed81cd6eSKent Russell .max_pasid_bits = 16, 295ed81cd6eSKent Russell .max_no_of_hqd = 24, 296ed81cd6eSKent Russell .doorbell_size = 4, 297ed81cd6eSKent Russell .ih_ring_entry_size = 4 * sizeof(uint32_t), 298ed81cd6eSKent Russell .event_interrupt_class = &event_interrupt_class_cik, 299ed81cd6eSKent Russell .num_of_watch_points = 4, 300ed81cd6eSKent Russell .mqd_size_aligned = MQD_SIZE_ALIGNED, 301ed81cd6eSKent Russell .supports_cwsr = true, 302ed81cd6eSKent Russell .needs_iommu_device = false, 303ed81cd6eSKent Russell .needs_pci_atomics = true, 304ed81cd6eSKent Russell .num_sdma_engines = 2, 305ed81cd6eSKent Russell .num_xgmi_sdma_engines = 0, 306a3084e6cSFelix Kuehling .num_sdma_queues_per_engine = 2, 307a3084e6cSFelix Kuehling }; 308a3084e6cSFelix Kuehling 309389056e5SFelix Kuehling static const struct kfd_device_info vega10_device_info = { 310389056e5SFelix Kuehling .asic_family = CHIP_VEGA10, 311c181159aSYong Zhao .asic_name = "vega10", 312389056e5SFelix Kuehling .max_pasid_bits = 16, 313389056e5SFelix Kuehling .max_no_of_hqd = 24, 314389056e5SFelix Kuehling .doorbell_size = 8, 315389056e5SFelix Kuehling .ih_ring_entry_size = 8 * sizeof(uint32_t), 316389056e5SFelix Kuehling .event_interrupt_class = &event_interrupt_class_v9, 317389056e5SFelix Kuehling .num_of_watch_points = 4, 318389056e5SFelix Kuehling .mqd_size_aligned = MQD_SIZE_ALIGNED, 319389056e5SFelix Kuehling .supports_cwsr = true, 320389056e5SFelix Kuehling .needs_iommu_device = false, 321389056e5SFelix Kuehling .needs_pci_atomics = false, 32298bb9222SYong Zhao .num_sdma_engines = 2, 3231b4670f6SOak Zeng .num_xgmi_sdma_engines = 0, 324d5094189SShaoyun Liu .num_sdma_queues_per_engine = 2, 325389056e5SFelix Kuehling }; 326389056e5SFelix Kuehling 327389056e5SFelix Kuehling static const struct kfd_device_info vega10_vf_device_info = { 328389056e5SFelix Kuehling .asic_family = CHIP_VEGA10, 329c181159aSYong Zhao .asic_name = "vega10", 330389056e5SFelix Kuehling .max_pasid_bits = 16, 331389056e5SFelix Kuehling .max_no_of_hqd = 24, 332389056e5SFelix Kuehling .doorbell_size = 8, 333389056e5SFelix Kuehling .ih_ring_entry_size = 8 * sizeof(uint32_t), 334389056e5SFelix Kuehling .event_interrupt_class = &event_interrupt_class_v9, 335389056e5SFelix Kuehling .num_of_watch_points = 4, 336389056e5SFelix Kuehling .mqd_size_aligned = MQD_SIZE_ALIGNED, 337389056e5SFelix Kuehling .supports_cwsr = true, 338389056e5SFelix Kuehling .needs_iommu_device = false, 339389056e5SFelix Kuehling .needs_pci_atomics = false, 34098bb9222SYong Zhao .num_sdma_engines = 2, 3411b4670f6SOak Zeng .num_xgmi_sdma_engines = 0, 342d5094189SShaoyun Liu .num_sdma_queues_per_engine = 2, 343389056e5SFelix Kuehling }; 344389056e5SFelix Kuehling 345846a44d7SGang Ba static const struct kfd_device_info vega12_device_info = { 346846a44d7SGang Ba .asic_family = CHIP_VEGA12, 347c181159aSYong Zhao .asic_name = "vega12", 348846a44d7SGang Ba .max_pasid_bits = 16, 349846a44d7SGang Ba .max_no_of_hqd = 24, 350846a44d7SGang Ba .doorbell_size = 8, 351846a44d7SGang Ba .ih_ring_entry_size = 8 * sizeof(uint32_t), 352846a44d7SGang Ba .event_interrupt_class = &event_interrupt_class_v9, 353846a44d7SGang Ba .num_of_watch_points = 4, 354846a44d7SGang Ba .mqd_size_aligned = MQD_SIZE_ALIGNED, 355846a44d7SGang Ba .supports_cwsr = true, 356846a44d7SGang Ba .needs_iommu_device = false, 357846a44d7SGang Ba .needs_pci_atomics = false, 358846a44d7SGang Ba .num_sdma_engines = 2, 3591b4670f6SOak Zeng .num_xgmi_sdma_engines = 0, 360846a44d7SGang Ba .num_sdma_queues_per_engine = 2, 361846a44d7SGang Ba }; 362846a44d7SGang Ba 36322a3a294SShaoyun Liu static const struct kfd_device_info vega20_device_info = { 36422a3a294SShaoyun Liu .asic_family = CHIP_VEGA20, 365c181159aSYong Zhao .asic_name = "vega20", 36622a3a294SShaoyun Liu .max_pasid_bits = 16, 36722a3a294SShaoyun Liu .max_no_of_hqd = 24, 36822a3a294SShaoyun Liu .doorbell_size = 8, 36922a3a294SShaoyun Liu .ih_ring_entry_size = 8 * sizeof(uint32_t), 37022a3a294SShaoyun Liu .event_interrupt_class = &event_interrupt_class_v9, 37122a3a294SShaoyun Liu .num_of_watch_points = 4, 37222a3a294SShaoyun Liu .mqd_size_aligned = MQD_SIZE_ALIGNED, 37322a3a294SShaoyun Liu .supports_cwsr = true, 37422a3a294SShaoyun Liu .needs_iommu_device = false, 375006a0b3dSShaoyun Liu .needs_pci_atomics = false, 37622a3a294SShaoyun Liu .num_sdma_engines = 2, 3771b4670f6SOak Zeng .num_xgmi_sdma_engines = 0, 37822a3a294SShaoyun Liu .num_sdma_queues_per_engine = 8, 37922a3a294SShaoyun Liu }; 38022a3a294SShaoyun Liu 38149adcf8aSYong Zhao static const struct kfd_device_info arcturus_device_info = { 38249adcf8aSYong Zhao .asic_family = CHIP_ARCTURUS, 383c181159aSYong Zhao .asic_name = "arcturus", 38449adcf8aSYong Zhao .max_pasid_bits = 16, 38549adcf8aSYong Zhao .max_no_of_hqd = 24, 38649adcf8aSYong Zhao .doorbell_size = 8, 38749adcf8aSYong Zhao .ih_ring_entry_size = 8 * sizeof(uint32_t), 38849adcf8aSYong Zhao .event_interrupt_class = &event_interrupt_class_v9, 38949adcf8aSYong Zhao .num_of_watch_points = 4, 39049adcf8aSYong Zhao .mqd_size_aligned = MQD_SIZE_ALIGNED, 39149adcf8aSYong Zhao .supports_cwsr = true, 39249adcf8aSYong Zhao .needs_iommu_device = false, 39349adcf8aSYong Zhao .needs_pci_atomics = false, 394b6689cf7SOak Zeng .num_sdma_engines = 2, 395b6689cf7SOak Zeng .num_xgmi_sdma_engines = 6, 39649adcf8aSYong Zhao .num_sdma_queues_per_engine = 8, 39749adcf8aSYong Zhao }; 39849adcf8aSYong Zhao 39936e22d59SYong Zhao static const struct kfd_device_info aldebaran_device_info = { 40036e22d59SYong Zhao .asic_family = CHIP_ALDEBARAN, 40136e22d59SYong Zhao .asic_name = "aldebaran", 40236e22d59SYong Zhao .max_pasid_bits = 16, 40336e22d59SYong Zhao .max_no_of_hqd = 24, 40436e22d59SYong Zhao .doorbell_size = 8, 40536e22d59SYong Zhao .ih_ring_entry_size = 8 * sizeof(uint32_t), 40636e22d59SYong Zhao .event_interrupt_class = &event_interrupt_class_v9, 40736e22d59SYong Zhao .num_of_watch_points = 4, 40836e22d59SYong Zhao .mqd_size_aligned = MQD_SIZE_ALIGNED, 40936e22d59SYong Zhao .supports_cwsr = true, 41036e22d59SYong Zhao .needs_iommu_device = false, 41136e22d59SYong Zhao .needs_pci_atomics = false, 41236e22d59SYong Zhao .num_sdma_engines = 2, 41336e22d59SYong Zhao .num_xgmi_sdma_engines = 3, 41436e22d59SYong Zhao .num_sdma_queues_per_engine = 8, 41536e22d59SYong Zhao }; 41636e22d59SYong Zhao 4172b9c2211SHuang Rui static const struct kfd_device_info renoir_device_info = { 4182b9c2211SHuang Rui .asic_family = CHIP_RENOIR, 419acb9acbeSHuang Rui .asic_name = "renoir", 4202b9c2211SHuang Rui .max_pasid_bits = 16, 4212b9c2211SHuang Rui .max_no_of_hqd = 24, 4222b9c2211SHuang Rui .doorbell_size = 8, 4232b9c2211SHuang Rui .ih_ring_entry_size = 8 * sizeof(uint32_t), 4242b9c2211SHuang Rui .event_interrupt_class = &event_interrupt_class_v9, 4252b9c2211SHuang Rui .num_of_watch_points = 4, 4262b9c2211SHuang Rui .mqd_size_aligned = MQD_SIZE_ALIGNED, 4272b9c2211SHuang Rui .supports_cwsr = true, 4282b9c2211SHuang Rui .needs_iommu_device = false, 4292b9c2211SHuang Rui .needs_pci_atomics = false, 4302b9c2211SHuang Rui .num_sdma_engines = 1, 4312b9c2211SHuang Rui .num_xgmi_sdma_engines = 0, 4322b9c2211SHuang Rui .num_sdma_queues_per_engine = 2, 4332b9c2211SHuang Rui }; 4342b9c2211SHuang Rui 43514328aa5SPhilip Cox static const struct kfd_device_info navi10_device_info = { 43614328aa5SPhilip Cox .asic_family = CHIP_NAVI10, 437c181159aSYong Zhao .asic_name = "navi10", 43814328aa5SPhilip Cox .max_pasid_bits = 16, 43914328aa5SPhilip Cox .max_no_of_hqd = 24, 44014328aa5SPhilip Cox .doorbell_size = 8, 44114328aa5SPhilip Cox .ih_ring_entry_size = 8 * sizeof(uint32_t), 44214328aa5SPhilip Cox .event_interrupt_class = &event_interrupt_class_v9, 44314328aa5SPhilip Cox .num_of_watch_points = 4, 44414328aa5SPhilip Cox .mqd_size_aligned = MQD_SIZE_ALIGNED, 44514328aa5SPhilip Cox .needs_iommu_device = false, 44614328aa5SPhilip Cox .supports_cwsr = true, 4476cc980e3SHarish Kasiviswanathan .needs_pci_atomics = true, 44814328aa5SPhilip Cox .num_sdma_engines = 2, 44914328aa5SPhilip Cox .num_xgmi_sdma_engines = 0, 45014328aa5SPhilip Cox .num_sdma_queues_per_engine = 8, 45114328aa5SPhilip Cox }; 45214328aa5SPhilip Cox 453b77fb9d8Sshaoyunl static const struct kfd_device_info navi12_device_info = { 4540e94b564Sshaoyunl .asic_family = CHIP_NAVI12, 455b77fb9d8Sshaoyunl .asic_name = "navi12", 456b77fb9d8Sshaoyunl .max_pasid_bits = 16, 457b77fb9d8Sshaoyunl .max_no_of_hqd = 24, 458b77fb9d8Sshaoyunl .doorbell_size = 8, 459b77fb9d8Sshaoyunl .ih_ring_entry_size = 8 * sizeof(uint32_t), 460b77fb9d8Sshaoyunl .event_interrupt_class = &event_interrupt_class_v9, 461b77fb9d8Sshaoyunl .num_of_watch_points = 4, 462b77fb9d8Sshaoyunl .mqd_size_aligned = MQD_SIZE_ALIGNED, 463b77fb9d8Sshaoyunl .needs_iommu_device = false, 464b77fb9d8Sshaoyunl .supports_cwsr = true, 4656cc980e3SHarish Kasiviswanathan .needs_pci_atomics = true, 466b77fb9d8Sshaoyunl .num_sdma_engines = 2, 467b77fb9d8Sshaoyunl .num_xgmi_sdma_engines = 0, 468b77fb9d8Sshaoyunl .num_sdma_queues_per_engine = 8, 469b77fb9d8Sshaoyunl }; 470b77fb9d8Sshaoyunl 4718099ae40SYong Zhao static const struct kfd_device_info navi14_device_info = { 4728099ae40SYong Zhao .asic_family = CHIP_NAVI14, 4738099ae40SYong Zhao .asic_name = "navi14", 4748099ae40SYong Zhao .max_pasid_bits = 16, 4758099ae40SYong Zhao .max_no_of_hqd = 24, 4768099ae40SYong Zhao .doorbell_size = 8, 4778099ae40SYong Zhao .ih_ring_entry_size = 8 * sizeof(uint32_t), 4788099ae40SYong Zhao .event_interrupt_class = &event_interrupt_class_v9, 4798099ae40SYong Zhao .num_of_watch_points = 4, 4808099ae40SYong Zhao .mqd_size_aligned = MQD_SIZE_ALIGNED, 4818099ae40SYong Zhao .needs_iommu_device = false, 4828099ae40SYong Zhao .supports_cwsr = true, 4836cc980e3SHarish Kasiviswanathan .needs_pci_atomics = true, 4848099ae40SYong Zhao .num_sdma_engines = 2, 4858099ae40SYong Zhao .num_xgmi_sdma_engines = 0, 4868099ae40SYong Zhao .num_sdma_queues_per_engine = 8, 4878099ae40SYong Zhao }; 4888099ae40SYong Zhao 4893a2f0c81SYong Zhao static const struct kfd_device_info sienna_cichlid_device_info = { 4903a2f0c81SYong Zhao .asic_family = CHIP_SIENNA_CICHLID, 4913a2f0c81SYong Zhao .asic_name = "sienna_cichlid", 4923a2f0c81SYong Zhao .max_pasid_bits = 16, 4933a2f0c81SYong Zhao .max_no_of_hqd = 24, 4943a2f0c81SYong Zhao .doorbell_size = 8, 4953a2f0c81SYong Zhao .ih_ring_entry_size = 8 * sizeof(uint32_t), 4963a2f0c81SYong Zhao .event_interrupt_class = &event_interrupt_class_v9, 4973a2f0c81SYong Zhao .num_of_watch_points = 4, 4983a2f0c81SYong Zhao .mqd_size_aligned = MQD_SIZE_ALIGNED, 4993a2f0c81SYong Zhao .needs_iommu_device = false, 5003a2f0c81SYong Zhao .supports_cwsr = true, 5016cc980e3SHarish Kasiviswanathan .needs_pci_atomics = true, 5023a2f0c81SYong Zhao .num_sdma_engines = 4, 5033a2f0c81SYong Zhao .num_xgmi_sdma_engines = 0, 5043a2f0c81SYong Zhao .num_sdma_queues_per_engine = 8, 5053a2f0c81SYong Zhao }; 5063a2f0c81SYong Zhao 507de89b2e4SChengming Gui static const struct kfd_device_info navy_flounder_device_info = { 508de89b2e4SChengming Gui .asic_family = CHIP_NAVY_FLOUNDER, 509de89b2e4SChengming Gui .asic_name = "navy_flounder", 510de89b2e4SChengming Gui .max_pasid_bits = 16, 511de89b2e4SChengming Gui .max_no_of_hqd = 24, 512de89b2e4SChengming Gui .doorbell_size = 8, 513de89b2e4SChengming Gui .ih_ring_entry_size = 8 * sizeof(uint32_t), 514de89b2e4SChengming Gui .event_interrupt_class = &event_interrupt_class_v9, 515de89b2e4SChengming Gui .num_of_watch_points = 4, 516de89b2e4SChengming Gui .mqd_size_aligned = MQD_SIZE_ALIGNED, 517de89b2e4SChengming Gui .needs_iommu_device = false, 518de89b2e4SChengming Gui .supports_cwsr = true, 5196cc980e3SHarish Kasiviswanathan .needs_pci_atomics = true, 520de89b2e4SChengming Gui .num_sdma_engines = 2, 521de89b2e4SChengming Gui .num_xgmi_sdma_engines = 0, 522de89b2e4SChengming Gui .num_sdma_queues_per_engine = 8, 523de89b2e4SChengming Gui }; 524de89b2e4SChengming Gui 5253a5e715dSHuang Rui static const struct kfd_device_info vangogh_device_info = { 5263a5e715dSHuang Rui .asic_family = CHIP_VANGOGH, 5273a5e715dSHuang Rui .asic_name = "vangogh", 5283a5e715dSHuang Rui .max_pasid_bits = 16, 5293a5e715dSHuang Rui .max_no_of_hqd = 24, 5303a5e715dSHuang Rui .doorbell_size = 8, 5313a5e715dSHuang Rui .ih_ring_entry_size = 8 * sizeof(uint32_t), 5323a5e715dSHuang Rui .event_interrupt_class = &event_interrupt_class_v9, 5333a5e715dSHuang Rui .num_of_watch_points = 4, 5343a5e715dSHuang Rui .mqd_size_aligned = MQD_SIZE_ALIGNED, 5353a5e715dSHuang Rui .needs_iommu_device = false, 5363a5e715dSHuang Rui .supports_cwsr = true, 5373a5e715dSHuang Rui .needs_pci_atomics = false, 5383a5e715dSHuang Rui .num_sdma_engines = 1, 5393a5e715dSHuang Rui .num_xgmi_sdma_engines = 0, 5403a5e715dSHuang Rui .num_sdma_queues_per_engine = 2, 5413a5e715dSHuang Rui }; 5423a5e715dSHuang Rui 543eb5a34d4SChengming Gui static const struct kfd_device_info dimgrey_cavefish_device_info = { 544eb5a34d4SChengming Gui .asic_family = CHIP_DIMGREY_CAVEFISH, 545eb5a34d4SChengming Gui .asic_name = "dimgrey_cavefish", 546eb5a34d4SChengming Gui .max_pasid_bits = 16, 547eb5a34d4SChengming Gui .max_no_of_hqd = 24, 548eb5a34d4SChengming Gui .doorbell_size = 8, 549eb5a34d4SChengming Gui .ih_ring_entry_size = 8 * sizeof(uint32_t), 550eb5a34d4SChengming Gui .event_interrupt_class = &event_interrupt_class_v9, 551eb5a34d4SChengming Gui .num_of_watch_points = 4, 552eb5a34d4SChengming Gui .mqd_size_aligned = MQD_SIZE_ALIGNED, 553eb5a34d4SChengming Gui .needs_iommu_device = false, 554eb5a34d4SChengming Gui .supports_cwsr = true, 5556cc980e3SHarish Kasiviswanathan .needs_pci_atomics = true, 556eb5a34d4SChengming Gui .num_sdma_engines = 2, 557eb5a34d4SChengming Gui .num_xgmi_sdma_engines = 0, 558eb5a34d4SChengming Gui .num_sdma_queues_per_engine = 8, 559eb5a34d4SChengming Gui }; 560eb5a34d4SChengming Gui 561eb5a34d4SChengming Gui 562050091abSYong Zhao /* For each entry, [0] is regular and [1] is virtualisation device. */ 563050091abSYong Zhao static const struct kfd_device_info *kfd_supported_devices[][2] = { 56495a5bd1bSYong Zhao #ifdef KFD_SUPPORT_IOMMU_V2 565050091abSYong Zhao [CHIP_KAVERI] = {&kaveri_device_info, NULL}, 56695a5bd1bSYong Zhao [CHIP_CARRIZO] = {&carrizo_device_info, NULL}, 56795a5bd1bSYong Zhao #endif 5682b3bbf23SYueHaibing [CHIP_RAVEN] = {&raven_device_info, NULL}, 569050091abSYong Zhao [CHIP_HAWAII] = {&hawaii_device_info, NULL}, 570050091abSYong Zhao [CHIP_TONGA] = {&tonga_device_info, NULL}, 571050091abSYong Zhao [CHIP_FIJI] = {&fiji_device_info, &fiji_vf_device_info}, 572050091abSYong Zhao [CHIP_POLARIS10] = {&polaris10_device_info, &polaris10_vf_device_info}, 573050091abSYong Zhao [CHIP_POLARIS11] = {&polaris11_device_info, NULL}, 574050091abSYong Zhao [CHIP_POLARIS12] = {&polaris12_device_info, NULL}, 575050091abSYong Zhao [CHIP_VEGAM] = {&vegam_device_info, NULL}, 576050091abSYong Zhao [CHIP_VEGA10] = {&vega10_device_info, &vega10_vf_device_info}, 577050091abSYong Zhao [CHIP_VEGA12] = {&vega12_device_info, NULL}, 578050091abSYong Zhao [CHIP_VEGA20] = {&vega20_device_info, NULL}, 5792b9c2211SHuang Rui [CHIP_RENOIR] = {&renoir_device_info, NULL}, 580050091abSYong Zhao [CHIP_ARCTURUS] = {&arcturus_device_info, &arcturus_device_info}, 581*cecd91b4SZhigang Luo [CHIP_ALDEBARAN] = {&aldebaran_device_info, &aldebaran_device_info}, 582050091abSYong Zhao [CHIP_NAVI10] = {&navi10_device_info, NULL}, 583b77fb9d8Sshaoyunl [CHIP_NAVI12] = {&navi12_device_info, &navi12_device_info}, 5848099ae40SYong Zhao [CHIP_NAVI14] = {&navi14_device_info, NULL}, 585adab4dadSshaoyunl [CHIP_SIENNA_CICHLID] = {&sienna_cichlid_device_info, &sienna_cichlid_device_info}, 586de89b2e4SChengming Gui [CHIP_NAVY_FLOUNDER] = {&navy_flounder_device_info, &navy_flounder_device_info}, 5873a5e715dSHuang Rui [CHIP_VANGOGH] = {&vangogh_device_info, NULL}, 588eb5a34d4SChengming Gui [CHIP_DIMGREY_CAVEFISH] = {&dimgrey_cavefish_device_info, &dimgrey_cavefish_device_info}, 5894a488a7aSOded Gabbay }; 5904a488a7aSOded Gabbay 5916e81090bSOded Gabbay static int kfd_gtt_sa_init(struct kfd_dev *kfd, unsigned int buf_size, 5926e81090bSOded Gabbay unsigned int chunk_size); 5936e81090bSOded Gabbay static void kfd_gtt_sa_fini(struct kfd_dev *kfd); 5946e81090bSOded Gabbay 595b8935a7cSYong Zhao static int kfd_resume(struct kfd_dev *kfd); 596b8935a7cSYong Zhao 597cea405b1SXihan Zhang struct kfd_dev *kgd2kfd_probe(struct kgd_dev *kgd, 598e392c887SYong Zhao struct pci_dev *pdev, unsigned int asic_type, bool vf) 5994a488a7aSOded Gabbay { 6004a488a7aSOded Gabbay struct kfd_dev *kfd; 601050091abSYong Zhao const struct kfd_device_info *device_info; 602e392c887SYong Zhao const struct kfd2kgd_calls *f2g; 603050091abSYong Zhao 604e392c887SYong Zhao if (asic_type >= sizeof(kfd_supported_devices) / (sizeof(void *) * 2) 605e392c887SYong Zhao || asic_type >= sizeof(kfd2kgd_funcs) / sizeof(void *)) { 606050091abSYong Zhao dev_err(kfd_device, "asic_type %d out of range\n", asic_type); 607050091abSYong Zhao return NULL; /* asic_type out of range */ 608050091abSYong Zhao } 609050091abSYong Zhao 610050091abSYong Zhao device_info = kfd_supported_devices[asic_type][vf]; 611e392c887SYong Zhao f2g = kfd2kgd_funcs[asic_type]; 6124a488a7aSOded Gabbay 613aa5e899dSDan Carpenter if (!device_info || !f2g) { 614050091abSYong Zhao dev_err(kfd_device, "%s %s not supported in kfd\n", 615050091abSYong Zhao amdgpu_asic_name[asic_type], vf ? "VF" : ""); 6164a488a7aSOded Gabbay return NULL; 6174ebc7182SYong Zhao } 6184a488a7aSOded Gabbay 619d35f00d8SEric Huang kfd = kzalloc(sizeof(*kfd), GFP_KERNEL); 620d35f00d8SEric Huang if (!kfd) 621d35f00d8SEric Huang return NULL; 622d35f00d8SEric Huang 6236106dce9Swelu /* Allow BIF to recode atomics to PCIe 3.0 AtomicOps. 6246106dce9Swelu * 32 and 64-bit requests are possible and must be 6256106dce9Swelu * supported. 6263ee2d00cSFelix Kuehling */ 627aabf3a95SJack Xiao kfd->pci_atomic_requested = amdgpu_amdkfd_have_atomics_support(kgd); 628aabf3a95SJack Xiao if (device_info->needs_pci_atomics && 629aabf3a95SJack Xiao !kfd->pci_atomic_requested) { 6303ee2d00cSFelix Kuehling dev_info(kfd_device, 6316106dce9Swelu "skipped device %x:%x, PCI rejects atomics\n", 6323ee2d00cSFelix Kuehling pdev->vendor, pdev->device); 633d35f00d8SEric Huang kfree(kfd); 6343ee2d00cSFelix Kuehling return NULL; 635aabf3a95SJack Xiao } 6364a488a7aSOded Gabbay 6374a488a7aSOded Gabbay kfd->kgd = kgd; 6384a488a7aSOded Gabbay kfd->device_info = device_info; 6394a488a7aSOded Gabbay kfd->pdev = pdev; 64019f6d2a6SOded Gabbay kfd->init_complete = false; 641cea405b1SXihan Zhang kfd->kfd2kgd = f2g; 64243d8107fSHarish Kasiviswanathan atomic_set(&kfd->compute_profile, 0); 643cea405b1SXihan Zhang 644cea405b1SXihan Zhang mutex_init(&kfd->doorbell_mutex); 645cea405b1SXihan Zhang memset(&kfd->doorbell_available_index, 0, 646cea405b1SXihan Zhang sizeof(kfd->doorbell_available_index)); 6474a488a7aSOded Gabbay 6489b54d201SEric Huang atomic_set(&kfd->sram_ecc_flag, 0); 6499b54d201SEric Huang 65059d7115dSMukul Joshi ida_init(&kfd->doorbell_ida); 65159d7115dSMukul Joshi 6524a488a7aSOded Gabbay return kfd; 6534a488a7aSOded Gabbay } 6544a488a7aSOded Gabbay 655373d7080SFelix Kuehling static void kfd_cwsr_init(struct kfd_dev *kfd) 656373d7080SFelix Kuehling { 657373d7080SFelix Kuehling if (cwsr_enable && kfd->device_info->supports_cwsr) { 6583e76c239SFelix Kuehling if (kfd->device_info->asic_family < CHIP_VEGA10) { 659373d7080SFelix Kuehling BUILD_BUG_ON(sizeof(cwsr_trap_gfx8_hex) > PAGE_SIZE); 660373d7080SFelix Kuehling kfd->cwsr_isa = cwsr_trap_gfx8_hex; 661373d7080SFelix Kuehling kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx8_hex); 6620ef6845cSJay Cornwall } else if (kfd->device_info->asic_family == CHIP_ARCTURUS) { 6633baa24f0SOak Zeng BUILD_BUG_ON(sizeof(cwsr_trap_arcturus_hex) > PAGE_SIZE); 6643baa24f0SOak Zeng kfd->cwsr_isa = cwsr_trap_arcturus_hex; 6653baa24f0SOak Zeng kfd->cwsr_isa_size = sizeof(cwsr_trap_arcturus_hex); 6660ef6845cSJay Cornwall } else if (kfd->device_info->asic_family == CHIP_ALDEBARAN) { 6670ef6845cSJay Cornwall BUILD_BUG_ON(sizeof(cwsr_trap_aldebaran_hex) > PAGE_SIZE); 6680ef6845cSJay Cornwall kfd->cwsr_isa = cwsr_trap_aldebaran_hex; 6690ef6845cSJay Cornwall kfd->cwsr_isa_size = sizeof(cwsr_trap_aldebaran_hex); 67014328aa5SPhilip Cox } else if (kfd->device_info->asic_family < CHIP_NAVI10) { 6713e76c239SFelix Kuehling BUILD_BUG_ON(sizeof(cwsr_trap_gfx9_hex) > PAGE_SIZE); 6723e76c239SFelix Kuehling kfd->cwsr_isa = cwsr_trap_gfx9_hex; 6733e76c239SFelix Kuehling kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx9_hex); 67480b6cfedSJay Cornwall } else if (kfd->device_info->asic_family < CHIP_SIENNA_CICHLID) { 67580b6cfedSJay Cornwall BUILD_BUG_ON(sizeof(cwsr_trap_nv1x_hex) > PAGE_SIZE); 67680b6cfedSJay Cornwall kfd->cwsr_isa = cwsr_trap_nv1x_hex; 67780b6cfedSJay Cornwall kfd->cwsr_isa_size = sizeof(cwsr_trap_nv1x_hex); 67814328aa5SPhilip Cox } else { 67914328aa5SPhilip Cox BUILD_BUG_ON(sizeof(cwsr_trap_gfx10_hex) > PAGE_SIZE); 68014328aa5SPhilip Cox kfd->cwsr_isa = cwsr_trap_gfx10_hex; 68114328aa5SPhilip Cox kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx10_hex); 6823e76c239SFelix Kuehling } 6833e76c239SFelix Kuehling 684373d7080SFelix Kuehling kfd->cwsr_enabled = true; 685373d7080SFelix Kuehling } 686373d7080SFelix Kuehling } 687373d7080SFelix Kuehling 68829633d0eSJoseph Greathouse static int kfd_gws_init(struct kfd_dev *kfd) 68929633d0eSJoseph Greathouse { 69029633d0eSJoseph Greathouse int ret = 0; 69129633d0eSJoseph Greathouse 69229633d0eSJoseph Greathouse if (kfd->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) 69329633d0eSJoseph Greathouse return 0; 69429633d0eSJoseph Greathouse 69529633d0eSJoseph Greathouse if (hws_gws_support 696fea7d919SJoseph Greathouse || (kfd->device_info->asic_family == CHIP_VEGA10 697fea7d919SJoseph Greathouse && kfd->mec2_fw_version >= 0x81b3) 698fea7d919SJoseph Greathouse || (kfd->device_info->asic_family >= CHIP_VEGA12 69929633d0eSJoseph Greathouse && kfd->device_info->asic_family <= CHIP_RAVEN 700fea7d919SJoseph Greathouse && kfd->mec2_fw_version >= 0x1b3) 701fea7d919SJoseph Greathouse || (kfd->device_info->asic_family == CHIP_ARCTURUS 7028baa6018SHarish Kasiviswanathan && kfd->mec2_fw_version >= 0x30) 7038baa6018SHarish Kasiviswanathan || (kfd->device_info->asic_family == CHIP_ALDEBARAN 7048baa6018SHarish Kasiviswanathan && kfd->mec2_fw_version >= 0x28)) 70529633d0eSJoseph Greathouse ret = amdgpu_amdkfd_alloc_gws(kfd->kgd, 70629633d0eSJoseph Greathouse amdgpu_amdkfd_get_num_gws(kfd->kgd), &kfd->gws); 70729633d0eSJoseph Greathouse 70829633d0eSJoseph Greathouse return ret; 70929633d0eSJoseph Greathouse } 71029633d0eSJoseph Greathouse 711938a0650SAmber Lin static void kfd_smi_init(struct kfd_dev *dev) { 712938a0650SAmber Lin INIT_LIST_HEAD(&dev->smi_clients); 713938a0650SAmber Lin spin_lock_init(&dev->smi_lock); 714938a0650SAmber Lin } 715938a0650SAmber Lin 7164a488a7aSOded Gabbay bool kgd2kfd_device_init(struct kfd_dev *kfd, 7173a0c3423SHarish Kasiviswanathan struct drm_device *ddev, 7184a488a7aSOded Gabbay const struct kgd2kfd_shared_resources *gpu_resources) 7194a488a7aSOded Gabbay { 720fd6a440eSJonathan Kim unsigned int size, map_process_packet_size; 72119f6d2a6SOded Gabbay 7223a0c3423SHarish Kasiviswanathan kfd->ddev = ddev; 7230da8b10eSAmber Lin kfd->mec_fw_version = amdgpu_amdkfd_get_fw_version(kfd->kgd, 7245ade6c9cSFelix Kuehling KGD_ENGINE_MEC1); 72529633d0eSJoseph Greathouse kfd->mec2_fw_version = amdgpu_amdkfd_get_fw_version(kfd->kgd, 72629633d0eSJoseph Greathouse KGD_ENGINE_MEC2); 7270da8b10eSAmber Lin kfd->sdma_fw_version = amdgpu_amdkfd_get_fw_version(kfd->kgd, 7285ade6c9cSFelix Kuehling KGD_ENGINE_SDMA1); 7294a488a7aSOded Gabbay kfd->shared_resources = *gpu_resources; 7304a488a7aSOded Gabbay 73144008d7aSYong Zhao kfd->vm_info.first_vmid_kfd = ffs(gpu_resources->compute_vmid_bitmap)-1; 73244008d7aSYong Zhao kfd->vm_info.last_vmid_kfd = fls(gpu_resources->compute_vmid_bitmap)-1; 73344008d7aSYong Zhao kfd->vm_info.vmid_num_kfd = kfd->vm_info.last_vmid_kfd 73444008d7aSYong Zhao - kfd->vm_info.first_vmid_kfd + 1; 73544008d7aSYong Zhao 736a99c6d4fSFelix Kuehling /* Verify module parameters regarding mapped process number*/ 737a99c6d4fSFelix Kuehling if ((hws_max_conc_proc < 0) 738a99c6d4fSFelix Kuehling || (hws_max_conc_proc > kfd->vm_info.vmid_num_kfd)) { 739a99c6d4fSFelix Kuehling dev_err(kfd_device, 740a99c6d4fSFelix Kuehling "hws_max_conc_proc %d must be between 0 and %d, use %d instead\n", 741a99c6d4fSFelix Kuehling hws_max_conc_proc, kfd->vm_info.vmid_num_kfd, 742a99c6d4fSFelix Kuehling kfd->vm_info.vmid_num_kfd); 743a99c6d4fSFelix Kuehling kfd->max_proc_per_quantum = kfd->vm_info.vmid_num_kfd; 744a99c6d4fSFelix Kuehling } else 745a99c6d4fSFelix Kuehling kfd->max_proc_per_quantum = hws_max_conc_proc; 746a99c6d4fSFelix Kuehling 74719f6d2a6SOded Gabbay /* calculate max size of mqds needed for queues */ 748b8cbab04SOded Gabbay size = max_num_of_queues_per_device * 74919f6d2a6SOded Gabbay kfd->device_info->mqd_size_aligned; 75019f6d2a6SOded Gabbay 751e18e794eSOded Gabbay /* 752e18e794eSOded Gabbay * calculate max size of runlist packet. 753e18e794eSOded Gabbay * There can be only 2 packets at once 754e18e794eSOded Gabbay */ 755fd6a440eSJonathan Kim map_process_packet_size = 756fd6a440eSJonathan Kim kfd->device_info->asic_family == CHIP_ALDEBARAN ? 757fd6a440eSJonathan Kim sizeof(struct pm4_mes_map_process_aldebaran) : 758fd6a440eSJonathan Kim sizeof(struct pm4_mes_map_process); 759fd6a440eSJonathan Kim size += (KFD_MAX_NUM_OF_PROCESSES * map_process_packet_size + 760507968ddSFelix Kuehling max_num_of_queues_per_device * sizeof(struct pm4_mes_map_queues) 761507968ddSFelix Kuehling + sizeof(struct pm4_mes_runlist)) * 2; 762e18e794eSOded Gabbay 763e18e794eSOded Gabbay /* Add size of HIQ & DIQ */ 764e18e794eSOded Gabbay size += KFD_KERNEL_QUEUE_SIZE * 2; 765e18e794eSOded Gabbay 766e18e794eSOded Gabbay /* add another 512KB for all other allocations on gart (HPD, fences) */ 76719f6d2a6SOded Gabbay size += 512 * 1024; 76819f6d2a6SOded Gabbay 7697cd52c91SAmber Lin if (amdgpu_amdkfd_alloc_gtt_mem( 770cea405b1SXihan Zhang kfd->kgd, size, &kfd->gtt_mem, 77115426dbbSYong Zhao &kfd->gtt_start_gpu_addr, &kfd->gtt_start_cpu_ptr, 77215426dbbSYong Zhao false)) { 77379775b62SKent Russell dev_err(kfd_device, "Could not allocate %d bytes\n", size); 774e09d4fc8SOak Zeng goto alloc_gtt_mem_failure; 77519f6d2a6SOded Gabbay } 77619f6d2a6SOded Gabbay 77779775b62SKent Russell dev_info(kfd_device, "Allocated %d bytes on gart\n", size); 778e18e794eSOded Gabbay 77973a1da0bSOded Gabbay /* Initialize GTT sa with 512 byte chunk size */ 78073a1da0bSOded Gabbay if (kfd_gtt_sa_init(kfd, size, 512) != 0) { 78179775b62SKent Russell dev_err(kfd_device, "Error initializing gtt sub-allocator\n"); 78273a1da0bSOded Gabbay goto kfd_gtt_sa_init_error; 78373a1da0bSOded Gabbay } 78473a1da0bSOded Gabbay 785735df2baSFelix Kuehling if (kfd_doorbell_init(kfd)) { 786735df2baSFelix Kuehling dev_err(kfd_device, 787735df2baSFelix Kuehling "Error initializing doorbell aperture\n"); 788735df2baSFelix Kuehling goto kfd_doorbell_error; 789735df2baSFelix Kuehling } 79019f6d2a6SOded Gabbay 791332f6e1eSFelix Kuehling kfd->hive_id = amdgpu_amdkfd_get_hive_id(kfd->kgd); 7920c1690e3SShaoyun Liu 7939b498efaSAlex Deucher kfd->noretry = amdgpu_amdkfd_get_noretry(kfd->kgd); 7949b498efaSAlex Deucher 7952249d558SAndrew Lewycky if (kfd_interrupt_init(kfd)) { 79679775b62SKent Russell dev_err(kfd_device, "Error initializing interrupts\n"); 7972249d558SAndrew Lewycky goto kfd_interrupt_error; 7982249d558SAndrew Lewycky } 7992249d558SAndrew Lewycky 80064c7f8cfSBen Goz kfd->dqm = device_queue_manager_init(kfd); 80164c7f8cfSBen Goz if (!kfd->dqm) { 80279775b62SKent Russell dev_err(kfd_device, "Error initializing queue manager\n"); 80364c7f8cfSBen Goz goto device_queue_manager_error; 80464c7f8cfSBen Goz } 80564c7f8cfSBen Goz 80629633d0eSJoseph Greathouse /* If supported on this device, allocate global GWS that is shared 80729633d0eSJoseph Greathouse * by all KFD processes 80829633d0eSJoseph Greathouse */ 80929633d0eSJoseph Greathouse if (kfd_gws_init(kfd)) { 81029633d0eSJoseph Greathouse dev_err(kfd_device, "Could not allocate %d gws\n", 81129633d0eSJoseph Greathouse amdgpu_amdkfd_get_num_gws(kfd->kgd)); 81229633d0eSJoseph Greathouse goto gws_error; 81329633d0eSJoseph Greathouse } 81429633d0eSJoseph Greathouse 8156127896fSHuang Rui /* If CRAT is broken, won't set iommu enabled */ 8166127896fSHuang Rui kfd_double_confirm_iommu_support(kfd); 8176127896fSHuang Rui 81864d1c3a4SFelix Kuehling if (kfd_iommu_device_init(kfd)) { 81964d1c3a4SFelix Kuehling dev_err(kfd_device, "Error initializing iommuv2\n"); 82064d1c3a4SFelix Kuehling goto device_iommu_error; 82164c7f8cfSBen Goz } 82264c7f8cfSBen Goz 823373d7080SFelix Kuehling kfd_cwsr_init(kfd); 824373d7080SFelix Kuehling 825814ab993SPhilip Yang svm_migrate_init((struct amdgpu_device *)kfd->kgd); 826814ab993SPhilip Yang 827b8935a7cSYong Zhao if (kfd_resume(kfd)) 828b8935a7cSYong Zhao goto kfd_resume_error; 829b8935a7cSYong Zhao 830fbeb661bSYair Shachar kfd->dbgmgr = NULL; 831fbeb661bSYair Shachar 832465ab9e0SOak Zeng if (kfd_topology_add_device(kfd)) { 833465ab9e0SOak Zeng dev_err(kfd_device, "Error adding device to topology\n"); 834465ab9e0SOak Zeng goto kfd_topology_add_device_error; 835465ab9e0SOak Zeng } 836465ab9e0SOak Zeng 837938a0650SAmber Lin kfd_smi_init(kfd); 838938a0650SAmber Lin 8394a488a7aSOded Gabbay kfd->init_complete = true; 84079775b62SKent Russell dev_info(kfd_device, "added device %x:%x\n", kfd->pdev->vendor, 8414a488a7aSOded Gabbay kfd->pdev->device); 8424a488a7aSOded Gabbay 84379775b62SKent Russell pr_debug("Starting kfd with the following scheduling policy %d\n", 844d146c5a7SFelix Kuehling kfd->dqm->sched_policy); 84564c7f8cfSBen Goz 84619f6d2a6SOded Gabbay goto out; 84719f6d2a6SOded Gabbay 848465ab9e0SOak Zeng kfd_topology_add_device_error: 849b8935a7cSYong Zhao kfd_resume_error: 85064d1c3a4SFelix Kuehling device_iommu_error: 85129633d0eSJoseph Greathouse gws_error: 85264c7f8cfSBen Goz device_queue_manager_uninit(kfd->dqm); 85364c7f8cfSBen Goz device_queue_manager_error: 8542249d558SAndrew Lewycky kfd_interrupt_exit(kfd); 8552249d558SAndrew Lewycky kfd_interrupt_error: 856735df2baSFelix Kuehling kfd_doorbell_fini(kfd); 857735df2baSFelix Kuehling kfd_doorbell_error: 85873a1da0bSOded Gabbay kfd_gtt_sa_fini(kfd); 85973a1da0bSOded Gabbay kfd_gtt_sa_init_error: 8607cd52c91SAmber Lin amdgpu_amdkfd_free_gtt_mem(kfd->kgd, kfd->gtt_mem); 861e09d4fc8SOak Zeng alloc_gtt_mem_failure: 86229633d0eSJoseph Greathouse if (kfd->gws) 863e09d4fc8SOak Zeng amdgpu_amdkfd_free_gws(kfd->kgd, kfd->gws); 86419f6d2a6SOded Gabbay dev_err(kfd_device, 86579775b62SKent Russell "device %x:%x NOT added due to errors\n", 86619f6d2a6SOded Gabbay kfd->pdev->vendor, kfd->pdev->device); 86719f6d2a6SOded Gabbay out: 86819f6d2a6SOded Gabbay return kfd->init_complete; 8694a488a7aSOded Gabbay } 8704a488a7aSOded Gabbay 8714a488a7aSOded Gabbay void kgd2kfd_device_exit(struct kfd_dev *kfd) 8724a488a7aSOded Gabbay { 873b17f068aSOded Gabbay if (kfd->init_complete) { 8749593f4d6SRajneesh Bhardwaj kgd2kfd_suspend(kfd, false); 875814ab993SPhilip Yang svm_migrate_fini((struct amdgpu_device *)kfd->kgd); 87664c7f8cfSBen Goz device_queue_manager_uninit(kfd->dqm); 8772249d558SAndrew Lewycky kfd_interrupt_exit(kfd); 87819f6d2a6SOded Gabbay kfd_topology_remove_device(kfd); 879735df2baSFelix Kuehling kfd_doorbell_fini(kfd); 88059d7115dSMukul Joshi ida_destroy(&kfd->doorbell_ida); 88173a1da0bSOded Gabbay kfd_gtt_sa_fini(kfd); 8827cd52c91SAmber Lin amdgpu_amdkfd_free_gtt_mem(kfd->kgd, kfd->gtt_mem); 88329633d0eSJoseph Greathouse if (kfd->gws) 884e09d4fc8SOak Zeng amdgpu_amdkfd_free_gws(kfd->kgd, kfd->gws); 885b17f068aSOded Gabbay } 8865b5c4e40SEvgeny Pinchuk 8874a488a7aSOded Gabbay kfree(kfd); 8884a488a7aSOded Gabbay } 8894a488a7aSOded Gabbay 890e3b7a967SShaoyun Liu int kgd2kfd_pre_reset(struct kfd_dev *kfd) 891e3b7a967SShaoyun Liu { 892e42051d2SShaoyun Liu if (!kfd->init_complete) 893e42051d2SShaoyun Liu return 0; 89409c34e8dSFelix Kuehling 89555977744SMukul Joshi kfd_smi_event_update_gpu_reset(kfd, false); 89655977744SMukul Joshi 89709c34e8dSFelix Kuehling kfd->dqm->ops.pre_reset(kfd->dqm); 89809c34e8dSFelix Kuehling 8999593f4d6SRajneesh Bhardwaj kgd2kfd_suspend(kfd, false); 900e42051d2SShaoyun Liu 901e42051d2SShaoyun Liu kfd_signal_reset_event(kfd); 902e3b7a967SShaoyun Liu return 0; 903e3b7a967SShaoyun Liu } 904e3b7a967SShaoyun Liu 905e42051d2SShaoyun Liu /* 906e42051d2SShaoyun Liu * Fix me. KFD won't be able to resume existing process for now. 907e42051d2SShaoyun Liu * We will keep all existing process in a evicted state and 908e42051d2SShaoyun Liu * wait the process to be terminated. 909e42051d2SShaoyun Liu */ 910e42051d2SShaoyun Liu 911e3b7a967SShaoyun Liu int kgd2kfd_post_reset(struct kfd_dev *kfd) 912e3b7a967SShaoyun Liu { 913a1bd079fSyu kuai int ret; 914e42051d2SShaoyun Liu 915e42051d2SShaoyun Liu if (!kfd->init_complete) 916e3b7a967SShaoyun Liu return 0; 917e42051d2SShaoyun Liu 918e42051d2SShaoyun Liu ret = kfd_resume(kfd); 919e42051d2SShaoyun Liu if (ret) 920e42051d2SShaoyun Liu return ret; 921a1bd079fSyu kuai atomic_dec(&kfd_locked); 9229b54d201SEric Huang 9239b54d201SEric Huang atomic_set(&kfd->sram_ecc_flag, 0); 9249b54d201SEric Huang 92555977744SMukul Joshi kfd_smi_event_update_gpu_reset(kfd, true); 92655977744SMukul Joshi 927e42051d2SShaoyun Liu return 0; 928e42051d2SShaoyun Liu } 929e42051d2SShaoyun Liu 930e42051d2SShaoyun Liu bool kfd_is_locked(void) 931e42051d2SShaoyun Liu { 932e42051d2SShaoyun Liu return (atomic_read(&kfd_locked) > 0); 933e3b7a967SShaoyun Liu } 934e3b7a967SShaoyun Liu 9359593f4d6SRajneesh Bhardwaj void kgd2kfd_suspend(struct kfd_dev *kfd, bool run_pm) 9364a488a7aSOded Gabbay { 937733fa1f7SYong Zhao if (!kfd->init_complete) 938733fa1f7SYong Zhao return; 939733fa1f7SYong Zhao 9409593f4d6SRajneesh Bhardwaj /* for runtime suspend, skip locking kfd */ 9419593f4d6SRajneesh Bhardwaj if (!run_pm) { 94226103436SFelix Kuehling /* For first KFD device suspend all the KFD processes */ 943e42051d2SShaoyun Liu if (atomic_inc_return(&kfd_locked) == 1) 94426103436SFelix Kuehling kfd_suspend_all_processes(); 9459593f4d6SRajneesh Bhardwaj } 94626103436SFelix Kuehling 94745c9a5e4SOded Gabbay kfd->dqm->ops.stop(kfd->dqm); 94864d1c3a4SFelix Kuehling kfd_iommu_suspend(kfd); 9494a488a7aSOded Gabbay } 9504a488a7aSOded Gabbay 9519593f4d6SRajneesh Bhardwaj int kgd2kfd_resume(struct kfd_dev *kfd, bool run_pm) 9524a488a7aSOded Gabbay { 95326103436SFelix Kuehling int ret, count; 95426103436SFelix Kuehling 955b8935a7cSYong Zhao if (!kfd->init_complete) 956b8935a7cSYong Zhao return 0; 957b17f068aSOded Gabbay 95826103436SFelix Kuehling ret = kfd_resume(kfd); 95926103436SFelix Kuehling if (ret) 96026103436SFelix Kuehling return ret; 961b17f068aSOded Gabbay 9629593f4d6SRajneesh Bhardwaj /* for runtime resume, skip unlocking kfd */ 9639593f4d6SRajneesh Bhardwaj if (!run_pm) { 964e42051d2SShaoyun Liu count = atomic_dec_return(&kfd_locked); 96526103436SFelix Kuehling WARN_ONCE(count < 0, "KFD suspend / resume ref. error"); 96626103436SFelix Kuehling if (count == 0) 96726103436SFelix Kuehling ret = kfd_resume_all_processes(); 9689593f4d6SRajneesh Bhardwaj } 96926103436SFelix Kuehling 97026103436SFelix Kuehling return ret; 9714ebc7182SYong Zhao } 9724ebc7182SYong Zhao 973b8935a7cSYong Zhao static int kfd_resume(struct kfd_dev *kfd) 974b8935a7cSYong Zhao { 975b8935a7cSYong Zhao int err = 0; 976b8935a7cSYong Zhao 97764d1c3a4SFelix Kuehling err = kfd_iommu_resume(kfd); 97864d1c3a4SFelix Kuehling if (err) { 97964d1c3a4SFelix Kuehling dev_err(kfd_device, 98064d1c3a4SFelix Kuehling "Failed to resume IOMMU for device %x:%x\n", 98164d1c3a4SFelix Kuehling kfd->pdev->vendor, kfd->pdev->device); 98264d1c3a4SFelix Kuehling return err; 98364d1c3a4SFelix Kuehling } 984733fa1f7SYong Zhao 985b8935a7cSYong Zhao err = kfd->dqm->ops.start(kfd->dqm); 986b8935a7cSYong Zhao if (err) { 987b8935a7cSYong Zhao dev_err(kfd_device, 988b8935a7cSYong Zhao "Error starting queue manager for device %x:%x\n", 989b8935a7cSYong Zhao kfd->pdev->vendor, kfd->pdev->device); 990b8935a7cSYong Zhao goto dqm_start_error; 991b17f068aSOded Gabbay } 992b17f068aSOded Gabbay 993b8935a7cSYong Zhao return err; 994b8935a7cSYong Zhao 995b8935a7cSYong Zhao dqm_start_error: 99664d1c3a4SFelix Kuehling kfd_iommu_suspend(kfd); 997b8935a7cSYong Zhao return err; 9984a488a7aSOded Gabbay } 9994a488a7aSOded Gabbay 1000b3eca59dSPhilip Yang static inline void kfd_queue_work(struct workqueue_struct *wq, 1001b3eca59dSPhilip Yang struct work_struct *work) 1002b3eca59dSPhilip Yang { 1003b3eca59dSPhilip Yang int cpu, new_cpu; 1004b3eca59dSPhilip Yang 1005b3eca59dSPhilip Yang cpu = new_cpu = smp_processor_id(); 1006b3eca59dSPhilip Yang do { 1007b3eca59dSPhilip Yang new_cpu = cpumask_next(new_cpu, cpu_online_mask) % nr_cpu_ids; 1008b3eca59dSPhilip Yang if (cpu_to_node(new_cpu) == numa_node_id()) 1009b3eca59dSPhilip Yang break; 1010b3eca59dSPhilip Yang } while (cpu != new_cpu); 1011b3eca59dSPhilip Yang 1012b3eca59dSPhilip Yang queue_work_on(new_cpu, wq, work); 1013b3eca59dSPhilip Yang } 1014b3eca59dSPhilip Yang 1015b3f5e6b4SAndrew Lewycky /* This is called directly from KGD at ISR. */ 1016b3f5e6b4SAndrew Lewycky void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry) 10174a488a7aSOded Gabbay { 101858e69886SLan Xiao uint32_t patched_ihre[KFD_MAX_RING_ENTRY_SIZE]; 101958e69886SLan Xiao bool is_patched = false; 10202383a767SChristian König unsigned long flags; 102158e69886SLan Xiao 10222249d558SAndrew Lewycky if (!kfd->init_complete) 10232249d558SAndrew Lewycky return; 10242249d558SAndrew Lewycky 102558e69886SLan Xiao if (kfd->device_info->ih_ring_entry_size > sizeof(patched_ihre)) { 102658e69886SLan Xiao dev_err_once(kfd_device, "Ring entry too small\n"); 102758e69886SLan Xiao return; 102858e69886SLan Xiao } 102958e69886SLan Xiao 10302383a767SChristian König spin_lock_irqsave(&kfd->interrupt_lock, flags); 10312249d558SAndrew Lewycky 10322249d558SAndrew Lewycky if (kfd->interrupts_active 103358e69886SLan Xiao && interrupt_is_wanted(kfd, ih_ring_entry, 103458e69886SLan Xiao patched_ihre, &is_patched) 103558e69886SLan Xiao && enqueue_ih_ring_entry(kfd, 103658e69886SLan Xiao is_patched ? patched_ihre : ih_ring_entry)) 1037b3eca59dSPhilip Yang kfd_queue_work(kfd->ih_wq, &kfd->interrupt_work); 10382249d558SAndrew Lewycky 10392383a767SChristian König spin_unlock_irqrestore(&kfd->interrupt_lock, flags); 10404a488a7aSOded Gabbay } 10416e81090bSOded Gabbay 10426b95e797SFelix Kuehling int kgd2kfd_quiesce_mm(struct mm_struct *mm) 10436b95e797SFelix Kuehling { 10446b95e797SFelix Kuehling struct kfd_process *p; 10456b95e797SFelix Kuehling int r; 10466b95e797SFelix Kuehling 10476b95e797SFelix Kuehling /* Because we are called from arbitrary context (workqueue) as opposed 10486b95e797SFelix Kuehling * to process context, kfd_process could attempt to exit while we are 10496b95e797SFelix Kuehling * running so the lookup function increments the process ref count. 10506b95e797SFelix Kuehling */ 10516b95e797SFelix Kuehling p = kfd_lookup_process_by_mm(mm); 10526b95e797SFelix Kuehling if (!p) 10536b95e797SFelix Kuehling return -ESRCH; 10546b95e797SFelix Kuehling 1055b2057956SFelix Kuehling WARN(debug_evictions, "Evicting pid %d", p->lead_thread->pid); 10566b95e797SFelix Kuehling r = kfd_process_evict_queues(p); 10576b95e797SFelix Kuehling 10586b95e797SFelix Kuehling kfd_unref_process(p); 10596b95e797SFelix Kuehling return r; 10606b95e797SFelix Kuehling } 10616b95e797SFelix Kuehling 10626b95e797SFelix Kuehling int kgd2kfd_resume_mm(struct mm_struct *mm) 10636b95e797SFelix Kuehling { 10646b95e797SFelix Kuehling struct kfd_process *p; 10656b95e797SFelix Kuehling int r; 10666b95e797SFelix Kuehling 10676b95e797SFelix Kuehling /* Because we are called from arbitrary context (workqueue) as opposed 10686b95e797SFelix Kuehling * to process context, kfd_process could attempt to exit while we are 10696b95e797SFelix Kuehling * running so the lookup function increments the process ref count. 10706b95e797SFelix Kuehling */ 10716b95e797SFelix Kuehling p = kfd_lookup_process_by_mm(mm); 10726b95e797SFelix Kuehling if (!p) 10736b95e797SFelix Kuehling return -ESRCH; 10746b95e797SFelix Kuehling 10756b95e797SFelix Kuehling r = kfd_process_restore_queues(p); 10766b95e797SFelix Kuehling 10776b95e797SFelix Kuehling kfd_unref_process(p); 10786b95e797SFelix Kuehling return r; 10796b95e797SFelix Kuehling } 10806b95e797SFelix Kuehling 108126103436SFelix Kuehling /** kgd2kfd_schedule_evict_and_restore_process - Schedules work queue that will 108226103436SFelix Kuehling * prepare for safe eviction of KFD BOs that belong to the specified 108326103436SFelix Kuehling * process. 108426103436SFelix Kuehling * 108526103436SFelix Kuehling * @mm: mm_struct that identifies the specified KFD process 108626103436SFelix Kuehling * @fence: eviction fence attached to KFD process BOs 108726103436SFelix Kuehling * 108826103436SFelix Kuehling */ 108926103436SFelix Kuehling int kgd2kfd_schedule_evict_and_restore_process(struct mm_struct *mm, 109026103436SFelix Kuehling struct dma_fence *fence) 109126103436SFelix Kuehling { 109226103436SFelix Kuehling struct kfd_process *p; 109326103436SFelix Kuehling unsigned long active_time; 109426103436SFelix Kuehling unsigned long delay_jiffies = msecs_to_jiffies(PROCESS_ACTIVE_TIME_MS); 109526103436SFelix Kuehling 109626103436SFelix Kuehling if (!fence) 109726103436SFelix Kuehling return -EINVAL; 109826103436SFelix Kuehling 109926103436SFelix Kuehling if (dma_fence_is_signaled(fence)) 110026103436SFelix Kuehling return 0; 110126103436SFelix Kuehling 110226103436SFelix Kuehling p = kfd_lookup_process_by_mm(mm); 110326103436SFelix Kuehling if (!p) 110426103436SFelix Kuehling return -ENODEV; 110526103436SFelix Kuehling 110626103436SFelix Kuehling if (fence->seqno == p->last_eviction_seqno) 110726103436SFelix Kuehling goto out; 110826103436SFelix Kuehling 110926103436SFelix Kuehling p->last_eviction_seqno = fence->seqno; 111026103436SFelix Kuehling 111126103436SFelix Kuehling /* Avoid KFD process starvation. Wait for at least 111226103436SFelix Kuehling * PROCESS_ACTIVE_TIME_MS before evicting the process again 111326103436SFelix Kuehling */ 111426103436SFelix Kuehling active_time = get_jiffies_64() - p->last_restore_timestamp; 111526103436SFelix Kuehling if (delay_jiffies > active_time) 111626103436SFelix Kuehling delay_jiffies -= active_time; 111726103436SFelix Kuehling else 111826103436SFelix Kuehling delay_jiffies = 0; 111926103436SFelix Kuehling 112026103436SFelix Kuehling /* During process initialization eviction_work.dwork is initialized 112126103436SFelix Kuehling * to kfd_evict_bo_worker 112226103436SFelix Kuehling */ 1123b2057956SFelix Kuehling WARN(debug_evictions, "Scheduling eviction of pid %d in %ld jiffies", 1124b2057956SFelix Kuehling p->lead_thread->pid, delay_jiffies); 112526103436SFelix Kuehling schedule_delayed_work(&p->eviction_work, delay_jiffies); 112626103436SFelix Kuehling out: 112726103436SFelix Kuehling kfd_unref_process(p); 112826103436SFelix Kuehling return 0; 112926103436SFelix Kuehling } 113026103436SFelix Kuehling 11316e81090bSOded Gabbay static int kfd_gtt_sa_init(struct kfd_dev *kfd, unsigned int buf_size, 11326e81090bSOded Gabbay unsigned int chunk_size) 11336e81090bSOded Gabbay { 11348625ff9cSFelix Kuehling unsigned int num_of_longs; 11356e81090bSOded Gabbay 113632fa8219SFelix Kuehling if (WARN_ON(buf_size < chunk_size)) 113732fa8219SFelix Kuehling return -EINVAL; 113832fa8219SFelix Kuehling if (WARN_ON(buf_size == 0)) 113932fa8219SFelix Kuehling return -EINVAL; 114032fa8219SFelix Kuehling if (WARN_ON(chunk_size == 0)) 114132fa8219SFelix Kuehling return -EINVAL; 11426e81090bSOded Gabbay 11436e81090bSOded Gabbay kfd->gtt_sa_chunk_size = chunk_size; 11446e81090bSOded Gabbay kfd->gtt_sa_num_of_chunks = buf_size / chunk_size; 11456e81090bSOded Gabbay 11468625ff9cSFelix Kuehling num_of_longs = (kfd->gtt_sa_num_of_chunks + BITS_PER_LONG - 1) / 11478625ff9cSFelix Kuehling BITS_PER_LONG; 11486e81090bSOded Gabbay 11498625ff9cSFelix Kuehling kfd->gtt_sa_bitmap = kcalloc(num_of_longs, sizeof(long), GFP_KERNEL); 11506e81090bSOded Gabbay 11516e81090bSOded Gabbay if (!kfd->gtt_sa_bitmap) 11526e81090bSOded Gabbay return -ENOMEM; 11536e81090bSOded Gabbay 115479775b62SKent Russell pr_debug("gtt_sa_num_of_chunks = %d, gtt_sa_bitmap = %p\n", 11556e81090bSOded Gabbay kfd->gtt_sa_num_of_chunks, kfd->gtt_sa_bitmap); 11566e81090bSOded Gabbay 11576e81090bSOded Gabbay mutex_init(&kfd->gtt_sa_lock); 11586e81090bSOded Gabbay 11596e81090bSOded Gabbay return 0; 11606e81090bSOded Gabbay 11616e81090bSOded Gabbay } 11626e81090bSOded Gabbay 11636e81090bSOded Gabbay static void kfd_gtt_sa_fini(struct kfd_dev *kfd) 11646e81090bSOded Gabbay { 11656e81090bSOded Gabbay mutex_destroy(&kfd->gtt_sa_lock); 11666e81090bSOded Gabbay kfree(kfd->gtt_sa_bitmap); 11676e81090bSOded Gabbay } 11686e81090bSOded Gabbay 11696e81090bSOded Gabbay static inline uint64_t kfd_gtt_sa_calc_gpu_addr(uint64_t start_addr, 11706e81090bSOded Gabbay unsigned int bit_num, 11716e81090bSOded Gabbay unsigned int chunk_size) 11726e81090bSOded Gabbay { 11736e81090bSOded Gabbay return start_addr + bit_num * chunk_size; 11746e81090bSOded Gabbay } 11756e81090bSOded Gabbay 11766e81090bSOded Gabbay static inline uint32_t *kfd_gtt_sa_calc_cpu_addr(void *start_addr, 11776e81090bSOded Gabbay unsigned int bit_num, 11786e81090bSOded Gabbay unsigned int chunk_size) 11796e81090bSOded Gabbay { 11806e81090bSOded Gabbay return (uint32_t *) ((uint64_t) start_addr + bit_num * chunk_size); 11816e81090bSOded Gabbay } 11826e81090bSOded Gabbay 11836e81090bSOded Gabbay int kfd_gtt_sa_allocate(struct kfd_dev *kfd, unsigned int size, 11846e81090bSOded Gabbay struct kfd_mem_obj **mem_obj) 11856e81090bSOded Gabbay { 11866e81090bSOded Gabbay unsigned int found, start_search, cur_size; 11876e81090bSOded Gabbay 11886e81090bSOded Gabbay if (size == 0) 11896e81090bSOded Gabbay return -EINVAL; 11906e81090bSOded Gabbay 11916e81090bSOded Gabbay if (size > kfd->gtt_sa_num_of_chunks * kfd->gtt_sa_chunk_size) 11926e81090bSOded Gabbay return -ENOMEM; 11936e81090bSOded Gabbay 11941cd106ecSFelix Kuehling *mem_obj = kzalloc(sizeof(struct kfd_mem_obj), GFP_KERNEL); 11951cd106ecSFelix Kuehling if (!(*mem_obj)) 11966e81090bSOded Gabbay return -ENOMEM; 11976e81090bSOded Gabbay 119879775b62SKent Russell pr_debug("Allocated mem_obj = %p for size = %d\n", *mem_obj, size); 11996e81090bSOded Gabbay 12006e81090bSOded Gabbay start_search = 0; 12016e81090bSOded Gabbay 12026e81090bSOded Gabbay mutex_lock(&kfd->gtt_sa_lock); 12036e81090bSOded Gabbay 12046e81090bSOded Gabbay kfd_gtt_restart_search: 12056e81090bSOded Gabbay /* Find the first chunk that is free */ 12066e81090bSOded Gabbay found = find_next_zero_bit(kfd->gtt_sa_bitmap, 12076e81090bSOded Gabbay kfd->gtt_sa_num_of_chunks, 12086e81090bSOded Gabbay start_search); 12096e81090bSOded Gabbay 121079775b62SKent Russell pr_debug("Found = %d\n", found); 12116e81090bSOded Gabbay 12126e81090bSOded Gabbay /* If there wasn't any free chunk, bail out */ 12136e81090bSOded Gabbay if (found == kfd->gtt_sa_num_of_chunks) 12146e81090bSOded Gabbay goto kfd_gtt_no_free_chunk; 12156e81090bSOded Gabbay 12166e81090bSOded Gabbay /* Update fields of mem_obj */ 12176e81090bSOded Gabbay (*mem_obj)->range_start = found; 12186e81090bSOded Gabbay (*mem_obj)->range_end = found; 12196e81090bSOded Gabbay (*mem_obj)->gpu_addr = kfd_gtt_sa_calc_gpu_addr( 12206e81090bSOded Gabbay kfd->gtt_start_gpu_addr, 12216e81090bSOded Gabbay found, 12226e81090bSOded Gabbay kfd->gtt_sa_chunk_size); 12236e81090bSOded Gabbay (*mem_obj)->cpu_ptr = kfd_gtt_sa_calc_cpu_addr( 12246e81090bSOded Gabbay kfd->gtt_start_cpu_ptr, 12256e81090bSOded Gabbay found, 12266e81090bSOded Gabbay kfd->gtt_sa_chunk_size); 12276e81090bSOded Gabbay 122879775b62SKent Russell pr_debug("gpu_addr = %p, cpu_addr = %p\n", 12296e81090bSOded Gabbay (uint64_t *) (*mem_obj)->gpu_addr, (*mem_obj)->cpu_ptr); 12306e81090bSOded Gabbay 12316e81090bSOded Gabbay /* If we need only one chunk, mark it as allocated and get out */ 12326e81090bSOded Gabbay if (size <= kfd->gtt_sa_chunk_size) { 123379775b62SKent Russell pr_debug("Single bit\n"); 12346e81090bSOded Gabbay set_bit(found, kfd->gtt_sa_bitmap); 12356e81090bSOded Gabbay goto kfd_gtt_out; 12366e81090bSOded Gabbay } 12376e81090bSOded Gabbay 12386e81090bSOded Gabbay /* Otherwise, try to see if we have enough contiguous chunks */ 12396e81090bSOded Gabbay cur_size = size - kfd->gtt_sa_chunk_size; 12406e81090bSOded Gabbay do { 12416e81090bSOded Gabbay (*mem_obj)->range_end = 12426e81090bSOded Gabbay find_next_zero_bit(kfd->gtt_sa_bitmap, 12436e81090bSOded Gabbay kfd->gtt_sa_num_of_chunks, ++found); 12446e81090bSOded Gabbay /* 12456e81090bSOded Gabbay * If next free chunk is not contiguous than we need to 12466e81090bSOded Gabbay * restart our search from the last free chunk we found (which 12476e81090bSOded Gabbay * wasn't contiguous to the previous ones 12486e81090bSOded Gabbay */ 12496e81090bSOded Gabbay if ((*mem_obj)->range_end != found) { 12506e81090bSOded Gabbay start_search = found; 12516e81090bSOded Gabbay goto kfd_gtt_restart_search; 12526e81090bSOded Gabbay } 12536e81090bSOded Gabbay 12546e81090bSOded Gabbay /* 12556e81090bSOded Gabbay * If we reached end of buffer, bail out with error 12566e81090bSOded Gabbay */ 12576e81090bSOded Gabbay if (found == kfd->gtt_sa_num_of_chunks) 12586e81090bSOded Gabbay goto kfd_gtt_no_free_chunk; 12596e81090bSOded Gabbay 12606e81090bSOded Gabbay /* Check if we don't need another chunk */ 12616e81090bSOded Gabbay if (cur_size <= kfd->gtt_sa_chunk_size) 12626e81090bSOded Gabbay cur_size = 0; 12636e81090bSOded Gabbay else 12646e81090bSOded Gabbay cur_size -= kfd->gtt_sa_chunk_size; 12656e81090bSOded Gabbay 12666e81090bSOded Gabbay } while (cur_size > 0); 12676e81090bSOded Gabbay 126879775b62SKent Russell pr_debug("range_start = %d, range_end = %d\n", 12696e81090bSOded Gabbay (*mem_obj)->range_start, (*mem_obj)->range_end); 12706e81090bSOded Gabbay 12716e81090bSOded Gabbay /* Mark the chunks as allocated */ 12726e81090bSOded Gabbay for (found = (*mem_obj)->range_start; 12736e81090bSOded Gabbay found <= (*mem_obj)->range_end; 12746e81090bSOded Gabbay found++) 12756e81090bSOded Gabbay set_bit(found, kfd->gtt_sa_bitmap); 12766e81090bSOded Gabbay 12776e81090bSOded Gabbay kfd_gtt_out: 12786e81090bSOded Gabbay mutex_unlock(&kfd->gtt_sa_lock); 12796e81090bSOded Gabbay return 0; 12806e81090bSOded Gabbay 12816e81090bSOded Gabbay kfd_gtt_no_free_chunk: 12823148a6a0SJack Zhang pr_debug("Allocation failed with mem_obj = %p\n", *mem_obj); 12836e81090bSOded Gabbay mutex_unlock(&kfd->gtt_sa_lock); 12843148a6a0SJack Zhang kfree(*mem_obj); 12856e81090bSOded Gabbay return -ENOMEM; 12866e81090bSOded Gabbay } 12876e81090bSOded Gabbay 12886e81090bSOded Gabbay int kfd_gtt_sa_free(struct kfd_dev *kfd, struct kfd_mem_obj *mem_obj) 12896e81090bSOded Gabbay { 12906e81090bSOded Gabbay unsigned int bit; 12916e81090bSOded Gabbay 12929216ed29SOded Gabbay /* Act like kfree when trying to free a NULL object */ 12939216ed29SOded Gabbay if (!mem_obj) 12949216ed29SOded Gabbay return 0; 12956e81090bSOded Gabbay 129679775b62SKent Russell pr_debug("Free mem_obj = %p, range_start = %d, range_end = %d\n", 12976e81090bSOded Gabbay mem_obj, mem_obj->range_start, mem_obj->range_end); 12986e81090bSOded Gabbay 12996e81090bSOded Gabbay mutex_lock(&kfd->gtt_sa_lock); 13006e81090bSOded Gabbay 13016e81090bSOded Gabbay /* Mark the chunks as free */ 13026e81090bSOded Gabbay for (bit = mem_obj->range_start; 13036e81090bSOded Gabbay bit <= mem_obj->range_end; 13046e81090bSOded Gabbay bit++) 13056e81090bSOded Gabbay clear_bit(bit, kfd->gtt_sa_bitmap); 13066e81090bSOded Gabbay 13076e81090bSOded Gabbay mutex_unlock(&kfd->gtt_sa_lock); 13086e81090bSOded Gabbay 13096e81090bSOded Gabbay kfree(mem_obj); 13106e81090bSOded Gabbay return 0; 13116e81090bSOded Gabbay } 1312a29ec470SShaoyun Liu 13139b54d201SEric Huang void kgd2kfd_set_sram_ecc_flag(struct kfd_dev *kfd) 13149b54d201SEric Huang { 13159b54d201SEric Huang if (kfd) 13169b54d201SEric Huang atomic_inc(&kfd->sram_ecc_flag); 13179b54d201SEric Huang } 13189b54d201SEric Huang 131943d8107fSHarish Kasiviswanathan void kfd_inc_compute_active(struct kfd_dev *kfd) 132043d8107fSHarish Kasiviswanathan { 132143d8107fSHarish Kasiviswanathan if (atomic_inc_return(&kfd->compute_profile) == 1) 132243d8107fSHarish Kasiviswanathan amdgpu_amdkfd_set_compute_idle(kfd->kgd, false); 132343d8107fSHarish Kasiviswanathan } 132443d8107fSHarish Kasiviswanathan 132543d8107fSHarish Kasiviswanathan void kfd_dec_compute_active(struct kfd_dev *kfd) 132643d8107fSHarish Kasiviswanathan { 132743d8107fSHarish Kasiviswanathan int count = atomic_dec_return(&kfd->compute_profile); 132843d8107fSHarish Kasiviswanathan 132943d8107fSHarish Kasiviswanathan if (count == 0) 133043d8107fSHarish Kasiviswanathan amdgpu_amdkfd_set_compute_idle(kfd->kgd, true); 133143d8107fSHarish Kasiviswanathan WARN_ONCE(count < 0, "Compute profile ref. count error"); 133243d8107fSHarish Kasiviswanathan } 133343d8107fSHarish Kasiviswanathan 13342c2b0d88SMukul Joshi void kgd2kfd_smi_event_throttle(struct kfd_dev *kfd, uint32_t throttle_bitmask) 13352c2b0d88SMukul Joshi { 1336158fc08dSAmber Lin if (kfd && kfd->init_complete) 13372c2b0d88SMukul Joshi kfd_smi_event_update_thermal_throttling(kfd, throttle_bitmask); 13382c2b0d88SMukul Joshi } 13392c2b0d88SMukul Joshi 1340a29ec470SShaoyun Liu #if defined(CONFIG_DEBUG_FS) 1341a29ec470SShaoyun Liu 1342a29ec470SShaoyun Liu /* This function will send a package to HIQ to hang the HWS 1343a29ec470SShaoyun Liu * which will trigger a GPU reset and bring the HWS back to normal state 1344a29ec470SShaoyun Liu */ 1345a29ec470SShaoyun Liu int kfd_debugfs_hang_hws(struct kfd_dev *dev) 1346a29ec470SShaoyun Liu { 1347a29ec470SShaoyun Liu int r = 0; 1348a29ec470SShaoyun Liu 1349a29ec470SShaoyun Liu if (dev->dqm->sched_policy != KFD_SCHED_POLICY_HWS) { 1350a29ec470SShaoyun Liu pr_err("HWS is not enabled"); 1351a29ec470SShaoyun Liu return -EINVAL; 1352a29ec470SShaoyun Liu } 1353a29ec470SShaoyun Liu 1354a29ec470SShaoyun Liu r = pm_debugfs_hang_hws(&dev->dqm->packets); 1355a29ec470SShaoyun Liu if (!r) 1356a29ec470SShaoyun Liu r = dqm_debugfs_execute_queues(dev->dqm); 1357a29ec470SShaoyun Liu 1358a29ec470SShaoyun Liu return r; 1359a29ec470SShaoyun Liu } 1360a29ec470SShaoyun Liu 1361a29ec470SShaoyun Liu #endif 1362