1f3a39818SAndrew Lewycky /* 2f3a39818SAndrew Lewycky * Copyright 2014 Advanced Micro Devices, Inc. 3f3a39818SAndrew Lewycky * 4f3a39818SAndrew Lewycky * Permission is hereby granted, free of charge, to any person obtaining a 5f3a39818SAndrew Lewycky * copy of this software and associated documentation files (the "Software"), 6f3a39818SAndrew Lewycky * to deal in the Software without restriction, including without limitation 7f3a39818SAndrew Lewycky * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8f3a39818SAndrew Lewycky * and/or sell copies of the Software, and to permit persons to whom the 9f3a39818SAndrew Lewycky * Software is furnished to do so, subject to the following conditions: 10f3a39818SAndrew Lewycky * 11f3a39818SAndrew Lewycky * The above copyright notice and this permission notice shall be included in 12f3a39818SAndrew Lewycky * all copies or substantial portions of the Software. 13f3a39818SAndrew Lewycky * 14f3a39818SAndrew Lewycky * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15f3a39818SAndrew Lewycky * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16f3a39818SAndrew Lewycky * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17f3a39818SAndrew Lewycky * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18f3a39818SAndrew Lewycky * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19f3a39818SAndrew Lewycky * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20f3a39818SAndrew Lewycky * OTHER DEALINGS IN THE SOFTWARE. 21f3a39818SAndrew Lewycky */ 22f3a39818SAndrew Lewycky 23f3a39818SAndrew Lewycky #include "kfd_priv.h" 24f3a39818SAndrew Lewycky #include "kfd_events.h" 25f3a39818SAndrew Lewycky #include "cik_int.h" 265b87245fSAmber Lin #include "amdgpu_amdkfd.h" 27938a0650SAmber Lin #include "kfd_smi_events.h" 28f3a39818SAndrew Lewycky 29f3a39818SAndrew Lewycky static bool cik_event_interrupt_isr(struct kfd_dev *dev, 3058e69886SLan Xiao const uint32_t *ih_ring_entry, 3158e69886SLan Xiao uint32_t *patched_ihre, 3258e69886SLan Xiao bool *patched_flag) 33f3a39818SAndrew Lewycky { 34f3a39818SAndrew Lewycky const struct cik_ih_ring_entry *ihre = 35f3a39818SAndrew Lewycky (const struct cik_ih_ring_entry *)ih_ring_entry; 3658e69886SLan Xiao const struct kfd2kgd_calls *f2g = dev->kfd2kgd; 3756fc40abSYong Zhao unsigned int vmid; 3856fc40abSYong Zhao uint16_t pasid; 3956fc40abSYong Zhao bool ret; 40f3a39818SAndrew Lewycky 4158e69886SLan Xiao /* This workaround is due to HW/FW limitation on Hawaii that 4258e69886SLan Xiao * VMID and PASID are not written into ih_ring_entry 4358e69886SLan Xiao */ 4458e69886SLan Xiao if ((ihre->source_id == CIK_INTSRC_GFX_PAGE_INV_FAULT || 4558e69886SLan Xiao ihre->source_id == CIK_INTSRC_GFX_MEM_PROT_FAULT) && 46*7eb0502aSGraham Sider dev->adev->asic_type == CHIP_HAWAII) { 4758e69886SLan Xiao struct cik_ih_ring_entry *tmp_ihre = 4858e69886SLan Xiao (struct cik_ih_ring_entry *)patched_ihre; 4958e69886SLan Xiao 5058e69886SLan Xiao *patched_flag = true; 5158e69886SLan Xiao *tmp_ihre = *ihre; 5258e69886SLan Xiao 533356c38dSGraham Sider vmid = f2g->read_vmid_from_vmfault_reg(dev->adev); 543356c38dSGraham Sider ret = f2g->get_atc_vmid_pasid_mapping_info(dev->adev, vmid, &pasid); 5558e69886SLan Xiao 5658e69886SLan Xiao tmp_ihre->ring_id &= 0x000000ff; 5758e69886SLan Xiao tmp_ihre->ring_id |= vmid << 8; 5858e69886SLan Xiao tmp_ihre->ring_id |= pasid << 16; 5958e69886SLan Xiao 6056fc40abSYong Zhao return ret && (pasid != 0) && 6158e69886SLan Xiao vmid >= dev->vm_info.first_vmid_kfd && 6258e69886SLan Xiao vmid <= dev->vm_info.last_vmid_kfd; 6358e69886SLan Xiao } 6458e69886SLan Xiao 65c129db12SFelix Kuehling /* Only handle interrupts from KFD VMIDs */ 66c129db12SFelix Kuehling vmid = (ihre->ring_id & 0x0000ff00) >> 8; 67c129db12SFelix Kuehling if (vmid < dev->vm_info.first_vmid_kfd || 68c129db12SFelix Kuehling vmid > dev->vm_info.last_vmid_kfd) 6986b6624aSSumera Priyadarsini return false; 70c129db12SFelix Kuehling 71c129db12SFelix Kuehling /* If there is no valid PASID, it's likely a firmware bug */ 72f3a39818SAndrew Lewycky pasid = (ihre->ring_id & 0xffff0000) >> 16; 73c129db12SFelix Kuehling if (WARN_ONCE(pasid == 0, "FW bug: No PASID in KFD interrupt")) 7486b6624aSSumera Priyadarsini return false; 75f3a39818SAndrew Lewycky 76c129db12SFelix Kuehling /* Interrupt types we care about: various signals and faults. 77c129db12SFelix Kuehling * They will be forwarded to a work queue (see below). 78c129db12SFelix Kuehling */ 79c129db12SFelix Kuehling return ihre->source_id == CIK_INTSRC_CP_END_OF_PIPE || 8066b783b4SBesar Wicaksono ihre->source_id == CIK_INTSRC_SDMA_TRAP || 81930c5ff4SAlexey Skidanov ihre->source_id == CIK_INTSRC_SQ_INTERRUPT_MSG || 822640c3faSshaoyunl ihre->source_id == CIK_INTSRC_CP_BAD_OPCODE || 836d909c5dSOak Zeng ((ihre->source_id == CIK_INTSRC_GFX_PAGE_INV_FAULT || 846d909c5dSOak Zeng ihre->source_id == CIK_INTSRC_GFX_MEM_PROT_FAULT) && 856d909c5dSOak Zeng !amdgpu_no_queue_eviction_on_vm_fault); 86f3a39818SAndrew Lewycky } 87f3a39818SAndrew Lewycky 88f3a39818SAndrew Lewycky static void cik_event_interrupt_wq(struct kfd_dev *dev, 89f3a39818SAndrew Lewycky const uint32_t *ih_ring_entry) 90f3a39818SAndrew Lewycky { 91f3a39818SAndrew Lewycky const struct cik_ih_ring_entry *ihre = 92f3a39818SAndrew Lewycky (const struct cik_ih_ring_entry *)ih_ring_entry; 933f04f961SFelix Kuehling uint32_t context_id = ihre->data & 0xfffffff; 942640c3faSshaoyunl unsigned int vmid = (ihre->ring_id & 0x0000ff00) >> 8; 95c7b6bac9SFenghua Yu u32 pasid = (ihre->ring_id & 0xffff0000) >> 16; 96f3a39818SAndrew Lewycky 97f3a39818SAndrew Lewycky if (pasid == 0) 98f3a39818SAndrew Lewycky return; 99f3a39818SAndrew Lewycky 100f3a39818SAndrew Lewycky if (ihre->source_id == CIK_INTSRC_CP_END_OF_PIPE) 1013f04f961SFelix Kuehling kfd_signal_event_interrupt(pasid, context_id, 28); 10266b783b4SBesar Wicaksono else if (ihre->source_id == CIK_INTSRC_SDMA_TRAP) 1033f04f961SFelix Kuehling kfd_signal_event_interrupt(pasid, context_id, 28); 104f3a39818SAndrew Lewycky else if (ihre->source_id == CIK_INTSRC_SQ_INTERRUPT_MSG) 1053f04f961SFelix Kuehling kfd_signal_event_interrupt(pasid, context_id & 0xff, 8); 106930c5ff4SAlexey Skidanov else if (ihre->source_id == CIK_INTSRC_CP_BAD_OPCODE) 107930c5ff4SAlexey Skidanov kfd_signal_hw_exception_event(pasid); 1082640c3faSshaoyunl else if (ihre->source_id == CIK_INTSRC_GFX_PAGE_INV_FAULT || 1092640c3faSshaoyunl ihre->source_id == CIK_INTSRC_GFX_MEM_PROT_FAULT) { 1102640c3faSshaoyunl struct kfd_vm_fault_info info; 1112640c3faSshaoyunl 112938a0650SAmber Lin kfd_smi_event_update_vmfault(dev, pasid); 1132640c3faSshaoyunl kfd_process_vm_fault(dev->dqm, pasid); 1142640c3faSshaoyunl 1152640c3faSshaoyunl memset(&info, 0, sizeof(info)); 116dff63da9SGraham Sider amdgpu_amdkfd_gpuvm_get_vm_fault_info(dev->adev, &info); 1172640c3faSshaoyunl if (!info.page_addr && !info.status) 1182640c3faSshaoyunl return; 1192640c3faSshaoyunl 1202640c3faSshaoyunl if (info.vmid == vmid) 1212640c3faSshaoyunl kfd_signal_vm_fault_event(dev, pasid, &info); 1222640c3faSshaoyunl else 1232640c3faSshaoyunl kfd_signal_vm_fault_event(dev, pasid, NULL); 1242640c3faSshaoyunl } 125f3a39818SAndrew Lewycky } 126f3a39818SAndrew Lewycky 127f3a39818SAndrew Lewycky const struct kfd_event_interrupt_class event_interrupt_class_cik = { 128f3a39818SAndrew Lewycky .interrupt_isr = cik_event_interrupt_isr, 129f3a39818SAndrew Lewycky .interrupt_wq = cik_event_interrupt_wq, 130f3a39818SAndrew Lewycky }; 131