1f204e0b8SIan Munsie /* 2f204e0b8SIan Munsie * Copyright 2014 IBM Corp. 3f204e0b8SIan Munsie * 4f204e0b8SIan Munsie * This program is free software; you can redistribute it and/or 5f204e0b8SIan Munsie * modify it under the terms of the GNU General Public License 6f204e0b8SIan Munsie * as published by the Free Software Foundation; either version 7f204e0b8SIan Munsie * 2 of the License, or (at your option) any later version. 8f204e0b8SIan Munsie */ 9f204e0b8SIan Munsie 10f204e0b8SIan Munsie #include <linux/workqueue.h> 11f204e0b8SIan Munsie #include <linux/sched.h> 12f204e0b8SIan Munsie #include <linux/pid.h> 13f204e0b8SIan Munsie #include <linux/mm.h> 14f204e0b8SIan Munsie #include <linux/moduleparam.h> 15f204e0b8SIan Munsie 16f204e0b8SIan Munsie #undef MODULE_PARAM_PREFIX 17f204e0b8SIan Munsie #define MODULE_PARAM_PREFIX "cxl" "." 18f204e0b8SIan Munsie #include <asm/current.h> 19f204e0b8SIan Munsie #include <asm/copro.h> 20f204e0b8SIan Munsie #include <asm/mmu.h> 21f204e0b8SIan Munsie 22f204e0b8SIan Munsie #include "cxl.h" 23f204e0b8SIan Munsie 24eb01d4c2SIan Munsie static bool sste_matches(struct cxl_sste *sste, struct copro_slb *slb) 25eb01d4c2SIan Munsie { 26eb01d4c2SIan Munsie return ((sste->vsid_data == cpu_to_be64(slb->vsid)) && 27eb01d4c2SIan Munsie (sste->esid_data == cpu_to_be64(slb->esid))); 28eb01d4c2SIan Munsie } 29eb01d4c2SIan Munsie 30eb01d4c2SIan Munsie /* 31eb01d4c2SIan Munsie * This finds a free SSTE for the given SLB, or returns NULL if it's already in 32eb01d4c2SIan Munsie * the segment table. 33eb01d4c2SIan Munsie */ 34b03a7f57SIan Munsie static struct cxl_sste* find_free_sste(struct cxl_context *ctx, 35b03a7f57SIan Munsie struct copro_slb *slb) 36f204e0b8SIan Munsie { 37eb01d4c2SIan Munsie struct cxl_sste *primary, *sste, *ret = NULL; 38b03a7f57SIan Munsie unsigned int mask = (ctx->sst_size >> 7) - 1; /* SSTP0[SegTableSize] */ 395100a9d6SIan Munsie unsigned int entry; 40b03a7f57SIan Munsie unsigned int hash; 41f204e0b8SIan Munsie 42b03a7f57SIan Munsie if (slb->vsid & SLB_VSID_B_1T) 43b03a7f57SIan Munsie hash = (slb->esid >> SID_SHIFT_1T) & mask; 44b03a7f57SIan Munsie else /* 256M */ 45b03a7f57SIan Munsie hash = (slb->esid >> SID_SHIFT) & mask; 46b03a7f57SIan Munsie 47b03a7f57SIan Munsie primary = ctx->sstp + (hash << 3); 48b03a7f57SIan Munsie 49b03a7f57SIan Munsie for (entry = 0, sste = primary; entry < 8; entry++, sste++) { 50eb01d4c2SIan Munsie if (!ret && !(be64_to_cpu(sste->esid_data) & SLB_ESID_V)) 51eb01d4c2SIan Munsie ret = sste; 52eb01d4c2SIan Munsie if (sste_matches(sste, slb)) 53eb01d4c2SIan Munsie return NULL; 54f204e0b8SIan Munsie } 55eb01d4c2SIan Munsie if (ret) 56eb01d4c2SIan Munsie return ret; 57b03a7f57SIan Munsie 58f204e0b8SIan Munsie /* Nothing free, select an entry to cast out */ 59eb01d4c2SIan Munsie ret = primary + ctx->sst_lru; 60b03a7f57SIan Munsie ctx->sst_lru = (ctx->sst_lru + 1) & 0x7; 61f204e0b8SIan Munsie 62eb01d4c2SIan Munsie return ret; 63f204e0b8SIan Munsie } 64f204e0b8SIan Munsie 65f204e0b8SIan Munsie static void cxl_load_segment(struct cxl_context *ctx, struct copro_slb *slb) 66f204e0b8SIan Munsie { 67f204e0b8SIan Munsie /* mask is the group index, we search primary and secondary here. */ 68f204e0b8SIan Munsie struct cxl_sste *sste; 69f204e0b8SIan Munsie unsigned long flags; 70f204e0b8SIan Munsie 71f204e0b8SIan Munsie spin_lock_irqsave(&ctx->sste_lock, flags); 72b03a7f57SIan Munsie sste = find_free_sste(ctx, slb); 73eb01d4c2SIan Munsie if (!sste) 74eb01d4c2SIan Munsie goto out_unlock; 75f204e0b8SIan Munsie 76f204e0b8SIan Munsie pr_devel("CXL Populating SST[%li]: %#llx %#llx\n", 77f204e0b8SIan Munsie sste - ctx->sstp, slb->vsid, slb->esid); 78f204e0b8SIan Munsie 79f204e0b8SIan Munsie sste->vsid_data = cpu_to_be64(slb->vsid); 80f204e0b8SIan Munsie sste->esid_data = cpu_to_be64(slb->esid); 81eb01d4c2SIan Munsie out_unlock: 82f204e0b8SIan Munsie spin_unlock_irqrestore(&ctx->sste_lock, flags); 83f204e0b8SIan Munsie } 84f204e0b8SIan Munsie 85f204e0b8SIan Munsie static int cxl_fault_segment(struct cxl_context *ctx, struct mm_struct *mm, 86f204e0b8SIan Munsie u64 ea) 87f204e0b8SIan Munsie { 88f204e0b8SIan Munsie struct copro_slb slb = {0,0}; 89f204e0b8SIan Munsie int rc; 90f204e0b8SIan Munsie 91f204e0b8SIan Munsie if (!(rc = copro_calculate_slb(mm, ea, &slb))) { 92f204e0b8SIan Munsie cxl_load_segment(ctx, &slb); 93f204e0b8SIan Munsie } 94f204e0b8SIan Munsie 95f204e0b8SIan Munsie return rc; 96f204e0b8SIan Munsie } 97f204e0b8SIan Munsie 98f204e0b8SIan Munsie static void cxl_ack_ae(struct cxl_context *ctx) 99f204e0b8SIan Munsie { 100f204e0b8SIan Munsie unsigned long flags; 101f204e0b8SIan Munsie 102f204e0b8SIan Munsie cxl_ack_irq(ctx, CXL_PSL_TFC_An_AE, 0); 103f204e0b8SIan Munsie 104f204e0b8SIan Munsie spin_lock_irqsave(&ctx->lock, flags); 105f204e0b8SIan Munsie ctx->pending_fault = true; 106f204e0b8SIan Munsie ctx->fault_addr = ctx->dar; 107f204e0b8SIan Munsie ctx->fault_dsisr = ctx->dsisr; 108f204e0b8SIan Munsie spin_unlock_irqrestore(&ctx->lock, flags); 109f204e0b8SIan Munsie 110f204e0b8SIan Munsie wake_up_all(&ctx->wq); 111f204e0b8SIan Munsie } 112f204e0b8SIan Munsie 113f204e0b8SIan Munsie static int cxl_handle_segment_miss(struct cxl_context *ctx, 114f204e0b8SIan Munsie struct mm_struct *mm, u64 ea) 115f204e0b8SIan Munsie { 116f204e0b8SIan Munsie int rc; 117f204e0b8SIan Munsie 118f204e0b8SIan Munsie pr_devel("CXL interrupt: Segment fault pe: %i ea: %#llx\n", ctx->pe, ea); 119f204e0b8SIan Munsie 120f204e0b8SIan Munsie if ((rc = cxl_fault_segment(ctx, mm, ea))) 121f204e0b8SIan Munsie cxl_ack_ae(ctx); 122f204e0b8SIan Munsie else { 123f204e0b8SIan Munsie 124f204e0b8SIan Munsie mb(); /* Order seg table write to TFC MMIO write */ 125f204e0b8SIan Munsie cxl_ack_irq(ctx, CXL_PSL_TFC_An_R, 0); 126f204e0b8SIan Munsie } 127f204e0b8SIan Munsie 128f204e0b8SIan Munsie return IRQ_HANDLED; 129f204e0b8SIan Munsie } 130f204e0b8SIan Munsie 131f204e0b8SIan Munsie static void cxl_handle_page_fault(struct cxl_context *ctx, 132f204e0b8SIan Munsie struct mm_struct *mm, u64 dsisr, u64 dar) 133f204e0b8SIan Munsie { 134f204e0b8SIan Munsie unsigned flt = 0; 135f204e0b8SIan Munsie int result; 136aefa5688SAneesh Kumar K.V unsigned long access, flags, inv_flags = 0; 137f204e0b8SIan Munsie 138f204e0b8SIan Munsie if ((result = copro_handle_mm_fault(mm, dar, dsisr, &flt))) { 139f204e0b8SIan Munsie pr_devel("copro_handle_mm_fault failed: %#x\n", result); 140f204e0b8SIan Munsie return cxl_ack_ae(ctx); 141f204e0b8SIan Munsie } 142f204e0b8SIan Munsie 143f204e0b8SIan Munsie /* 144f204e0b8SIan Munsie * update_mmu_cache() will not have loaded the hash since current->trap 145f204e0b8SIan Munsie * is not a 0x400 or 0x300, so just call hash_page_mm() here. 146f204e0b8SIan Munsie */ 147f204e0b8SIan Munsie access = _PAGE_PRESENT; 148f204e0b8SIan Munsie if (dsisr & CXL_PSL_DSISR_An_S) 149f204e0b8SIan Munsie access |= _PAGE_RW; 150f204e0b8SIan Munsie if ((!ctx->kernel) || ~(dar & (1ULL << 63))) 151f204e0b8SIan Munsie access |= _PAGE_USER; 152aefa5688SAneesh Kumar K.V 153aefa5688SAneesh Kumar K.V if (dsisr & DSISR_NOHPTE) 154aefa5688SAneesh Kumar K.V inv_flags |= HPTE_NOHPTE_UPDATE; 155aefa5688SAneesh Kumar K.V 156f204e0b8SIan Munsie local_irq_save(flags); 157aefa5688SAneesh Kumar K.V hash_page_mm(mm, dar, access, 0x300, inv_flags); 158f204e0b8SIan Munsie local_irq_restore(flags); 159f204e0b8SIan Munsie 160f204e0b8SIan Munsie pr_devel("Page fault successfully handled for pe: %i!\n", ctx->pe); 161f204e0b8SIan Munsie cxl_ack_irq(ctx, CXL_PSL_TFC_An_R, 0); 162f204e0b8SIan Munsie } 163f204e0b8SIan Munsie 164f204e0b8SIan Munsie void cxl_handle_fault(struct work_struct *fault_work) 165f204e0b8SIan Munsie { 166f204e0b8SIan Munsie struct cxl_context *ctx = 167f204e0b8SIan Munsie container_of(fault_work, struct cxl_context, fault_work); 168f204e0b8SIan Munsie u64 dsisr = ctx->dsisr; 169f204e0b8SIan Munsie u64 dar = ctx->dar; 170f204e0b8SIan Munsie struct task_struct *task; 171f204e0b8SIan Munsie struct mm_struct *mm; 172f204e0b8SIan Munsie 173f204e0b8SIan Munsie if (cxl_p2n_read(ctx->afu, CXL_PSL_DSISR_An) != dsisr || 174f204e0b8SIan Munsie cxl_p2n_read(ctx->afu, CXL_PSL_DAR_An) != dar || 175f204e0b8SIan Munsie cxl_p2n_read(ctx->afu, CXL_PSL_PEHandle_An) != ctx->pe) { 176f204e0b8SIan Munsie /* Most likely explanation is harmless - a dedicated process 177f204e0b8SIan Munsie * has detached and these were cleared by the PSL purge, but 178f204e0b8SIan Munsie * warn about it just in case */ 179f204e0b8SIan Munsie dev_notice(&ctx->afu->dev, "cxl_handle_fault: Translation fault regs changed\n"); 180f204e0b8SIan Munsie return; 181f204e0b8SIan Munsie } 182f204e0b8SIan Munsie 18313da7046SIan Munsie /* Early return if the context is being / has been detached */ 18413da7046SIan Munsie if (ctx->status == CLOSED) { 18513da7046SIan Munsie cxl_ack_ae(ctx); 18613da7046SIan Munsie return; 18713da7046SIan Munsie } 18813da7046SIan Munsie 189f204e0b8SIan Munsie pr_devel("CXL BOTTOM HALF handling fault for afu pe: %i. " 190f204e0b8SIan Munsie "DSISR: %#llx DAR: %#llx\n", ctx->pe, dsisr, dar); 191f204e0b8SIan Munsie 192f204e0b8SIan Munsie if (!(task = get_pid_task(ctx->pid, PIDTYPE_PID))) { 193f204e0b8SIan Munsie pr_devel("cxl_handle_fault unable to get task %i\n", 194f204e0b8SIan Munsie pid_nr(ctx->pid)); 195f204e0b8SIan Munsie cxl_ack_ae(ctx); 196f204e0b8SIan Munsie return; 197f204e0b8SIan Munsie } 198f204e0b8SIan Munsie if (!(mm = get_task_mm(task))) { 199f204e0b8SIan Munsie pr_devel("cxl_handle_fault unable to get mm %i\n", 200f204e0b8SIan Munsie pid_nr(ctx->pid)); 201f204e0b8SIan Munsie cxl_ack_ae(ctx); 202f204e0b8SIan Munsie goto out; 203f204e0b8SIan Munsie } 204f204e0b8SIan Munsie 205f204e0b8SIan Munsie if (dsisr & CXL_PSL_DSISR_An_DS) 206f204e0b8SIan Munsie cxl_handle_segment_miss(ctx, mm, dar); 207f204e0b8SIan Munsie else if (dsisr & CXL_PSL_DSISR_An_DM) 208f204e0b8SIan Munsie cxl_handle_page_fault(ctx, mm, dsisr, dar); 209f204e0b8SIan Munsie else 210f204e0b8SIan Munsie WARN(1, "cxl_handle_fault has nothing to handle\n"); 211f204e0b8SIan Munsie 212f204e0b8SIan Munsie mmput(mm); 213f204e0b8SIan Munsie out: 214f204e0b8SIan Munsie put_task_struct(task); 215f204e0b8SIan Munsie } 216f204e0b8SIan Munsie 217f204e0b8SIan Munsie static void cxl_prefault_one(struct cxl_context *ctx, u64 ea) 218f204e0b8SIan Munsie { 219f204e0b8SIan Munsie int rc; 220f204e0b8SIan Munsie struct task_struct *task; 221f204e0b8SIan Munsie struct mm_struct *mm; 222f204e0b8SIan Munsie 223f204e0b8SIan Munsie if (!(task = get_pid_task(ctx->pid, PIDTYPE_PID))) { 224f204e0b8SIan Munsie pr_devel("cxl_prefault_one unable to get task %i\n", 225f204e0b8SIan Munsie pid_nr(ctx->pid)); 226f204e0b8SIan Munsie return; 227f204e0b8SIan Munsie } 228f204e0b8SIan Munsie if (!(mm = get_task_mm(task))) { 229f204e0b8SIan Munsie pr_devel("cxl_prefault_one unable to get mm %i\n", 230f204e0b8SIan Munsie pid_nr(ctx->pid)); 231f204e0b8SIan Munsie put_task_struct(task); 232f204e0b8SIan Munsie return; 233f204e0b8SIan Munsie } 234f204e0b8SIan Munsie 235f204e0b8SIan Munsie rc = cxl_fault_segment(ctx, mm, ea); 236f204e0b8SIan Munsie 237f204e0b8SIan Munsie mmput(mm); 238f204e0b8SIan Munsie put_task_struct(task); 239f204e0b8SIan Munsie } 240f204e0b8SIan Munsie 241f204e0b8SIan Munsie static u64 next_segment(u64 ea, u64 vsid) 242f204e0b8SIan Munsie { 243f204e0b8SIan Munsie if (vsid & SLB_VSID_B_1T) 244f204e0b8SIan Munsie ea |= (1ULL << 40) - 1; 245f204e0b8SIan Munsie else 246f204e0b8SIan Munsie ea |= (1ULL << 28) - 1; 247f204e0b8SIan Munsie 248f204e0b8SIan Munsie return ea + 1; 249f204e0b8SIan Munsie } 250f204e0b8SIan Munsie 251f204e0b8SIan Munsie static void cxl_prefault_vma(struct cxl_context *ctx) 252f204e0b8SIan Munsie { 253f204e0b8SIan Munsie u64 ea, last_esid = 0; 254f204e0b8SIan Munsie struct copro_slb slb; 255f204e0b8SIan Munsie struct vm_area_struct *vma; 256f204e0b8SIan Munsie int rc; 257f204e0b8SIan Munsie struct task_struct *task; 258f204e0b8SIan Munsie struct mm_struct *mm; 259f204e0b8SIan Munsie 260f204e0b8SIan Munsie if (!(task = get_pid_task(ctx->pid, PIDTYPE_PID))) { 261f204e0b8SIan Munsie pr_devel("cxl_prefault_vma unable to get task %i\n", 262f204e0b8SIan Munsie pid_nr(ctx->pid)); 263f204e0b8SIan Munsie return; 264f204e0b8SIan Munsie } 265f204e0b8SIan Munsie if (!(mm = get_task_mm(task))) { 266f204e0b8SIan Munsie pr_devel("cxl_prefault_vm unable to get mm %i\n", 267f204e0b8SIan Munsie pid_nr(ctx->pid)); 268f204e0b8SIan Munsie goto out1; 269f204e0b8SIan Munsie } 270f204e0b8SIan Munsie 271f204e0b8SIan Munsie down_read(&mm->mmap_sem); 272f204e0b8SIan Munsie for (vma = mm->mmap; vma; vma = vma->vm_next) { 273f204e0b8SIan Munsie for (ea = vma->vm_start; ea < vma->vm_end; 274f204e0b8SIan Munsie ea = next_segment(ea, slb.vsid)) { 275f204e0b8SIan Munsie rc = copro_calculate_slb(mm, ea, &slb); 276f204e0b8SIan Munsie if (rc) 277f204e0b8SIan Munsie continue; 278f204e0b8SIan Munsie 279f204e0b8SIan Munsie if (last_esid == slb.esid) 280f204e0b8SIan Munsie continue; 281f204e0b8SIan Munsie 282f204e0b8SIan Munsie cxl_load_segment(ctx, &slb); 283f204e0b8SIan Munsie last_esid = slb.esid; 284f204e0b8SIan Munsie } 285f204e0b8SIan Munsie } 286f204e0b8SIan Munsie up_read(&mm->mmap_sem); 287f204e0b8SIan Munsie 288f204e0b8SIan Munsie mmput(mm); 289f204e0b8SIan Munsie out1: 290f204e0b8SIan Munsie put_task_struct(task); 291f204e0b8SIan Munsie } 292f204e0b8SIan Munsie 293f204e0b8SIan Munsie void cxl_prefault(struct cxl_context *ctx, u64 wed) 294f204e0b8SIan Munsie { 295f204e0b8SIan Munsie switch (ctx->afu->prefault_mode) { 296f204e0b8SIan Munsie case CXL_PREFAULT_WED: 297f204e0b8SIan Munsie cxl_prefault_one(ctx, wed); 298f204e0b8SIan Munsie break; 299f204e0b8SIan Munsie case CXL_PREFAULT_ALL: 300f204e0b8SIan Munsie cxl_prefault_vma(ctx); 301f204e0b8SIan Munsie break; 302f204e0b8SIan Munsie default: 303f204e0b8SIan Munsie break; 304f204e0b8SIan Munsie } 305f204e0b8SIan Munsie } 306