1 /* 2 * Copyright 2014 IBM Corp. 3 * 4 * This program is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU General Public License 6 * as published by the Free Software Foundation; either version 7 * 2 of the License, or (at your option) any later version. 8 */ 9 10 #include <linux/workqueue.h> 11 #include <linux/sched.h> 12 #include <linux/pid.h> 13 #include <linux/mm.h> 14 #include <linux/moduleparam.h> 15 16 #undef MODULE_PARAM_PREFIX 17 #define MODULE_PARAM_PREFIX "cxl" "." 18 #include <asm/current.h> 19 #include <asm/copro.h> 20 #include <asm/mmu.h> 21 22 #include "cxl.h" 23 24 static struct cxl_sste* find_free_sste(struct cxl_sste *primary_group, 25 bool sec_hash, 26 struct cxl_sste *secondary_group, 27 unsigned int *lru) 28 { 29 unsigned int i, entry; 30 struct cxl_sste *sste, *group = primary_group; 31 32 for (i = 0; i < 2; i++) { 33 for (entry = 0; entry < 8; entry++) { 34 sste = group + entry; 35 if (!(be64_to_cpu(sste->esid_data) & SLB_ESID_V)) 36 return sste; 37 } 38 if (!sec_hash) 39 break; 40 group = secondary_group; 41 } 42 /* Nothing free, select an entry to cast out */ 43 if (sec_hash && (*lru & 0x8)) 44 sste = secondary_group + (*lru & 0x7); 45 else 46 sste = primary_group + (*lru & 0x7); 47 *lru = (*lru + 1) & 0xf; 48 49 return sste; 50 } 51 52 static void cxl_load_segment(struct cxl_context *ctx, struct copro_slb *slb) 53 { 54 /* mask is the group index, we search primary and secondary here. */ 55 unsigned int mask = (ctx->sst_size >> 7)-1; /* SSTP0[SegTableSize] */ 56 bool sec_hash = 1; 57 struct cxl_sste *sste; 58 unsigned int hash; 59 unsigned long flags; 60 61 62 sec_hash = !!(cxl_p1n_read(ctx->afu, CXL_PSL_SR_An) & CXL_PSL_SR_An_SC); 63 64 if (slb->vsid & SLB_VSID_B_1T) 65 hash = (slb->esid >> SID_SHIFT_1T) & mask; 66 else /* 256M */ 67 hash = (slb->esid >> SID_SHIFT) & mask; 68 69 spin_lock_irqsave(&ctx->sste_lock, flags); 70 sste = find_free_sste(ctx->sstp + (hash << 3), sec_hash, 71 ctx->sstp + ((~hash & mask) << 3), &ctx->sst_lru); 72 73 pr_devel("CXL Populating SST[%li]: %#llx %#llx\n", 74 sste - ctx->sstp, slb->vsid, slb->esid); 75 76 sste->vsid_data = cpu_to_be64(slb->vsid); 77 sste->esid_data = cpu_to_be64(slb->esid); 78 spin_unlock_irqrestore(&ctx->sste_lock, flags); 79 } 80 81 static int cxl_fault_segment(struct cxl_context *ctx, struct mm_struct *mm, 82 u64 ea) 83 { 84 struct copro_slb slb = {0,0}; 85 int rc; 86 87 if (!(rc = copro_calculate_slb(mm, ea, &slb))) { 88 cxl_load_segment(ctx, &slb); 89 } 90 91 return rc; 92 } 93 94 static void cxl_ack_ae(struct cxl_context *ctx) 95 { 96 unsigned long flags; 97 98 cxl_ack_irq(ctx, CXL_PSL_TFC_An_AE, 0); 99 100 spin_lock_irqsave(&ctx->lock, flags); 101 ctx->pending_fault = true; 102 ctx->fault_addr = ctx->dar; 103 ctx->fault_dsisr = ctx->dsisr; 104 spin_unlock_irqrestore(&ctx->lock, flags); 105 106 wake_up_all(&ctx->wq); 107 } 108 109 static int cxl_handle_segment_miss(struct cxl_context *ctx, 110 struct mm_struct *mm, u64 ea) 111 { 112 int rc; 113 114 pr_devel("CXL interrupt: Segment fault pe: %i ea: %#llx\n", ctx->pe, ea); 115 116 if ((rc = cxl_fault_segment(ctx, mm, ea))) 117 cxl_ack_ae(ctx); 118 else { 119 120 mb(); /* Order seg table write to TFC MMIO write */ 121 cxl_ack_irq(ctx, CXL_PSL_TFC_An_R, 0); 122 } 123 124 return IRQ_HANDLED; 125 } 126 127 static void cxl_handle_page_fault(struct cxl_context *ctx, 128 struct mm_struct *mm, u64 dsisr, u64 dar) 129 { 130 unsigned flt = 0; 131 int result; 132 unsigned long access, flags; 133 134 if ((result = copro_handle_mm_fault(mm, dar, dsisr, &flt))) { 135 pr_devel("copro_handle_mm_fault failed: %#x\n", result); 136 return cxl_ack_ae(ctx); 137 } 138 139 /* 140 * update_mmu_cache() will not have loaded the hash since current->trap 141 * is not a 0x400 or 0x300, so just call hash_page_mm() here. 142 */ 143 access = _PAGE_PRESENT; 144 if (dsisr & CXL_PSL_DSISR_An_S) 145 access |= _PAGE_RW; 146 if ((!ctx->kernel) || ~(dar & (1ULL << 63))) 147 access |= _PAGE_USER; 148 local_irq_save(flags); 149 hash_page_mm(mm, dar, access, 0x300); 150 local_irq_restore(flags); 151 152 pr_devel("Page fault successfully handled for pe: %i!\n", ctx->pe); 153 cxl_ack_irq(ctx, CXL_PSL_TFC_An_R, 0); 154 } 155 156 void cxl_handle_fault(struct work_struct *fault_work) 157 { 158 struct cxl_context *ctx = 159 container_of(fault_work, struct cxl_context, fault_work); 160 u64 dsisr = ctx->dsisr; 161 u64 dar = ctx->dar; 162 struct task_struct *task; 163 struct mm_struct *mm; 164 165 if (cxl_p2n_read(ctx->afu, CXL_PSL_DSISR_An) != dsisr || 166 cxl_p2n_read(ctx->afu, CXL_PSL_DAR_An) != dar || 167 cxl_p2n_read(ctx->afu, CXL_PSL_PEHandle_An) != ctx->pe) { 168 /* Most likely explanation is harmless - a dedicated process 169 * has detached and these were cleared by the PSL purge, but 170 * warn about it just in case */ 171 dev_notice(&ctx->afu->dev, "cxl_handle_fault: Translation fault regs changed\n"); 172 return; 173 } 174 175 pr_devel("CXL BOTTOM HALF handling fault for afu pe: %i. " 176 "DSISR: %#llx DAR: %#llx\n", ctx->pe, dsisr, dar); 177 178 if (!(task = get_pid_task(ctx->pid, PIDTYPE_PID))) { 179 pr_devel("cxl_handle_fault unable to get task %i\n", 180 pid_nr(ctx->pid)); 181 cxl_ack_ae(ctx); 182 return; 183 } 184 if (!(mm = get_task_mm(task))) { 185 pr_devel("cxl_handle_fault unable to get mm %i\n", 186 pid_nr(ctx->pid)); 187 cxl_ack_ae(ctx); 188 goto out; 189 } 190 191 if (dsisr & CXL_PSL_DSISR_An_DS) 192 cxl_handle_segment_miss(ctx, mm, dar); 193 else if (dsisr & CXL_PSL_DSISR_An_DM) 194 cxl_handle_page_fault(ctx, mm, dsisr, dar); 195 else 196 WARN(1, "cxl_handle_fault has nothing to handle\n"); 197 198 mmput(mm); 199 out: 200 put_task_struct(task); 201 } 202 203 static void cxl_prefault_one(struct cxl_context *ctx, u64 ea) 204 { 205 int rc; 206 struct task_struct *task; 207 struct mm_struct *mm; 208 209 if (!(task = get_pid_task(ctx->pid, PIDTYPE_PID))) { 210 pr_devel("cxl_prefault_one unable to get task %i\n", 211 pid_nr(ctx->pid)); 212 return; 213 } 214 if (!(mm = get_task_mm(task))) { 215 pr_devel("cxl_prefault_one unable to get mm %i\n", 216 pid_nr(ctx->pid)); 217 put_task_struct(task); 218 return; 219 } 220 221 rc = cxl_fault_segment(ctx, mm, ea); 222 223 mmput(mm); 224 put_task_struct(task); 225 } 226 227 static u64 next_segment(u64 ea, u64 vsid) 228 { 229 if (vsid & SLB_VSID_B_1T) 230 ea |= (1ULL << 40) - 1; 231 else 232 ea |= (1ULL << 28) - 1; 233 234 return ea + 1; 235 } 236 237 static void cxl_prefault_vma(struct cxl_context *ctx) 238 { 239 u64 ea, last_esid = 0; 240 struct copro_slb slb; 241 struct vm_area_struct *vma; 242 int rc; 243 struct task_struct *task; 244 struct mm_struct *mm; 245 246 if (!(task = get_pid_task(ctx->pid, PIDTYPE_PID))) { 247 pr_devel("cxl_prefault_vma unable to get task %i\n", 248 pid_nr(ctx->pid)); 249 return; 250 } 251 if (!(mm = get_task_mm(task))) { 252 pr_devel("cxl_prefault_vm unable to get mm %i\n", 253 pid_nr(ctx->pid)); 254 goto out1; 255 } 256 257 down_read(&mm->mmap_sem); 258 for (vma = mm->mmap; vma; vma = vma->vm_next) { 259 for (ea = vma->vm_start; ea < vma->vm_end; 260 ea = next_segment(ea, slb.vsid)) { 261 rc = copro_calculate_slb(mm, ea, &slb); 262 if (rc) 263 continue; 264 265 if (last_esid == slb.esid) 266 continue; 267 268 cxl_load_segment(ctx, &slb); 269 last_esid = slb.esid; 270 } 271 } 272 up_read(&mm->mmap_sem); 273 274 mmput(mm); 275 out1: 276 put_task_struct(task); 277 } 278 279 void cxl_prefault(struct cxl_context *ctx, u64 wed) 280 { 281 switch (ctx->afu->prefault_mode) { 282 case CXL_PREFAULT_WED: 283 cxl_prefault_one(ctx, wed); 284 break; 285 case CXL_PREFAULT_ALL: 286 cxl_prefault_vma(ctx); 287 break; 288 default: 289 break; 290 } 291 } 292