1 // SPDX-License-Identifier: GPL-2.0+ 2 // Copyright 2017 IBM Corp. 3 #include <linux/sched/mm.h> 4 #include "trace.h" 5 #include "ocxl_internal.h" 6 7 int ocxl_context_alloc(struct ocxl_context **context, struct ocxl_afu *afu, 8 struct address_space *mapping) 9 { 10 int pasid; 11 struct ocxl_context *ctx; 12 13 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 14 if (!ctx) 15 return -ENOMEM; 16 17 ctx->afu = afu; 18 mutex_lock(&afu->contexts_lock); 19 pasid = idr_alloc(&afu->contexts_idr, ctx, afu->pasid_base, 20 afu->pasid_base + afu->pasid_max, GFP_KERNEL); 21 if (pasid < 0) { 22 mutex_unlock(&afu->contexts_lock); 23 kfree(ctx); 24 return pasid; 25 } 26 afu->pasid_count++; 27 mutex_unlock(&afu->contexts_lock); 28 29 ctx->pasid = pasid; 30 ctx->status = OPENED; 31 mutex_init(&ctx->status_mutex); 32 ctx->mapping = mapping; 33 mutex_init(&ctx->mapping_lock); 34 init_waitqueue_head(&ctx->events_wq); 35 mutex_init(&ctx->xsl_error_lock); 36 mutex_init(&ctx->irq_lock); 37 idr_init(&ctx->irq_idr); 38 ctx->tidr = 0; 39 40 /* 41 * Keep a reference on the AFU to make sure it's valid for the 42 * duration of the life of the context 43 */ 44 ocxl_afu_get(afu); 45 *context = ctx; 46 return 0; 47 } 48 EXPORT_SYMBOL_GPL(ocxl_context_alloc); 49 50 /* 51 * Callback for when a translation fault triggers an error 52 * data: a pointer to the context which triggered the fault 53 * addr: the address that triggered the error 54 * dsisr: the value of the PPC64 dsisr register 55 */ 56 static void xsl_fault_error(void *data, u64 addr, u64 dsisr) 57 { 58 struct ocxl_context *ctx = (struct ocxl_context *) data; 59 60 mutex_lock(&ctx->xsl_error_lock); 61 ctx->xsl_error.addr = addr; 62 ctx->xsl_error.dsisr = dsisr; 63 ctx->xsl_error.count++; 64 mutex_unlock(&ctx->xsl_error_lock); 65 66 wake_up_all(&ctx->events_wq); 67 } 68 69 int ocxl_context_attach(struct ocxl_context *ctx, u64 amr, struct mm_struct *mm) 70 { 71 int rc; 72 unsigned long pidr = 0; 73 74 // Locks both status & tidr 75 mutex_lock(&ctx->status_mutex); 76 if (ctx->status != OPENED) { 77 rc = -EIO; 78 goto out; 79 } 80 81 if (mm) 82 pidr = mm->context.id; 83 84 rc = ocxl_link_add_pe(ctx->afu->fn->link, ctx->pasid, pidr, ctx->tidr, 85 amr, mm, xsl_fault_error, ctx); 86 if (rc) 87 goto out; 88 89 ctx->status = ATTACHED; 90 out: 91 mutex_unlock(&ctx->status_mutex); 92 return rc; 93 } 94 EXPORT_SYMBOL_GPL(ocxl_context_attach); 95 96 static vm_fault_t map_afu_irq(struct vm_area_struct *vma, unsigned long address, 97 u64 offset, struct ocxl_context *ctx) 98 { 99 u64 trigger_addr; 100 int irq_id = ocxl_irq_offset_to_id(ctx, offset); 101 102 trigger_addr = ocxl_afu_irq_get_addr(ctx, irq_id); 103 if (!trigger_addr) 104 return VM_FAULT_SIGBUS; 105 106 return vmf_insert_pfn(vma, address, trigger_addr >> PAGE_SHIFT); 107 } 108 109 static vm_fault_t map_pp_mmio(struct vm_area_struct *vma, unsigned long address, 110 u64 offset, struct ocxl_context *ctx) 111 { 112 u64 pp_mmio_addr; 113 int pasid_off; 114 vm_fault_t ret; 115 116 if (offset >= ctx->afu->config.pp_mmio_stride) 117 return VM_FAULT_SIGBUS; 118 119 mutex_lock(&ctx->status_mutex); 120 if (ctx->status != ATTACHED) { 121 mutex_unlock(&ctx->status_mutex); 122 pr_debug("%s: Context not attached, failing mmio mmap\n", 123 __func__); 124 return VM_FAULT_SIGBUS; 125 } 126 127 pasid_off = ctx->pasid - ctx->afu->pasid_base; 128 pp_mmio_addr = ctx->afu->pp_mmio_start + 129 pasid_off * ctx->afu->config.pp_mmio_stride + 130 offset; 131 132 ret = vmf_insert_pfn(vma, address, pp_mmio_addr >> PAGE_SHIFT); 133 mutex_unlock(&ctx->status_mutex); 134 return ret; 135 } 136 137 static vm_fault_t ocxl_mmap_fault(struct vm_fault *vmf) 138 { 139 struct vm_area_struct *vma = vmf->vma; 140 struct ocxl_context *ctx = vma->vm_file->private_data; 141 u64 offset; 142 vm_fault_t ret; 143 144 offset = vmf->pgoff << PAGE_SHIFT; 145 pr_debug("%s: pasid %d address 0x%lx offset 0x%llx\n", __func__, 146 ctx->pasid, vmf->address, offset); 147 148 if (offset < ctx->afu->irq_base_offset) 149 ret = map_pp_mmio(vma, vmf->address, offset, ctx); 150 else 151 ret = map_afu_irq(vma, vmf->address, offset, ctx); 152 return ret; 153 } 154 155 static const struct vm_operations_struct ocxl_vmops = { 156 .fault = ocxl_mmap_fault, 157 }; 158 159 static int check_mmap_afu_irq(struct ocxl_context *ctx, 160 struct vm_area_struct *vma) 161 { 162 int irq_id = ocxl_irq_offset_to_id(ctx, vma->vm_pgoff << PAGE_SHIFT); 163 164 /* only one page */ 165 if (vma_pages(vma) != 1) 166 return -EINVAL; 167 168 /* check offset validty */ 169 if (!ocxl_afu_irq_get_addr(ctx, irq_id)) 170 return -EINVAL; 171 172 /* 173 * trigger page should only be accessible in write mode. 174 * 175 * It's a bit theoretical, as a page mmaped with only 176 * PROT_WRITE is currently readable, but it doesn't hurt. 177 */ 178 if ((vma->vm_flags & VM_READ) || (vma->vm_flags & VM_EXEC) || 179 !(vma->vm_flags & VM_WRITE)) 180 return -EINVAL; 181 vma->vm_flags &= ~(VM_MAYREAD | VM_MAYEXEC); 182 return 0; 183 } 184 185 static int check_mmap_mmio(struct ocxl_context *ctx, 186 struct vm_area_struct *vma) 187 { 188 if ((vma_pages(vma) + vma->vm_pgoff) > 189 (ctx->afu->config.pp_mmio_stride >> PAGE_SHIFT)) 190 return -EINVAL; 191 return 0; 192 } 193 194 int ocxl_context_mmap(struct ocxl_context *ctx, struct vm_area_struct *vma) 195 { 196 int rc; 197 198 if ((vma->vm_pgoff << PAGE_SHIFT) < ctx->afu->irq_base_offset) 199 rc = check_mmap_mmio(ctx, vma); 200 else 201 rc = check_mmap_afu_irq(ctx, vma); 202 if (rc) 203 return rc; 204 205 vma->vm_flags |= VM_IO | VM_PFNMAP; 206 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 207 vma->vm_ops = &ocxl_vmops; 208 return 0; 209 } 210 211 int ocxl_context_detach(struct ocxl_context *ctx) 212 { 213 struct pci_dev *dev; 214 int afu_control_pos; 215 enum ocxl_context_status status; 216 int rc; 217 218 mutex_lock(&ctx->status_mutex); 219 status = ctx->status; 220 ctx->status = CLOSED; 221 mutex_unlock(&ctx->status_mutex); 222 if (status != ATTACHED) 223 return 0; 224 225 dev = to_pci_dev(ctx->afu->fn->dev.parent); 226 afu_control_pos = ctx->afu->config.dvsec_afu_control_pos; 227 228 mutex_lock(&ctx->afu->afu_control_lock); 229 rc = ocxl_config_terminate_pasid(dev, afu_control_pos, ctx->pasid); 230 mutex_unlock(&ctx->afu->afu_control_lock); 231 trace_ocxl_terminate_pasid(ctx->pasid, rc); 232 if (rc) { 233 /* 234 * If we timeout waiting for the AFU to terminate the 235 * pasid, then it's dangerous to clean up the Process 236 * Element entry in the SPA, as it may be referenced 237 * in the future by the AFU. In which case, we would 238 * checkstop because of an invalid PE access (FIR 239 * register 2, bit 42). So leave the PE 240 * defined. Caller shouldn't free the context so that 241 * PASID remains allocated. 242 * 243 * A link reset will be required to cleanup the AFU 244 * and the SPA. 245 */ 246 if (rc == -EBUSY) 247 return rc; 248 } 249 rc = ocxl_link_remove_pe(ctx->afu->fn->link, ctx->pasid); 250 if (rc) { 251 dev_warn(&dev->dev, 252 "Couldn't remove PE entry cleanly: %d\n", rc); 253 } 254 return 0; 255 } 256 EXPORT_SYMBOL_GPL(ocxl_context_detach); 257 258 void ocxl_context_detach_all(struct ocxl_afu *afu) 259 { 260 struct ocxl_context *ctx; 261 int tmp; 262 263 mutex_lock(&afu->contexts_lock); 264 idr_for_each_entry(&afu->contexts_idr, ctx, tmp) { 265 ocxl_context_detach(ctx); 266 /* 267 * We are force detaching - remove any active mmio 268 * mappings so userspace cannot interfere with the 269 * card if it comes back. Easiest way to exercise 270 * this is to unbind and rebind the driver via sysfs 271 * while it is in use. 272 */ 273 mutex_lock(&ctx->mapping_lock); 274 if (ctx->mapping) 275 unmap_mapping_range(ctx->mapping, 0, 0, 1); 276 mutex_unlock(&ctx->mapping_lock); 277 } 278 mutex_unlock(&afu->contexts_lock); 279 } 280 281 void ocxl_context_free(struct ocxl_context *ctx) 282 { 283 mutex_lock(&ctx->afu->contexts_lock); 284 ctx->afu->pasid_count--; 285 idr_remove(&ctx->afu->contexts_idr, ctx->pasid); 286 mutex_unlock(&ctx->afu->contexts_lock); 287 288 ocxl_afu_irq_free_all(ctx); 289 idr_destroy(&ctx->irq_idr); 290 /* reference to the AFU taken in ocxl_context_alloc() */ 291 ocxl_afu_put(ctx->afu); 292 kfree(ctx); 293 } 294 EXPORT_SYMBOL_GPL(ocxl_context_free); 295