1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2016-2018 Intel Corporation. All rights reserved. */ 3 #include <linux/memremap.h> 4 #include <linux/pagemap.h> 5 #include <linux/module.h> 6 #include <linux/device.h> 7 #include <linux/pfn_t.h> 8 #include <linux/cdev.h> 9 #include <linux/slab.h> 10 #include <linux/dax.h> 11 #include <linux/fs.h> 12 #include <linux/mm.h> 13 #include <linux/mman.h> 14 #include "dax-private.h" 15 #include "bus.h" 16 17 static int check_vma(struct dev_dax *dev_dax, struct vm_area_struct *vma, 18 const char *func) 19 { 20 struct device *dev = &dev_dax->dev; 21 unsigned long mask; 22 23 if (!dax_alive(dev_dax->dax_dev)) 24 return -ENXIO; 25 26 /* prevent private mappings from being established */ 27 if ((vma->vm_flags & VM_MAYSHARE) != VM_MAYSHARE) { 28 dev_info_ratelimited(dev, 29 "%s: %s: fail, attempted private mapping\n", 30 current->comm, func); 31 return -EINVAL; 32 } 33 34 mask = dev_dax->align - 1; 35 if (vma->vm_start & mask || vma->vm_end & mask) { 36 dev_info_ratelimited(dev, 37 "%s: %s: fail, unaligned vma (%#lx - %#lx, %#lx)\n", 38 current->comm, func, vma->vm_start, vma->vm_end, 39 mask); 40 return -EINVAL; 41 } 42 43 if (!vma_is_dax(vma)) { 44 dev_info_ratelimited(dev, 45 "%s: %s: fail, vma is not DAX capable\n", 46 current->comm, func); 47 return -EINVAL; 48 } 49 50 return 0; 51 } 52 53 /* see "strong" declaration in tools/testing/nvdimm/dax-dev.c */ 54 __weak phys_addr_t dax_pgoff_to_phys(struct dev_dax *dev_dax, pgoff_t pgoff, 55 unsigned long size) 56 { 57 int i; 58 59 for (i = 0; i < dev_dax->nr_range; i++) { 60 struct dev_dax_range *dax_range = &dev_dax->ranges[i]; 61 struct range *range = &dax_range->range; 62 unsigned long long pgoff_end; 63 phys_addr_t phys; 64 65 pgoff_end = dax_range->pgoff + PHYS_PFN(range_len(range)) - 1; 66 if (pgoff < dax_range->pgoff || pgoff > pgoff_end) 67 continue; 68 phys = PFN_PHYS(pgoff - dax_range->pgoff) + range->start; 69 if (phys + size - 1 <= range->end) 70 return phys; 71 break; 72 } 73 return -1; 74 } 75 76 static void dax_set_mapping(struct vm_fault *vmf, pfn_t pfn, 77 unsigned long fault_size) 78 { 79 unsigned long i, nr_pages = fault_size / PAGE_SIZE; 80 struct file *filp = vmf->vma->vm_file; 81 struct dev_dax *dev_dax = filp->private_data; 82 pgoff_t pgoff; 83 84 /* mapping is only set on the head */ 85 if (dev_dax->pgmap->vmemmap_shift) 86 nr_pages = 1; 87 88 pgoff = linear_page_index(vmf->vma, 89 ALIGN(vmf->address, fault_size)); 90 91 for (i = 0; i < nr_pages; i++) { 92 struct page *page = pfn_to_page(pfn_t_to_pfn(pfn) + i); 93 94 page = compound_head(page); 95 if (page->mapping) 96 continue; 97 98 page->mapping = filp->f_mapping; 99 page->index = pgoff + i; 100 } 101 } 102 103 static vm_fault_t __dev_dax_pte_fault(struct dev_dax *dev_dax, 104 struct vm_fault *vmf) 105 { 106 struct device *dev = &dev_dax->dev; 107 phys_addr_t phys; 108 pfn_t pfn; 109 unsigned int fault_size = PAGE_SIZE; 110 111 if (check_vma(dev_dax, vmf->vma, __func__)) 112 return VM_FAULT_SIGBUS; 113 114 if (dev_dax->align > PAGE_SIZE) { 115 dev_dbg(dev, "alignment (%#x) > fault size (%#x)\n", 116 dev_dax->align, fault_size); 117 return VM_FAULT_SIGBUS; 118 } 119 120 if (fault_size != dev_dax->align) 121 return VM_FAULT_SIGBUS; 122 123 phys = dax_pgoff_to_phys(dev_dax, vmf->pgoff, PAGE_SIZE); 124 if (phys == -1) { 125 dev_dbg(dev, "pgoff_to_phys(%#lx) failed\n", vmf->pgoff); 126 return VM_FAULT_SIGBUS; 127 } 128 129 pfn = phys_to_pfn_t(phys, PFN_DEV|PFN_MAP); 130 131 dax_set_mapping(vmf, pfn, fault_size); 132 133 return vmf_insert_mixed(vmf->vma, vmf->address, pfn); 134 } 135 136 static vm_fault_t __dev_dax_pmd_fault(struct dev_dax *dev_dax, 137 struct vm_fault *vmf) 138 { 139 unsigned long pmd_addr = vmf->address & PMD_MASK; 140 struct device *dev = &dev_dax->dev; 141 phys_addr_t phys; 142 pgoff_t pgoff; 143 pfn_t pfn; 144 unsigned int fault_size = PMD_SIZE; 145 146 if (check_vma(dev_dax, vmf->vma, __func__)) 147 return VM_FAULT_SIGBUS; 148 149 if (dev_dax->align > PMD_SIZE) { 150 dev_dbg(dev, "alignment (%#x) > fault size (%#x)\n", 151 dev_dax->align, fault_size); 152 return VM_FAULT_SIGBUS; 153 } 154 155 if (fault_size < dev_dax->align) 156 return VM_FAULT_SIGBUS; 157 else if (fault_size > dev_dax->align) 158 return VM_FAULT_FALLBACK; 159 160 /* if we are outside of the VMA */ 161 if (pmd_addr < vmf->vma->vm_start || 162 (pmd_addr + PMD_SIZE) > vmf->vma->vm_end) 163 return VM_FAULT_SIGBUS; 164 165 pgoff = linear_page_index(vmf->vma, pmd_addr); 166 phys = dax_pgoff_to_phys(dev_dax, pgoff, PMD_SIZE); 167 if (phys == -1) { 168 dev_dbg(dev, "pgoff_to_phys(%#lx) failed\n", pgoff); 169 return VM_FAULT_SIGBUS; 170 } 171 172 pfn = phys_to_pfn_t(phys, PFN_DEV|PFN_MAP); 173 174 dax_set_mapping(vmf, pfn, fault_size); 175 176 return vmf_insert_pfn_pmd(vmf, pfn, vmf->flags & FAULT_FLAG_WRITE); 177 } 178 179 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 180 static vm_fault_t __dev_dax_pud_fault(struct dev_dax *dev_dax, 181 struct vm_fault *vmf) 182 { 183 unsigned long pud_addr = vmf->address & PUD_MASK; 184 struct device *dev = &dev_dax->dev; 185 phys_addr_t phys; 186 pgoff_t pgoff; 187 pfn_t pfn; 188 unsigned int fault_size = PUD_SIZE; 189 190 191 if (check_vma(dev_dax, vmf->vma, __func__)) 192 return VM_FAULT_SIGBUS; 193 194 if (dev_dax->align > PUD_SIZE) { 195 dev_dbg(dev, "alignment (%#x) > fault size (%#x)\n", 196 dev_dax->align, fault_size); 197 return VM_FAULT_SIGBUS; 198 } 199 200 if (fault_size < dev_dax->align) 201 return VM_FAULT_SIGBUS; 202 else if (fault_size > dev_dax->align) 203 return VM_FAULT_FALLBACK; 204 205 /* if we are outside of the VMA */ 206 if (pud_addr < vmf->vma->vm_start || 207 (pud_addr + PUD_SIZE) > vmf->vma->vm_end) 208 return VM_FAULT_SIGBUS; 209 210 pgoff = linear_page_index(vmf->vma, pud_addr); 211 phys = dax_pgoff_to_phys(dev_dax, pgoff, PUD_SIZE); 212 if (phys == -1) { 213 dev_dbg(dev, "pgoff_to_phys(%#lx) failed\n", pgoff); 214 return VM_FAULT_SIGBUS; 215 } 216 217 pfn = phys_to_pfn_t(phys, PFN_DEV|PFN_MAP); 218 219 dax_set_mapping(vmf, pfn, fault_size); 220 221 return vmf_insert_pfn_pud(vmf, pfn, vmf->flags & FAULT_FLAG_WRITE); 222 } 223 #else 224 static vm_fault_t __dev_dax_pud_fault(struct dev_dax *dev_dax, 225 struct vm_fault *vmf) 226 { 227 return VM_FAULT_FALLBACK; 228 } 229 #endif /* !CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 230 231 static vm_fault_t dev_dax_huge_fault(struct vm_fault *vmf, 232 enum page_entry_size pe_size) 233 { 234 struct file *filp = vmf->vma->vm_file; 235 vm_fault_t rc = VM_FAULT_SIGBUS; 236 int id; 237 struct dev_dax *dev_dax = filp->private_data; 238 239 dev_dbg(&dev_dax->dev, "%s: %s (%#lx - %#lx) size = %d\n", current->comm, 240 (vmf->flags & FAULT_FLAG_WRITE) ? "write" : "read", 241 vmf->vma->vm_start, vmf->vma->vm_end, pe_size); 242 243 id = dax_read_lock(); 244 switch (pe_size) { 245 case PE_SIZE_PTE: 246 rc = __dev_dax_pte_fault(dev_dax, vmf); 247 break; 248 case PE_SIZE_PMD: 249 rc = __dev_dax_pmd_fault(dev_dax, vmf); 250 break; 251 case PE_SIZE_PUD: 252 rc = __dev_dax_pud_fault(dev_dax, vmf); 253 break; 254 default: 255 rc = VM_FAULT_SIGBUS; 256 } 257 258 dax_read_unlock(id); 259 260 return rc; 261 } 262 263 static vm_fault_t dev_dax_fault(struct vm_fault *vmf) 264 { 265 return dev_dax_huge_fault(vmf, PE_SIZE_PTE); 266 } 267 268 static int dev_dax_may_split(struct vm_area_struct *vma, unsigned long addr) 269 { 270 struct file *filp = vma->vm_file; 271 struct dev_dax *dev_dax = filp->private_data; 272 273 if (!IS_ALIGNED(addr, dev_dax->align)) 274 return -EINVAL; 275 return 0; 276 } 277 278 static unsigned long dev_dax_pagesize(struct vm_area_struct *vma) 279 { 280 struct file *filp = vma->vm_file; 281 struct dev_dax *dev_dax = filp->private_data; 282 283 return dev_dax->align; 284 } 285 286 static const struct vm_operations_struct dax_vm_ops = { 287 .fault = dev_dax_fault, 288 .huge_fault = dev_dax_huge_fault, 289 .may_split = dev_dax_may_split, 290 .pagesize = dev_dax_pagesize, 291 }; 292 293 static int dax_mmap(struct file *filp, struct vm_area_struct *vma) 294 { 295 struct dev_dax *dev_dax = filp->private_data; 296 int rc, id; 297 298 dev_dbg(&dev_dax->dev, "trace\n"); 299 300 /* 301 * We lock to check dax_dev liveness and will re-check at 302 * fault time. 303 */ 304 id = dax_read_lock(); 305 rc = check_vma(dev_dax, vma, __func__); 306 dax_read_unlock(id); 307 if (rc) 308 return rc; 309 310 vma->vm_ops = &dax_vm_ops; 311 vma->vm_flags |= VM_HUGEPAGE; 312 return 0; 313 } 314 315 /* return an unmapped area aligned to the dax region specified alignment */ 316 static unsigned long dax_get_unmapped_area(struct file *filp, 317 unsigned long addr, unsigned long len, unsigned long pgoff, 318 unsigned long flags) 319 { 320 unsigned long off, off_end, off_align, len_align, addr_align, align; 321 struct dev_dax *dev_dax = filp ? filp->private_data : NULL; 322 323 if (!dev_dax || addr) 324 goto out; 325 326 align = dev_dax->align; 327 off = pgoff << PAGE_SHIFT; 328 off_end = off + len; 329 off_align = round_up(off, align); 330 331 if ((off_end <= off_align) || ((off_end - off_align) < align)) 332 goto out; 333 334 len_align = len + align; 335 if ((off + len_align) < off) 336 goto out; 337 338 addr_align = current->mm->get_unmapped_area(filp, addr, len_align, 339 pgoff, flags); 340 if (!IS_ERR_VALUE(addr_align)) { 341 addr_align += (off - addr_align) & (align - 1); 342 return addr_align; 343 } 344 out: 345 return current->mm->get_unmapped_area(filp, addr, len, pgoff, flags); 346 } 347 348 static const struct address_space_operations dev_dax_aops = { 349 .set_page_dirty = __set_page_dirty_no_writeback, 350 .invalidatepage = noop_invalidatepage, 351 }; 352 353 static int dax_open(struct inode *inode, struct file *filp) 354 { 355 struct dax_device *dax_dev = inode_dax(inode); 356 struct inode *__dax_inode = dax_inode(dax_dev); 357 struct dev_dax *dev_dax = dax_get_private(dax_dev); 358 359 dev_dbg(&dev_dax->dev, "trace\n"); 360 inode->i_mapping = __dax_inode->i_mapping; 361 inode->i_mapping->host = __dax_inode; 362 inode->i_mapping->a_ops = &dev_dax_aops; 363 filp->f_mapping = inode->i_mapping; 364 filp->f_wb_err = filemap_sample_wb_err(filp->f_mapping); 365 filp->f_sb_err = file_sample_sb_err(filp); 366 filp->private_data = dev_dax; 367 inode->i_flags = S_DAX; 368 369 return 0; 370 } 371 372 static int dax_release(struct inode *inode, struct file *filp) 373 { 374 struct dev_dax *dev_dax = filp->private_data; 375 376 dev_dbg(&dev_dax->dev, "trace\n"); 377 return 0; 378 } 379 380 static const struct file_operations dax_fops = { 381 .llseek = noop_llseek, 382 .owner = THIS_MODULE, 383 .open = dax_open, 384 .release = dax_release, 385 .get_unmapped_area = dax_get_unmapped_area, 386 .mmap = dax_mmap, 387 .mmap_supported_flags = MAP_SYNC, 388 }; 389 390 static void dev_dax_cdev_del(void *cdev) 391 { 392 cdev_del(cdev); 393 } 394 395 static void dev_dax_kill(void *dev_dax) 396 { 397 kill_dev_dax(dev_dax); 398 } 399 400 int dev_dax_probe(struct dev_dax *dev_dax) 401 { 402 struct dax_device *dax_dev = dev_dax->dax_dev; 403 struct device *dev = &dev_dax->dev; 404 struct dev_pagemap *pgmap; 405 struct inode *inode; 406 struct cdev *cdev; 407 void *addr; 408 int rc, i; 409 410 if (static_dev_dax(dev_dax)) { 411 if (dev_dax->nr_range > 1) { 412 dev_warn(dev, 413 "static pgmap / multi-range device conflict\n"); 414 return -EINVAL; 415 } 416 417 pgmap = dev_dax->pgmap; 418 } else { 419 if (dev_dax->pgmap) { 420 dev_warn(dev, 421 "dynamic-dax with pre-populated page map\n"); 422 return -EINVAL; 423 } 424 425 pgmap = devm_kzalloc(dev, 426 struct_size(pgmap, ranges, dev_dax->nr_range - 1), 427 GFP_KERNEL); 428 if (!pgmap) 429 return -ENOMEM; 430 431 pgmap->nr_range = dev_dax->nr_range; 432 dev_dax->pgmap = pgmap; 433 434 for (i = 0; i < dev_dax->nr_range; i++) { 435 struct range *range = &dev_dax->ranges[i].range; 436 pgmap->ranges[i] = *range; 437 } 438 } 439 440 for (i = 0; i < dev_dax->nr_range; i++) { 441 struct range *range = &dev_dax->ranges[i].range; 442 443 if (!devm_request_mem_region(dev, range->start, 444 range_len(range), dev_name(dev))) { 445 dev_warn(dev, "mapping%d: %#llx-%#llx could not reserve range\n", 446 i, range->start, range->end); 447 return -EBUSY; 448 } 449 } 450 451 pgmap->type = MEMORY_DEVICE_GENERIC; 452 if (dev_dax->align > PAGE_SIZE) 453 pgmap->vmemmap_shift = 454 order_base_2(dev_dax->align >> PAGE_SHIFT); 455 addr = devm_memremap_pages(dev, pgmap); 456 if (IS_ERR(addr)) 457 return PTR_ERR(addr); 458 459 inode = dax_inode(dax_dev); 460 cdev = inode->i_cdev; 461 cdev_init(cdev, &dax_fops); 462 cdev->owner = dev->driver->owner; 463 cdev_set_parent(cdev, &dev->kobj); 464 rc = cdev_add(cdev, dev->devt, 1); 465 if (rc) 466 return rc; 467 468 rc = devm_add_action_or_reset(dev, dev_dax_cdev_del, cdev); 469 if (rc) 470 return rc; 471 472 run_dax(dax_dev); 473 return devm_add_action_or_reset(dev, dev_dax_kill, dev_dax); 474 } 475 EXPORT_SYMBOL_GPL(dev_dax_probe); 476 477 static struct dax_device_driver device_dax_driver = { 478 .probe = dev_dax_probe, 479 /* all probe actions are unwound by devm, so .remove isn't necessary */ 480 .match_always = 1, 481 }; 482 483 static int __init dax_init(void) 484 { 485 return dax_driver_register(&device_dax_driver); 486 } 487 488 static void __exit dax_exit(void) 489 { 490 dax_driver_unregister(&device_dax_driver); 491 } 492 493 MODULE_AUTHOR("Intel Corporation"); 494 MODULE_LICENSE("GPL v2"); 495 module_init(dax_init); 496 module_exit(dax_exit); 497 MODULE_ALIAS_DAX_DEVICE(0); 498