1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2018 Christoph Hellwig. 4 * 5 * DMA operations that map physical memory directly without using an IOMMU. 6 */ 7 #include <linux/memblock.h> /* for max_pfn */ 8 #include <linux/export.h> 9 #include <linux/mm.h> 10 #include <linux/dma-direct.h> 11 #include <linux/scatterlist.h> 12 #include <linux/dma-contiguous.h> 13 #include <linux/dma-noncoherent.h> 14 #include <linux/pfn.h> 15 #include <linux/vmalloc.h> 16 #include <linux/set_memory.h> 17 #include <linux/swiotlb.h> 18 19 /* 20 * Most architectures use ZONE_DMA for the first 16 Megabytes, but some use it 21 * it for entirely different regions. In that case the arch code needs to 22 * override the variable below for dma-direct to work properly. 23 */ 24 unsigned int zone_dma_bits __ro_after_init = 24; 25 26 static inline dma_addr_t phys_to_dma_direct(struct device *dev, 27 phys_addr_t phys) 28 { 29 if (force_dma_unencrypted(dev)) 30 return __phys_to_dma(dev, phys); 31 return phys_to_dma(dev, phys); 32 } 33 34 static inline struct page *dma_direct_to_page(struct device *dev, 35 dma_addr_t dma_addr) 36 { 37 return pfn_to_page(PHYS_PFN(dma_to_phys(dev, dma_addr))); 38 } 39 40 u64 dma_direct_get_required_mask(struct device *dev) 41 { 42 phys_addr_t phys = (phys_addr_t)(max_pfn - 1) << PAGE_SHIFT; 43 u64 max_dma = phys_to_dma_direct(dev, phys); 44 45 return (1ULL << (fls64(max_dma) - 1)) * 2 - 1; 46 } 47 48 gfp_t dma_direct_optimal_gfp_mask(struct device *dev, u64 dma_mask, 49 u64 *phys_limit) 50 { 51 u64 dma_limit = min_not_zero(dma_mask, dev->bus_dma_limit); 52 53 if (force_dma_unencrypted(dev)) 54 *phys_limit = __dma_to_phys(dev, dma_limit); 55 else 56 *phys_limit = dma_to_phys(dev, dma_limit); 57 58 /* 59 * Optimistically try the zone that the physical address mask falls 60 * into first. If that returns memory that isn't actually addressable 61 * we will fallback to the next lower zone and try again. 62 * 63 * Note that GFP_DMA32 and GFP_DMA are no ops without the corresponding 64 * zones. 65 */ 66 if (*phys_limit <= DMA_BIT_MASK(zone_dma_bits)) 67 return GFP_DMA; 68 if (*phys_limit <= DMA_BIT_MASK(32)) 69 return GFP_DMA32; 70 return 0; 71 } 72 73 static bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size) 74 { 75 return phys_to_dma_direct(dev, phys) + size - 1 <= 76 min_not_zero(dev->coherent_dma_mask, dev->bus_dma_limit); 77 } 78 79 /* 80 * Decrypting memory is allowed to block, so if this device requires 81 * unencrypted memory it must come from atomic pools. 82 */ 83 static inline bool dma_should_alloc_from_pool(struct device *dev, gfp_t gfp, 84 unsigned long attrs) 85 { 86 if (!IS_ENABLED(CONFIG_DMA_COHERENT_POOL)) 87 return false; 88 if (gfpflags_allow_blocking(gfp)) 89 return false; 90 if (force_dma_unencrypted(dev)) 91 return true; 92 if (!IS_ENABLED(CONFIG_DMA_DIRECT_REMAP)) 93 return false; 94 if (dma_alloc_need_uncached(dev, attrs)) 95 return true; 96 return false; 97 } 98 99 static inline bool dma_should_free_from_pool(struct device *dev, 100 unsigned long attrs) 101 { 102 if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL)) 103 return true; 104 if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) && 105 !force_dma_unencrypted(dev)) 106 return false; 107 if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP)) 108 return true; 109 return false; 110 } 111 112 struct page *__dma_direct_alloc_pages(struct device *dev, size_t size, 113 gfp_t gfp, unsigned long attrs) 114 { 115 size_t alloc_size = PAGE_ALIGN(size); 116 int node = dev_to_node(dev); 117 struct page *page = NULL; 118 u64 phys_limit; 119 120 if (attrs & DMA_ATTR_NO_WARN) 121 gfp |= __GFP_NOWARN; 122 123 /* we always manually zero the memory once we are done: */ 124 gfp &= ~__GFP_ZERO; 125 gfp |= dma_direct_optimal_gfp_mask(dev, dev->coherent_dma_mask, 126 &phys_limit); 127 page = dma_alloc_contiguous(dev, alloc_size, gfp); 128 if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) { 129 dma_free_contiguous(dev, page, alloc_size); 130 page = NULL; 131 } 132 again: 133 if (!page) 134 page = alloc_pages_node(node, gfp, get_order(alloc_size)); 135 if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) { 136 dma_free_contiguous(dev, page, size); 137 page = NULL; 138 139 if (IS_ENABLED(CONFIG_ZONE_DMA32) && 140 phys_limit < DMA_BIT_MASK(64) && 141 !(gfp & (GFP_DMA32 | GFP_DMA))) { 142 gfp |= GFP_DMA32; 143 goto again; 144 } 145 146 if (IS_ENABLED(CONFIG_ZONE_DMA) && !(gfp & GFP_DMA)) { 147 gfp = (gfp & ~GFP_DMA32) | GFP_DMA; 148 goto again; 149 } 150 } 151 152 return page; 153 } 154 155 void *dma_direct_alloc_pages(struct device *dev, size_t size, 156 dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) 157 { 158 struct page *page; 159 void *ret; 160 161 if (dma_should_alloc_from_pool(dev, gfp, attrs)) { 162 ret = dma_alloc_from_pool(dev, PAGE_ALIGN(size), &page, gfp); 163 if (!ret) 164 return NULL; 165 goto done; 166 } 167 168 page = __dma_direct_alloc_pages(dev, size, gfp, attrs); 169 if (!page) 170 return NULL; 171 172 if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) && 173 !force_dma_unencrypted(dev)) { 174 /* remove any dirty cache lines on the kernel alias */ 175 if (!PageHighMem(page)) 176 arch_dma_prep_coherent(page, size); 177 /* return the page pointer as the opaque cookie */ 178 ret = page; 179 goto done; 180 } 181 182 if ((IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) && 183 dma_alloc_need_uncached(dev, attrs)) || 184 (IS_ENABLED(CONFIG_DMA_REMAP) && PageHighMem(page))) { 185 /* remove any dirty cache lines on the kernel alias */ 186 arch_dma_prep_coherent(page, PAGE_ALIGN(size)); 187 188 /* create a coherent mapping */ 189 ret = dma_common_contiguous_remap(page, PAGE_ALIGN(size), 190 dma_pgprot(dev, PAGE_KERNEL, attrs), 191 __builtin_return_address(0)); 192 if (!ret) 193 goto out_free_pages; 194 memset(ret, 0, size); 195 goto done; 196 } 197 198 if (PageHighMem(page)) { 199 /* 200 * Depending on the cma= arguments and per-arch setup 201 * dma_alloc_contiguous could return highmem pages. 202 * Without remapping there is no way to return them here, 203 * so log an error and fail. 204 */ 205 dev_info(dev, "Rejecting highmem page from CMA.\n"); 206 goto out_free_pages; 207 } 208 209 ret = page_address(page); 210 if (force_dma_unencrypted(dev)) 211 set_memory_decrypted((unsigned long)ret, 1 << get_order(size)); 212 213 memset(ret, 0, size); 214 215 if (IS_ENABLED(CONFIG_ARCH_HAS_DMA_SET_UNCACHED) && 216 dma_alloc_need_uncached(dev, attrs)) { 217 arch_dma_prep_coherent(page, size); 218 ret = arch_dma_set_uncached(ret, size); 219 if (IS_ERR(ret)) 220 goto out_free_pages; 221 } 222 done: 223 if (force_dma_unencrypted(dev)) 224 *dma_handle = __phys_to_dma(dev, page_to_phys(page)); 225 else 226 *dma_handle = phys_to_dma(dev, page_to_phys(page)); 227 return ret; 228 out_free_pages: 229 dma_free_contiguous(dev, page, size); 230 return NULL; 231 } 232 233 void dma_direct_free_pages(struct device *dev, size_t size, void *cpu_addr, 234 dma_addr_t dma_addr, unsigned long attrs) 235 { 236 unsigned int page_order = get_order(size); 237 238 /* If cpu_addr is not from an atomic pool, dma_free_from_pool() fails */ 239 if (dma_should_free_from_pool(dev, attrs) && 240 dma_free_from_pool(dev, cpu_addr, PAGE_ALIGN(size))) 241 return; 242 243 if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) && 244 !force_dma_unencrypted(dev)) { 245 /* cpu_addr is a struct page cookie, not a kernel address */ 246 dma_free_contiguous(dev, cpu_addr, size); 247 return; 248 } 249 250 if (force_dma_unencrypted(dev)) 251 set_memory_encrypted((unsigned long)cpu_addr, 1 << page_order); 252 253 if (IS_ENABLED(CONFIG_DMA_REMAP) && is_vmalloc_addr(cpu_addr)) 254 vunmap(cpu_addr); 255 else if (IS_ENABLED(CONFIG_ARCH_HAS_DMA_CLEAR_UNCACHED)) 256 arch_dma_clear_uncached(cpu_addr, size); 257 258 dma_free_contiguous(dev, dma_direct_to_page(dev, dma_addr), size); 259 } 260 261 void *dma_direct_alloc(struct device *dev, size_t size, 262 dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) 263 { 264 if (!IS_ENABLED(CONFIG_ARCH_HAS_DMA_SET_UNCACHED) && 265 !IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) && 266 dma_alloc_need_uncached(dev, attrs)) 267 return arch_dma_alloc(dev, size, dma_handle, gfp, attrs); 268 return dma_direct_alloc_pages(dev, size, dma_handle, gfp, attrs); 269 } 270 271 void dma_direct_free(struct device *dev, size_t size, 272 void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs) 273 { 274 if (!IS_ENABLED(CONFIG_ARCH_HAS_DMA_SET_UNCACHED) && 275 !IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) && 276 dma_alloc_need_uncached(dev, attrs)) 277 arch_dma_free(dev, size, cpu_addr, dma_addr, attrs); 278 else 279 dma_direct_free_pages(dev, size, cpu_addr, dma_addr, attrs); 280 } 281 282 #if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \ 283 defined(CONFIG_SWIOTLB) 284 void dma_direct_sync_single_for_device(struct device *dev, 285 dma_addr_t addr, size_t size, enum dma_data_direction dir) 286 { 287 phys_addr_t paddr = dma_to_phys(dev, addr); 288 289 if (unlikely(is_swiotlb_buffer(paddr))) 290 swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_DEVICE); 291 292 if (!dev_is_dma_coherent(dev)) 293 arch_sync_dma_for_device(paddr, size, dir); 294 } 295 EXPORT_SYMBOL(dma_direct_sync_single_for_device); 296 297 void dma_direct_sync_sg_for_device(struct device *dev, 298 struct scatterlist *sgl, int nents, enum dma_data_direction dir) 299 { 300 struct scatterlist *sg; 301 int i; 302 303 for_each_sg(sgl, sg, nents, i) { 304 phys_addr_t paddr = dma_to_phys(dev, sg_dma_address(sg)); 305 306 if (unlikely(is_swiotlb_buffer(paddr))) 307 swiotlb_tbl_sync_single(dev, paddr, sg->length, 308 dir, SYNC_FOR_DEVICE); 309 310 if (!dev_is_dma_coherent(dev)) 311 arch_sync_dma_for_device(paddr, sg->length, 312 dir); 313 } 314 } 315 EXPORT_SYMBOL(dma_direct_sync_sg_for_device); 316 #endif 317 318 #if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \ 319 defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) || \ 320 defined(CONFIG_SWIOTLB) 321 void dma_direct_sync_single_for_cpu(struct device *dev, 322 dma_addr_t addr, size_t size, enum dma_data_direction dir) 323 { 324 phys_addr_t paddr = dma_to_phys(dev, addr); 325 326 if (!dev_is_dma_coherent(dev)) { 327 arch_sync_dma_for_cpu(paddr, size, dir); 328 arch_sync_dma_for_cpu_all(); 329 } 330 331 if (unlikely(is_swiotlb_buffer(paddr))) 332 swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_CPU); 333 } 334 EXPORT_SYMBOL(dma_direct_sync_single_for_cpu); 335 336 void dma_direct_sync_sg_for_cpu(struct device *dev, 337 struct scatterlist *sgl, int nents, enum dma_data_direction dir) 338 { 339 struct scatterlist *sg; 340 int i; 341 342 for_each_sg(sgl, sg, nents, i) { 343 phys_addr_t paddr = dma_to_phys(dev, sg_dma_address(sg)); 344 345 if (!dev_is_dma_coherent(dev)) 346 arch_sync_dma_for_cpu(paddr, sg->length, dir); 347 348 if (unlikely(is_swiotlb_buffer(paddr))) 349 swiotlb_tbl_sync_single(dev, paddr, sg->length, dir, 350 SYNC_FOR_CPU); 351 } 352 353 if (!dev_is_dma_coherent(dev)) 354 arch_sync_dma_for_cpu_all(); 355 } 356 EXPORT_SYMBOL(dma_direct_sync_sg_for_cpu); 357 358 void dma_direct_unmap_page(struct device *dev, dma_addr_t addr, 359 size_t size, enum dma_data_direction dir, unsigned long attrs) 360 { 361 phys_addr_t phys = dma_to_phys(dev, addr); 362 363 if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) 364 dma_direct_sync_single_for_cpu(dev, addr, size, dir); 365 366 if (unlikely(is_swiotlb_buffer(phys))) 367 swiotlb_tbl_unmap_single(dev, phys, size, size, dir, attrs); 368 } 369 EXPORT_SYMBOL(dma_direct_unmap_page); 370 371 void dma_direct_unmap_sg(struct device *dev, struct scatterlist *sgl, 372 int nents, enum dma_data_direction dir, unsigned long attrs) 373 { 374 struct scatterlist *sg; 375 int i; 376 377 for_each_sg(sgl, sg, nents, i) 378 dma_direct_unmap_page(dev, sg->dma_address, sg_dma_len(sg), dir, 379 attrs); 380 } 381 EXPORT_SYMBOL(dma_direct_unmap_sg); 382 #endif 383 384 dma_addr_t dma_direct_map_page(struct device *dev, struct page *page, 385 unsigned long offset, size_t size, enum dma_data_direction dir, 386 unsigned long attrs) 387 { 388 phys_addr_t phys = page_to_phys(page) + offset; 389 dma_addr_t dma_addr = phys_to_dma(dev, phys); 390 391 if (unlikely(swiotlb_force == SWIOTLB_FORCE)) 392 return swiotlb_map(dev, phys, size, dir, attrs); 393 394 if (unlikely(!dma_capable(dev, dma_addr, size, true))) { 395 if (swiotlb_force != SWIOTLB_NO_FORCE) 396 return swiotlb_map(dev, phys, size, dir, attrs); 397 398 dev_WARN_ONCE(dev, 1, 399 "DMA addr %pad+%zu overflow (mask %llx, bus limit %llx).\n", 400 &dma_addr, size, *dev->dma_mask, dev->bus_dma_limit); 401 return DMA_MAPPING_ERROR; 402 } 403 404 if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) 405 arch_sync_dma_for_device(phys, size, dir); 406 return dma_addr; 407 } 408 EXPORT_SYMBOL(dma_direct_map_page); 409 410 int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents, 411 enum dma_data_direction dir, unsigned long attrs) 412 { 413 int i; 414 struct scatterlist *sg; 415 416 for_each_sg(sgl, sg, nents, i) { 417 sg->dma_address = dma_direct_map_page(dev, sg_page(sg), 418 sg->offset, sg->length, dir, attrs); 419 if (sg->dma_address == DMA_MAPPING_ERROR) 420 goto out_unmap; 421 sg_dma_len(sg) = sg->length; 422 } 423 424 return nents; 425 426 out_unmap: 427 dma_direct_unmap_sg(dev, sgl, i, dir, attrs | DMA_ATTR_SKIP_CPU_SYNC); 428 return 0; 429 } 430 EXPORT_SYMBOL(dma_direct_map_sg); 431 432 dma_addr_t dma_direct_map_resource(struct device *dev, phys_addr_t paddr, 433 size_t size, enum dma_data_direction dir, unsigned long attrs) 434 { 435 dma_addr_t dma_addr = paddr; 436 437 if (unlikely(!dma_capable(dev, dma_addr, size, false))) { 438 dev_err_once(dev, 439 "DMA addr %pad+%zu overflow (mask %llx, bus limit %llx).\n", 440 &dma_addr, size, *dev->dma_mask, dev->bus_dma_limit); 441 WARN_ON_ONCE(1); 442 return DMA_MAPPING_ERROR; 443 } 444 445 return dma_addr; 446 } 447 EXPORT_SYMBOL(dma_direct_map_resource); 448 449 int dma_direct_get_sgtable(struct device *dev, struct sg_table *sgt, 450 void *cpu_addr, dma_addr_t dma_addr, size_t size, 451 unsigned long attrs) 452 { 453 struct page *page = dma_direct_to_page(dev, dma_addr); 454 int ret; 455 456 ret = sg_alloc_table(sgt, 1, GFP_KERNEL); 457 if (!ret) 458 sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0); 459 return ret; 460 } 461 462 #ifdef CONFIG_MMU 463 bool dma_direct_can_mmap(struct device *dev) 464 { 465 return dev_is_dma_coherent(dev) || 466 IS_ENABLED(CONFIG_DMA_NONCOHERENT_MMAP); 467 } 468 469 int dma_direct_mmap(struct device *dev, struct vm_area_struct *vma, 470 void *cpu_addr, dma_addr_t dma_addr, size_t size, 471 unsigned long attrs) 472 { 473 unsigned long user_count = vma_pages(vma); 474 unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; 475 unsigned long pfn = PHYS_PFN(dma_to_phys(dev, dma_addr)); 476 int ret = -ENXIO; 477 478 vma->vm_page_prot = dma_pgprot(dev, vma->vm_page_prot, attrs); 479 480 if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret)) 481 return ret; 482 483 if (vma->vm_pgoff >= count || user_count > count - vma->vm_pgoff) 484 return -ENXIO; 485 return remap_pfn_range(vma, vma->vm_start, pfn + vma->vm_pgoff, 486 user_count << PAGE_SHIFT, vma->vm_page_prot); 487 } 488 #else /* CONFIG_MMU */ 489 bool dma_direct_can_mmap(struct device *dev) 490 { 491 return false; 492 } 493 494 int dma_direct_mmap(struct device *dev, struct vm_area_struct *vma, 495 void *cpu_addr, dma_addr_t dma_addr, size_t size, 496 unsigned long attrs) 497 { 498 return -ENXIO; 499 } 500 #endif /* CONFIG_MMU */ 501 502 int dma_direct_supported(struct device *dev, u64 mask) 503 { 504 u64 min_mask = (max_pfn - 1) << PAGE_SHIFT; 505 506 /* 507 * Because 32-bit DMA masks are so common we expect every architecture 508 * to be able to satisfy them - either by not supporting more physical 509 * memory, or by providing a ZONE_DMA32. If neither is the case, the 510 * architecture needs to use an IOMMU instead of the direct mapping. 511 */ 512 if (mask >= DMA_BIT_MASK(32)) 513 return 1; 514 515 /* 516 * This check needs to be against the actual bit mask value, so 517 * use __phys_to_dma() here so that the SME encryption mask isn't 518 * part of the check. 519 */ 520 if (IS_ENABLED(CONFIG_ZONE_DMA)) 521 min_mask = min_t(u64, min_mask, DMA_BIT_MASK(zone_dma_bits)); 522 return mask >= __phys_to_dma(dev, min_mask); 523 } 524 525 size_t dma_direct_max_mapping_size(struct device *dev) 526 { 527 /* If SWIOTLB is active, use its maximum mapping size */ 528 if (is_swiotlb_active() && 529 (dma_addressing_limited(dev) || swiotlb_force == SWIOTLB_FORCE)) 530 return swiotlb_max_mapping_size(dev); 531 return SIZE_MAX; 532 } 533