Lines Matching +full:mem +full:- +full:io
1 // SPDX-License-Identifier: GPL-2.0-only
9 * Copyright (C) 2000, 2003 Hewlett-Packard Co
10 * David Mosberger-Tang <davidm@hpl.hp.com>
12 * 03/05/07 davidm Switch from PCI-DMA to generic device DMA API.
14 * unnecessary i-cache flushing.
21 #define pr_fmt(fmt) "software IO TLB: " fmt
27 #include <linux/dma-direct.h>
28 #include <linux/dma-map-ops.h>
32 #include <linux/io.h>
33 #include <linux/iommu-helper.h>
55 #define SLABS_PER_PAGE (1 << (PAGE_SHIFT - IO_TLB_SHIFT))
58 * Minimum IO TLB size to bother booting with. Systems with mainly
67 * struct io_tlb_slot - IO TLB slot descriptor
73 * allocated non-padding slot.
106 * struct io_tlb_area - IO TLB memory area descriptor
110 * @used: The number of used IO TLB block.
145 * swiotlb_adjust_nareas() - adjust the number of areas and slots
168 * limit_nareas() - get the maximum number of areas for a given memory pool size
231 struct io_tlb_pool *mem = &io_tlb_default_mem.defpool; in swiotlb_print_info() local
233 if (!mem->nslabs) { in swiotlb_print_info()
234 pr_warn("No low mem\n"); in swiotlb_print_info()
238 pr_info("mapped [mem %pa-%pa] (%luMB)\n", &mem->start, &mem->end, in swiotlb_print_info()
239 (mem->nslabs << IO_TLB_SHIFT) >> 20); in swiotlb_print_info()
244 return val & (IO_TLB_SEGSIZE - 1); in io_tlb_offset()
260 struct io_tlb_pool *mem = &io_tlb_default_mem.defpool; in swiotlb_update_mem_attributes() local
263 if (!mem->nslabs || mem->late_alloc) in swiotlb_update_mem_attributes()
265 bytes = PAGE_ALIGN(mem->nslabs << IO_TLB_SHIFT); in swiotlb_update_mem_attributes()
266 set_memory_decrypted((unsigned long)mem->vaddr, bytes >> PAGE_SHIFT); in swiotlb_update_mem_attributes()
269 static void swiotlb_init_io_tlb_pool(struct io_tlb_pool *mem, phys_addr_t start, in swiotlb_init_io_tlb_pool() argument
275 mem->nslabs = nslabs; in swiotlb_init_io_tlb_pool()
276 mem->start = start; in swiotlb_init_io_tlb_pool()
277 mem->end = mem->start + bytes; in swiotlb_init_io_tlb_pool()
278 mem->late_alloc = late_alloc; in swiotlb_init_io_tlb_pool()
279 mem->nareas = nareas; in swiotlb_init_io_tlb_pool()
280 mem->area_nslabs = nslabs / mem->nareas; in swiotlb_init_io_tlb_pool()
282 for (i = 0; i < mem->nareas; i++) { in swiotlb_init_io_tlb_pool()
283 spin_lock_init(&mem->areas[i].lock); in swiotlb_init_io_tlb_pool()
284 mem->areas[i].index = 0; in swiotlb_init_io_tlb_pool()
285 mem->areas[i].used = 0; in swiotlb_init_io_tlb_pool()
288 for (i = 0; i < mem->nslabs; i++) { in swiotlb_init_io_tlb_pool()
289 mem->slots[i].list = min(IO_TLB_SEGSIZE - io_tlb_offset(i), in swiotlb_init_io_tlb_pool()
290 mem->nslabs - i); in swiotlb_init_io_tlb_pool()
291 mem->slots[i].orig_addr = INVALID_PHYS_ADDR; in swiotlb_init_io_tlb_pool()
292 mem->slots[i].alloc_size = 0; in swiotlb_init_io_tlb_pool()
293 mem->slots[i].pad_slots = 0; in swiotlb_init_io_tlb_pool()
297 mem->vaddr = vaddr; in swiotlb_init_io_tlb_pool()
302 * add_mem_pool() - add a memory pool to the allocator
303 * @mem: Software IO TLB allocator.
306 static void add_mem_pool(struct io_tlb_mem *mem, struct io_tlb_pool *pool) in add_mem_pool() argument
309 spin_lock(&mem->lock); in add_mem_pool()
310 list_add_rcu(&pool->node, &mem->pools); in add_mem_pool()
311 mem->nslabs += pool->nslabs; in add_mem_pool()
312 spin_unlock(&mem->lock); in add_mem_pool()
314 mem->nslabs = pool->nslabs; in add_mem_pool()
352 * structures for the software IO TLB used to implement the DMA API.
357 struct io_tlb_pool *mem = &io_tlb_default_mem.defpool; in swiotlb_init_remap() local
375 io_tlb_default_mem.phys_limit = virt_to_phys(high_memory - 1); in swiotlb_init_remap()
393 pr_info("SWIOTLB bounce buffer size adjusted %lu -> %lu slabs", in swiotlb_init_remap()
398 alloc_size = PAGE_ALIGN(array_size(sizeof(*mem->slots), nslabs)); in swiotlb_init_remap()
399 mem->slots = memblock_alloc(alloc_size, PAGE_SIZE); in swiotlb_init_remap()
400 if (!mem->slots) { in swiotlb_init_remap()
406 mem->areas = memblock_alloc(array_size(sizeof(struct io_tlb_area), in swiotlb_init_remap()
408 if (!mem->areas) { in swiotlb_init_remap()
409 pr_warn("%s: Failed to allocate mem->areas.\n", __func__); in swiotlb_init_remap()
413 swiotlb_init_io_tlb_pool(mem, __pa(tlb), nslabs, false, nareas); in swiotlb_init_remap()
414 add_mem_pool(&io_tlb_default_mem, mem); in swiotlb_init_remap()
433 struct io_tlb_pool *mem = &io_tlb_default_mem.defpool; in swiotlb_init_late() local
457 io_tlb_default_mem.phys_limit = virt_to_phys(high_memory - 1); in swiotlb_init_late()
472 order--; in swiotlb_init_late()
478 return -ENOMEM; in swiotlb_init_late()
498 area_order = get_order(array_size(sizeof(*mem->areas), nareas)); in swiotlb_init_late()
499 mem->areas = (struct io_tlb_area *) in swiotlb_init_late()
501 if (!mem->areas) in swiotlb_init_late()
504 mem->slots = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, in swiotlb_init_late()
505 get_order(array_size(sizeof(*mem->slots), nslabs))); in swiotlb_init_late()
506 if (!mem->slots) in swiotlb_init_late()
511 swiotlb_init_io_tlb_pool(mem, virt_to_phys(vstart), nslabs, true, in swiotlb_init_late()
513 add_mem_pool(&io_tlb_default_mem, mem); in swiotlb_init_late()
519 free_pages((unsigned long)mem->areas, area_order); in swiotlb_init_late()
522 return -ENOMEM; in swiotlb_init_late()
527 struct io_tlb_pool *mem = &io_tlb_default_mem.defpool; in swiotlb_exit() local
535 if (!mem->nslabs) in swiotlb_exit()
539 tbl_vaddr = (unsigned long)phys_to_virt(mem->start); in swiotlb_exit()
540 tbl_size = PAGE_ALIGN(mem->end - mem->start); in swiotlb_exit()
541 slots_size = PAGE_ALIGN(array_size(sizeof(*mem->slots), mem->nslabs)); in swiotlb_exit()
544 if (mem->late_alloc) { in swiotlb_exit()
545 area_order = get_order(array_size(sizeof(*mem->areas), in swiotlb_exit()
546 mem->nareas)); in swiotlb_exit()
547 free_pages((unsigned long)mem->areas, area_order); in swiotlb_exit()
549 free_pages((unsigned long)mem->slots, get_order(slots_size)); in swiotlb_exit()
551 memblock_free_late(__pa(mem->areas), in swiotlb_exit()
552 array_size(sizeof(*mem->areas), mem->nareas)); in swiotlb_exit()
553 memblock_free_late(mem->start, tbl_size); in swiotlb_exit()
554 memblock_free_late(__pa(mem->slots), slots_size); in swiotlb_exit()
557 memset(mem, 0, sizeof(*mem)); in swiotlb_exit()
563 * alloc_dma_pages() - allocate pages to be used for DMA
571 * Return: Decrypted pages, %NULL on allocation failure, or ERR_PTR(-EAGAIN)
586 if (paddr + bytes - 1 > phys_limit) { in alloc_dma_pages()
588 return ERR_PTR(-EAGAIN); in alloc_dma_pages()
604 * swiotlb_alloc_tlb() - allocate a dynamic IO TLB buffer
653 * swiotlb_free_tlb() - free a dynamically allocated IO TLB buffer
669 * swiotlb_alloc_pool() - allocate a new IO TLB memory pool
677 * Allocate and initialize a new IO TLB memory pool. The actual number of
698 pool_size = sizeof(*pool) + array_size(sizeof(*pool->areas), nareas); in swiotlb_alloc_pool()
702 pool->areas = (void *)pool + sizeof(*pool); in swiotlb_alloc_pool()
713 slot_order = get_order(array_size(sizeof(*pool->slots), nslabs)); in swiotlb_alloc_pool()
714 pool->slots = (struct io_tlb_slot *) in swiotlb_alloc_pool()
716 if (!pool->slots) in swiotlb_alloc_pool()
731 * swiotlb_dyn_alloc() - dynamic memory pool allocation worker
736 struct io_tlb_mem *mem = in swiotlb_dyn_alloc() local
741 default_nareas, mem->phys_limit, GFP_KERNEL); in swiotlb_dyn_alloc()
747 add_mem_pool(mem, pool); in swiotlb_dyn_alloc()
751 * swiotlb_dyn_free() - RCU callback to free a memory pool
757 size_t slots_size = array_size(sizeof(*pool->slots), pool->nslabs); in swiotlb_dyn_free()
758 size_t tlb_size = pool->end - pool->start; in swiotlb_dyn_free()
760 free_pages((unsigned long)pool->slots, get_order(slots_size)); in swiotlb_dyn_free()
761 swiotlb_free_tlb(pool->vaddr, tlb_size); in swiotlb_dyn_free()
766 * swiotlb_find_pool() - find the IO TLB pool for a physical address
770 * Find the IO TLB memory pool descriptor which contains the given physical
777 struct io_tlb_mem *mem = dev->dma_io_tlb_mem; in swiotlb_find_pool() local
781 list_for_each_entry_rcu(pool, &mem->pools, node) { in swiotlb_find_pool()
782 if (paddr >= pool->start && paddr < pool->end) in swiotlb_find_pool()
786 list_for_each_entry_rcu(pool, &dev->dma_io_tlb_pools, node) { in swiotlb_find_pool()
787 if (paddr >= pool->start && paddr < pool->end) in swiotlb_find_pool()
797 * swiotlb_del_pool() - remove an IO TLB pool from a device
805 spin_lock_irqsave(&dev->dma_io_tlb_lock, flags); in swiotlb_del_pool()
806 list_del_rcu(&pool->node); in swiotlb_del_pool()
807 spin_unlock_irqrestore(&dev->dma_io_tlb_lock, flags); in swiotlb_del_pool()
809 call_rcu(&pool->rcu, swiotlb_dyn_free); in swiotlb_del_pool()
815 * swiotlb_dev_init() - initialize swiotlb fields in &struct device
820 dev->dma_io_tlb_mem = &io_tlb_default_mem; in swiotlb_dev_init()
822 INIT_LIST_HEAD(&dev->dma_io_tlb_pools); in swiotlb_dev_init()
823 spin_lock_init(&dev->dma_io_tlb_lock); in swiotlb_dev_init()
824 dev->dma_uses_io_tlb = false; in swiotlb_dev_init()
829 * swiotlb_align_offset() - Get required offset into an IO TLB allocation.
834 * Return the minimum offset from the start of an IO TLB allocation which is
840 * choosing a suitable IO TLB slot, but bits below IO_TLB_SHIFT require extra
851 (align_mask | (IO_TLB_SIZE - 1)); in swiotlb_align_offset()
860 struct io_tlb_pool *mem = swiotlb_find_pool(dev, tlb_addr); in swiotlb_bounce() local
861 int index = (tlb_addr - mem->start) >> IO_TLB_SHIFT; in swiotlb_bounce()
862 phys_addr_t orig_addr = mem->slots[index].orig_addr; in swiotlb_bounce()
863 size_t alloc_size = mem->slots[index].alloc_size; in swiotlb_bounce()
865 unsigned char *vaddr = mem->vaddr + tlb_addr - mem->start; in swiotlb_bounce()
871 tlb_offset = tlb_addr & (IO_TLB_SIZE - 1); in swiotlb_bounce()
880 tlb_offset -= orig_addr_offset; in swiotlb_bounce()
889 alloc_size -= tlb_offset; in swiotlb_bounce()
905 sz = min_t(size_t, PAGE_SIZE - offset, size); in swiotlb_bounce()
915 size -= sz; in swiotlb_bounce()
940 static unsigned int wrap_area_index(struct io_tlb_pool *mem, unsigned int index) in wrap_area_index() argument
942 if (index >= mem->area_nslabs) in wrap_area_index()
954 static void inc_used_and_hiwater(struct io_tlb_mem *mem, unsigned int nslots) in inc_used_and_hiwater() argument
958 new_used = atomic_long_add_return(nslots, &mem->total_used); in inc_used_and_hiwater()
959 old_hiwater = atomic_long_read(&mem->used_hiwater); in inc_used_and_hiwater()
963 } while (!atomic_long_try_cmpxchg(&mem->used_hiwater, in inc_used_and_hiwater()
967 static void dec_used(struct io_tlb_mem *mem, unsigned int nslots) in dec_used() argument
969 atomic_long_sub(nslots, &mem->total_used); in dec_used()
973 static void inc_used_and_hiwater(struct io_tlb_mem *mem, unsigned int nslots) in inc_used_and_hiwater() argument
976 static void dec_used(struct io_tlb_mem *mem, unsigned int nslots) in dec_used() argument
982 * swiotlb_area_find_slots() - search for slots in one IO TLB memory area
985 * @area_index: Index of the IO TLB memory area to be searched.
986 * @orig_addr: Original (non-bounced) IO buffer address.
991 * Find a suitable sequence of IO TLB entries for the request and allocate
992 * a buffer from the given IO TLB memory area.
995 * Return: Index of the first allocated slot, or -1 on error.
1001 struct io_tlb_area *area = pool->areas + area_index; in swiotlb_area_find_slots()
1004 phys_to_dma_unencrypted(dev, pool->start) & boundary_mask; in swiotlb_area_find_slots()
1015 BUG_ON(area_index >= pool->nareas); in swiotlb_area_find_slots()
1019 * page-aligned in the absence of any other alignment requirements. in swiotlb_area_find_slots()
1026 alloc_align_mask = PAGE_SIZE - 1; in swiotlb_area_find_slots()
1029 * Ensure that the allocation is at least slot-aligned and update in swiotlb_area_find_slots()
1033 alloc_align_mask |= (IO_TLB_SIZE - 1); in swiotlb_area_find_slots()
1042 spin_lock_irqsave(&area->lock, flags); in swiotlb_area_find_slots()
1043 if (unlikely(nslots > pool->area_nslabs - area->used)) in swiotlb_area_find_slots()
1046 slot_base = area_index * pool->area_nslabs; in swiotlb_area_find_slots()
1047 index = area->index; in swiotlb_area_find_slots()
1049 for (slots_checked = 0; slots_checked < pool->area_nslabs; ) { in swiotlb_area_find_slots()
1066 if (pool->slots[slot_index].list >= nslots) in swiotlb_area_find_slots()
1074 spin_unlock_irqrestore(&area->lock, flags); in swiotlb_area_find_slots()
1075 return -1; in swiotlb_area_find_slots()
1084 pool->slots[i].list = 0; in swiotlb_area_find_slots()
1085 pool->slots[i].alloc_size = alloc_size - (offset + in swiotlb_area_find_slots()
1086 ((i - slot_index) << IO_TLB_SHIFT)); in swiotlb_area_find_slots()
1088 for (i = slot_index - 1; in swiotlb_area_find_slots()
1089 io_tlb_offset(i) != IO_TLB_SEGSIZE - 1 && in swiotlb_area_find_slots()
1090 pool->slots[i].list; i--) in swiotlb_area_find_slots()
1091 pool->slots[i].list = ++count; in swiotlb_area_find_slots()
1096 area->index = wrap_area_index(pool, index + nslots); in swiotlb_area_find_slots()
1097 area->used += nslots; in swiotlb_area_find_slots()
1098 spin_unlock_irqrestore(&area->lock, flags); in swiotlb_area_find_slots()
1100 inc_used_and_hiwater(dev->dma_io_tlb_mem, nslots); in swiotlb_area_find_slots()
1105 * swiotlb_pool_find_slots() - search for slots in one memory pool
1108 * @orig_addr: Original (non-bounced) IO buffer address.
1116 * Return: Index of the first allocated slot, or -1 on error.
1122 int start = raw_smp_processor_id() & (pool->nareas - 1); in swiotlb_pool_find_slots()
1130 if (++i >= pool->nareas) in swiotlb_pool_find_slots()
1134 return -1; in swiotlb_pool_find_slots()
1140 * swiotlb_find_slots() - search for slots in the whole swiotlb
1142 * @orig_addr: Original (non-bounced) IO buffer address.
1148 * Search through the whole software IO TLB to find a sequence of slots that
1151 * Return: Index of the first allocated slot, or -1 on error.
1157 struct io_tlb_mem *mem = dev->dma_io_tlb_mem; in swiotlb_find_slots() local
1165 list_for_each_entry_rcu(pool, &mem->pools, node) { in swiotlb_find_slots()
1174 if (!mem->can_grow) in swiotlb_find_slots()
1175 return -1; in swiotlb_find_slots()
1177 schedule_work(&mem->dyn_alloc); in swiotlb_find_slots()
1180 phys_limit = min_not_zero(*dev->dma_mask, dev->bus_dma_limit); in swiotlb_find_slots()
1184 return -1; in swiotlb_find_slots()
1189 swiotlb_dyn_free(&pool->rcu); in swiotlb_find_slots()
1190 return -1; in swiotlb_find_slots()
1193 pool->transient = true; in swiotlb_find_slots()
1194 spin_lock_irqsave(&dev->dma_io_tlb_lock, flags); in swiotlb_find_slots()
1195 list_add_rcu(&pool->node, &dev->dma_io_tlb_pools); in swiotlb_find_slots()
1196 spin_unlock_irqrestore(&dev->dma_io_tlb_lock, flags); in swiotlb_find_slots()
1199 WRITE_ONCE(dev->dma_uses_io_tlb, true); in swiotlb_find_slots()
1206 * First, the store to dev->dma_uses_io_tlb must be ordered before the in swiotlb_find_slots()
1208 * cannot be passed to another CPU before updating dev->dma_uses_io_tlb. in swiotlb_find_slots()
1210 * Second, the load from mem->pools must be ordered before the same in swiotlb_find_slots()
1230 *retpool = &dev->dma_io_tlb_mem->defpool; in swiotlb_find_slots()
1240 * mem_used() - get number of used slots in an allocator
1241 * @mem: Software IO TLB allocator.
1248 static unsigned long mem_used(struct io_tlb_mem *mem) in mem_used() argument
1250 return atomic_long_read(&mem->total_used); in mem_used()
1256 * mem_pool_used() - get number of used slots in a memory pool
1257 * @pool: Software IO TLB memory pool.
1268 for (i = 0; i < pool->nareas; i++) in mem_pool_used()
1269 used += pool->areas[i].used; in mem_pool_used()
1274 * mem_used() - get number of used slots in an allocator
1275 * @mem: Software IO TLB allocator.
1282 static unsigned long mem_used(struct io_tlb_mem *mem) in mem_used() argument
1289 list_for_each_entry_rcu(pool, &mem->pools, node) in mem_used()
1295 return mem_pool_used(&mem->defpool); in mem_used()
1306 struct io_tlb_mem *mem = dev->dma_io_tlb_mem; in swiotlb_tbl_map_single() local
1314 if (!mem || !mem->nslabs) { in swiotlb_tbl_map_single()
1332 if (index == -1) { in swiotlb_tbl_map_single()
1336 alloc_size, mem->nslabs, mem_used(mem)); in swiotlb_tbl_map_single()
1346 offset &= (IO_TLB_SIZE - 1); in swiotlb_tbl_map_single()
1348 pool->slots[index].pad_slots = pad_slots; in swiotlb_tbl_map_single()
1350 pool->slots[index + i].orig_addr = slot_addr(orig_addr, i); in swiotlb_tbl_map_single()
1351 tlb_addr = slot_addr(pool->start, index) + offset; in swiotlb_tbl_map_single()
1357 * kernel memory) to user-space. in swiotlb_tbl_map_single()
1365 struct io_tlb_pool *mem = swiotlb_find_pool(dev, tlb_addr); in swiotlb_release_slots() local
1372 index = (tlb_addr - offset - mem->start) >> IO_TLB_SHIFT; in swiotlb_release_slots()
1373 index -= mem->slots[index].pad_slots; in swiotlb_release_slots()
1374 nslots = nr_slots(mem->slots[index].alloc_size + offset); in swiotlb_release_slots()
1375 aindex = index / mem->area_nslabs; in swiotlb_release_slots()
1376 area = &mem->areas[aindex]; in swiotlb_release_slots()
1384 BUG_ON(aindex >= mem->nareas); in swiotlb_release_slots()
1386 spin_lock_irqsave(&area->lock, flags); in swiotlb_release_slots()
1388 count = mem->slots[index + nslots].list; in swiotlb_release_slots()
1396 for (i = index + nslots - 1; i >= index; i--) { in swiotlb_release_slots()
1397 mem->slots[i].list = ++count; in swiotlb_release_slots()
1398 mem->slots[i].orig_addr = INVALID_PHYS_ADDR; in swiotlb_release_slots()
1399 mem->slots[i].alloc_size = 0; in swiotlb_release_slots()
1400 mem->slots[i].pad_slots = 0; in swiotlb_release_slots()
1407 for (i = index - 1; in swiotlb_release_slots()
1408 io_tlb_offset(i) != IO_TLB_SEGSIZE - 1 && mem->slots[i].list; in swiotlb_release_slots()
1409 i--) in swiotlb_release_slots()
1410 mem->slots[i].list = ++count; in swiotlb_release_slots()
1411 area->used -= nslots; in swiotlb_release_slots()
1412 spin_unlock_irqrestore(&area->lock, flags); in swiotlb_release_slots()
1414 dec_used(dev->dma_io_tlb_mem, nslots); in swiotlb_release_slots()
1420 * swiotlb_del_transient() - delete a transient memory pool
1434 if (!pool->transient) in swiotlb_del_transient()
1437 dec_used(dev->dma_io_tlb_mem, pool->nslabs); in swiotlb_del_transient()
1513 &dma_addr, size, *dev->dma_mask, dev->bus_dma_limit); in swiotlb_map()
1535 return ((size_t)IO_TLB_SIZE) * IO_TLB_SEGSIZE - min_align; in swiotlb_max_mapping_size()
1539 * is_swiotlb_allocated() - check if the default software IO TLB is initialized
1548 struct io_tlb_mem *mem = dev->dma_io_tlb_mem; in is_swiotlb_active() local
1550 return mem && mem->nslabs; in is_swiotlb_active()
1554 * default_swiotlb_base() - get the base address of the default SWIOTLB
1556 * Get the lowest physical address used by the default software IO TLB pool.
1567 * default_swiotlb_limit() - get the address limit of the default SWIOTLB
1569 * Get the highest physical address used by the default software IO TLB pool.
1576 return io_tlb_default_mem.defpool.end - 1; in default_swiotlb_limit()
1584 struct io_tlb_mem *mem = data; in io_tlb_used_get() local
1586 *val = mem_used(mem); in io_tlb_used_get()
1592 struct io_tlb_mem *mem = data; in io_tlb_hiwater_get() local
1594 *val = atomic_long_read(&mem->used_hiwater); in io_tlb_hiwater_get()
1600 struct io_tlb_mem *mem = data; in io_tlb_hiwater_set() local
1604 return -EINVAL; in io_tlb_hiwater_set()
1606 atomic_long_set(&mem->used_hiwater, val); in io_tlb_hiwater_set()
1614 static void swiotlb_create_debugfs_files(struct io_tlb_mem *mem, in swiotlb_create_debugfs_files() argument
1617 atomic_long_set(&mem->total_used, 0); in swiotlb_create_debugfs_files()
1618 atomic_long_set(&mem->used_hiwater, 0); in swiotlb_create_debugfs_files()
1620 mem->debugfs = debugfs_create_dir(dirname, io_tlb_default_mem.debugfs); in swiotlb_create_debugfs_files()
1621 if (!mem->nslabs) in swiotlb_create_debugfs_files()
1624 debugfs_create_ulong("io_tlb_nslabs", 0400, mem->debugfs, &mem->nslabs); in swiotlb_create_debugfs_files()
1625 debugfs_create_file("io_tlb_used", 0400, mem->debugfs, mem, in swiotlb_create_debugfs_files()
1627 debugfs_create_file("io_tlb_used_hiwater", 0600, mem->debugfs, mem, in swiotlb_create_debugfs_files()
1641 static inline void swiotlb_create_debugfs_files(struct io_tlb_mem *mem, in swiotlb_create_debugfs_files() argument
1652 struct io_tlb_mem *mem = dev->dma_io_tlb_mem; in swiotlb_alloc() local
1658 if (!mem) in swiotlb_alloc()
1661 align = (1 << (get_order(size) + PAGE_SHIFT)) - 1; in swiotlb_alloc()
1663 if (index == -1) in swiotlb_alloc()
1666 tlb_addr = slot_addr(pool->start, index); in swiotlb_alloc()
1668 dev_WARN_ONCE(dev, 1, "Cannot allocate pages from non page-aligned swiotlb addr 0x%pa.\n", in swiotlb_alloc()
1692 struct io_tlb_mem *mem = rmem->priv; in rmem_swiotlb_device_init() local
1693 unsigned long nslabs = rmem->size >> IO_TLB_SHIFT; in rmem_swiotlb_device_init()
1695 /* Set Per-device io tlb area to one */ in rmem_swiotlb_device_init()
1698 if (PageHighMem(pfn_to_page(PHYS_PFN(rmem->base)))) { in rmem_swiotlb_device_init()
1700 return -EINVAL; in rmem_swiotlb_device_init()
1708 if (!mem) { in rmem_swiotlb_device_init()
1711 mem = kzalloc(sizeof(*mem), GFP_KERNEL); in rmem_swiotlb_device_init()
1712 if (!mem) in rmem_swiotlb_device_init()
1713 return -ENOMEM; in rmem_swiotlb_device_init()
1714 pool = &mem->defpool; in rmem_swiotlb_device_init()
1716 pool->slots = kcalloc(nslabs, sizeof(*pool->slots), GFP_KERNEL); in rmem_swiotlb_device_init()
1717 if (!pool->slots) { in rmem_swiotlb_device_init()
1718 kfree(mem); in rmem_swiotlb_device_init()
1719 return -ENOMEM; in rmem_swiotlb_device_init()
1722 pool->areas = kcalloc(nareas, sizeof(*pool->areas), in rmem_swiotlb_device_init()
1724 if (!pool->areas) { in rmem_swiotlb_device_init()
1725 kfree(pool->slots); in rmem_swiotlb_device_init()
1726 kfree(mem); in rmem_swiotlb_device_init()
1727 return -ENOMEM; in rmem_swiotlb_device_init()
1730 set_memory_decrypted((unsigned long)phys_to_virt(rmem->base), in rmem_swiotlb_device_init()
1731 rmem->size >> PAGE_SHIFT); in rmem_swiotlb_device_init()
1732 swiotlb_init_io_tlb_pool(pool, rmem->base, nslabs, in rmem_swiotlb_device_init()
1734 mem->force_bounce = true; in rmem_swiotlb_device_init()
1735 mem->for_alloc = true; in rmem_swiotlb_device_init()
1737 spin_lock_init(&mem->lock); in rmem_swiotlb_device_init()
1738 INIT_LIST_HEAD_RCU(&mem->pools); in rmem_swiotlb_device_init()
1740 add_mem_pool(mem, pool); in rmem_swiotlb_device_init()
1742 rmem->priv = mem; in rmem_swiotlb_device_init()
1744 swiotlb_create_debugfs_files(mem, rmem->name); in rmem_swiotlb_device_init()
1747 dev->dma_io_tlb_mem = mem; in rmem_swiotlb_device_init()
1755 dev->dma_io_tlb_mem = &io_tlb_default_mem; in rmem_swiotlb_device_release()
1765 unsigned long node = rmem->fdt_node; in rmem_swiotlb_setup()
1768 of_get_flat_dt_prop(node, "linux,cma-default", NULL) || in rmem_swiotlb_setup()
1769 of_get_flat_dt_prop(node, "linux,dma-default", NULL) || in rmem_swiotlb_setup()
1770 of_get_flat_dt_prop(node, "no-map", NULL)) in rmem_swiotlb_setup()
1771 return -EINVAL; in rmem_swiotlb_setup()
1773 rmem->ops = &rmem_swiotlb_ops; in rmem_swiotlb_setup()
1775 &rmem->base, (unsigned long)rmem->size / SZ_1M); in rmem_swiotlb_setup()
1779 RESERVEDMEM_OF_DECLARE(dma, "restricted-dma-pool", rmem_swiotlb_setup);