54  * @vaddr: original vaddr return for the mapping and order in the lower bits
78 /* Allocate pages of size 1 << order with the given gfp_flags */
80 					unsigned int order)  in ttm_pool_alloc_page()  argument
87 	/* Don't set the __GFP_COMP flag for higher order allocations.  in ttm_pool_alloc_page()
91 	if (order)  in ttm_pool_alloc_page()
96 		p = alloc_pages_node(pool->nid, gfp_flags, order);  in ttm_pool_alloc_page()
98 			p->private = order;  in ttm_pool_alloc_page()
106 	if (order)  in ttm_pool_alloc_page()
109 	vaddr = dma_alloc_attrs(pool->dev, (1ULL << order) * PAGE_SIZE,  in ttm_pool_alloc_page()
122 	dma->vaddr = (unsigned long)vaddr | order;  in ttm_pool_alloc_page()
131 /* Reset the caching and pages of size 1 << order */
133 			       unsigned int order, struct page *p)  in ttm_pool_free_page()  argument
144 		set_pages_wb(p, 1 << order);  in ttm_pool_free_page()
148 		__free_pages(p, order);  in ttm_pool_free_page()
152 	if (order)  in ttm_pool_free_page()
157 	dma_free_attrs(pool->dev, (1UL << order) * PAGE_SIZE, vaddr, dma->addr,  in ttm_pool_free_page()
184 /* Map pages of 1 << order size and fill the DMA address array  */
185 static int ttm_pool_map(struct ttm_pool *pool, unsigned int order,  in ttm_pool_map()  argument
196 		size_t size = (1ULL << order) * PAGE_SIZE;  in ttm_pool_map()
203 	for (i = 1 << order; i ; --i) {  in ttm_pool_map()
211 /* Unmap pages of 1 << order size */
226 	unsigned int i, num_pages = 1 << pt->order;  in ttm_pool_type_give()
238 	atomic_long_add(1 << pt->order, &allocated_pages);  in ttm_pool_type_give()
249 		atomic_long_sub(1 << pt->order, &allocated_pages);  in ttm_pool_type_take()
259 			       enum ttm_caching caching, unsigned int order)  in ttm_pool_type_init()  argument
263 	pt->order = order;  in ttm_pool_type_init()
282 		ttm_pool_free_page(pt->pool, pt->caching, pt->order, p);  in ttm_pool_type_fini()
285 /* Return the pool_type to use for the given caching and order */
288 						  unsigned int order)  in ttm_pool_select_type()  argument
291 		return &pool->caching[caching].orders[order];  in ttm_pool_select_type()
297 			return &pool->caching[caching].orders[order];  in ttm_pool_select_type()
300 			return &global_dma32_write_combined[order];  in ttm_pool_select_type()
302 		return &global_write_combined[order];  in ttm_pool_select_type()
305 			return &pool->caching[caching].orders[order];  in ttm_pool_select_type()
308 			return &global_dma32_uncached[order];  in ttm_pool_select_type()
310 		return &global_uncached[order];  in ttm_pool_select_type()
333 		ttm_pool_free_page(pt->pool, pt->caching, pt->order, p);  in ttm_pool_shrink()
334 		num_pages = 1 << pt->order;  in ttm_pool_shrink()
342 /* Return the allocation order based for a page */
355 static int ttm_pool_page_allocated(struct ttm_pool *pool, unsigned int order,  in ttm_pool_page_allocated()  argument
364 		r = ttm_pool_map(pool, order, p, dma_addr);  in ttm_pool_page_allocated()
369 	*num_pages -= 1 << order;  in ttm_pool_page_allocated()
370 	for (i = 1 << order; i; --i, ++(*pages), ++p)  in ttm_pool_page_allocated()
394 	unsigned int order;  in ttm_pool_free_range()  local
400 		order = ttm_pool_page_order(pool, *pages);  in ttm_pool_free_range()
401 		nr = (1UL << order);  in ttm_pool_free_range()
405 		pt = ttm_pool_select_type(pool, caching, order);  in ttm_pool_free_range()
409 			ttm_pool_free_page(pool, caching, order, *pages);  in ttm_pool_free_range()
435 	unsigned int order;  in ttm_pool_alloc()  local
453 	for (order = min_t(unsigned int, MAX_ORDER, __fls(num_pages));  in ttm_pool_alloc()
455 	     order = min_t(unsigned int, order, __fls(num_pages))) {  in ttm_pool_alloc()
459 		pt = ttm_pool_select_type(pool, tt->caching, order);  in ttm_pool_alloc()
469 				r = ttm_pool_page_allocated(pool, order, p,  in ttm_pool_alloc()
477 				if (num_pages < (1 << order))  in ttm_pool_alloc()
485 		while (num_pages >= (1 << order) &&  in ttm_pool_alloc()
486 		       (p = ttm_pool_alloc_page(pool, gfp_flags, order))) {  in ttm_pool_alloc()
495 			r = ttm_pool_page_allocated(pool, order, p, &dma_addr,  in ttm_pool_alloc()
504 			if (order) {  in ttm_pool_alloc()
505 				--order;  in ttm_pool_alloc()
520 	ttm_pool_free_page(pool, page_caching, order, p);  in ttm_pool_alloc()
656 /* Print a nice header for the order */