1e126ba97SEli Cohen /* 2e126ba97SEli Cohen * Copyright (c) 2013, Mellanox Technologies inc. All rights reserved. 3e126ba97SEli Cohen * 4e126ba97SEli Cohen * This software is available to you under a choice of one of two 5e126ba97SEli Cohen * licenses. You may choose to be licensed under the terms of the GNU 6e126ba97SEli Cohen * General Public License (GPL) Version 2, available from the file 7e126ba97SEli Cohen * COPYING in the main directory of this source tree, or the 8e126ba97SEli Cohen * OpenIB.org BSD license below: 9e126ba97SEli Cohen * 10e126ba97SEli Cohen * Redistribution and use in source and binary forms, with or 11e126ba97SEli Cohen * without modification, are permitted provided that the following 12e126ba97SEli Cohen * conditions are met: 13e126ba97SEli Cohen * 14e126ba97SEli Cohen * - Redistributions of source code must retain the above 15e126ba97SEli Cohen * copyright notice, this list of conditions and the following 16e126ba97SEli Cohen * disclaimer. 17e126ba97SEli Cohen * 18e126ba97SEli Cohen * - Redistributions in binary form must reproduce the above 19e126ba97SEli Cohen * copyright notice, this list of conditions and the following 20e126ba97SEli Cohen * disclaimer in the documentation and/or other materials 21e126ba97SEli Cohen * provided with the distribution. 22e126ba97SEli Cohen * 23e126ba97SEli Cohen * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24e126ba97SEli Cohen * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25e126ba97SEli Cohen * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26e126ba97SEli Cohen * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27e126ba97SEli Cohen * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28e126ba97SEli Cohen * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29e126ba97SEli Cohen * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30e126ba97SEli Cohen * SOFTWARE. 31e126ba97SEli Cohen */ 32e126ba97SEli Cohen 33e126ba97SEli Cohen #include <linux/module.h> 34e126ba97SEli Cohen #include <rdma/ib_umem.h> 35e126ba97SEli Cohen #include "mlx5_ib.h" 36e126ba97SEli Cohen 37e126ba97SEli Cohen /* @umem: umem object to scan 38e126ba97SEli Cohen * @addr: ib virtual address requested by the user 39e126ba97SEli Cohen * @count: number of PAGE_SIZE pages covered by umem 40e126ba97SEli Cohen * @shift: page shift for the compound pages found in the region 41e126ba97SEli Cohen * @ncont: number of compund pages 42e126ba97SEli Cohen * @order: log2 of the number of compound pages 43e126ba97SEli Cohen */ 44e126ba97SEli Cohen void mlx5_ib_cont_pages(struct ib_umem *umem, u64 addr, int *count, int *shift, 45e126ba97SEli Cohen int *ncont, int *order) 46e126ba97SEli Cohen { 47e126ba97SEli Cohen struct ib_umem_chunk *chunk; 48e126ba97SEli Cohen unsigned long tmp; 49e126ba97SEli Cohen unsigned long m; 50e126ba97SEli Cohen int i, j, k; 51e126ba97SEli Cohen u64 base = 0; 52e126ba97SEli Cohen int p = 0; 53e126ba97SEli Cohen int skip; 54e126ba97SEli Cohen int mask; 55e126ba97SEli Cohen u64 len; 56e126ba97SEli Cohen u64 pfn; 57e126ba97SEli Cohen 58e126ba97SEli Cohen addr = addr >> PAGE_SHIFT; 59e126ba97SEli Cohen tmp = (unsigned long)addr; 60e126ba97SEli Cohen m = find_first_bit(&tmp, sizeof(tmp)); 61e126ba97SEli Cohen skip = 1 << m; 62e126ba97SEli Cohen mask = skip - 1; 63e126ba97SEli Cohen i = 0; 64e126ba97SEli Cohen list_for_each_entry(chunk, &umem->chunk_list, list) 65e126ba97SEli Cohen for (j = 0; j < chunk->nmap; j++) { 66e126ba97SEli Cohen len = sg_dma_len(&chunk->page_list[j]) >> PAGE_SHIFT; 67e126ba97SEli Cohen pfn = sg_dma_address(&chunk->page_list[j]) >> PAGE_SHIFT; 68e126ba97SEli Cohen for (k = 0; k < len; k++) { 69e126ba97SEli Cohen if (!(i & mask)) { 70e126ba97SEli Cohen tmp = (unsigned long)pfn; 71e126ba97SEli Cohen m = min(m, find_first_bit(&tmp, sizeof(tmp))); 72e126ba97SEli Cohen skip = 1 << m; 73e126ba97SEli Cohen mask = skip - 1; 74e126ba97SEli Cohen base = pfn; 75e126ba97SEli Cohen p = 0; 76e126ba97SEli Cohen } else { 77e126ba97SEli Cohen if (base + p != pfn) { 78e126ba97SEli Cohen tmp = (unsigned long)p; 79e126ba97SEli Cohen m = find_first_bit(&tmp, sizeof(tmp)); 80e126ba97SEli Cohen skip = 1 << m; 81e126ba97SEli Cohen mask = skip - 1; 82e126ba97SEli Cohen base = pfn; 83e126ba97SEli Cohen p = 0; 84e126ba97SEli Cohen } 85e126ba97SEli Cohen } 86e126ba97SEli Cohen p++; 87e126ba97SEli Cohen i++; 88e126ba97SEli Cohen } 89e126ba97SEli Cohen } 90e126ba97SEli Cohen 91e126ba97SEli Cohen if (i) { 92e126ba97SEli Cohen m = min_t(unsigned long, ilog2(roundup_pow_of_two(i)), m); 93e126ba97SEli Cohen 94e126ba97SEli Cohen if (order) 95e126ba97SEli Cohen *order = ilog2(roundup_pow_of_two(i) >> m); 96e126ba97SEli Cohen 97e126ba97SEli Cohen *ncont = DIV_ROUND_UP(i, (1 << m)); 98e126ba97SEli Cohen } else { 99e126ba97SEli Cohen m = 0; 100e126ba97SEli Cohen 101e126ba97SEli Cohen if (order) 102e126ba97SEli Cohen *order = 0; 103e126ba97SEli Cohen 104e126ba97SEli Cohen *ncont = 0; 105e126ba97SEli Cohen } 106e126ba97SEli Cohen *shift = PAGE_SHIFT + m; 107e126ba97SEli Cohen *count = i; 108e126ba97SEli Cohen } 109e126ba97SEli Cohen 110e126ba97SEli Cohen void mlx5_ib_populate_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem, 111e126ba97SEli Cohen int page_shift, __be64 *pas, int umr) 112e126ba97SEli Cohen { 113e126ba97SEli Cohen int shift = page_shift - PAGE_SHIFT; 114e126ba97SEli Cohen int mask = (1 << shift) - 1; 115e126ba97SEli Cohen struct ib_umem_chunk *chunk; 116e126ba97SEli Cohen int i, j, k; 117e126ba97SEli Cohen u64 cur = 0; 118e126ba97SEli Cohen u64 base; 119e126ba97SEli Cohen int len; 120e126ba97SEli Cohen 121e126ba97SEli Cohen i = 0; 122e126ba97SEli Cohen list_for_each_entry(chunk, &umem->chunk_list, list) 123e126ba97SEli Cohen for (j = 0; j < chunk->nmap; j++) { 124e126ba97SEli Cohen len = sg_dma_len(&chunk->page_list[j]) >> PAGE_SHIFT; 125e126ba97SEli Cohen base = sg_dma_address(&chunk->page_list[j]); 126e126ba97SEli Cohen for (k = 0; k < len; k++) { 127e126ba97SEli Cohen if (!(i & mask)) { 128e126ba97SEli Cohen cur = base + (k << PAGE_SHIFT); 129e126ba97SEli Cohen if (umr) 130e126ba97SEli Cohen cur |= 3; 131e126ba97SEli Cohen 132e126ba97SEli Cohen pas[i >> shift] = cpu_to_be64(cur); 133e126ba97SEli Cohen mlx5_ib_dbg(dev, "pas[%d] 0x%llx\n", 134e126ba97SEli Cohen i >> shift, be64_to_cpu(pas[i >> shift])); 135e126ba97SEli Cohen } else 136e126ba97SEli Cohen mlx5_ib_dbg(dev, "=====> 0x%llx\n", 137e126ba97SEli Cohen base + (k << PAGE_SHIFT)); 138e126ba97SEli Cohen i++; 139e126ba97SEli Cohen } 140e126ba97SEli Cohen } 141e126ba97SEli Cohen } 142e126ba97SEli Cohen 143e126ba97SEli Cohen int mlx5_ib_get_buf_offset(u64 addr, int page_shift, u32 *offset) 144e126ba97SEli Cohen { 145e126ba97SEli Cohen u64 page_size; 146e126ba97SEli Cohen u64 page_mask; 147e126ba97SEli Cohen u64 off_size; 148e126ba97SEli Cohen u64 off_mask; 149e126ba97SEli Cohen u64 buf_off; 150e126ba97SEli Cohen 151e126ba97SEli Cohen page_size = 1 << page_shift; 152e126ba97SEli Cohen page_mask = page_size - 1; 153e126ba97SEli Cohen buf_off = addr & page_mask; 154e126ba97SEli Cohen off_size = page_size >> 6; 155e126ba97SEli Cohen off_mask = off_size - 1; 156e126ba97SEli Cohen 157e126ba97SEli Cohen if (buf_off & off_mask) 158e126ba97SEli Cohen return -EINVAL; 159e126ba97SEli Cohen 160e126ba97SEli Cohen *offset = buf_off >> ilog2(off_size); 161e126ba97SEli Cohen return 0; 162e126ba97SEli Cohen } 163