1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2019 HiSilicon Limited. */ 3 #include <linux/align.h> 4 #include <linux/dma-mapping.h> 5 #include <linux/hisi_acc_qm.h> 6 #include <linux/module.h> 7 #include <linux/slab.h> 8 9 #define HISI_ACC_SGL_SGE_NR_MIN 1 10 #define HISI_ACC_SGL_NR_MAX 256 11 #define HISI_ACC_SGL_ALIGN_SIZE 64 12 #define HISI_ACC_MEM_BLOCK_NR 5 13 14 struct acc_hw_sge { 15 dma_addr_t buf; 16 void *page_ctrl; 17 __le32 len; 18 __le32 pad; 19 __le32 pad0; 20 __le32 pad1; 21 }; 22 23 /* use default sgl head size 64B */ 24 struct hisi_acc_hw_sgl { 25 dma_addr_t next_dma; 26 __le16 entry_sum_in_chain; 27 __le16 entry_sum_in_sgl; 28 __le16 entry_length_in_sgl; 29 __le16 pad0; 30 __le64 pad1[5]; 31 struct hisi_acc_hw_sgl *next; 32 struct acc_hw_sge sge_entries[]; 33 } __aligned(1); 34 35 struct hisi_acc_sgl_pool { 36 struct mem_block { 37 struct hisi_acc_hw_sgl *sgl; 38 dma_addr_t sgl_dma; 39 size_t size; 40 } mem_block[HISI_ACC_MEM_BLOCK_NR]; 41 u32 sgl_num_per_block; 42 u32 block_num; 43 u32 count; 44 u32 sge_nr; 45 size_t sgl_size; 46 }; 47 48 /** 49 * hisi_acc_create_sgl_pool() - Create a hw sgl pool. 50 * @dev: The device which hw sgl pool belongs to. 51 * @count: Count of hisi_acc_hw_sgl in pool. 52 * @sge_nr: The count of sge in hw_sgl 53 * 54 * This function creates a hw sgl pool, after this user can get hw sgl memory 55 * from it. 56 */ 57 struct hisi_acc_sgl_pool *hisi_acc_create_sgl_pool(struct device *dev, 58 u32 count, u32 sge_nr) 59 { 60 u32 sgl_size, block_size, sgl_num_per_block, block_num, remain_sgl; 61 struct hisi_acc_sgl_pool *pool; 62 struct mem_block *block; 63 u32 i, j; 64 65 if (!dev || !count || !sge_nr || sge_nr > HISI_ACC_SGL_SGE_NR_MAX) 66 return ERR_PTR(-EINVAL); 67 68 sgl_size = ALIGN(sizeof(struct acc_hw_sge) * sge_nr + 69 sizeof(struct hisi_acc_hw_sgl), 70 HISI_ACC_SGL_ALIGN_SIZE); 71 72 /* 73 * the pool may allocate a block of memory of size PAGE_SIZE * 2^(MAX_ORDER - 1), 74 * block size may exceed 2^31 on ia64, so the max of block size is 2^31 75 */ 76 block_size = 1 << (PAGE_SHIFT + MAX_ORDER <= 32 ? 77 PAGE_SHIFT + MAX_ORDER - 1 : 31); 78 sgl_num_per_block = block_size / sgl_size; 79 block_num = count / sgl_num_per_block; 80 remain_sgl = count % sgl_num_per_block; 81 82 if ((!remain_sgl && block_num > HISI_ACC_MEM_BLOCK_NR) || 83 (remain_sgl > 0 && block_num > HISI_ACC_MEM_BLOCK_NR - 1)) 84 return ERR_PTR(-EINVAL); 85 86 pool = kzalloc(sizeof(*pool), GFP_KERNEL); 87 if (!pool) 88 return ERR_PTR(-ENOMEM); 89 block = pool->mem_block; 90 91 for (i = 0; i < block_num; i++) { 92 block[i].sgl = dma_alloc_coherent(dev, block_size, 93 &block[i].sgl_dma, 94 GFP_KERNEL); 95 if (!block[i].sgl) { 96 dev_err(dev, "Fail to allocate hw SG buffer!\n"); 97 goto err_free_mem; 98 } 99 100 block[i].size = block_size; 101 } 102 103 if (remain_sgl > 0) { 104 block[i].sgl = dma_alloc_coherent(dev, remain_sgl * sgl_size, 105 &block[i].sgl_dma, 106 GFP_KERNEL); 107 if (!block[i].sgl) { 108 dev_err(dev, "Fail to allocate remained hw SG buffer!\n"); 109 goto err_free_mem; 110 } 111 112 block[i].size = remain_sgl * sgl_size; 113 } 114 115 pool->sgl_num_per_block = sgl_num_per_block; 116 pool->block_num = remain_sgl ? block_num + 1 : block_num; 117 pool->count = count; 118 pool->sgl_size = sgl_size; 119 pool->sge_nr = sge_nr; 120 121 return pool; 122 123 err_free_mem: 124 for (j = 0; j < i; j++) { 125 dma_free_coherent(dev, block_size, block[j].sgl, 126 block[j].sgl_dma); 127 memset(block + j, 0, sizeof(*block)); 128 } 129 kfree(pool); 130 return ERR_PTR(-ENOMEM); 131 } 132 EXPORT_SYMBOL_GPL(hisi_acc_create_sgl_pool); 133 134 /** 135 * hisi_acc_free_sgl_pool() - Free a hw sgl pool. 136 * @dev: The device which hw sgl pool belongs to. 137 * @pool: Pointer of pool. 138 * 139 * This function frees memory of a hw sgl pool. 140 */ 141 void hisi_acc_free_sgl_pool(struct device *dev, struct hisi_acc_sgl_pool *pool) 142 { 143 struct mem_block *block; 144 int i; 145 146 if (!dev || !pool) 147 return; 148 149 block = pool->mem_block; 150 151 for (i = 0; i < pool->block_num; i++) 152 dma_free_coherent(dev, block[i].size, block[i].sgl, 153 block[i].sgl_dma); 154 155 kfree(pool); 156 } 157 EXPORT_SYMBOL_GPL(hisi_acc_free_sgl_pool); 158 159 static struct hisi_acc_hw_sgl *acc_get_sgl(struct hisi_acc_sgl_pool *pool, 160 u32 index, dma_addr_t *hw_sgl_dma) 161 { 162 struct mem_block *block; 163 u32 block_index, offset; 164 165 if (!pool || !hw_sgl_dma || index >= pool->count) 166 return ERR_PTR(-EINVAL); 167 168 block = pool->mem_block; 169 block_index = index / pool->sgl_num_per_block; 170 offset = index % pool->sgl_num_per_block; 171 172 *hw_sgl_dma = block[block_index].sgl_dma + pool->sgl_size * offset; 173 return (void *)block[block_index].sgl + pool->sgl_size * offset; 174 } 175 176 static void sg_map_to_hw_sg(struct scatterlist *sgl, 177 struct acc_hw_sge *hw_sge) 178 { 179 hw_sge->buf = sg_dma_address(sgl); 180 hw_sge->len = cpu_to_le32(sg_dma_len(sgl)); 181 hw_sge->page_ctrl = sg_virt(sgl); 182 } 183 184 static void inc_hw_sgl_sge(struct hisi_acc_hw_sgl *hw_sgl) 185 { 186 u16 var = le16_to_cpu(hw_sgl->entry_sum_in_sgl); 187 188 var++; 189 hw_sgl->entry_sum_in_sgl = cpu_to_le16(var); 190 } 191 192 static void update_hw_sgl_sum_sge(struct hisi_acc_hw_sgl *hw_sgl, u16 sum) 193 { 194 hw_sgl->entry_sum_in_chain = cpu_to_le16(sum); 195 } 196 197 static void clear_hw_sgl_sge(struct hisi_acc_hw_sgl *hw_sgl) 198 { 199 struct acc_hw_sge *hw_sge = hw_sgl->sge_entries; 200 int i; 201 202 for (i = 0; i < le16_to_cpu(hw_sgl->entry_sum_in_sgl); i++) { 203 hw_sge[i].page_ctrl = NULL; 204 hw_sge[i].buf = 0; 205 hw_sge[i].len = 0; 206 } 207 } 208 209 /** 210 * hisi_acc_sg_buf_map_to_hw_sgl - Map a scatterlist to a hw sgl. 211 * @dev: The device which hw sgl belongs to. 212 * @sgl: Scatterlist which will be mapped to hw sgl. 213 * @pool: Pool which hw sgl memory will be allocated in. 214 * @index: Index of hisi_acc_hw_sgl in pool. 215 * @hw_sgl_dma: The dma address of allocated hw sgl. 216 * 217 * This function builds hw sgl according input sgl, user can use hw_sgl_dma 218 * as src/dst in its BD. Only support single hw sgl currently. 219 */ 220 struct hisi_acc_hw_sgl * 221 hisi_acc_sg_buf_map_to_hw_sgl(struct device *dev, 222 struct scatterlist *sgl, 223 struct hisi_acc_sgl_pool *pool, 224 u32 index, dma_addr_t *hw_sgl_dma) 225 { 226 struct hisi_acc_hw_sgl *curr_hw_sgl; 227 dma_addr_t curr_sgl_dma = 0; 228 struct acc_hw_sge *curr_hw_sge; 229 struct scatterlist *sg; 230 int i, sg_n, sg_n_mapped; 231 232 if (!dev || !sgl || !pool || !hw_sgl_dma) 233 return ERR_PTR(-EINVAL); 234 235 sg_n = sg_nents(sgl); 236 237 sg_n_mapped = dma_map_sg(dev, sgl, sg_n, DMA_BIDIRECTIONAL); 238 if (!sg_n_mapped) { 239 dev_err(dev, "DMA mapping for SG error!\n"); 240 return ERR_PTR(-EINVAL); 241 } 242 243 if (sg_n_mapped > pool->sge_nr) { 244 dev_err(dev, "the number of entries in input scatterlist is bigger than SGL pool setting.\n"); 245 return ERR_PTR(-EINVAL); 246 } 247 248 curr_hw_sgl = acc_get_sgl(pool, index, &curr_sgl_dma); 249 if (IS_ERR(curr_hw_sgl)) { 250 dev_err(dev, "Get SGL error!\n"); 251 dma_unmap_sg(dev, sgl, sg_n, DMA_BIDIRECTIONAL); 252 return ERR_PTR(-ENOMEM); 253 254 } 255 curr_hw_sgl->entry_length_in_sgl = cpu_to_le16(pool->sge_nr); 256 curr_hw_sge = curr_hw_sgl->sge_entries; 257 258 for_each_sg(sgl, sg, sg_n_mapped, i) { 259 sg_map_to_hw_sg(sg, curr_hw_sge); 260 inc_hw_sgl_sge(curr_hw_sgl); 261 curr_hw_sge++; 262 } 263 264 update_hw_sgl_sum_sge(curr_hw_sgl, pool->sge_nr); 265 *hw_sgl_dma = curr_sgl_dma; 266 267 return curr_hw_sgl; 268 } 269 EXPORT_SYMBOL_GPL(hisi_acc_sg_buf_map_to_hw_sgl); 270 271 /** 272 * hisi_acc_sg_buf_unmap() - Unmap allocated hw sgl. 273 * @dev: The device which hw sgl belongs to. 274 * @sgl: Related scatterlist. 275 * @hw_sgl: Virtual address of hw sgl. 276 * 277 * This function unmaps allocated hw sgl. 278 */ 279 void hisi_acc_sg_buf_unmap(struct device *dev, struct scatterlist *sgl, 280 struct hisi_acc_hw_sgl *hw_sgl) 281 { 282 if (!dev || !sgl || !hw_sgl) 283 return; 284 285 dma_unmap_sg(dev, sgl, sg_nents(sgl), DMA_BIDIRECTIONAL); 286 clear_hw_sgl_sge(hw_sgl); 287 hw_sgl->entry_sum_in_chain = 0; 288 hw_sgl->entry_sum_in_sgl = 0; 289 hw_sgl->entry_length_in_sgl = 0; 290 } 291 EXPORT_SYMBOL_GPL(hisi_acc_sg_buf_unmap); 292