1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) 2 /* Copyright (c) 2020 Marvell International Ltd. */ 3 4 #include <linux/dma-mapping.h> 5 #include <linux/qed/qed_chain.h> 6 #include <linux/vmalloc.h> 7 8 #include "qed_dev_api.h" 9 10 static void qed_chain_init(struct qed_chain *chain, 11 const struct qed_chain_init_params *params, 12 u32 page_cnt) 13 { 14 memset(chain, 0, sizeof(*chain)); 15 16 chain->elem_size = params->elem_size; 17 chain->intended_use = params->intended_use; 18 chain->mode = params->mode; 19 chain->cnt_type = params->cnt_type; 20 21 chain->elem_per_page = ELEMS_PER_PAGE(params->elem_size, 22 params->page_size); 23 chain->usable_per_page = USABLE_ELEMS_PER_PAGE(params->elem_size, 24 params->page_size, 25 params->mode); 26 chain->elem_unusable = UNUSABLE_ELEMS_PER_PAGE(params->elem_size, 27 params->mode); 28 29 chain->elem_per_page_mask = chain->elem_per_page - 1; 30 chain->next_page_mask = chain->usable_per_page & 31 chain->elem_per_page_mask; 32 33 chain->page_size = params->page_size; 34 chain->page_cnt = page_cnt; 35 chain->capacity = chain->usable_per_page * page_cnt; 36 chain->size = chain->elem_per_page * page_cnt; 37 38 if (params->ext_pbl_virt) { 39 chain->pbl_sp.table_virt = params->ext_pbl_virt; 40 chain->pbl_sp.table_phys = params->ext_pbl_phys; 41 42 chain->b_external_pbl = true; 43 } 44 } 45 46 static void qed_chain_init_next_ptr_elem(const struct qed_chain *chain, 47 void *virt_curr, void *virt_next, 48 dma_addr_t phys_next) 49 { 50 struct qed_chain_next *next; 51 u32 size; 52 53 size = chain->elem_size * chain->usable_per_page; 54 next = virt_curr + size; 55 56 DMA_REGPAIR_LE(next->next_phys, phys_next); 57 next->next_virt = virt_next; 58 } 59 60 static void qed_chain_init_mem(struct qed_chain *chain, void *virt_addr, 61 dma_addr_t phys_addr) 62 { 63 chain->p_virt_addr = virt_addr; 64 chain->p_phys_addr = phys_addr; 65 } 66 67 static void qed_chain_free_next_ptr(struct qed_dev *cdev, 68 struct qed_chain *chain) 69 { 70 struct device *dev = &cdev->pdev->dev; 71 struct qed_chain_next *next; 72 dma_addr_t phys, phys_next; 73 void *virt, *virt_next; 74 u32 size, i; 75 76 size = chain->elem_size * chain->usable_per_page; 77 virt = chain->p_virt_addr; 78 phys = chain->p_phys_addr; 79 80 for (i = 0; i < chain->page_cnt; i++) { 81 if (!virt) 82 break; 83 84 next = virt + size; 85 virt_next = next->next_virt; 86 phys_next = HILO_DMA_REGPAIR(next->next_phys); 87 88 dma_free_coherent(dev, chain->page_size, virt, phys); 89 90 virt = virt_next; 91 phys = phys_next; 92 } 93 } 94 95 static void qed_chain_free_single(struct qed_dev *cdev, 96 struct qed_chain *chain) 97 { 98 if (!chain->p_virt_addr) 99 return; 100 101 dma_free_coherent(&cdev->pdev->dev, chain->page_size, 102 chain->p_virt_addr, chain->p_phys_addr); 103 } 104 105 static void qed_chain_free_pbl(struct qed_dev *cdev, struct qed_chain *chain) 106 { 107 struct device *dev = &cdev->pdev->dev; 108 struct addr_tbl_entry *entry; 109 u32 i; 110 111 if (!chain->pbl.pp_addr_tbl) 112 return; 113 114 for (i = 0; i < chain->page_cnt; i++) { 115 entry = chain->pbl.pp_addr_tbl + i; 116 if (!entry->virt_addr) 117 break; 118 119 dma_free_coherent(dev, chain->page_size, entry->virt_addr, 120 entry->dma_map); 121 } 122 123 if (!chain->b_external_pbl) 124 dma_free_coherent(dev, chain->pbl_sp.table_size, 125 chain->pbl_sp.table_virt, 126 chain->pbl_sp.table_phys); 127 128 vfree(chain->pbl.pp_addr_tbl); 129 chain->pbl.pp_addr_tbl = NULL; 130 } 131 132 /** 133 * qed_chain_free() - Free chain DMA memory. 134 * 135 * @cdev: Main device structure. 136 * @chain: Chain to free. 137 */ 138 void qed_chain_free(struct qed_dev *cdev, struct qed_chain *chain) 139 { 140 switch (chain->mode) { 141 case QED_CHAIN_MODE_NEXT_PTR: 142 qed_chain_free_next_ptr(cdev, chain); 143 break; 144 case QED_CHAIN_MODE_SINGLE: 145 qed_chain_free_single(cdev, chain); 146 break; 147 case QED_CHAIN_MODE_PBL: 148 qed_chain_free_pbl(cdev, chain); 149 break; 150 default: 151 return; 152 } 153 154 qed_chain_init_mem(chain, NULL, 0); 155 } 156 157 static int 158 qed_chain_alloc_sanity_check(struct qed_dev *cdev, 159 const struct qed_chain_init_params *params, 160 u32 page_cnt) 161 { 162 u64 chain_size; 163 164 chain_size = ELEMS_PER_PAGE(params->elem_size, params->page_size); 165 chain_size *= page_cnt; 166 167 if (!chain_size) 168 return -EINVAL; 169 170 /* The actual chain size can be larger than the maximal possible value 171 * after rounding up the requested elements number to pages, and after 172 * taking into account the unusuable elements (next-ptr elements). 173 * The size of a "u16" chain can be (U16_MAX + 1) since the chain 174 * size/capacity fields are of u32 type. 175 */ 176 switch (params->cnt_type) { 177 case QED_CHAIN_CNT_TYPE_U16: 178 if (chain_size > U16_MAX + 1) 179 break; 180 181 return 0; 182 case QED_CHAIN_CNT_TYPE_U32: 183 if (chain_size > U32_MAX) 184 break; 185 186 return 0; 187 default: 188 return -EINVAL; 189 } 190 191 DP_NOTICE(cdev, 192 "The actual chain size (0x%llx) is larger than the maximal possible value\n", 193 chain_size); 194 195 return -EINVAL; 196 } 197 198 static int qed_chain_alloc_next_ptr(struct qed_dev *cdev, 199 struct qed_chain *chain) 200 { 201 struct device *dev = &cdev->pdev->dev; 202 void *virt, *virt_prev = NULL; 203 dma_addr_t phys; 204 u32 i; 205 206 for (i = 0; i < chain->page_cnt; i++) { 207 virt = dma_alloc_coherent(dev, chain->page_size, &phys, 208 GFP_KERNEL); 209 if (!virt) 210 return -ENOMEM; 211 212 if (i == 0) { 213 qed_chain_init_mem(chain, virt, phys); 214 qed_chain_reset(chain); 215 } else { 216 qed_chain_init_next_ptr_elem(chain, virt_prev, virt, 217 phys); 218 } 219 220 virt_prev = virt; 221 } 222 223 /* Last page's next element should point to the beginning of the 224 * chain. 225 */ 226 qed_chain_init_next_ptr_elem(chain, virt_prev, chain->p_virt_addr, 227 chain->p_phys_addr); 228 229 return 0; 230 } 231 232 static int qed_chain_alloc_single(struct qed_dev *cdev, 233 struct qed_chain *chain) 234 { 235 dma_addr_t phys; 236 void *virt; 237 238 virt = dma_alloc_coherent(&cdev->pdev->dev, chain->page_size, 239 &phys, GFP_KERNEL); 240 if (!virt) 241 return -ENOMEM; 242 243 qed_chain_init_mem(chain, virt, phys); 244 qed_chain_reset(chain); 245 246 return 0; 247 } 248 249 static int qed_chain_alloc_pbl(struct qed_dev *cdev, struct qed_chain *chain) 250 { 251 struct device *dev = &cdev->pdev->dev; 252 struct addr_tbl_entry *addr_tbl; 253 dma_addr_t phys, pbl_phys; 254 __le64 *pbl_virt; 255 u32 page_cnt, i; 256 size_t size; 257 void *virt; 258 259 page_cnt = chain->page_cnt; 260 261 size = array_size(page_cnt, sizeof(*addr_tbl)); 262 if (unlikely(size == SIZE_MAX)) 263 return -EOVERFLOW; 264 265 addr_tbl = vzalloc(size); 266 if (!addr_tbl) 267 return -ENOMEM; 268 269 chain->pbl.pp_addr_tbl = addr_tbl; 270 271 if (chain->b_external_pbl) 272 goto alloc_pages; 273 274 size = array_size(page_cnt, sizeof(*pbl_virt)); 275 if (unlikely(size == SIZE_MAX)) 276 return -EOVERFLOW; 277 278 pbl_virt = dma_alloc_coherent(dev, size, &pbl_phys, GFP_KERNEL); 279 if (!pbl_virt) 280 return -ENOMEM; 281 282 chain->pbl_sp.table_virt = pbl_virt; 283 chain->pbl_sp.table_phys = pbl_phys; 284 chain->pbl_sp.table_size = size; 285 286 alloc_pages: 287 for (i = 0; i < page_cnt; i++) { 288 virt = dma_alloc_coherent(dev, chain->page_size, &phys, 289 GFP_KERNEL); 290 if (!virt) 291 return -ENOMEM; 292 293 if (i == 0) { 294 qed_chain_init_mem(chain, virt, phys); 295 qed_chain_reset(chain); 296 } 297 298 /* Fill the PBL table with the physical address of the page */ 299 pbl_virt[i] = cpu_to_le64(phys); 300 301 /* Keep the virtual address of the page */ 302 addr_tbl[i].virt_addr = virt; 303 addr_tbl[i].dma_map = phys; 304 } 305 306 return 0; 307 } 308 309 /** 310 * qed_chain_alloc() - Allocate and initialize a chain. 311 * 312 * @cdev: Main device structure. 313 * @chain: Chain to be processed. 314 * @params: Chain initialization parameters. 315 * 316 * Return: 0 on success, negative errno otherwise. 317 */ 318 int qed_chain_alloc(struct qed_dev *cdev, struct qed_chain *chain, 319 struct qed_chain_init_params *params) 320 { 321 u32 page_cnt; 322 int rc; 323 324 if (!params->page_size) 325 params->page_size = QED_CHAIN_PAGE_SIZE; 326 327 if (params->mode == QED_CHAIN_MODE_SINGLE) 328 page_cnt = 1; 329 else 330 page_cnt = QED_CHAIN_PAGE_CNT(params->num_elems, 331 params->elem_size, 332 params->page_size, 333 params->mode); 334 335 rc = qed_chain_alloc_sanity_check(cdev, params, page_cnt); 336 if (rc) { 337 DP_NOTICE(cdev, 338 "Cannot allocate a chain with the given arguments:\n"); 339 DP_NOTICE(cdev, 340 "[use_mode %d, mode %d, cnt_type %d, num_elems %d, elem_size %zu, page_size %u]\n", 341 params->intended_use, params->mode, params->cnt_type, 342 params->num_elems, params->elem_size, 343 params->page_size); 344 return rc; 345 } 346 347 qed_chain_init(chain, params, page_cnt); 348 349 switch (params->mode) { 350 case QED_CHAIN_MODE_NEXT_PTR: 351 rc = qed_chain_alloc_next_ptr(cdev, chain); 352 break; 353 case QED_CHAIN_MODE_SINGLE: 354 rc = qed_chain_alloc_single(cdev, chain); 355 break; 356 case QED_CHAIN_MODE_PBL: 357 rc = qed_chain_alloc_pbl(cdev, chain); 358 break; 359 default: 360 return -EINVAL; 361 } 362 363 if (!rc) 364 return 0; 365 366 qed_chain_free(cdev, chain); 367 368 return rc; 369 } 370