1 // SPDX-License-Identifier: GPL-2.0 2 3 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved. 4 * Copyright (C) 2019-2023 Linaro Ltd. 5 */ 6 7 #include <linux/types.h> 8 #include <linux/bitfield.h> 9 #include <linux/bug.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/iommu.h> 12 #include <linux/io.h> 13 #include <linux/soc/qcom/smem.h> 14 15 #include "ipa.h" 16 #include "ipa_reg.h" 17 #include "ipa_data.h" 18 #include "ipa_cmd.h" 19 #include "ipa_mem.h" 20 #include "ipa_table.h" 21 #include "gsi_trans.h" 22 23 /* "Canary" value placed between memory regions to detect overflow */ 24 #define IPA_MEM_CANARY_VAL cpu_to_le32(0xdeadbeef) 25 26 /* SMEM host id representing the modem. */ 27 #define QCOM_SMEM_HOST_MODEM 1 28 29 const struct ipa_mem *ipa_mem_find(struct ipa *ipa, enum ipa_mem_id mem_id) 30 { 31 u32 i; 32 33 for (i = 0; i < ipa->mem_count; i++) { 34 const struct ipa_mem *mem = &ipa->mem[i]; 35 36 if (mem->id == mem_id) 37 return mem; 38 } 39 40 return NULL; 41 } 42 43 /* Add an immediate command to a transaction that zeroes a memory region */ 44 static void 45 ipa_mem_zero_region_add(struct gsi_trans *trans, enum ipa_mem_id mem_id) 46 { 47 struct ipa *ipa = container_of(trans->gsi, struct ipa, gsi); 48 const struct ipa_mem *mem = ipa_mem_find(ipa, mem_id); 49 dma_addr_t addr = ipa->zero_addr; 50 51 if (!mem->size) 52 return; 53 54 ipa_cmd_dma_shared_mem_add(trans, mem->offset, mem->size, addr, true); 55 } 56 57 /** 58 * ipa_mem_setup() - Set up IPA AP and modem shared memory areas 59 * @ipa: IPA pointer 60 * 61 * Set up the shared memory regions in IPA local memory. This involves 62 * zero-filling memory regions, and in the case of header memory, telling 63 * the IPA where it's located. 64 * 65 * This function performs the initial setup of this memory. If the modem 66 * crashes, its regions are re-zeroed in ipa_mem_zero_modem(). 67 * 68 * The AP informs the modem where its portions of memory are located 69 * in a QMI exchange that occurs at modem startup. 70 * 71 * There is no need for a matching ipa_mem_teardown() function. 72 * 73 * Return: 0 if successful, or a negative error code 74 */ 75 int ipa_mem_setup(struct ipa *ipa) 76 { 77 dma_addr_t addr = ipa->zero_addr; 78 const struct reg *reg; 79 const struct ipa_mem *mem; 80 struct gsi_trans *trans; 81 u32 offset; 82 u16 size; 83 u32 val; 84 85 /* Get a transaction to define the header memory region and to zero 86 * the processing context and modem memory regions. 87 */ 88 trans = ipa_cmd_trans_alloc(ipa, 4); 89 if (!trans) { 90 dev_err(&ipa->pdev->dev, "no transaction for memory setup\n"); 91 return -EBUSY; 92 } 93 94 /* Initialize IPA-local header memory. The AP header region, if 95 * present, is contiguous with and follows the modem header region, 96 * and they are initialized together. 97 */ 98 mem = ipa_mem_find(ipa, IPA_MEM_MODEM_HEADER); 99 offset = mem->offset; 100 size = mem->size; 101 mem = ipa_mem_find(ipa, IPA_MEM_AP_HEADER); 102 if (mem) 103 size += mem->size; 104 105 ipa_cmd_hdr_init_local_add(trans, offset, size, addr); 106 107 ipa_mem_zero_region_add(trans, IPA_MEM_MODEM_PROC_CTX); 108 ipa_mem_zero_region_add(trans, IPA_MEM_AP_PROC_CTX); 109 ipa_mem_zero_region_add(trans, IPA_MEM_MODEM); 110 111 gsi_trans_commit_wait(trans); 112 113 /* Tell the hardware where the processing context area is located */ 114 mem = ipa_mem_find(ipa, IPA_MEM_MODEM_PROC_CTX); 115 offset = ipa->mem_offset + mem->offset; 116 117 reg = ipa_reg(ipa, LOCAL_PKT_PROC_CNTXT); 118 val = reg_encode(reg, IPA_BASE_ADDR, offset); 119 iowrite32(val, ipa->reg_virt + reg_offset(reg)); 120 121 return 0; 122 } 123 124 /* Is the given memory region ID is valid for the current IPA version? */ 125 static bool ipa_mem_id_valid(struct ipa *ipa, enum ipa_mem_id mem_id) 126 { 127 enum ipa_version version = ipa->version; 128 129 switch (mem_id) { 130 case IPA_MEM_UC_SHARED: 131 case IPA_MEM_UC_INFO: 132 case IPA_MEM_V4_FILTER_HASHED: 133 case IPA_MEM_V4_FILTER: 134 case IPA_MEM_V6_FILTER_HASHED: 135 case IPA_MEM_V6_FILTER: 136 case IPA_MEM_V4_ROUTE_HASHED: 137 case IPA_MEM_V4_ROUTE: 138 case IPA_MEM_V6_ROUTE_HASHED: 139 case IPA_MEM_V6_ROUTE: 140 case IPA_MEM_MODEM_HEADER: 141 case IPA_MEM_AP_HEADER: 142 case IPA_MEM_MODEM_PROC_CTX: 143 case IPA_MEM_AP_PROC_CTX: 144 case IPA_MEM_MODEM: 145 case IPA_MEM_UC_EVENT_RING: 146 case IPA_MEM_PDN_CONFIG: 147 case IPA_MEM_STATS_QUOTA_MODEM: 148 case IPA_MEM_STATS_QUOTA_AP: 149 case IPA_MEM_END_MARKER: /* pseudo region */ 150 break; 151 152 case IPA_MEM_STATS_TETHERING: 153 case IPA_MEM_STATS_DROP: 154 if (version < IPA_VERSION_4_0) 155 return false; 156 break; 157 158 case IPA_MEM_STATS_V4_FILTER: 159 case IPA_MEM_STATS_V6_FILTER: 160 case IPA_MEM_STATS_V4_ROUTE: 161 case IPA_MEM_STATS_V6_ROUTE: 162 if (version < IPA_VERSION_4_0 || version > IPA_VERSION_4_2) 163 return false; 164 break; 165 166 case IPA_MEM_AP_V4_FILTER: 167 case IPA_MEM_AP_V6_FILTER: 168 if (version != IPA_VERSION_5_0) 169 return false; 170 break; 171 172 case IPA_MEM_NAT_TABLE: 173 case IPA_MEM_STATS_FILTER_ROUTE: 174 if (version < IPA_VERSION_4_5) 175 return false; 176 break; 177 178 default: 179 return false; 180 } 181 182 return true; 183 } 184 185 /* Must the given memory region be present in the configuration? */ 186 static bool ipa_mem_id_required(struct ipa *ipa, enum ipa_mem_id mem_id) 187 { 188 switch (mem_id) { 189 case IPA_MEM_UC_SHARED: 190 case IPA_MEM_UC_INFO: 191 case IPA_MEM_V4_FILTER_HASHED: 192 case IPA_MEM_V4_FILTER: 193 case IPA_MEM_V6_FILTER_HASHED: 194 case IPA_MEM_V6_FILTER: 195 case IPA_MEM_V4_ROUTE_HASHED: 196 case IPA_MEM_V4_ROUTE: 197 case IPA_MEM_V6_ROUTE_HASHED: 198 case IPA_MEM_V6_ROUTE: 199 case IPA_MEM_MODEM_HEADER: 200 case IPA_MEM_MODEM_PROC_CTX: 201 case IPA_MEM_AP_PROC_CTX: 202 case IPA_MEM_MODEM: 203 return true; 204 205 case IPA_MEM_PDN_CONFIG: 206 case IPA_MEM_STATS_QUOTA_MODEM: 207 return ipa->version >= IPA_VERSION_4_0; 208 209 case IPA_MEM_STATS_TETHERING: 210 return ipa->version >= IPA_VERSION_4_0 && 211 ipa->version != IPA_VERSION_5_0; 212 213 default: 214 return false; /* Anything else is optional */ 215 } 216 } 217 218 static bool ipa_mem_valid_one(struct ipa *ipa, const struct ipa_mem *mem) 219 { 220 struct device *dev = &ipa->pdev->dev; 221 enum ipa_mem_id mem_id = mem->id; 222 u16 size_multiple; 223 224 /* Make sure the memory region is valid for this version of IPA */ 225 if (!ipa_mem_id_valid(ipa, mem_id)) { 226 dev_err(dev, "region id %u not valid\n", mem_id); 227 return false; 228 } 229 230 if (!mem->size && !mem->canary_count) { 231 dev_err(dev, "empty memory region %u\n", mem_id); 232 return false; 233 } 234 235 /* Other than modem memory, sizes must be a multiple of 8 */ 236 size_multiple = mem_id == IPA_MEM_MODEM ? 4 : 8; 237 if (mem->size % size_multiple) 238 dev_err(dev, "region %u size not a multiple of %u bytes\n", 239 mem_id, size_multiple); 240 else if (mem->offset % 8) 241 dev_err(dev, "region %u offset not 8-byte aligned\n", mem_id); 242 else if (mem->offset < mem->canary_count * sizeof(__le32)) 243 dev_err(dev, "region %u offset too small for %hu canaries\n", 244 mem_id, mem->canary_count); 245 else if (mem_id == IPA_MEM_END_MARKER && mem->size) 246 dev_err(dev, "non-zero end marker region size\n"); 247 else 248 return true; 249 250 return false; 251 } 252 253 /* Verify each defined memory region is valid. */ 254 static bool ipa_mem_valid(struct ipa *ipa, const struct ipa_mem_data *mem_data) 255 { 256 DECLARE_BITMAP(regions, IPA_MEM_COUNT) = { }; 257 struct device *dev = &ipa->pdev->dev; 258 enum ipa_mem_id mem_id; 259 u32 i; 260 261 if (mem_data->local_count > IPA_MEM_COUNT) { 262 dev_err(dev, "too many memory regions (%u > %u)\n", 263 mem_data->local_count, IPA_MEM_COUNT); 264 return false; 265 } 266 267 for (i = 0; i < mem_data->local_count; i++) { 268 const struct ipa_mem *mem = &mem_data->local[i]; 269 270 if (__test_and_set_bit(mem->id, regions)) { 271 dev_err(dev, "duplicate memory region %u\n", mem->id); 272 return false; 273 } 274 275 /* Defined regions have non-zero size and/or canary count */ 276 if (!ipa_mem_valid_one(ipa, mem)) 277 return false; 278 } 279 280 /* Now see if any required regions are not defined */ 281 for_each_clear_bit(mem_id, regions, IPA_MEM_COUNT) { 282 if (ipa_mem_id_required(ipa, mem_id)) 283 dev_err(dev, "required memory region %u missing\n", 284 mem_id); 285 } 286 287 return true; 288 } 289 290 /* Do all memory regions fit within the IPA local memory? */ 291 static bool ipa_mem_size_valid(struct ipa *ipa) 292 { 293 struct device *dev = &ipa->pdev->dev; 294 u32 limit = ipa->mem_size; 295 u32 i; 296 297 for (i = 0; i < ipa->mem_count; i++) { 298 const struct ipa_mem *mem = &ipa->mem[i]; 299 300 if (mem->offset + mem->size <= limit) 301 continue; 302 303 dev_err(dev, "region %u ends beyond memory limit (0x%08x)\n", 304 mem->id, limit); 305 306 return false; 307 } 308 309 return true; 310 } 311 312 /** 313 * ipa_mem_config() - Configure IPA shared memory 314 * @ipa: IPA pointer 315 * 316 * Return: 0 if successful, or a negative error code 317 */ 318 int ipa_mem_config(struct ipa *ipa) 319 { 320 struct device *dev = &ipa->pdev->dev; 321 const struct ipa_mem *mem; 322 const struct reg *reg; 323 dma_addr_t addr; 324 u32 mem_size; 325 void *virt; 326 u32 val; 327 u32 i; 328 329 /* Check the advertised location and size of the shared memory area */ 330 reg = ipa_reg(ipa, SHARED_MEM_SIZE); 331 val = ioread32(ipa->reg_virt + reg_offset(reg)); 332 333 /* The fields in the register are in 8 byte units */ 334 ipa->mem_offset = 8 * reg_decode(reg, MEM_BADDR, val); 335 336 /* Make sure the end is within the region's mapped space */ 337 mem_size = 8 * reg_decode(reg, MEM_SIZE, val); 338 339 /* If the sizes don't match, issue a warning */ 340 if (ipa->mem_offset + mem_size < ipa->mem_size) { 341 dev_warn(dev, "limiting IPA memory size to 0x%08x\n", 342 mem_size); 343 ipa->mem_size = mem_size; 344 } else if (ipa->mem_offset + mem_size > ipa->mem_size) { 345 dev_dbg(dev, "ignoring larger reported memory size: 0x%08x\n", 346 mem_size); 347 } 348 349 /* We know our memory size; make sure regions are all in range */ 350 if (!ipa_mem_size_valid(ipa)) 351 return -EINVAL; 352 353 /* Prealloc DMA memory for zeroing regions */ 354 virt = dma_alloc_coherent(dev, IPA_MEM_MAX, &addr, GFP_KERNEL); 355 if (!virt) 356 return -ENOMEM; 357 ipa->zero_addr = addr; 358 ipa->zero_virt = virt; 359 ipa->zero_size = IPA_MEM_MAX; 360 361 /* For each defined region, write "canary" values in the 362 * space prior to the region's base address if indicated. 363 */ 364 for (i = 0; i < ipa->mem_count; i++) { 365 u16 canary_count = ipa->mem[i].canary_count; 366 __le32 *canary; 367 368 if (!canary_count) 369 continue; 370 371 /* Write canary values in the space before the region */ 372 canary = ipa->mem_virt + ipa->mem_offset + ipa->mem[i].offset; 373 do 374 *--canary = IPA_MEM_CANARY_VAL; 375 while (--canary_count); 376 } 377 378 /* Verify the microcontroller ring alignment (if defined) */ 379 mem = ipa_mem_find(ipa, IPA_MEM_UC_EVENT_RING); 380 if (mem && mem->offset % 1024) { 381 dev_err(dev, "microcontroller ring not 1024-byte aligned\n"); 382 goto err_dma_free; 383 } 384 385 return 0; 386 387 err_dma_free: 388 dma_free_coherent(dev, IPA_MEM_MAX, ipa->zero_virt, ipa->zero_addr); 389 390 return -EINVAL; 391 } 392 393 /* Inverse of ipa_mem_config() */ 394 void ipa_mem_deconfig(struct ipa *ipa) 395 { 396 struct device *dev = &ipa->pdev->dev; 397 398 dma_free_coherent(dev, ipa->zero_size, ipa->zero_virt, ipa->zero_addr); 399 ipa->zero_size = 0; 400 ipa->zero_virt = NULL; 401 ipa->zero_addr = 0; 402 } 403 404 /** 405 * ipa_mem_zero_modem() - Zero IPA-local memory regions owned by the modem 406 * @ipa: IPA pointer 407 * 408 * Zero regions of IPA-local memory used by the modem. These are configured 409 * (and initially zeroed) by ipa_mem_setup(), but if the modem crashes and 410 * restarts via SSR we need to re-initialize them. A QMI message tells the 411 * modem where to find regions of IPA local memory it needs to know about 412 * (these included). 413 */ 414 int ipa_mem_zero_modem(struct ipa *ipa) 415 { 416 struct gsi_trans *trans; 417 418 /* Get a transaction to zero the modem memory, modem header, 419 * and modem processing context regions. 420 */ 421 trans = ipa_cmd_trans_alloc(ipa, 3); 422 if (!trans) { 423 dev_err(&ipa->pdev->dev, 424 "no transaction to zero modem memory\n"); 425 return -EBUSY; 426 } 427 428 ipa_mem_zero_region_add(trans, IPA_MEM_MODEM_HEADER); 429 ipa_mem_zero_region_add(trans, IPA_MEM_MODEM_PROC_CTX); 430 ipa_mem_zero_region_add(trans, IPA_MEM_MODEM); 431 432 gsi_trans_commit_wait(trans); 433 434 return 0; 435 } 436 437 /** 438 * ipa_imem_init() - Initialize IMEM memory used by the IPA 439 * @ipa: IPA pointer 440 * @addr: Physical address of the IPA region in IMEM 441 * @size: Size (bytes) of the IPA region in IMEM 442 * 443 * IMEM is a block of shared memory separate from system DRAM, and 444 * a portion of this memory is available for the IPA to use. The 445 * modem accesses this memory directly, but the IPA accesses it 446 * via the IOMMU, using the AP's credentials. 447 * 448 * If this region exists (size > 0) we map it for read/write access 449 * through the IOMMU using the IPA device. 450 * 451 * Note: @addr and @size are not guaranteed to be page-aligned. 452 */ 453 static int ipa_imem_init(struct ipa *ipa, unsigned long addr, size_t size) 454 { 455 struct device *dev = &ipa->pdev->dev; 456 struct iommu_domain *domain; 457 unsigned long iova; 458 phys_addr_t phys; 459 int ret; 460 461 if (!size) 462 return 0; /* IMEM memory not used */ 463 464 domain = iommu_get_domain_for_dev(dev); 465 if (!domain) { 466 dev_err(dev, "no IOMMU domain found for IMEM\n"); 467 return -EINVAL; 468 } 469 470 /* Align the address down and the size up to page boundaries */ 471 phys = addr & PAGE_MASK; 472 size = PAGE_ALIGN(size + addr - phys); 473 iova = phys; /* We just want a direct mapping */ 474 475 ret = iommu_map(domain, iova, phys, size, IOMMU_READ | IOMMU_WRITE, 476 GFP_KERNEL); 477 if (ret) 478 return ret; 479 480 ipa->imem_iova = iova; 481 ipa->imem_size = size; 482 483 return 0; 484 } 485 486 static void ipa_imem_exit(struct ipa *ipa) 487 { 488 struct iommu_domain *domain; 489 struct device *dev; 490 491 if (!ipa->imem_size) 492 return; 493 494 dev = &ipa->pdev->dev; 495 domain = iommu_get_domain_for_dev(dev); 496 if (domain) { 497 size_t size; 498 499 size = iommu_unmap(domain, ipa->imem_iova, ipa->imem_size); 500 if (size != ipa->imem_size) 501 dev_warn(dev, "unmapped %zu IMEM bytes, expected %zu\n", 502 size, ipa->imem_size); 503 } else { 504 dev_err(dev, "couldn't get IPA IOMMU domain for IMEM\n"); 505 } 506 507 ipa->imem_size = 0; 508 ipa->imem_iova = 0; 509 } 510 511 /** 512 * ipa_smem_init() - Initialize SMEM memory used by the IPA 513 * @ipa: IPA pointer 514 * @item: Item ID of SMEM memory 515 * @size: Size (bytes) of SMEM memory region 516 * 517 * SMEM is a managed block of shared DRAM, from which numbered "items" 518 * can be allocated. One item is designated for use by the IPA. 519 * 520 * The modem accesses SMEM memory directly, but the IPA accesses it 521 * via the IOMMU, using the AP's credentials. 522 * 523 * If size provided is non-zero, we allocate it and map it for 524 * access through the IOMMU. 525 * 526 * Note: @size and the item address are is not guaranteed to be page-aligned. 527 */ 528 static int ipa_smem_init(struct ipa *ipa, u32 item, size_t size) 529 { 530 struct device *dev = &ipa->pdev->dev; 531 struct iommu_domain *domain; 532 unsigned long iova; 533 phys_addr_t phys; 534 phys_addr_t addr; 535 size_t actual; 536 void *virt; 537 int ret; 538 539 if (!size) 540 return 0; /* SMEM memory not used */ 541 542 /* SMEM is memory shared between the AP and another system entity 543 * (in this case, the modem). An allocation from SMEM is persistent 544 * until the AP reboots; there is no way to free an allocated SMEM 545 * region. Allocation only reserves the space; to use it you need 546 * to "get" a pointer it (this does not imply reference counting). 547 * The item might have already been allocated, in which case we 548 * use it unless the size isn't what we expect. 549 */ 550 ret = qcom_smem_alloc(QCOM_SMEM_HOST_MODEM, item, size); 551 if (ret && ret != -EEXIST) { 552 dev_err(dev, "error %d allocating size %zu SMEM item %u\n", 553 ret, size, item); 554 return ret; 555 } 556 557 /* Now get the address of the SMEM memory region */ 558 virt = qcom_smem_get(QCOM_SMEM_HOST_MODEM, item, &actual); 559 if (IS_ERR(virt)) { 560 ret = PTR_ERR(virt); 561 dev_err(dev, "error %d getting SMEM item %u\n", ret, item); 562 return ret; 563 } 564 565 /* In case the region was already allocated, verify the size */ 566 if (ret && actual != size) { 567 dev_err(dev, "SMEM item %u has size %zu, expected %zu\n", 568 item, actual, size); 569 return -EINVAL; 570 } 571 572 domain = iommu_get_domain_for_dev(dev); 573 if (!domain) { 574 dev_err(dev, "no IOMMU domain found for SMEM\n"); 575 return -EINVAL; 576 } 577 578 /* Align the address down and the size up to a page boundary */ 579 addr = qcom_smem_virt_to_phys(virt); 580 phys = addr & PAGE_MASK; 581 size = PAGE_ALIGN(size + addr - phys); 582 iova = phys; /* We just want a direct mapping */ 583 584 ret = iommu_map(domain, iova, phys, size, IOMMU_READ | IOMMU_WRITE, 585 GFP_KERNEL); 586 if (ret) 587 return ret; 588 589 ipa->smem_iova = iova; 590 ipa->smem_size = size; 591 592 return 0; 593 } 594 595 static void ipa_smem_exit(struct ipa *ipa) 596 { 597 struct device *dev = &ipa->pdev->dev; 598 struct iommu_domain *domain; 599 600 domain = iommu_get_domain_for_dev(dev); 601 if (domain) { 602 size_t size; 603 604 size = iommu_unmap(domain, ipa->smem_iova, ipa->smem_size); 605 if (size != ipa->smem_size) 606 dev_warn(dev, "unmapped %zu SMEM bytes, expected %zu\n", 607 size, ipa->smem_size); 608 609 } else { 610 dev_err(dev, "couldn't get IPA IOMMU domain for SMEM\n"); 611 } 612 613 ipa->smem_size = 0; 614 ipa->smem_iova = 0; 615 } 616 617 /* Perform memory region-related initialization */ 618 int ipa_mem_init(struct ipa *ipa, const struct ipa_mem_data *mem_data) 619 { 620 struct device *dev = &ipa->pdev->dev; 621 struct resource *res; 622 int ret; 623 624 /* Make sure the set of defined memory regions is valid */ 625 if (!ipa_mem_valid(ipa, mem_data)) 626 return -EINVAL; 627 628 ipa->mem_count = mem_data->local_count; 629 ipa->mem = mem_data->local; 630 631 /* Check the route and filter table memory regions */ 632 if (!ipa_table_mem_valid(ipa, false)) 633 return -EINVAL; 634 if (!ipa_table_mem_valid(ipa, true)) 635 return -EINVAL; 636 637 ret = dma_set_mask_and_coherent(&ipa->pdev->dev, DMA_BIT_MASK(64)); 638 if (ret) { 639 dev_err(dev, "error %d setting DMA mask\n", ret); 640 return ret; 641 } 642 643 res = platform_get_resource_byname(ipa->pdev, IORESOURCE_MEM, 644 "ipa-shared"); 645 if (!res) { 646 dev_err(dev, 647 "DT error getting \"ipa-shared\" memory property\n"); 648 return -ENODEV; 649 } 650 651 ipa->mem_virt = memremap(res->start, resource_size(res), MEMREMAP_WC); 652 if (!ipa->mem_virt) { 653 dev_err(dev, "unable to remap \"ipa-shared\" memory\n"); 654 return -ENOMEM; 655 } 656 657 ipa->mem_addr = res->start; 658 ipa->mem_size = resource_size(res); 659 660 ret = ipa_imem_init(ipa, mem_data->imem_addr, mem_data->imem_size); 661 if (ret) 662 goto err_unmap; 663 664 ret = ipa_smem_init(ipa, mem_data->smem_id, mem_data->smem_size); 665 if (ret) 666 goto err_imem_exit; 667 668 return 0; 669 670 err_imem_exit: 671 ipa_imem_exit(ipa); 672 err_unmap: 673 memunmap(ipa->mem_virt); 674 675 return ret; 676 } 677 678 /* Inverse of ipa_mem_init() */ 679 void ipa_mem_exit(struct ipa *ipa) 680 { 681 ipa_smem_exit(ipa); 682 ipa_imem_exit(ipa); 683 memunmap(ipa->mem_virt); 684 } 685