1ba764c4dSAlex Elder // SPDX-License-Identifier: GPL-2.0 2ba764c4dSAlex Elder 3ba764c4dSAlex Elder /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved. 4*5157d6bfSAlex Elder * Copyright (C) 2019-2023 Linaro Ltd. 5ba764c4dSAlex Elder */ 6ba764c4dSAlex Elder 7ba764c4dSAlex Elder #include <linux/types.h> 8ba764c4dSAlex Elder #include <linux/bitfield.h> 9ba764c4dSAlex Elder #include <linux/bug.h> 10ba764c4dSAlex Elder #include <linux/dma-mapping.h> 113e313c3fSAlex Elder #include <linux/iommu.h> 12ba764c4dSAlex Elder #include <linux/io.h> 13a0036bb4SAlex Elder #include <linux/soc/qcom/smem.h> 14ba764c4dSAlex Elder 15ba764c4dSAlex Elder #include "ipa.h" 16ba764c4dSAlex Elder #include "ipa_reg.h" 173128aae8SAlex Elder #include "ipa_data.h" 18ba764c4dSAlex Elder #include "ipa_cmd.h" 19ba764c4dSAlex Elder #include "ipa_mem.h" 20ba764c4dSAlex Elder #include "ipa_table.h" 21ba764c4dSAlex Elder #include "gsi_trans.h" 22ba764c4dSAlex Elder 23ba764c4dSAlex Elder /* "Canary" value placed between memory regions to detect overflow */ 24ba764c4dSAlex Elder #define IPA_MEM_CANARY_VAL cpu_to_le32(0xdeadbeef) 25ba764c4dSAlex Elder 26a0036bb4SAlex Elder /* SMEM host id representing the modem. */ 27a0036bb4SAlex Elder #define QCOM_SMEM_HOST_MODEM 1 28a0036bb4SAlex Elder 295e3bc1e5SAlex Elder const struct ipa_mem *ipa_mem_find(struct ipa *ipa, enum ipa_mem_id mem_id) 305e3bc1e5SAlex Elder { 31c61cfb94SAlex Elder u32 i; 32c61cfb94SAlex Elder 33c61cfb94SAlex Elder for (i = 0; i < ipa->mem_count; i++) { 34c61cfb94SAlex Elder const struct ipa_mem *mem = &ipa->mem[i]; 35c61cfb94SAlex Elder 36c61cfb94SAlex Elder if (mem->id == mem_id) 37c61cfb94SAlex Elder return mem; 38c61cfb94SAlex Elder } 395e3bc1e5SAlex Elder 405e3bc1e5SAlex Elder return NULL; 415e3bc1e5SAlex Elder } 425e3bc1e5SAlex Elder 43ba764c4dSAlex Elder /* Add an immediate command to a transaction that zeroes a memory region */ 44ba764c4dSAlex Elder static void 45ce928bf8SAlex Elder ipa_mem_zero_region_add(struct gsi_trans *trans, enum ipa_mem_id mem_id) 46ba764c4dSAlex Elder { 47ba764c4dSAlex Elder struct ipa *ipa = container_of(trans->gsi, struct ipa, gsi); 485e3bc1e5SAlex Elder const struct ipa_mem *mem = ipa_mem_find(ipa, mem_id); 49ba764c4dSAlex Elder dma_addr_t addr = ipa->zero_addr; 50ba764c4dSAlex Elder 51ba764c4dSAlex Elder if (!mem->size) 52ba764c4dSAlex Elder return; 53ba764c4dSAlex Elder 54ba764c4dSAlex Elder ipa_cmd_dma_shared_mem_add(trans, mem->offset, mem->size, addr, true); 55ba764c4dSAlex Elder } 56ba764c4dSAlex Elder 57ba764c4dSAlex Elder /** 58ba764c4dSAlex Elder * ipa_mem_setup() - Set up IPA AP and modem shared memory areas 59e3eea08eSAlex Elder * @ipa: IPA pointer 60ba764c4dSAlex Elder * 61ba764c4dSAlex Elder * Set up the shared memory regions in IPA local memory. This involves 62ba764c4dSAlex Elder * zero-filling memory regions, and in the case of header memory, telling 63ba764c4dSAlex Elder * the IPA where it's located. 64ba764c4dSAlex Elder * 65ba764c4dSAlex Elder * This function performs the initial setup of this memory. If the modem 66ba764c4dSAlex Elder * crashes, its regions are re-zeroed in ipa_mem_zero_modem(). 67ba764c4dSAlex Elder * 68ba764c4dSAlex Elder * The AP informs the modem where its portions of memory are located 69ba764c4dSAlex Elder * in a QMI exchange that occurs at modem startup. 70ba764c4dSAlex Elder * 7174858b63SAlex Elder * There is no need for a matching ipa_mem_teardown() function. 7274858b63SAlex Elder * 73e3eea08eSAlex Elder * Return: 0 if successful, or a negative error code 74ba764c4dSAlex Elder */ 75ba764c4dSAlex Elder int ipa_mem_setup(struct ipa *ipa) 76ba764c4dSAlex Elder { 77ba764c4dSAlex Elder dma_addr_t addr = ipa->zero_addr; 786a244b75SAlex Elder const struct ipa_reg *reg; 795e3bc1e5SAlex Elder const struct ipa_mem *mem; 80ba764c4dSAlex Elder struct gsi_trans *trans; 81ba764c4dSAlex Elder u32 offset; 82ba764c4dSAlex Elder u16 size; 83e6e49e43SAlex Elder u32 val; 84ba764c4dSAlex Elder 85ba764c4dSAlex Elder /* Get a transaction to define the header memory region and to zero 86ba764c4dSAlex Elder * the processing context and modem memory regions. 87ba764c4dSAlex Elder */ 88ba764c4dSAlex Elder trans = ipa_cmd_trans_alloc(ipa, 4); 89ba764c4dSAlex Elder if (!trans) { 90ba764c4dSAlex Elder dev_err(&ipa->pdev->dev, "no transaction for memory setup\n"); 91ba764c4dSAlex Elder return -EBUSY; 92ba764c4dSAlex Elder } 93ba764c4dSAlex Elder 945e3bc1e5SAlex Elder /* Initialize IPA-local header memory. The AP header region, if 955e3bc1e5SAlex Elder * present, is contiguous with and follows the modem header region, 965e3bc1e5SAlex Elder * and they are initialized together. 97ba764c4dSAlex Elder */ 985e3bc1e5SAlex Elder mem = ipa_mem_find(ipa, IPA_MEM_MODEM_HEADER); 995e3bc1e5SAlex Elder offset = mem->offset; 1005e3bc1e5SAlex Elder size = mem->size; 1015e3bc1e5SAlex Elder mem = ipa_mem_find(ipa, IPA_MEM_AP_HEADER); 1025e3bc1e5SAlex Elder if (mem) 1035e3bc1e5SAlex Elder size += mem->size; 104ba764c4dSAlex Elder 105ba764c4dSAlex Elder ipa_cmd_hdr_init_local_add(trans, offset, size, addr); 106ba764c4dSAlex Elder 107ce928bf8SAlex Elder ipa_mem_zero_region_add(trans, IPA_MEM_MODEM_PROC_CTX); 108ce928bf8SAlex Elder ipa_mem_zero_region_add(trans, IPA_MEM_AP_PROC_CTX); 109ce928bf8SAlex Elder ipa_mem_zero_region_add(trans, IPA_MEM_MODEM); 110ba764c4dSAlex Elder 111ba764c4dSAlex Elder gsi_trans_commit_wait(trans); 112ba764c4dSAlex Elder 113ba764c4dSAlex Elder /* Tell the hardware where the processing context area is located */ 1145e3bc1e5SAlex Elder mem = ipa_mem_find(ipa, IPA_MEM_MODEM_PROC_CTX); 1155e3bc1e5SAlex Elder offset = ipa->mem_offset + mem->offset; 1166bfb7538SAlex Elder 1176a244b75SAlex Elder reg = ipa_reg(ipa, LOCAL_PKT_PROC_CNTXT); 118b5c35fa4SAlex Elder val = ipa_reg_encode(reg, IPA_BASE_ADDR, offset); 1196a244b75SAlex Elder iowrite32(val, ipa->reg_virt + ipa_reg_offset(reg)); 120ba764c4dSAlex Elder 121ba764c4dSAlex Elder return 0; 122ba764c4dSAlex Elder } 123ba764c4dSAlex Elder 12475bcfde6SAlex Elder /* Is the given memory region ID is valid for the current IPA version? */ 12575bcfde6SAlex Elder static bool ipa_mem_id_valid(struct ipa *ipa, enum ipa_mem_id mem_id) 12675bcfde6SAlex Elder { 12775bcfde6SAlex Elder enum ipa_version version = ipa->version; 12875bcfde6SAlex Elder 12975bcfde6SAlex Elder switch (mem_id) { 13075bcfde6SAlex Elder case IPA_MEM_UC_SHARED: 13175bcfde6SAlex Elder case IPA_MEM_UC_INFO: 13275bcfde6SAlex Elder case IPA_MEM_V4_FILTER_HASHED: 13375bcfde6SAlex Elder case IPA_MEM_V4_FILTER: 13475bcfde6SAlex Elder case IPA_MEM_V6_FILTER_HASHED: 13575bcfde6SAlex Elder case IPA_MEM_V6_FILTER: 13675bcfde6SAlex Elder case IPA_MEM_V4_ROUTE_HASHED: 13775bcfde6SAlex Elder case IPA_MEM_V4_ROUTE: 13875bcfde6SAlex Elder case IPA_MEM_V6_ROUTE_HASHED: 13975bcfde6SAlex Elder case IPA_MEM_V6_ROUTE: 14075bcfde6SAlex Elder case IPA_MEM_MODEM_HEADER: 14175bcfde6SAlex Elder case IPA_MEM_AP_HEADER: 14275bcfde6SAlex Elder case IPA_MEM_MODEM_PROC_CTX: 14375bcfde6SAlex Elder case IPA_MEM_AP_PROC_CTX: 14475bcfde6SAlex Elder case IPA_MEM_MODEM: 14575bcfde6SAlex Elder case IPA_MEM_UC_EVENT_RING: 14675bcfde6SAlex Elder case IPA_MEM_PDN_CONFIG: 14775bcfde6SAlex Elder case IPA_MEM_STATS_QUOTA_MODEM: 14875bcfde6SAlex Elder case IPA_MEM_STATS_QUOTA_AP: 14975bcfde6SAlex Elder case IPA_MEM_END_MARKER: /* pseudo region */ 15075bcfde6SAlex Elder break; 15175bcfde6SAlex Elder 15275bcfde6SAlex Elder case IPA_MEM_STATS_TETHERING: 15375bcfde6SAlex Elder case IPA_MEM_STATS_DROP: 15475bcfde6SAlex Elder if (version < IPA_VERSION_4_0) 15575bcfde6SAlex Elder return false; 15675bcfde6SAlex Elder break; 15775bcfde6SAlex Elder 15875bcfde6SAlex Elder case IPA_MEM_STATS_V4_FILTER: 15975bcfde6SAlex Elder case IPA_MEM_STATS_V6_FILTER: 16075bcfde6SAlex Elder case IPA_MEM_STATS_V4_ROUTE: 16175bcfde6SAlex Elder case IPA_MEM_STATS_V6_ROUTE: 16275bcfde6SAlex Elder if (version < IPA_VERSION_4_0 || version > IPA_VERSION_4_2) 16375bcfde6SAlex Elder return false; 16475bcfde6SAlex Elder break; 16575bcfde6SAlex Elder 166*5157d6bfSAlex Elder case IPA_MEM_AP_V4_FILTER: 167*5157d6bfSAlex Elder case IPA_MEM_AP_V6_FILTER: 168*5157d6bfSAlex Elder if (version != IPA_VERSION_5_0) 169*5157d6bfSAlex Elder return false; 170*5157d6bfSAlex Elder break; 171*5157d6bfSAlex Elder 17275bcfde6SAlex Elder case IPA_MEM_NAT_TABLE: 17375bcfde6SAlex Elder case IPA_MEM_STATS_FILTER_ROUTE: 17475bcfde6SAlex Elder if (version < IPA_VERSION_4_5) 17575bcfde6SAlex Elder return false; 17675bcfde6SAlex Elder break; 17775bcfde6SAlex Elder 17875bcfde6SAlex Elder default: 17975bcfde6SAlex Elder return false; 18075bcfde6SAlex Elder } 18175bcfde6SAlex Elder 18275bcfde6SAlex Elder return true; 18375bcfde6SAlex Elder } 18475bcfde6SAlex Elder 185d39ffb97SAlex Elder /* Must the given memory region be present in the configuration? */ 186d39ffb97SAlex Elder static bool ipa_mem_id_required(struct ipa *ipa, enum ipa_mem_id mem_id) 187d39ffb97SAlex Elder { 188d39ffb97SAlex Elder switch (mem_id) { 189d39ffb97SAlex Elder case IPA_MEM_UC_SHARED: 190d39ffb97SAlex Elder case IPA_MEM_UC_INFO: 191d39ffb97SAlex Elder case IPA_MEM_V4_FILTER_HASHED: 192d39ffb97SAlex Elder case IPA_MEM_V4_FILTER: 193d39ffb97SAlex Elder case IPA_MEM_V6_FILTER_HASHED: 194d39ffb97SAlex Elder case IPA_MEM_V6_FILTER: 195d39ffb97SAlex Elder case IPA_MEM_V4_ROUTE_HASHED: 196d39ffb97SAlex Elder case IPA_MEM_V4_ROUTE: 197d39ffb97SAlex Elder case IPA_MEM_V6_ROUTE_HASHED: 198d39ffb97SAlex Elder case IPA_MEM_V6_ROUTE: 199d39ffb97SAlex Elder case IPA_MEM_MODEM_HEADER: 200d39ffb97SAlex Elder case IPA_MEM_MODEM_PROC_CTX: 201d39ffb97SAlex Elder case IPA_MEM_AP_PROC_CTX: 202d39ffb97SAlex Elder case IPA_MEM_MODEM: 203d39ffb97SAlex Elder return true; 204d39ffb97SAlex Elder 205d39ffb97SAlex Elder case IPA_MEM_PDN_CONFIG: 206d39ffb97SAlex Elder case IPA_MEM_STATS_QUOTA_MODEM: 207d39ffb97SAlex Elder return ipa->version >= IPA_VERSION_4_0; 208d39ffb97SAlex Elder 2095ba5faa2SAlex Elder case IPA_MEM_STATS_TETHERING: 2105ba5faa2SAlex Elder return ipa->version >= IPA_VERSION_4_0 && 2115ba5faa2SAlex Elder ipa->version != IPA_VERSION_5_0; 2125ba5faa2SAlex Elder 213d39ffb97SAlex Elder default: 214d39ffb97SAlex Elder return false; /* Anything else is optional */ 215d39ffb97SAlex Elder } 216d39ffb97SAlex Elder } 217d39ffb97SAlex Elder 21898334d2aSAlex Elder static bool ipa_mem_valid_one(struct ipa *ipa, const struct ipa_mem *mem) 219ba764c4dSAlex Elder { 220ba764c4dSAlex Elder struct device *dev = &ipa->pdev->dev; 2210300df2dSAlex Elder enum ipa_mem_id mem_id = mem->id; 222ba764c4dSAlex Elder u16 size_multiple; 223ba764c4dSAlex Elder 22475bcfde6SAlex Elder /* Make sure the memory region is valid for this version of IPA */ 22575bcfde6SAlex Elder if (!ipa_mem_id_valid(ipa, mem_id)) { 22675bcfde6SAlex Elder dev_err(dev, "region id %u not valid\n", mem_id); 22775bcfde6SAlex Elder return false; 22875bcfde6SAlex Elder } 22975bcfde6SAlex Elder 230c61cfb94SAlex Elder if (!mem->size && !mem->canary_count) { 231c61cfb94SAlex Elder dev_err(dev, "empty memory region %u\n", mem_id); 232c61cfb94SAlex Elder return false; 233c61cfb94SAlex Elder } 234c61cfb94SAlex Elder 235ba764c4dSAlex Elder /* Other than modem memory, sizes must be a multiple of 8 */ 236ba764c4dSAlex Elder size_multiple = mem_id == IPA_MEM_MODEM ? 4 : 8; 237ba764c4dSAlex Elder if (mem->size % size_multiple) 238ba764c4dSAlex Elder dev_err(dev, "region %u size not a multiple of %u bytes\n", 239ba764c4dSAlex Elder mem_id, size_multiple); 240ba764c4dSAlex Elder else if (mem->offset % 8) 241ba764c4dSAlex Elder dev_err(dev, "region %u offset not 8-byte aligned\n", mem_id); 242ba764c4dSAlex Elder else if (mem->offset < mem->canary_count * sizeof(__le32)) 243ba764c4dSAlex Elder dev_err(dev, "region %u offset too small for %hu canaries\n", 244ba764c4dSAlex Elder mem_id, mem->canary_count); 245f636a836SAlex Elder else if (mem_id == IPA_MEM_END_MARKER && mem->size) 246f636a836SAlex Elder dev_err(dev, "non-zero end marker region size\n"); 247ba764c4dSAlex Elder else 248ba764c4dSAlex Elder return true; 249ba764c4dSAlex Elder 250ba764c4dSAlex Elder return false; 251ba764c4dSAlex Elder } 252ba764c4dSAlex Elder 25398334d2aSAlex Elder /* Verify each defined memory region is valid. */ 2541eec7677SAlex Elder static bool ipa_mem_valid(struct ipa *ipa, const struct ipa_mem_data *mem_data) 25598334d2aSAlex Elder { 256eadf7f93SAlex Elder DECLARE_BITMAP(regions, IPA_MEM_COUNT) = { }; 25798334d2aSAlex Elder struct device *dev = &ipa->pdev->dev; 25898334d2aSAlex Elder enum ipa_mem_id mem_id; 2598cc7ebbfSAlex Elder u32 i; 26098334d2aSAlex Elder 2611eec7677SAlex Elder if (mem_data->local_count > IPA_MEM_COUNT) { 2621eec7677SAlex Elder dev_err(dev, "too many memory regions (%u > %u)\n", 2631eec7677SAlex Elder mem_data->local_count, IPA_MEM_COUNT); 2641eec7677SAlex Elder return false; 2651eec7677SAlex Elder } 2661eec7677SAlex Elder 2678cc7ebbfSAlex Elder for (i = 0; i < mem_data->local_count; i++) { 2688cc7ebbfSAlex Elder const struct ipa_mem *mem = &mem_data->local[i]; 26998334d2aSAlex Elder 270eadf7f93SAlex Elder if (__test_and_set_bit(mem->id, regions)) { 271eadf7f93SAlex Elder dev_err(dev, "duplicate memory region %u\n", mem->id); 272eadf7f93SAlex Elder return false; 273eadf7f93SAlex Elder } 274eadf7f93SAlex Elder 27598334d2aSAlex Elder /* Defined regions have non-zero size and/or canary count */ 276c61cfb94SAlex Elder if (!ipa_mem_valid_one(ipa, mem)) 27798334d2aSAlex Elder return false; 27898334d2aSAlex Elder } 27998334d2aSAlex Elder 280d39ffb97SAlex Elder /* Now see if any required regions are not defined */ 281a6366b13SChristophe JAILLET for_each_clear_bit(mem_id, regions, IPA_MEM_COUNT) { 2826857b023SAlex Elder if (ipa_mem_id_required(ipa, mem_id)) 283d39ffb97SAlex Elder dev_err(dev, "required memory region %u missing\n", 284d39ffb97SAlex Elder mem_id); 2856857b023SAlex Elder } 28698334d2aSAlex Elder 28798334d2aSAlex Elder return true; 28898334d2aSAlex Elder } 28998334d2aSAlex Elder 2905e57c6c5SAlex Elder /* Do all memory regions fit within the IPA local memory? */ 2915e57c6c5SAlex Elder static bool ipa_mem_size_valid(struct ipa *ipa) 2925e57c6c5SAlex Elder { 2935e57c6c5SAlex Elder struct device *dev = &ipa->pdev->dev; 2945e57c6c5SAlex Elder u32 limit = ipa->mem_size; 2958cc7ebbfSAlex Elder u32 i; 2965e57c6c5SAlex Elder 2978cc7ebbfSAlex Elder for (i = 0; i < ipa->mem_count; i++) { 2988cc7ebbfSAlex Elder const struct ipa_mem *mem = &ipa->mem[i]; 2995e57c6c5SAlex Elder 3005e57c6c5SAlex Elder if (mem->offset + mem->size <= limit) 3015e57c6c5SAlex Elder continue; 3025e57c6c5SAlex Elder 3035e57c6c5SAlex Elder dev_err(dev, "region %u ends beyond memory limit (0x%08x)\n", 3048cc7ebbfSAlex Elder mem->id, limit); 3055e57c6c5SAlex Elder 3065e57c6c5SAlex Elder return false; 3075e57c6c5SAlex Elder } 3085e57c6c5SAlex Elder 3095e57c6c5SAlex Elder return true; 3105e57c6c5SAlex Elder } 3115e57c6c5SAlex Elder 312ba764c4dSAlex Elder /** 313ba764c4dSAlex Elder * ipa_mem_config() - Configure IPA shared memory 314e3eea08eSAlex Elder * @ipa: IPA pointer 315ba764c4dSAlex Elder * 316e3eea08eSAlex Elder * Return: 0 if successful, or a negative error code 317ba764c4dSAlex Elder */ 318ba764c4dSAlex Elder int ipa_mem_config(struct ipa *ipa) 319ba764c4dSAlex Elder { 320ba764c4dSAlex Elder struct device *dev = &ipa->pdev->dev; 3216a244b75SAlex Elder const struct ipa_reg *reg; 3225e3bc1e5SAlex Elder const struct ipa_mem *mem; 323ba764c4dSAlex Elder dma_addr_t addr; 324ba764c4dSAlex Elder u32 mem_size; 325ba764c4dSAlex Elder void *virt; 326ba764c4dSAlex Elder u32 val; 3278cc7ebbfSAlex Elder u32 i; 328ba764c4dSAlex Elder 329ba764c4dSAlex Elder /* Check the advertised location and size of the shared memory area */ 3306a244b75SAlex Elder reg = ipa_reg(ipa, SHARED_MEM_SIZE); 3316a244b75SAlex Elder val = ioread32(ipa->reg_virt + ipa_reg_offset(reg)); 332ba764c4dSAlex Elder 333ba764c4dSAlex Elder /* The fields in the register are in 8 byte units */ 33462b9c009SAlex Elder ipa->mem_offset = 8 * ipa_reg_decode(reg, MEM_BADDR, val); 33562b9c009SAlex Elder 336ba764c4dSAlex Elder /* Make sure the end is within the region's mapped space */ 33762b9c009SAlex Elder mem_size = 8 * ipa_reg_decode(reg, MEM_SIZE, val); 338ba764c4dSAlex Elder 339ba764c4dSAlex Elder /* If the sizes don't match, issue a warning */ 3402c642c48SAlex Elder if (ipa->mem_offset + mem_size < ipa->mem_size) { 341ba764c4dSAlex Elder dev_warn(dev, "limiting IPA memory size to 0x%08x\n", 342ba764c4dSAlex Elder mem_size); 343ba764c4dSAlex Elder ipa->mem_size = mem_size; 3442c642c48SAlex Elder } else if (ipa->mem_offset + mem_size > ipa->mem_size) { 3452c642c48SAlex Elder dev_dbg(dev, "ignoring larger reported memory size: 0x%08x\n", 3462c642c48SAlex Elder mem_size); 347ba764c4dSAlex Elder } 348ba764c4dSAlex Elder 3495e57c6c5SAlex Elder /* We know our memory size; make sure regions are all in range */ 3505e57c6c5SAlex Elder if (!ipa_mem_size_valid(ipa)) 3515e57c6c5SAlex Elder return -EINVAL; 3525e57c6c5SAlex Elder 353ba764c4dSAlex Elder /* Prealloc DMA memory for zeroing regions */ 354ba764c4dSAlex Elder virt = dma_alloc_coherent(dev, IPA_MEM_MAX, &addr, GFP_KERNEL); 355ba764c4dSAlex Elder if (!virt) 356ba764c4dSAlex Elder return -ENOMEM; 357ba764c4dSAlex Elder ipa->zero_addr = addr; 358ba764c4dSAlex Elder ipa->zero_virt = virt; 359ba764c4dSAlex Elder ipa->zero_size = IPA_MEM_MAX; 360ba764c4dSAlex Elder 3618cc7ebbfSAlex Elder /* For each defined region, write "canary" values in the 3628cc7ebbfSAlex Elder * space prior to the region's base address if indicated. 363ba764c4dSAlex Elder */ 3648cc7ebbfSAlex Elder for (i = 0; i < ipa->mem_count; i++) { 365c61cfb94SAlex Elder u16 canary_count = ipa->mem[i].canary_count; 366ba764c4dSAlex Elder __le32 *canary; 367ba764c4dSAlex Elder 368ba764c4dSAlex Elder if (!canary_count) 369ba764c4dSAlex Elder continue; 370ba764c4dSAlex Elder 371ba764c4dSAlex Elder /* Write canary values in the space before the region */ 372c61cfb94SAlex Elder canary = ipa->mem_virt + ipa->mem_offset + ipa->mem[i].offset; 373ba764c4dSAlex Elder do 374ba764c4dSAlex Elder *--canary = IPA_MEM_CANARY_VAL; 375ba764c4dSAlex Elder while (--canary_count); 376ba764c4dSAlex Elder } 377ba764c4dSAlex Elder 3785e3bc1e5SAlex Elder /* Verify the microcontroller ring alignment (if defined) */ 3795e3bc1e5SAlex Elder mem = ipa_mem_find(ipa, IPA_MEM_UC_EVENT_RING); 3805e3bc1e5SAlex Elder if (mem && mem->offset % 1024) { 381ba764c4dSAlex Elder dev_err(dev, "microcontroller ring not 1024-byte aligned\n"); 382ba764c4dSAlex Elder goto err_dma_free; 383ba764c4dSAlex Elder } 384ba764c4dSAlex Elder 385ba764c4dSAlex Elder return 0; 386ba764c4dSAlex Elder 387ba764c4dSAlex Elder err_dma_free: 388ba764c4dSAlex Elder dma_free_coherent(dev, IPA_MEM_MAX, ipa->zero_virt, ipa->zero_addr); 389ba764c4dSAlex Elder 390ba764c4dSAlex Elder return -EINVAL; 391ba764c4dSAlex Elder } 392ba764c4dSAlex Elder 393ba764c4dSAlex Elder /* Inverse of ipa_mem_config() */ 394ba764c4dSAlex Elder void ipa_mem_deconfig(struct ipa *ipa) 395ba764c4dSAlex Elder { 396ba764c4dSAlex Elder struct device *dev = &ipa->pdev->dev; 397ba764c4dSAlex Elder 398ba764c4dSAlex Elder dma_free_coherent(dev, ipa->zero_size, ipa->zero_virt, ipa->zero_addr); 399ba764c4dSAlex Elder ipa->zero_size = 0; 400ba764c4dSAlex Elder ipa->zero_virt = NULL; 401ba764c4dSAlex Elder ipa->zero_addr = 0; 402ba764c4dSAlex Elder } 403ba764c4dSAlex Elder 404ba764c4dSAlex Elder /** 405ba764c4dSAlex Elder * ipa_mem_zero_modem() - Zero IPA-local memory regions owned by the modem 406e3eea08eSAlex Elder * @ipa: IPA pointer 407ba764c4dSAlex Elder * 408ba764c4dSAlex Elder * Zero regions of IPA-local memory used by the modem. These are configured 409ba764c4dSAlex Elder * (and initially zeroed) by ipa_mem_setup(), but if the modem crashes and 410ba764c4dSAlex Elder * restarts via SSR we need to re-initialize them. A QMI message tells the 411ba764c4dSAlex Elder * modem where to find regions of IPA local memory it needs to know about 412ba764c4dSAlex Elder * (these included). 413ba764c4dSAlex Elder */ 414ba764c4dSAlex Elder int ipa_mem_zero_modem(struct ipa *ipa) 415ba764c4dSAlex Elder { 416ba764c4dSAlex Elder struct gsi_trans *trans; 417ba764c4dSAlex Elder 418ba764c4dSAlex Elder /* Get a transaction to zero the modem memory, modem header, 419ba764c4dSAlex Elder * and modem processing context regions. 420ba764c4dSAlex Elder */ 421ba764c4dSAlex Elder trans = ipa_cmd_trans_alloc(ipa, 3); 422ba764c4dSAlex Elder if (!trans) { 423ba764c4dSAlex Elder dev_err(&ipa->pdev->dev, 424ba764c4dSAlex Elder "no transaction to zero modem memory\n"); 425ba764c4dSAlex Elder return -EBUSY; 426ba764c4dSAlex Elder } 427ba764c4dSAlex Elder 428ce928bf8SAlex Elder ipa_mem_zero_region_add(trans, IPA_MEM_MODEM_HEADER); 429ce928bf8SAlex Elder ipa_mem_zero_region_add(trans, IPA_MEM_MODEM_PROC_CTX); 430ce928bf8SAlex Elder ipa_mem_zero_region_add(trans, IPA_MEM_MODEM); 431ba764c4dSAlex Elder 432ba764c4dSAlex Elder gsi_trans_commit_wait(trans); 433ba764c4dSAlex Elder 434ba764c4dSAlex Elder return 0; 435ba764c4dSAlex Elder } 436ba764c4dSAlex Elder 4373e313c3fSAlex Elder /** 4383e313c3fSAlex Elder * ipa_imem_init() - Initialize IMEM memory used by the IPA 4393e313c3fSAlex Elder * @ipa: IPA pointer 4403e313c3fSAlex Elder * @addr: Physical address of the IPA region in IMEM 4413e313c3fSAlex Elder * @size: Size (bytes) of the IPA region in IMEM 4423e313c3fSAlex Elder * 4433e313c3fSAlex Elder * IMEM is a block of shared memory separate from system DRAM, and 4443e313c3fSAlex Elder * a portion of this memory is available for the IPA to use. The 4453e313c3fSAlex Elder * modem accesses this memory directly, but the IPA accesses it 4463e313c3fSAlex Elder * via the IOMMU, using the AP's credentials. 4473e313c3fSAlex Elder * 4483e313c3fSAlex Elder * If this region exists (size > 0) we map it for read/write access 4493e313c3fSAlex Elder * through the IOMMU using the IPA device. 4503e313c3fSAlex Elder * 4513e313c3fSAlex Elder * Note: @addr and @size are not guaranteed to be page-aligned. 4523e313c3fSAlex Elder */ 4533e313c3fSAlex Elder static int ipa_imem_init(struct ipa *ipa, unsigned long addr, size_t size) 4543e313c3fSAlex Elder { 4553e313c3fSAlex Elder struct device *dev = &ipa->pdev->dev; 4563e313c3fSAlex Elder struct iommu_domain *domain; 4573e313c3fSAlex Elder unsigned long iova; 4583e313c3fSAlex Elder phys_addr_t phys; 4593e313c3fSAlex Elder int ret; 4603e313c3fSAlex Elder 4613e313c3fSAlex Elder if (!size) 4623e313c3fSAlex Elder return 0; /* IMEM memory not used */ 4633e313c3fSAlex Elder 4643e313c3fSAlex Elder domain = iommu_get_domain_for_dev(dev); 4653e313c3fSAlex Elder if (!domain) { 4663e313c3fSAlex Elder dev_err(dev, "no IOMMU domain found for IMEM\n"); 4673e313c3fSAlex Elder return -EINVAL; 4683e313c3fSAlex Elder } 4693e313c3fSAlex Elder 4703e313c3fSAlex Elder /* Align the address down and the size up to page boundaries */ 4713e313c3fSAlex Elder phys = addr & PAGE_MASK; 4723e313c3fSAlex Elder size = PAGE_ALIGN(size + addr - phys); 4733e313c3fSAlex Elder iova = phys; /* We just want a direct mapping */ 4743e313c3fSAlex Elder 4753e313c3fSAlex Elder ret = iommu_map(domain, iova, phys, size, IOMMU_READ | IOMMU_WRITE); 4763e313c3fSAlex Elder if (ret) 4773e313c3fSAlex Elder return ret; 4783e313c3fSAlex Elder 4793e313c3fSAlex Elder ipa->imem_iova = iova; 4803e313c3fSAlex Elder ipa->imem_size = size; 4813e313c3fSAlex Elder 4823e313c3fSAlex Elder return 0; 4833e313c3fSAlex Elder } 4843e313c3fSAlex Elder 4853e313c3fSAlex Elder static void ipa_imem_exit(struct ipa *ipa) 4863e313c3fSAlex Elder { 4873e313c3fSAlex Elder struct iommu_domain *domain; 4883e313c3fSAlex Elder struct device *dev; 4893e313c3fSAlex Elder 4903e313c3fSAlex Elder if (!ipa->imem_size) 4913e313c3fSAlex Elder return; 4923e313c3fSAlex Elder 4933e313c3fSAlex Elder dev = &ipa->pdev->dev; 4943e313c3fSAlex Elder domain = iommu_get_domain_for_dev(dev); 4953e313c3fSAlex Elder if (domain) { 4963e313c3fSAlex Elder size_t size; 4973e313c3fSAlex Elder 4983e313c3fSAlex Elder size = iommu_unmap(domain, ipa->imem_iova, ipa->imem_size); 4993e313c3fSAlex Elder if (size != ipa->imem_size) 500113b6ea0SAlex Elder dev_warn(dev, "unmapped %zu IMEM bytes, expected %zu\n", 5013e313c3fSAlex Elder size, ipa->imem_size); 5023e313c3fSAlex Elder } else { 5033e313c3fSAlex Elder dev_err(dev, "couldn't get IPA IOMMU domain for IMEM\n"); 5043e313c3fSAlex Elder } 5053e313c3fSAlex Elder 5063e313c3fSAlex Elder ipa->imem_size = 0; 5073e313c3fSAlex Elder ipa->imem_iova = 0; 5083e313c3fSAlex Elder } 5093e313c3fSAlex Elder 510a0036bb4SAlex Elder /** 511a0036bb4SAlex Elder * ipa_smem_init() - Initialize SMEM memory used by the IPA 512a0036bb4SAlex Elder * @ipa: IPA pointer 513a0036bb4SAlex Elder * @item: Item ID of SMEM memory 514a0036bb4SAlex Elder * @size: Size (bytes) of SMEM memory region 515a0036bb4SAlex Elder * 516a0036bb4SAlex Elder * SMEM is a managed block of shared DRAM, from which numbered "items" 517a0036bb4SAlex Elder * can be allocated. One item is designated for use by the IPA. 518a0036bb4SAlex Elder * 519a0036bb4SAlex Elder * The modem accesses SMEM memory directly, but the IPA accesses it 520a0036bb4SAlex Elder * via the IOMMU, using the AP's credentials. 521a0036bb4SAlex Elder * 522a0036bb4SAlex Elder * If size provided is non-zero, we allocate it and map it for 523a0036bb4SAlex Elder * access through the IOMMU. 524a0036bb4SAlex Elder * 525a0036bb4SAlex Elder * Note: @size and the item address are is not guaranteed to be page-aligned. 526a0036bb4SAlex Elder */ 527a0036bb4SAlex Elder static int ipa_smem_init(struct ipa *ipa, u32 item, size_t size) 528a0036bb4SAlex Elder { 529a0036bb4SAlex Elder struct device *dev = &ipa->pdev->dev; 530a0036bb4SAlex Elder struct iommu_domain *domain; 531a0036bb4SAlex Elder unsigned long iova; 532a0036bb4SAlex Elder phys_addr_t phys; 533a0036bb4SAlex Elder phys_addr_t addr; 534a0036bb4SAlex Elder size_t actual; 535a0036bb4SAlex Elder void *virt; 536a0036bb4SAlex Elder int ret; 537a0036bb4SAlex Elder 538a0036bb4SAlex Elder if (!size) 539a0036bb4SAlex Elder return 0; /* SMEM memory not used */ 540a0036bb4SAlex Elder 541a0036bb4SAlex Elder /* SMEM is memory shared between the AP and another system entity 542a0036bb4SAlex Elder * (in this case, the modem). An allocation from SMEM is persistent 543a0036bb4SAlex Elder * until the AP reboots; there is no way to free an allocated SMEM 544a0036bb4SAlex Elder * region. Allocation only reserves the space; to use it you need 5455e3bc1e5SAlex Elder * to "get" a pointer it (this does not imply reference counting). 546a0036bb4SAlex Elder * The item might have already been allocated, in which case we 547a0036bb4SAlex Elder * use it unless the size isn't what we expect. 548a0036bb4SAlex Elder */ 549a0036bb4SAlex Elder ret = qcom_smem_alloc(QCOM_SMEM_HOST_MODEM, item, size); 550a0036bb4SAlex Elder if (ret && ret != -EEXIST) { 551a0036bb4SAlex Elder dev_err(dev, "error %d allocating size %zu SMEM item %u\n", 552a0036bb4SAlex Elder ret, size, item); 553a0036bb4SAlex Elder return ret; 554a0036bb4SAlex Elder } 555a0036bb4SAlex Elder 556a0036bb4SAlex Elder /* Now get the address of the SMEM memory region */ 557a0036bb4SAlex Elder virt = qcom_smem_get(QCOM_SMEM_HOST_MODEM, item, &actual); 558a0036bb4SAlex Elder if (IS_ERR(virt)) { 559a0036bb4SAlex Elder ret = PTR_ERR(virt); 560a0036bb4SAlex Elder dev_err(dev, "error %d getting SMEM item %u\n", ret, item); 561a0036bb4SAlex Elder return ret; 562a0036bb4SAlex Elder } 563a0036bb4SAlex Elder 564a0036bb4SAlex Elder /* In case the region was already allocated, verify the size */ 565a0036bb4SAlex Elder if (ret && actual != size) { 566a0036bb4SAlex Elder dev_err(dev, "SMEM item %u has size %zu, expected %zu\n", 567a0036bb4SAlex Elder item, actual, size); 568a0036bb4SAlex Elder return -EINVAL; 569a0036bb4SAlex Elder } 570a0036bb4SAlex Elder 571a0036bb4SAlex Elder domain = iommu_get_domain_for_dev(dev); 572a0036bb4SAlex Elder if (!domain) { 573a0036bb4SAlex Elder dev_err(dev, "no IOMMU domain found for SMEM\n"); 574a0036bb4SAlex Elder return -EINVAL; 575a0036bb4SAlex Elder } 576a0036bb4SAlex Elder 577a0036bb4SAlex Elder /* Align the address down and the size up to a page boundary */ 578b8d43803SAlex Elder addr = qcom_smem_virt_to_phys(virt); 579a0036bb4SAlex Elder phys = addr & PAGE_MASK; 580a0036bb4SAlex Elder size = PAGE_ALIGN(size + addr - phys); 581a0036bb4SAlex Elder iova = phys; /* We just want a direct mapping */ 582a0036bb4SAlex Elder 583a0036bb4SAlex Elder ret = iommu_map(domain, iova, phys, size, IOMMU_READ | IOMMU_WRITE); 584a0036bb4SAlex Elder if (ret) 585a0036bb4SAlex Elder return ret; 586a0036bb4SAlex Elder 587a0036bb4SAlex Elder ipa->smem_iova = iova; 588a0036bb4SAlex Elder ipa->smem_size = size; 589a0036bb4SAlex Elder 590a0036bb4SAlex Elder return 0; 591a0036bb4SAlex Elder } 592a0036bb4SAlex Elder 593a0036bb4SAlex Elder static void ipa_smem_exit(struct ipa *ipa) 594a0036bb4SAlex Elder { 595a0036bb4SAlex Elder struct device *dev = &ipa->pdev->dev; 596a0036bb4SAlex Elder struct iommu_domain *domain; 597a0036bb4SAlex Elder 598a0036bb4SAlex Elder domain = iommu_get_domain_for_dev(dev); 599a0036bb4SAlex Elder if (domain) { 600a0036bb4SAlex Elder size_t size; 601a0036bb4SAlex Elder 602a0036bb4SAlex Elder size = iommu_unmap(domain, ipa->smem_iova, ipa->smem_size); 603a0036bb4SAlex Elder if (size != ipa->smem_size) 604113b6ea0SAlex Elder dev_warn(dev, "unmapped %zu SMEM bytes, expected %zu\n", 605a0036bb4SAlex Elder size, ipa->smem_size); 606a0036bb4SAlex Elder 607a0036bb4SAlex Elder } else { 608a0036bb4SAlex Elder dev_err(dev, "couldn't get IPA IOMMU domain for SMEM\n"); 609a0036bb4SAlex Elder } 610a0036bb4SAlex Elder 611a0036bb4SAlex Elder ipa->smem_size = 0; 612a0036bb4SAlex Elder ipa->smem_iova = 0; 613a0036bb4SAlex Elder } 614a0036bb4SAlex Elder 615ba764c4dSAlex Elder /* Perform memory region-related initialization */ 6163128aae8SAlex Elder int ipa_mem_init(struct ipa *ipa, const struct ipa_mem_data *mem_data) 617ba764c4dSAlex Elder { 618ba764c4dSAlex Elder struct device *dev = &ipa->pdev->dev; 619ba764c4dSAlex Elder struct resource *res; 620ba764c4dSAlex Elder int ret; 621ba764c4dSAlex Elder 6221eec7677SAlex Elder /* Make sure the set of defined memory regions is valid */ 6231eec7677SAlex Elder if (!ipa_mem_valid(ipa, mem_data)) 624ba764c4dSAlex Elder return -EINVAL; 6251eec7677SAlex Elder 6261eec7677SAlex Elder ipa->mem_count = mem_data->local_count; 6271eec7677SAlex Elder ipa->mem = mem_data->local; 628ba764c4dSAlex Elder 62973da9cacSAlex Elder /* Check the route and filter table memory regions */ 6308defab8bSAlex Elder if (!ipa_table_mem_valid(ipa, false)) 63173da9cacSAlex Elder return -EINVAL; 6328defab8bSAlex Elder if (!ipa_table_mem_valid(ipa, true)) 63373da9cacSAlex Elder return -EINVAL; 63473da9cacSAlex Elder 635ba764c4dSAlex Elder ret = dma_set_mask_and_coherent(&ipa->pdev->dev, DMA_BIT_MASK(64)); 636ba764c4dSAlex Elder if (ret) { 637ba764c4dSAlex Elder dev_err(dev, "error %d setting DMA mask\n", ret); 638ba764c4dSAlex Elder return ret; 639ba764c4dSAlex Elder } 640ba764c4dSAlex Elder 641ba764c4dSAlex Elder res = platform_get_resource_byname(ipa->pdev, IORESOURCE_MEM, 642ba764c4dSAlex Elder "ipa-shared"); 643ba764c4dSAlex Elder if (!res) { 644ba764c4dSAlex Elder dev_err(dev, 645ba764c4dSAlex Elder "DT error getting \"ipa-shared\" memory property\n"); 646ba764c4dSAlex Elder return -ENODEV; 647ba764c4dSAlex Elder } 648ba764c4dSAlex Elder 649ba764c4dSAlex Elder ipa->mem_virt = memremap(res->start, resource_size(res), MEMREMAP_WC); 650ba764c4dSAlex Elder if (!ipa->mem_virt) { 651ba764c4dSAlex Elder dev_err(dev, "unable to remap \"ipa-shared\" memory\n"); 652ba764c4dSAlex Elder return -ENOMEM; 653ba764c4dSAlex Elder } 654ba764c4dSAlex Elder 655ba764c4dSAlex Elder ipa->mem_addr = res->start; 656ba764c4dSAlex Elder ipa->mem_size = resource_size(res); 657ba764c4dSAlex Elder 6583e313c3fSAlex Elder ret = ipa_imem_init(ipa, mem_data->imem_addr, mem_data->imem_size); 6593e313c3fSAlex Elder if (ret) 6603e313c3fSAlex Elder goto err_unmap; 6613e313c3fSAlex Elder 662a0036bb4SAlex Elder ret = ipa_smem_init(ipa, mem_data->smem_id, mem_data->smem_size); 663a0036bb4SAlex Elder if (ret) 664a0036bb4SAlex Elder goto err_imem_exit; 665a0036bb4SAlex Elder 666ba764c4dSAlex Elder return 0; 6673e313c3fSAlex Elder 668a0036bb4SAlex Elder err_imem_exit: 669a0036bb4SAlex Elder ipa_imem_exit(ipa); 6703e313c3fSAlex Elder err_unmap: 6713e313c3fSAlex Elder memunmap(ipa->mem_virt); 6723e313c3fSAlex Elder 6733e313c3fSAlex Elder return ret; 674ba764c4dSAlex Elder } 675ba764c4dSAlex Elder 676ba764c4dSAlex Elder /* Inverse of ipa_mem_init() */ 677ba764c4dSAlex Elder void ipa_mem_exit(struct ipa *ipa) 678ba764c4dSAlex Elder { 679a0036bb4SAlex Elder ipa_smem_exit(ipa); 6803e313c3fSAlex Elder ipa_imem_exit(ipa); 681ba764c4dSAlex Elder memunmap(ipa->mem_virt); 682ba764c4dSAlex Elder } 683