1ba764c4dSAlex Elder // SPDX-License-Identifier: GPL-2.0 2ba764c4dSAlex Elder 3ba764c4dSAlex Elder /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved. 474858b63SAlex Elder * Copyright (C) 2019-2021 Linaro Ltd. 5ba764c4dSAlex Elder */ 6ba764c4dSAlex Elder 7ba764c4dSAlex Elder #include <linux/types.h> 8ba764c4dSAlex Elder #include <linux/bitfield.h> 9ba764c4dSAlex Elder #include <linux/bug.h> 10ba764c4dSAlex Elder #include <linux/dma-mapping.h> 113e313c3fSAlex Elder #include <linux/iommu.h> 12ba764c4dSAlex Elder #include <linux/io.h> 13a0036bb4SAlex Elder #include <linux/soc/qcom/smem.h> 14ba764c4dSAlex Elder 15ba764c4dSAlex Elder #include "ipa.h" 16ba764c4dSAlex Elder #include "ipa_reg.h" 173128aae8SAlex Elder #include "ipa_data.h" 18ba764c4dSAlex Elder #include "ipa_cmd.h" 19ba764c4dSAlex Elder #include "ipa_mem.h" 20ba764c4dSAlex Elder #include "ipa_table.h" 21ba764c4dSAlex Elder #include "gsi_trans.h" 22ba764c4dSAlex Elder 23ba764c4dSAlex Elder /* "Canary" value placed between memory regions to detect overflow */ 24ba764c4dSAlex Elder #define IPA_MEM_CANARY_VAL cpu_to_le32(0xdeadbeef) 25ba764c4dSAlex Elder 26a0036bb4SAlex Elder /* SMEM host id representing the modem. */ 27a0036bb4SAlex Elder #define QCOM_SMEM_HOST_MODEM 1 28a0036bb4SAlex Elder 29*5e3bc1e5SAlex Elder const struct ipa_mem *ipa_mem_find(struct ipa *ipa, enum ipa_mem_id mem_id) 30*5e3bc1e5SAlex Elder { 31*5e3bc1e5SAlex Elder if (mem_id < IPA_MEM_COUNT) 32*5e3bc1e5SAlex Elder return &ipa->mem[mem_id]; 33*5e3bc1e5SAlex Elder 34*5e3bc1e5SAlex Elder return NULL; 35*5e3bc1e5SAlex Elder } 36*5e3bc1e5SAlex Elder 37ba764c4dSAlex Elder /* Add an immediate command to a transaction that zeroes a memory region */ 38ba764c4dSAlex Elder static void 39ce928bf8SAlex Elder ipa_mem_zero_region_add(struct gsi_trans *trans, enum ipa_mem_id mem_id) 40ba764c4dSAlex Elder { 41ba764c4dSAlex Elder struct ipa *ipa = container_of(trans->gsi, struct ipa, gsi); 42*5e3bc1e5SAlex Elder const struct ipa_mem *mem = ipa_mem_find(ipa, mem_id); 43ba764c4dSAlex Elder dma_addr_t addr = ipa->zero_addr; 44ba764c4dSAlex Elder 45ba764c4dSAlex Elder if (!mem->size) 46ba764c4dSAlex Elder return; 47ba764c4dSAlex Elder 48ba764c4dSAlex Elder ipa_cmd_dma_shared_mem_add(trans, mem->offset, mem->size, addr, true); 49ba764c4dSAlex Elder } 50ba764c4dSAlex Elder 51ba764c4dSAlex Elder /** 52ba764c4dSAlex Elder * ipa_mem_setup() - Set up IPA AP and modem shared memory areas 53e3eea08eSAlex Elder * @ipa: IPA pointer 54ba764c4dSAlex Elder * 55ba764c4dSAlex Elder * Set up the shared memory regions in IPA local memory. This involves 56ba764c4dSAlex Elder * zero-filling memory regions, and in the case of header memory, telling 57ba764c4dSAlex Elder * the IPA where it's located. 58ba764c4dSAlex Elder * 59ba764c4dSAlex Elder * This function performs the initial setup of this memory. If the modem 60ba764c4dSAlex Elder * crashes, its regions are re-zeroed in ipa_mem_zero_modem(). 61ba764c4dSAlex Elder * 62ba764c4dSAlex Elder * The AP informs the modem where its portions of memory are located 63ba764c4dSAlex Elder * in a QMI exchange that occurs at modem startup. 64ba764c4dSAlex Elder * 6574858b63SAlex Elder * There is no need for a matching ipa_mem_teardown() function. 6674858b63SAlex Elder * 67e3eea08eSAlex Elder * Return: 0 if successful, or a negative error code 68ba764c4dSAlex Elder */ 69ba764c4dSAlex Elder int ipa_mem_setup(struct ipa *ipa) 70ba764c4dSAlex Elder { 71ba764c4dSAlex Elder dma_addr_t addr = ipa->zero_addr; 72*5e3bc1e5SAlex Elder const struct ipa_mem *mem; 73ba764c4dSAlex Elder struct gsi_trans *trans; 74ba764c4dSAlex Elder u32 offset; 75ba764c4dSAlex Elder u16 size; 76e6e49e43SAlex Elder u32 val; 77ba764c4dSAlex Elder 78ba764c4dSAlex Elder /* Get a transaction to define the header memory region and to zero 79ba764c4dSAlex Elder * the processing context and modem memory regions. 80ba764c4dSAlex Elder */ 81ba764c4dSAlex Elder trans = ipa_cmd_trans_alloc(ipa, 4); 82ba764c4dSAlex Elder if (!trans) { 83ba764c4dSAlex Elder dev_err(&ipa->pdev->dev, "no transaction for memory setup\n"); 84ba764c4dSAlex Elder return -EBUSY; 85ba764c4dSAlex Elder } 86ba764c4dSAlex Elder 87*5e3bc1e5SAlex Elder /* Initialize IPA-local header memory. The AP header region, if 88*5e3bc1e5SAlex Elder * present, is contiguous with and follows the modem header region, 89*5e3bc1e5SAlex Elder * and they are initialized together. 90ba764c4dSAlex Elder */ 91*5e3bc1e5SAlex Elder mem = ipa_mem_find(ipa, IPA_MEM_MODEM_HEADER); 92*5e3bc1e5SAlex Elder offset = mem->offset; 93*5e3bc1e5SAlex Elder size = mem->size; 94*5e3bc1e5SAlex Elder mem = ipa_mem_find(ipa, IPA_MEM_AP_HEADER); 95*5e3bc1e5SAlex Elder if (mem) 96*5e3bc1e5SAlex Elder size += mem->size; 97ba764c4dSAlex Elder 98ba764c4dSAlex Elder ipa_cmd_hdr_init_local_add(trans, offset, size, addr); 99ba764c4dSAlex Elder 100ce928bf8SAlex Elder ipa_mem_zero_region_add(trans, IPA_MEM_MODEM_PROC_CTX); 101ce928bf8SAlex Elder ipa_mem_zero_region_add(trans, IPA_MEM_AP_PROC_CTX); 102ce928bf8SAlex Elder ipa_mem_zero_region_add(trans, IPA_MEM_MODEM); 103ba764c4dSAlex Elder 104ba764c4dSAlex Elder gsi_trans_commit_wait(trans); 105ba764c4dSAlex Elder 106ba764c4dSAlex Elder /* Tell the hardware where the processing context area is located */ 107*5e3bc1e5SAlex Elder mem = ipa_mem_find(ipa, IPA_MEM_MODEM_PROC_CTX); 108*5e3bc1e5SAlex Elder offset = ipa->mem_offset + mem->offset; 109e6e49e43SAlex Elder val = proc_cntxt_base_addr_encoded(ipa->version, offset); 110e6e49e43SAlex Elder iowrite32(val, ipa->reg_virt + IPA_REG_LOCAL_PKT_PROC_CNTXT_OFFSET); 111ba764c4dSAlex Elder 112ba764c4dSAlex Elder return 0; 113ba764c4dSAlex Elder } 114ba764c4dSAlex Elder 11575bcfde6SAlex Elder /* Is the given memory region ID is valid for the current IPA version? */ 11675bcfde6SAlex Elder static bool ipa_mem_id_valid(struct ipa *ipa, enum ipa_mem_id mem_id) 11775bcfde6SAlex Elder { 11875bcfde6SAlex Elder enum ipa_version version = ipa->version; 11975bcfde6SAlex Elder 12075bcfde6SAlex Elder switch (mem_id) { 12175bcfde6SAlex Elder case IPA_MEM_UC_SHARED: 12275bcfde6SAlex Elder case IPA_MEM_UC_INFO: 12375bcfde6SAlex Elder case IPA_MEM_V4_FILTER_HASHED: 12475bcfde6SAlex Elder case IPA_MEM_V4_FILTER: 12575bcfde6SAlex Elder case IPA_MEM_V6_FILTER_HASHED: 12675bcfde6SAlex Elder case IPA_MEM_V6_FILTER: 12775bcfde6SAlex Elder case IPA_MEM_V4_ROUTE_HASHED: 12875bcfde6SAlex Elder case IPA_MEM_V4_ROUTE: 12975bcfde6SAlex Elder case IPA_MEM_V6_ROUTE_HASHED: 13075bcfde6SAlex Elder case IPA_MEM_V6_ROUTE: 13175bcfde6SAlex Elder case IPA_MEM_MODEM_HEADER: 13275bcfde6SAlex Elder case IPA_MEM_AP_HEADER: 13375bcfde6SAlex Elder case IPA_MEM_MODEM_PROC_CTX: 13475bcfde6SAlex Elder case IPA_MEM_AP_PROC_CTX: 13575bcfde6SAlex Elder case IPA_MEM_MODEM: 13675bcfde6SAlex Elder case IPA_MEM_UC_EVENT_RING: 13775bcfde6SAlex Elder case IPA_MEM_PDN_CONFIG: 13875bcfde6SAlex Elder case IPA_MEM_STATS_QUOTA_MODEM: 13975bcfde6SAlex Elder case IPA_MEM_STATS_QUOTA_AP: 14075bcfde6SAlex Elder case IPA_MEM_END_MARKER: /* pseudo region */ 14175bcfde6SAlex Elder break; 14275bcfde6SAlex Elder 14375bcfde6SAlex Elder case IPA_MEM_STATS_TETHERING: 14475bcfde6SAlex Elder case IPA_MEM_STATS_DROP: 14575bcfde6SAlex Elder if (version < IPA_VERSION_4_0) 14675bcfde6SAlex Elder return false; 14775bcfde6SAlex Elder break; 14875bcfde6SAlex Elder 14975bcfde6SAlex Elder case IPA_MEM_STATS_V4_FILTER: 15075bcfde6SAlex Elder case IPA_MEM_STATS_V6_FILTER: 15175bcfde6SAlex Elder case IPA_MEM_STATS_V4_ROUTE: 15275bcfde6SAlex Elder case IPA_MEM_STATS_V6_ROUTE: 15375bcfde6SAlex Elder if (version < IPA_VERSION_4_0 || version > IPA_VERSION_4_2) 15475bcfde6SAlex Elder return false; 15575bcfde6SAlex Elder break; 15675bcfde6SAlex Elder 15775bcfde6SAlex Elder case IPA_MEM_NAT_TABLE: 15875bcfde6SAlex Elder case IPA_MEM_STATS_FILTER_ROUTE: 15975bcfde6SAlex Elder if (version < IPA_VERSION_4_5) 16075bcfde6SAlex Elder return false; 16175bcfde6SAlex Elder break; 16275bcfde6SAlex Elder 16375bcfde6SAlex Elder default: 16475bcfde6SAlex Elder return false; 16575bcfde6SAlex Elder } 16675bcfde6SAlex Elder 16775bcfde6SAlex Elder return true; 16875bcfde6SAlex Elder } 16975bcfde6SAlex Elder 170d39ffb97SAlex Elder /* Must the given memory region be present in the configuration? */ 171d39ffb97SAlex Elder static bool ipa_mem_id_required(struct ipa *ipa, enum ipa_mem_id mem_id) 172d39ffb97SAlex Elder { 173d39ffb97SAlex Elder switch (mem_id) { 174d39ffb97SAlex Elder case IPA_MEM_UC_SHARED: 175d39ffb97SAlex Elder case IPA_MEM_UC_INFO: 176d39ffb97SAlex Elder case IPA_MEM_V4_FILTER_HASHED: 177d39ffb97SAlex Elder case IPA_MEM_V4_FILTER: 178d39ffb97SAlex Elder case IPA_MEM_V6_FILTER_HASHED: 179d39ffb97SAlex Elder case IPA_MEM_V6_FILTER: 180d39ffb97SAlex Elder case IPA_MEM_V4_ROUTE_HASHED: 181d39ffb97SAlex Elder case IPA_MEM_V4_ROUTE: 182d39ffb97SAlex Elder case IPA_MEM_V6_ROUTE_HASHED: 183d39ffb97SAlex Elder case IPA_MEM_V6_ROUTE: 184d39ffb97SAlex Elder case IPA_MEM_MODEM_HEADER: 185d39ffb97SAlex Elder case IPA_MEM_MODEM_PROC_CTX: 186d39ffb97SAlex Elder case IPA_MEM_AP_PROC_CTX: 187d39ffb97SAlex Elder case IPA_MEM_MODEM: 188d39ffb97SAlex Elder return true; 189d39ffb97SAlex Elder 190d39ffb97SAlex Elder case IPA_MEM_PDN_CONFIG: 191d39ffb97SAlex Elder case IPA_MEM_STATS_QUOTA_MODEM: 192d39ffb97SAlex Elder case IPA_MEM_STATS_TETHERING: 193d39ffb97SAlex Elder return ipa->version >= IPA_VERSION_4_0; 194d39ffb97SAlex Elder 195d39ffb97SAlex Elder default: 196d39ffb97SAlex Elder return false; /* Anything else is optional */ 197d39ffb97SAlex Elder } 198d39ffb97SAlex Elder } 199d39ffb97SAlex Elder 20098334d2aSAlex Elder static bool ipa_mem_valid_one(struct ipa *ipa, const struct ipa_mem *mem) 201ba764c4dSAlex Elder { 202ba764c4dSAlex Elder struct device *dev = &ipa->pdev->dev; 2030300df2dSAlex Elder enum ipa_mem_id mem_id = mem->id; 204ba764c4dSAlex Elder u16 size_multiple; 205ba764c4dSAlex Elder 20675bcfde6SAlex Elder /* Make sure the memory region is valid for this version of IPA */ 20775bcfde6SAlex Elder if (!ipa_mem_id_valid(ipa, mem_id)) { 20875bcfde6SAlex Elder dev_err(dev, "region id %u not valid\n", mem_id); 20975bcfde6SAlex Elder return false; 21075bcfde6SAlex Elder } 21175bcfde6SAlex Elder 212ba764c4dSAlex Elder /* Other than modem memory, sizes must be a multiple of 8 */ 213ba764c4dSAlex Elder size_multiple = mem_id == IPA_MEM_MODEM ? 4 : 8; 214ba764c4dSAlex Elder if (mem->size % size_multiple) 215ba764c4dSAlex Elder dev_err(dev, "region %u size not a multiple of %u bytes\n", 216ba764c4dSAlex Elder mem_id, size_multiple); 217ba764c4dSAlex Elder else if (mem->offset % 8) 218ba764c4dSAlex Elder dev_err(dev, "region %u offset not 8-byte aligned\n", mem_id); 219ba764c4dSAlex Elder else if (mem->offset < mem->canary_count * sizeof(__le32)) 220ba764c4dSAlex Elder dev_err(dev, "region %u offset too small for %hu canaries\n", 221ba764c4dSAlex Elder mem_id, mem->canary_count); 222f636a836SAlex Elder else if (mem_id == IPA_MEM_END_MARKER && mem->size) 223f636a836SAlex Elder dev_err(dev, "non-zero end marker region size\n"); 224ba764c4dSAlex Elder else 225ba764c4dSAlex Elder return true; 226ba764c4dSAlex Elder 227ba764c4dSAlex Elder return false; 228ba764c4dSAlex Elder } 229ba764c4dSAlex Elder 23098334d2aSAlex Elder /* Verify each defined memory region is valid. */ 2311eec7677SAlex Elder static bool ipa_mem_valid(struct ipa *ipa, const struct ipa_mem_data *mem_data) 23298334d2aSAlex Elder { 233eadf7f93SAlex Elder DECLARE_BITMAP(regions, IPA_MEM_COUNT) = { }; 23498334d2aSAlex Elder struct device *dev = &ipa->pdev->dev; 23598334d2aSAlex Elder enum ipa_mem_id mem_id; 2368cc7ebbfSAlex Elder u32 i; 23798334d2aSAlex Elder 2381eec7677SAlex Elder if (mem_data->local_count > IPA_MEM_COUNT) { 2391eec7677SAlex Elder dev_err(dev, "too many memory regions (%u > %u)\n", 2401eec7677SAlex Elder mem_data->local_count, IPA_MEM_COUNT); 2411eec7677SAlex Elder return false; 2421eec7677SAlex Elder } 2431eec7677SAlex Elder 2448cc7ebbfSAlex Elder for (i = 0; i < mem_data->local_count; i++) { 2458cc7ebbfSAlex Elder const struct ipa_mem *mem = &mem_data->local[i]; 24698334d2aSAlex Elder 2478cc7ebbfSAlex Elder if (mem->id == IPA_MEM_UNDEFINED) 248eadf7f93SAlex Elder continue; 249eadf7f93SAlex Elder 250eadf7f93SAlex Elder if (__test_and_set_bit(mem->id, regions)) { 251eadf7f93SAlex Elder dev_err(dev, "duplicate memory region %u\n", mem->id); 252eadf7f93SAlex Elder return false; 253eadf7f93SAlex Elder } 254eadf7f93SAlex Elder 25598334d2aSAlex Elder /* Defined regions have non-zero size and/or canary count */ 25698334d2aSAlex Elder if (mem->size || mem->canary_count) { 25798334d2aSAlex Elder if (ipa_mem_valid_one(ipa, mem)) 25898334d2aSAlex Elder continue; 25998334d2aSAlex Elder return false; 26098334d2aSAlex Elder } 26198334d2aSAlex Elder 26298334d2aSAlex Elder /* It's harmless, but warn if an offset is provided */ 26398334d2aSAlex Elder if (mem->offset) 26498334d2aSAlex Elder dev_warn(dev, "empty region %u has non-zero offset\n", 2658cc7ebbfSAlex Elder mem->id); 266d39ffb97SAlex Elder } 267d39ffb97SAlex Elder 268d39ffb97SAlex Elder /* Now see if any required regions are not defined */ 2696857b023SAlex Elder for (mem_id = find_first_zero_bit(regions, IPA_MEM_COUNT); 2706857b023SAlex Elder mem_id < IPA_MEM_COUNT; 2716857b023SAlex Elder mem_id = find_next_zero_bit(regions, IPA_MEM_COUNT, mem_id + 1)) { 2726857b023SAlex Elder if (ipa_mem_id_required(ipa, mem_id)) 273d39ffb97SAlex Elder dev_err(dev, "required memory region %u missing\n", 274d39ffb97SAlex Elder mem_id); 2756857b023SAlex Elder } 27698334d2aSAlex Elder 27798334d2aSAlex Elder return true; 27898334d2aSAlex Elder } 27998334d2aSAlex Elder 2805e57c6c5SAlex Elder /* Do all memory regions fit within the IPA local memory? */ 2815e57c6c5SAlex Elder static bool ipa_mem_size_valid(struct ipa *ipa) 2825e57c6c5SAlex Elder { 2835e57c6c5SAlex Elder struct device *dev = &ipa->pdev->dev; 2845e57c6c5SAlex Elder u32 limit = ipa->mem_size; 2858cc7ebbfSAlex Elder u32 i; 2865e57c6c5SAlex Elder 2878cc7ebbfSAlex Elder for (i = 0; i < ipa->mem_count; i++) { 2888cc7ebbfSAlex Elder const struct ipa_mem *mem = &ipa->mem[i]; 2895e57c6c5SAlex Elder 2905e57c6c5SAlex Elder if (mem->offset + mem->size <= limit) 2915e57c6c5SAlex Elder continue; 2925e57c6c5SAlex Elder 2935e57c6c5SAlex Elder dev_err(dev, "region %u ends beyond memory limit (0x%08x)\n", 2948cc7ebbfSAlex Elder mem->id, limit); 2955e57c6c5SAlex Elder 2965e57c6c5SAlex Elder return false; 2975e57c6c5SAlex Elder } 2985e57c6c5SAlex Elder 2995e57c6c5SAlex Elder return true; 3005e57c6c5SAlex Elder } 3015e57c6c5SAlex Elder 302ba764c4dSAlex Elder /** 303ba764c4dSAlex Elder * ipa_mem_config() - Configure IPA shared memory 304e3eea08eSAlex Elder * @ipa: IPA pointer 305ba764c4dSAlex Elder * 306e3eea08eSAlex Elder * Return: 0 if successful, or a negative error code 307ba764c4dSAlex Elder */ 308ba764c4dSAlex Elder int ipa_mem_config(struct ipa *ipa) 309ba764c4dSAlex Elder { 310ba764c4dSAlex Elder struct device *dev = &ipa->pdev->dev; 311*5e3bc1e5SAlex Elder const struct ipa_mem *mem; 312ba764c4dSAlex Elder dma_addr_t addr; 313ba764c4dSAlex Elder u32 mem_size; 314ba764c4dSAlex Elder void *virt; 315ba764c4dSAlex Elder u32 val; 3168cc7ebbfSAlex Elder u32 i; 317ba764c4dSAlex Elder 318ba764c4dSAlex Elder /* Check the advertised location and size of the shared memory area */ 319ba764c4dSAlex Elder val = ioread32(ipa->reg_virt + IPA_REG_SHARED_MEM_SIZE_OFFSET); 320ba764c4dSAlex Elder 321ba764c4dSAlex Elder /* The fields in the register are in 8 byte units */ 322ba764c4dSAlex Elder ipa->mem_offset = 8 * u32_get_bits(val, SHARED_MEM_BADDR_FMASK); 323ba764c4dSAlex Elder /* Make sure the end is within the region's mapped space */ 324ba764c4dSAlex Elder mem_size = 8 * u32_get_bits(val, SHARED_MEM_SIZE_FMASK); 325ba764c4dSAlex Elder 326ba764c4dSAlex Elder /* If the sizes don't match, issue a warning */ 3272c642c48SAlex Elder if (ipa->mem_offset + mem_size < ipa->mem_size) { 328ba764c4dSAlex Elder dev_warn(dev, "limiting IPA memory size to 0x%08x\n", 329ba764c4dSAlex Elder mem_size); 330ba764c4dSAlex Elder ipa->mem_size = mem_size; 3312c642c48SAlex Elder } else if (ipa->mem_offset + mem_size > ipa->mem_size) { 3322c642c48SAlex Elder dev_dbg(dev, "ignoring larger reported memory size: 0x%08x\n", 3332c642c48SAlex Elder mem_size); 334ba764c4dSAlex Elder } 335ba764c4dSAlex Elder 3365e57c6c5SAlex Elder /* We know our memory size; make sure regions are all in range */ 3375e57c6c5SAlex Elder if (!ipa_mem_size_valid(ipa)) 3385e57c6c5SAlex Elder return -EINVAL; 3395e57c6c5SAlex Elder 340ba764c4dSAlex Elder /* Prealloc DMA memory for zeroing regions */ 341ba764c4dSAlex Elder virt = dma_alloc_coherent(dev, IPA_MEM_MAX, &addr, GFP_KERNEL); 342ba764c4dSAlex Elder if (!virt) 343ba764c4dSAlex Elder return -ENOMEM; 344ba764c4dSAlex Elder ipa->zero_addr = addr; 345ba764c4dSAlex Elder ipa->zero_virt = virt; 346ba764c4dSAlex Elder ipa->zero_size = IPA_MEM_MAX; 347ba764c4dSAlex Elder 3488cc7ebbfSAlex Elder /* For each defined region, write "canary" values in the 3498cc7ebbfSAlex Elder * space prior to the region's base address if indicated. 350ba764c4dSAlex Elder */ 3518cc7ebbfSAlex Elder for (i = 0; i < ipa->mem_count; i++) { 352ba764c4dSAlex Elder u16 canary_count; 353ba764c4dSAlex Elder __le32 *canary; 354ba764c4dSAlex Elder 355ba764c4dSAlex Elder /* Skip over undefined regions */ 356*5e3bc1e5SAlex Elder mem = &ipa->mem[i]; 357ba764c4dSAlex Elder if (!mem->offset && !mem->size) 358ba764c4dSAlex Elder continue; 359ba764c4dSAlex Elder 360ba764c4dSAlex Elder canary_count = mem->canary_count; 361ba764c4dSAlex Elder if (!canary_count) 362ba764c4dSAlex Elder continue; 363ba764c4dSAlex Elder 364ba764c4dSAlex Elder /* Write canary values in the space before the region */ 365ba764c4dSAlex Elder canary = ipa->mem_virt + ipa->mem_offset + mem->offset; 366ba764c4dSAlex Elder do 367ba764c4dSAlex Elder *--canary = IPA_MEM_CANARY_VAL; 368ba764c4dSAlex Elder while (--canary_count); 369ba764c4dSAlex Elder } 370ba764c4dSAlex Elder 371ba764c4dSAlex Elder /* Make sure filter and route table memory regions are valid */ 372ba764c4dSAlex Elder if (!ipa_table_valid(ipa)) 373ba764c4dSAlex Elder goto err_dma_free; 374ba764c4dSAlex Elder 375ba764c4dSAlex Elder /* Validate memory-related properties relevant to immediate commands */ 376ba764c4dSAlex Elder if (!ipa_cmd_data_valid(ipa)) 377ba764c4dSAlex Elder goto err_dma_free; 378ba764c4dSAlex Elder 379*5e3bc1e5SAlex Elder /* Verify the microcontroller ring alignment (if defined) */ 380*5e3bc1e5SAlex Elder mem = ipa_mem_find(ipa, IPA_MEM_UC_EVENT_RING); 381*5e3bc1e5SAlex Elder if (mem && mem->offset % 1024) { 382ba764c4dSAlex Elder dev_err(dev, "microcontroller ring not 1024-byte aligned\n"); 383ba764c4dSAlex Elder goto err_dma_free; 384ba764c4dSAlex Elder } 385ba764c4dSAlex Elder 386ba764c4dSAlex Elder return 0; 387ba764c4dSAlex Elder 388ba764c4dSAlex Elder err_dma_free: 389ba764c4dSAlex Elder dma_free_coherent(dev, IPA_MEM_MAX, ipa->zero_virt, ipa->zero_addr); 390ba764c4dSAlex Elder 391ba764c4dSAlex Elder return -EINVAL; 392ba764c4dSAlex Elder } 393ba764c4dSAlex Elder 394ba764c4dSAlex Elder /* Inverse of ipa_mem_config() */ 395ba764c4dSAlex Elder void ipa_mem_deconfig(struct ipa *ipa) 396ba764c4dSAlex Elder { 397ba764c4dSAlex Elder struct device *dev = &ipa->pdev->dev; 398ba764c4dSAlex Elder 399ba764c4dSAlex Elder dma_free_coherent(dev, ipa->zero_size, ipa->zero_virt, ipa->zero_addr); 400ba764c4dSAlex Elder ipa->zero_size = 0; 401ba764c4dSAlex Elder ipa->zero_virt = NULL; 402ba764c4dSAlex Elder ipa->zero_addr = 0; 403ba764c4dSAlex Elder } 404ba764c4dSAlex Elder 405ba764c4dSAlex Elder /** 406ba764c4dSAlex Elder * ipa_mem_zero_modem() - Zero IPA-local memory regions owned by the modem 407e3eea08eSAlex Elder * @ipa: IPA pointer 408ba764c4dSAlex Elder * 409ba764c4dSAlex Elder * Zero regions of IPA-local memory used by the modem. These are configured 410ba764c4dSAlex Elder * (and initially zeroed) by ipa_mem_setup(), but if the modem crashes and 411ba764c4dSAlex Elder * restarts via SSR we need to re-initialize them. A QMI message tells the 412ba764c4dSAlex Elder * modem where to find regions of IPA local memory it needs to know about 413ba764c4dSAlex Elder * (these included). 414ba764c4dSAlex Elder */ 415ba764c4dSAlex Elder int ipa_mem_zero_modem(struct ipa *ipa) 416ba764c4dSAlex Elder { 417ba764c4dSAlex Elder struct gsi_trans *trans; 418ba764c4dSAlex Elder 419ba764c4dSAlex Elder /* Get a transaction to zero the modem memory, modem header, 420ba764c4dSAlex Elder * and modem processing context regions. 421ba764c4dSAlex Elder */ 422ba764c4dSAlex Elder trans = ipa_cmd_trans_alloc(ipa, 3); 423ba764c4dSAlex Elder if (!trans) { 424ba764c4dSAlex Elder dev_err(&ipa->pdev->dev, 425ba764c4dSAlex Elder "no transaction to zero modem memory\n"); 426ba764c4dSAlex Elder return -EBUSY; 427ba764c4dSAlex Elder } 428ba764c4dSAlex Elder 429ce928bf8SAlex Elder ipa_mem_zero_region_add(trans, IPA_MEM_MODEM_HEADER); 430ce928bf8SAlex Elder ipa_mem_zero_region_add(trans, IPA_MEM_MODEM_PROC_CTX); 431ce928bf8SAlex Elder ipa_mem_zero_region_add(trans, IPA_MEM_MODEM); 432ba764c4dSAlex Elder 433ba764c4dSAlex Elder gsi_trans_commit_wait(trans); 434ba764c4dSAlex Elder 435ba764c4dSAlex Elder return 0; 436ba764c4dSAlex Elder } 437ba764c4dSAlex Elder 4383e313c3fSAlex Elder /** 4393e313c3fSAlex Elder * ipa_imem_init() - Initialize IMEM memory used by the IPA 4403e313c3fSAlex Elder * @ipa: IPA pointer 4413e313c3fSAlex Elder * @addr: Physical address of the IPA region in IMEM 4423e313c3fSAlex Elder * @size: Size (bytes) of the IPA region in IMEM 4433e313c3fSAlex Elder * 4443e313c3fSAlex Elder * IMEM is a block of shared memory separate from system DRAM, and 4453e313c3fSAlex Elder * a portion of this memory is available for the IPA to use. The 4463e313c3fSAlex Elder * modem accesses this memory directly, but the IPA accesses it 4473e313c3fSAlex Elder * via the IOMMU, using the AP's credentials. 4483e313c3fSAlex Elder * 4493e313c3fSAlex Elder * If this region exists (size > 0) we map it for read/write access 4503e313c3fSAlex Elder * through the IOMMU using the IPA device. 4513e313c3fSAlex Elder * 4523e313c3fSAlex Elder * Note: @addr and @size are not guaranteed to be page-aligned. 4533e313c3fSAlex Elder */ 4543e313c3fSAlex Elder static int ipa_imem_init(struct ipa *ipa, unsigned long addr, size_t size) 4553e313c3fSAlex Elder { 4563e313c3fSAlex Elder struct device *dev = &ipa->pdev->dev; 4573e313c3fSAlex Elder struct iommu_domain *domain; 4583e313c3fSAlex Elder unsigned long iova; 4593e313c3fSAlex Elder phys_addr_t phys; 4603e313c3fSAlex Elder int ret; 4613e313c3fSAlex Elder 4623e313c3fSAlex Elder if (!size) 4633e313c3fSAlex Elder return 0; /* IMEM memory not used */ 4643e313c3fSAlex Elder 4653e313c3fSAlex Elder domain = iommu_get_domain_for_dev(dev); 4663e313c3fSAlex Elder if (!domain) { 4673e313c3fSAlex Elder dev_err(dev, "no IOMMU domain found for IMEM\n"); 4683e313c3fSAlex Elder return -EINVAL; 4693e313c3fSAlex Elder } 4703e313c3fSAlex Elder 4713e313c3fSAlex Elder /* Align the address down and the size up to page boundaries */ 4723e313c3fSAlex Elder phys = addr & PAGE_MASK; 4733e313c3fSAlex Elder size = PAGE_ALIGN(size + addr - phys); 4743e313c3fSAlex Elder iova = phys; /* We just want a direct mapping */ 4753e313c3fSAlex Elder 4763e313c3fSAlex Elder ret = iommu_map(domain, iova, phys, size, IOMMU_READ | IOMMU_WRITE); 4773e313c3fSAlex Elder if (ret) 4783e313c3fSAlex Elder return ret; 4793e313c3fSAlex Elder 4803e313c3fSAlex Elder ipa->imem_iova = iova; 4813e313c3fSAlex Elder ipa->imem_size = size; 4823e313c3fSAlex Elder 4833e313c3fSAlex Elder return 0; 4843e313c3fSAlex Elder } 4853e313c3fSAlex Elder 4863e313c3fSAlex Elder static void ipa_imem_exit(struct ipa *ipa) 4873e313c3fSAlex Elder { 4883e313c3fSAlex Elder struct iommu_domain *domain; 4893e313c3fSAlex Elder struct device *dev; 4903e313c3fSAlex Elder 4913e313c3fSAlex Elder if (!ipa->imem_size) 4923e313c3fSAlex Elder return; 4933e313c3fSAlex Elder 4943e313c3fSAlex Elder dev = &ipa->pdev->dev; 4953e313c3fSAlex Elder domain = iommu_get_domain_for_dev(dev); 4963e313c3fSAlex Elder if (domain) { 4973e313c3fSAlex Elder size_t size; 4983e313c3fSAlex Elder 4993e313c3fSAlex Elder size = iommu_unmap(domain, ipa->imem_iova, ipa->imem_size); 5003e313c3fSAlex Elder if (size != ipa->imem_size) 501113b6ea0SAlex Elder dev_warn(dev, "unmapped %zu IMEM bytes, expected %zu\n", 5023e313c3fSAlex Elder size, ipa->imem_size); 5033e313c3fSAlex Elder } else { 5043e313c3fSAlex Elder dev_err(dev, "couldn't get IPA IOMMU domain for IMEM\n"); 5053e313c3fSAlex Elder } 5063e313c3fSAlex Elder 5073e313c3fSAlex Elder ipa->imem_size = 0; 5083e313c3fSAlex Elder ipa->imem_iova = 0; 5093e313c3fSAlex Elder } 5103e313c3fSAlex Elder 511a0036bb4SAlex Elder /** 512a0036bb4SAlex Elder * ipa_smem_init() - Initialize SMEM memory used by the IPA 513a0036bb4SAlex Elder * @ipa: IPA pointer 514a0036bb4SAlex Elder * @item: Item ID of SMEM memory 515a0036bb4SAlex Elder * @size: Size (bytes) of SMEM memory region 516a0036bb4SAlex Elder * 517a0036bb4SAlex Elder * SMEM is a managed block of shared DRAM, from which numbered "items" 518a0036bb4SAlex Elder * can be allocated. One item is designated for use by the IPA. 519a0036bb4SAlex Elder * 520a0036bb4SAlex Elder * The modem accesses SMEM memory directly, but the IPA accesses it 521a0036bb4SAlex Elder * via the IOMMU, using the AP's credentials. 522a0036bb4SAlex Elder * 523a0036bb4SAlex Elder * If size provided is non-zero, we allocate it and map it for 524a0036bb4SAlex Elder * access through the IOMMU. 525a0036bb4SAlex Elder * 526a0036bb4SAlex Elder * Note: @size and the item address are is not guaranteed to be page-aligned. 527a0036bb4SAlex Elder */ 528a0036bb4SAlex Elder static int ipa_smem_init(struct ipa *ipa, u32 item, size_t size) 529a0036bb4SAlex Elder { 530a0036bb4SAlex Elder struct device *dev = &ipa->pdev->dev; 531a0036bb4SAlex Elder struct iommu_domain *domain; 532a0036bb4SAlex Elder unsigned long iova; 533a0036bb4SAlex Elder phys_addr_t phys; 534a0036bb4SAlex Elder phys_addr_t addr; 535a0036bb4SAlex Elder size_t actual; 536a0036bb4SAlex Elder void *virt; 537a0036bb4SAlex Elder int ret; 538a0036bb4SAlex Elder 539a0036bb4SAlex Elder if (!size) 540a0036bb4SAlex Elder return 0; /* SMEM memory not used */ 541a0036bb4SAlex Elder 542a0036bb4SAlex Elder /* SMEM is memory shared between the AP and another system entity 543a0036bb4SAlex Elder * (in this case, the modem). An allocation from SMEM is persistent 544a0036bb4SAlex Elder * until the AP reboots; there is no way to free an allocated SMEM 545a0036bb4SAlex Elder * region. Allocation only reserves the space; to use it you need 546*5e3bc1e5SAlex Elder * to "get" a pointer it (this does not imply reference counting). 547a0036bb4SAlex Elder * The item might have already been allocated, in which case we 548a0036bb4SAlex Elder * use it unless the size isn't what we expect. 549a0036bb4SAlex Elder */ 550a0036bb4SAlex Elder ret = qcom_smem_alloc(QCOM_SMEM_HOST_MODEM, item, size); 551a0036bb4SAlex Elder if (ret && ret != -EEXIST) { 552a0036bb4SAlex Elder dev_err(dev, "error %d allocating size %zu SMEM item %u\n", 553a0036bb4SAlex Elder ret, size, item); 554a0036bb4SAlex Elder return ret; 555a0036bb4SAlex Elder } 556a0036bb4SAlex Elder 557a0036bb4SAlex Elder /* Now get the address of the SMEM memory region */ 558a0036bb4SAlex Elder virt = qcom_smem_get(QCOM_SMEM_HOST_MODEM, item, &actual); 559a0036bb4SAlex Elder if (IS_ERR(virt)) { 560a0036bb4SAlex Elder ret = PTR_ERR(virt); 561a0036bb4SAlex Elder dev_err(dev, "error %d getting SMEM item %u\n", ret, item); 562a0036bb4SAlex Elder return ret; 563a0036bb4SAlex Elder } 564a0036bb4SAlex Elder 565a0036bb4SAlex Elder /* In case the region was already allocated, verify the size */ 566a0036bb4SAlex Elder if (ret && actual != size) { 567a0036bb4SAlex Elder dev_err(dev, "SMEM item %u has size %zu, expected %zu\n", 568a0036bb4SAlex Elder item, actual, size); 569a0036bb4SAlex Elder return -EINVAL; 570a0036bb4SAlex Elder } 571a0036bb4SAlex Elder 572a0036bb4SAlex Elder domain = iommu_get_domain_for_dev(dev); 573a0036bb4SAlex Elder if (!domain) { 574a0036bb4SAlex Elder dev_err(dev, "no IOMMU domain found for SMEM\n"); 575a0036bb4SAlex Elder return -EINVAL; 576a0036bb4SAlex Elder } 577a0036bb4SAlex Elder 578a0036bb4SAlex Elder /* Align the address down and the size up to a page boundary */ 579a0036bb4SAlex Elder addr = qcom_smem_virt_to_phys(virt) & PAGE_MASK; 580a0036bb4SAlex Elder phys = addr & PAGE_MASK; 581a0036bb4SAlex Elder size = PAGE_ALIGN(size + addr - phys); 582a0036bb4SAlex Elder iova = phys; /* We just want a direct mapping */ 583a0036bb4SAlex Elder 584a0036bb4SAlex Elder ret = iommu_map(domain, iova, phys, size, IOMMU_READ | IOMMU_WRITE); 585a0036bb4SAlex Elder if (ret) 586a0036bb4SAlex Elder return ret; 587a0036bb4SAlex Elder 588a0036bb4SAlex Elder ipa->smem_iova = iova; 589a0036bb4SAlex Elder ipa->smem_size = size; 590a0036bb4SAlex Elder 591a0036bb4SAlex Elder return 0; 592a0036bb4SAlex Elder } 593a0036bb4SAlex Elder 594a0036bb4SAlex Elder static void ipa_smem_exit(struct ipa *ipa) 595a0036bb4SAlex Elder { 596a0036bb4SAlex Elder struct device *dev = &ipa->pdev->dev; 597a0036bb4SAlex Elder struct iommu_domain *domain; 598a0036bb4SAlex Elder 599a0036bb4SAlex Elder domain = iommu_get_domain_for_dev(dev); 600a0036bb4SAlex Elder if (domain) { 601a0036bb4SAlex Elder size_t size; 602a0036bb4SAlex Elder 603a0036bb4SAlex Elder size = iommu_unmap(domain, ipa->smem_iova, ipa->smem_size); 604a0036bb4SAlex Elder if (size != ipa->smem_size) 605113b6ea0SAlex Elder dev_warn(dev, "unmapped %zu SMEM bytes, expected %zu\n", 606a0036bb4SAlex Elder size, ipa->smem_size); 607a0036bb4SAlex Elder 608a0036bb4SAlex Elder } else { 609a0036bb4SAlex Elder dev_err(dev, "couldn't get IPA IOMMU domain for SMEM\n"); 610a0036bb4SAlex Elder } 611a0036bb4SAlex Elder 612a0036bb4SAlex Elder ipa->smem_size = 0; 613a0036bb4SAlex Elder ipa->smem_iova = 0; 614a0036bb4SAlex Elder } 615a0036bb4SAlex Elder 616ba764c4dSAlex Elder /* Perform memory region-related initialization */ 6173128aae8SAlex Elder int ipa_mem_init(struct ipa *ipa, const struct ipa_mem_data *mem_data) 618ba764c4dSAlex Elder { 619ba764c4dSAlex Elder struct device *dev = &ipa->pdev->dev; 620ba764c4dSAlex Elder struct resource *res; 621ba764c4dSAlex Elder int ret; 622ba764c4dSAlex Elder 6231eec7677SAlex Elder /* Make sure the set of defined memory regions is valid */ 6241eec7677SAlex Elder if (!ipa_mem_valid(ipa, mem_data)) 625ba764c4dSAlex Elder return -EINVAL; 6261eec7677SAlex Elder 6271eec7677SAlex Elder ipa->mem_count = mem_data->local_count; 6281eec7677SAlex Elder ipa->mem = mem_data->local; 629ba764c4dSAlex Elder 630ba764c4dSAlex Elder ret = dma_set_mask_and_coherent(&ipa->pdev->dev, DMA_BIT_MASK(64)); 631ba764c4dSAlex Elder if (ret) { 632ba764c4dSAlex Elder dev_err(dev, "error %d setting DMA mask\n", ret); 633ba764c4dSAlex Elder return ret; 634ba764c4dSAlex Elder } 635ba764c4dSAlex Elder 636ba764c4dSAlex Elder res = platform_get_resource_byname(ipa->pdev, IORESOURCE_MEM, 637ba764c4dSAlex Elder "ipa-shared"); 638ba764c4dSAlex Elder if (!res) { 639ba764c4dSAlex Elder dev_err(dev, 640ba764c4dSAlex Elder "DT error getting \"ipa-shared\" memory property\n"); 641ba764c4dSAlex Elder return -ENODEV; 642ba764c4dSAlex Elder } 643ba764c4dSAlex Elder 644ba764c4dSAlex Elder ipa->mem_virt = memremap(res->start, resource_size(res), MEMREMAP_WC); 645ba764c4dSAlex Elder if (!ipa->mem_virt) { 646ba764c4dSAlex Elder dev_err(dev, "unable to remap \"ipa-shared\" memory\n"); 647ba764c4dSAlex Elder return -ENOMEM; 648ba764c4dSAlex Elder } 649ba764c4dSAlex Elder 650ba764c4dSAlex Elder ipa->mem_addr = res->start; 651ba764c4dSAlex Elder ipa->mem_size = resource_size(res); 652ba764c4dSAlex Elder 6533e313c3fSAlex Elder ret = ipa_imem_init(ipa, mem_data->imem_addr, mem_data->imem_size); 6543e313c3fSAlex Elder if (ret) 6553e313c3fSAlex Elder goto err_unmap; 6563e313c3fSAlex Elder 657a0036bb4SAlex Elder ret = ipa_smem_init(ipa, mem_data->smem_id, mem_data->smem_size); 658a0036bb4SAlex Elder if (ret) 659a0036bb4SAlex Elder goto err_imem_exit; 660a0036bb4SAlex Elder 661ba764c4dSAlex Elder return 0; 6623e313c3fSAlex Elder 663a0036bb4SAlex Elder err_imem_exit: 664a0036bb4SAlex Elder ipa_imem_exit(ipa); 6653e313c3fSAlex Elder err_unmap: 6663e313c3fSAlex Elder memunmap(ipa->mem_virt); 6673e313c3fSAlex Elder 6683e313c3fSAlex Elder return ret; 669ba764c4dSAlex Elder } 670ba764c4dSAlex Elder 671ba764c4dSAlex Elder /* Inverse of ipa_mem_init() */ 672ba764c4dSAlex Elder void ipa_mem_exit(struct ipa *ipa) 673ba764c4dSAlex Elder { 674a0036bb4SAlex Elder ipa_smem_exit(ipa); 6753e313c3fSAlex Elder ipa_imem_exit(ipa); 676ba764c4dSAlex Elder memunmap(ipa->mem_virt); 677ba764c4dSAlex Elder } 678