1ba764c4dSAlex Elder // SPDX-License-Identifier: GPL-2.0 2ba764c4dSAlex Elder 3ba764c4dSAlex Elder /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved. 474858b63SAlex Elder * Copyright (C) 2019-2021 Linaro Ltd. 5ba764c4dSAlex Elder */ 6ba764c4dSAlex Elder 7ba764c4dSAlex Elder #include <linux/types.h> 8ba764c4dSAlex Elder #include <linux/bitfield.h> 9ba764c4dSAlex Elder #include <linux/bug.h> 10ba764c4dSAlex Elder #include <linux/dma-mapping.h> 113e313c3fSAlex Elder #include <linux/iommu.h> 12ba764c4dSAlex Elder #include <linux/io.h> 13a0036bb4SAlex Elder #include <linux/soc/qcom/smem.h> 14ba764c4dSAlex Elder 15ba764c4dSAlex Elder #include "ipa.h" 16ba764c4dSAlex Elder #include "ipa_reg.h" 173128aae8SAlex Elder #include "ipa_data.h" 18ba764c4dSAlex Elder #include "ipa_cmd.h" 19ba764c4dSAlex Elder #include "ipa_mem.h" 20ba764c4dSAlex Elder #include "ipa_table.h" 21ba764c4dSAlex Elder #include "gsi_trans.h" 22ba764c4dSAlex Elder 23ba764c4dSAlex Elder /* "Canary" value placed between memory regions to detect overflow */ 24ba764c4dSAlex Elder #define IPA_MEM_CANARY_VAL cpu_to_le32(0xdeadbeef) 25ba764c4dSAlex Elder 26a0036bb4SAlex Elder /* SMEM host id representing the modem. */ 27a0036bb4SAlex Elder #define QCOM_SMEM_HOST_MODEM 1 28a0036bb4SAlex Elder 29ba764c4dSAlex Elder /* Add an immediate command to a transaction that zeroes a memory region */ 30ba764c4dSAlex Elder static void 31ba764c4dSAlex Elder ipa_mem_zero_region_add(struct gsi_trans *trans, const struct ipa_mem *mem) 32ba764c4dSAlex Elder { 33ba764c4dSAlex Elder struct ipa *ipa = container_of(trans->gsi, struct ipa, gsi); 34ba764c4dSAlex Elder dma_addr_t addr = ipa->zero_addr; 35ba764c4dSAlex Elder 36ba764c4dSAlex Elder if (!mem->size) 37ba764c4dSAlex Elder return; 38ba764c4dSAlex Elder 39ba764c4dSAlex Elder ipa_cmd_dma_shared_mem_add(trans, mem->offset, mem->size, addr, true); 40ba764c4dSAlex Elder } 41ba764c4dSAlex Elder 42ba764c4dSAlex Elder /** 43ba764c4dSAlex Elder * ipa_mem_setup() - Set up IPA AP and modem shared memory areas 44e3eea08eSAlex Elder * @ipa: IPA pointer 45ba764c4dSAlex Elder * 46ba764c4dSAlex Elder * Set up the shared memory regions in IPA local memory. This involves 47ba764c4dSAlex Elder * zero-filling memory regions, and in the case of header memory, telling 48ba764c4dSAlex Elder * the IPA where it's located. 49ba764c4dSAlex Elder * 50ba764c4dSAlex Elder * This function performs the initial setup of this memory. If the modem 51ba764c4dSAlex Elder * crashes, its regions are re-zeroed in ipa_mem_zero_modem(). 52ba764c4dSAlex Elder * 53ba764c4dSAlex Elder * The AP informs the modem where its portions of memory are located 54ba764c4dSAlex Elder * in a QMI exchange that occurs at modem startup. 55ba764c4dSAlex Elder * 5674858b63SAlex Elder * There is no need for a matching ipa_mem_teardown() function. 5774858b63SAlex Elder * 58e3eea08eSAlex Elder * Return: 0 if successful, or a negative error code 59ba764c4dSAlex Elder */ 60ba764c4dSAlex Elder int ipa_mem_setup(struct ipa *ipa) 61ba764c4dSAlex Elder { 62ba764c4dSAlex Elder dma_addr_t addr = ipa->zero_addr; 63ba764c4dSAlex Elder struct gsi_trans *trans; 64ba764c4dSAlex Elder u32 offset; 65ba764c4dSAlex Elder u16 size; 66e6e49e43SAlex Elder u32 val; 67ba764c4dSAlex Elder 68ba764c4dSAlex Elder /* Get a transaction to define the header memory region and to zero 69ba764c4dSAlex Elder * the processing context and modem memory regions. 70ba764c4dSAlex Elder */ 71ba764c4dSAlex Elder trans = ipa_cmd_trans_alloc(ipa, 4); 72ba764c4dSAlex Elder if (!trans) { 73ba764c4dSAlex Elder dev_err(&ipa->pdev->dev, "no transaction for memory setup\n"); 74ba764c4dSAlex Elder return -EBUSY; 75ba764c4dSAlex Elder } 76ba764c4dSAlex Elder 77ba764c4dSAlex Elder /* Initialize IPA-local header memory. The modem and AP header 78ba764c4dSAlex Elder * regions are contiguous, and initialized together. 79ba764c4dSAlex Elder */ 80ba764c4dSAlex Elder offset = ipa->mem[IPA_MEM_MODEM_HEADER].offset; 81ba764c4dSAlex Elder size = ipa->mem[IPA_MEM_MODEM_HEADER].size; 82ba764c4dSAlex Elder size += ipa->mem[IPA_MEM_AP_HEADER].size; 83ba764c4dSAlex Elder 84ba764c4dSAlex Elder ipa_cmd_hdr_init_local_add(trans, offset, size, addr); 85ba764c4dSAlex Elder 86ba764c4dSAlex Elder ipa_mem_zero_region_add(trans, &ipa->mem[IPA_MEM_MODEM_PROC_CTX]); 87ba764c4dSAlex Elder 88ba764c4dSAlex Elder ipa_mem_zero_region_add(trans, &ipa->mem[IPA_MEM_AP_PROC_CTX]); 89ba764c4dSAlex Elder 90ba764c4dSAlex Elder ipa_mem_zero_region_add(trans, &ipa->mem[IPA_MEM_MODEM]); 91ba764c4dSAlex Elder 92ba764c4dSAlex Elder gsi_trans_commit_wait(trans); 93ba764c4dSAlex Elder 94ba764c4dSAlex Elder /* Tell the hardware where the processing context area is located */ 95e6e49e43SAlex Elder offset = ipa->mem_offset + ipa->mem[IPA_MEM_MODEM_PROC_CTX].offset; 96e6e49e43SAlex Elder val = proc_cntxt_base_addr_encoded(ipa->version, offset); 97e6e49e43SAlex Elder iowrite32(val, ipa->reg_virt + IPA_REG_LOCAL_PKT_PROC_CNTXT_OFFSET); 98ba764c4dSAlex Elder 99ba764c4dSAlex Elder return 0; 100ba764c4dSAlex Elder } 101ba764c4dSAlex Elder 10275bcfde6SAlex Elder /* Is the given memory region ID is valid for the current IPA version? */ 10375bcfde6SAlex Elder static bool ipa_mem_id_valid(struct ipa *ipa, enum ipa_mem_id mem_id) 10475bcfde6SAlex Elder { 10575bcfde6SAlex Elder enum ipa_version version = ipa->version; 10675bcfde6SAlex Elder 10775bcfde6SAlex Elder switch (mem_id) { 10875bcfde6SAlex Elder case IPA_MEM_UC_SHARED: 10975bcfde6SAlex Elder case IPA_MEM_UC_INFO: 11075bcfde6SAlex Elder case IPA_MEM_V4_FILTER_HASHED: 11175bcfde6SAlex Elder case IPA_MEM_V4_FILTER: 11275bcfde6SAlex Elder case IPA_MEM_V6_FILTER_HASHED: 11375bcfde6SAlex Elder case IPA_MEM_V6_FILTER: 11475bcfde6SAlex Elder case IPA_MEM_V4_ROUTE_HASHED: 11575bcfde6SAlex Elder case IPA_MEM_V4_ROUTE: 11675bcfde6SAlex Elder case IPA_MEM_V6_ROUTE_HASHED: 11775bcfde6SAlex Elder case IPA_MEM_V6_ROUTE: 11875bcfde6SAlex Elder case IPA_MEM_MODEM_HEADER: 11975bcfde6SAlex Elder case IPA_MEM_AP_HEADER: 12075bcfde6SAlex Elder case IPA_MEM_MODEM_PROC_CTX: 12175bcfde6SAlex Elder case IPA_MEM_AP_PROC_CTX: 12275bcfde6SAlex Elder case IPA_MEM_MODEM: 12375bcfde6SAlex Elder case IPA_MEM_UC_EVENT_RING: 12475bcfde6SAlex Elder case IPA_MEM_PDN_CONFIG: 12575bcfde6SAlex Elder case IPA_MEM_STATS_QUOTA_MODEM: 12675bcfde6SAlex Elder case IPA_MEM_STATS_QUOTA_AP: 12775bcfde6SAlex Elder case IPA_MEM_END_MARKER: /* pseudo region */ 12875bcfde6SAlex Elder break; 12975bcfde6SAlex Elder 13075bcfde6SAlex Elder case IPA_MEM_STATS_TETHERING: 13175bcfde6SAlex Elder case IPA_MEM_STATS_DROP: 13275bcfde6SAlex Elder if (version < IPA_VERSION_4_0) 13375bcfde6SAlex Elder return false; 13475bcfde6SAlex Elder break; 13575bcfde6SAlex Elder 13675bcfde6SAlex Elder case IPA_MEM_STATS_V4_FILTER: 13775bcfde6SAlex Elder case IPA_MEM_STATS_V6_FILTER: 13875bcfde6SAlex Elder case IPA_MEM_STATS_V4_ROUTE: 13975bcfde6SAlex Elder case IPA_MEM_STATS_V6_ROUTE: 14075bcfde6SAlex Elder if (version < IPA_VERSION_4_0 || version > IPA_VERSION_4_2) 14175bcfde6SAlex Elder return false; 14275bcfde6SAlex Elder break; 14375bcfde6SAlex Elder 14475bcfde6SAlex Elder case IPA_MEM_NAT_TABLE: 14575bcfde6SAlex Elder case IPA_MEM_STATS_FILTER_ROUTE: 14675bcfde6SAlex Elder if (version < IPA_VERSION_4_5) 14775bcfde6SAlex Elder return false; 14875bcfde6SAlex Elder break; 14975bcfde6SAlex Elder 15075bcfde6SAlex Elder default: 15175bcfde6SAlex Elder return false; 15275bcfde6SAlex Elder } 15375bcfde6SAlex Elder 15475bcfde6SAlex Elder return true; 15575bcfde6SAlex Elder } 15675bcfde6SAlex Elder 157d39ffb97SAlex Elder /* Must the given memory region be present in the configuration? */ 158d39ffb97SAlex Elder static bool ipa_mem_id_required(struct ipa *ipa, enum ipa_mem_id mem_id) 159d39ffb97SAlex Elder { 160d39ffb97SAlex Elder switch (mem_id) { 161d39ffb97SAlex Elder case IPA_MEM_UC_SHARED: 162d39ffb97SAlex Elder case IPA_MEM_UC_INFO: 163d39ffb97SAlex Elder case IPA_MEM_V4_FILTER_HASHED: 164d39ffb97SAlex Elder case IPA_MEM_V4_FILTER: 165d39ffb97SAlex Elder case IPA_MEM_V6_FILTER_HASHED: 166d39ffb97SAlex Elder case IPA_MEM_V6_FILTER: 167d39ffb97SAlex Elder case IPA_MEM_V4_ROUTE_HASHED: 168d39ffb97SAlex Elder case IPA_MEM_V4_ROUTE: 169d39ffb97SAlex Elder case IPA_MEM_V6_ROUTE_HASHED: 170d39ffb97SAlex Elder case IPA_MEM_V6_ROUTE: 171d39ffb97SAlex Elder case IPA_MEM_MODEM_HEADER: 172d39ffb97SAlex Elder case IPA_MEM_MODEM_PROC_CTX: 173d39ffb97SAlex Elder case IPA_MEM_AP_PROC_CTX: 174d39ffb97SAlex Elder case IPA_MEM_MODEM: 175d39ffb97SAlex Elder return true; 176d39ffb97SAlex Elder 177d39ffb97SAlex Elder case IPA_MEM_PDN_CONFIG: 178d39ffb97SAlex Elder case IPA_MEM_STATS_QUOTA_MODEM: 179d39ffb97SAlex Elder case IPA_MEM_STATS_TETHERING: 180d39ffb97SAlex Elder return ipa->version >= IPA_VERSION_4_0; 181d39ffb97SAlex Elder 182d39ffb97SAlex Elder default: 183d39ffb97SAlex Elder return false; /* Anything else is optional */ 184d39ffb97SAlex Elder } 185d39ffb97SAlex Elder } 186d39ffb97SAlex Elder 18798334d2aSAlex Elder static bool ipa_mem_valid_one(struct ipa *ipa, const struct ipa_mem *mem) 188ba764c4dSAlex Elder { 189ba764c4dSAlex Elder struct device *dev = &ipa->pdev->dev; 1900300df2dSAlex Elder enum ipa_mem_id mem_id = mem->id; 191ba764c4dSAlex Elder u16 size_multiple; 192ba764c4dSAlex Elder 19375bcfde6SAlex Elder /* Make sure the memory region is valid for this version of IPA */ 19475bcfde6SAlex Elder if (!ipa_mem_id_valid(ipa, mem_id)) { 19575bcfde6SAlex Elder dev_err(dev, "region id %u not valid\n", mem_id); 19675bcfde6SAlex Elder return false; 19775bcfde6SAlex Elder } 19875bcfde6SAlex Elder 199ba764c4dSAlex Elder /* Other than modem memory, sizes must be a multiple of 8 */ 200ba764c4dSAlex Elder size_multiple = mem_id == IPA_MEM_MODEM ? 4 : 8; 201ba764c4dSAlex Elder if (mem->size % size_multiple) 202ba764c4dSAlex Elder dev_err(dev, "region %u size not a multiple of %u bytes\n", 203ba764c4dSAlex Elder mem_id, size_multiple); 204ba764c4dSAlex Elder else if (mem->offset % 8) 205ba764c4dSAlex Elder dev_err(dev, "region %u offset not 8-byte aligned\n", mem_id); 206ba764c4dSAlex Elder else if (mem->offset < mem->canary_count * sizeof(__le32)) 207ba764c4dSAlex Elder dev_err(dev, "region %u offset too small for %hu canaries\n", 208ba764c4dSAlex Elder mem_id, mem->canary_count); 209f636a836SAlex Elder else if (mem_id == IPA_MEM_END_MARKER && mem->size) 210f636a836SAlex Elder dev_err(dev, "non-zero end marker region size\n"); 211ba764c4dSAlex Elder else 212ba764c4dSAlex Elder return true; 213ba764c4dSAlex Elder 214ba764c4dSAlex Elder return false; 215ba764c4dSAlex Elder } 216ba764c4dSAlex Elder 21798334d2aSAlex Elder /* Verify each defined memory region is valid. */ 2181eec7677SAlex Elder static bool ipa_mem_valid(struct ipa *ipa, const struct ipa_mem_data *mem_data) 21998334d2aSAlex Elder { 220eadf7f93SAlex Elder DECLARE_BITMAP(regions, IPA_MEM_COUNT) = { }; 22198334d2aSAlex Elder struct device *dev = &ipa->pdev->dev; 22298334d2aSAlex Elder enum ipa_mem_id mem_id; 223*8cc7ebbfSAlex Elder u32 i; 22498334d2aSAlex Elder 2251eec7677SAlex Elder if (mem_data->local_count > IPA_MEM_COUNT) { 2261eec7677SAlex Elder dev_err(dev, "too many memory regions (%u > %u)\n", 2271eec7677SAlex Elder mem_data->local_count, IPA_MEM_COUNT); 2281eec7677SAlex Elder return false; 2291eec7677SAlex Elder } 2301eec7677SAlex Elder 231*8cc7ebbfSAlex Elder for (i = 0; i < mem_data->local_count; i++) { 232*8cc7ebbfSAlex Elder const struct ipa_mem *mem = &mem_data->local[i]; 23398334d2aSAlex Elder 234*8cc7ebbfSAlex Elder if (mem->id == IPA_MEM_UNDEFINED) 235eadf7f93SAlex Elder continue; 236eadf7f93SAlex Elder 237eadf7f93SAlex Elder if (__test_and_set_bit(mem->id, regions)) { 238eadf7f93SAlex Elder dev_err(dev, "duplicate memory region %u\n", mem->id); 239eadf7f93SAlex Elder return false; 240eadf7f93SAlex Elder } 241eadf7f93SAlex Elder 24298334d2aSAlex Elder /* Defined regions have non-zero size and/or canary count */ 24398334d2aSAlex Elder if (mem->size || mem->canary_count) { 24498334d2aSAlex Elder if (ipa_mem_valid_one(ipa, mem)) 24598334d2aSAlex Elder continue; 24698334d2aSAlex Elder return false; 24798334d2aSAlex Elder } 24898334d2aSAlex Elder 24998334d2aSAlex Elder /* It's harmless, but warn if an offset is provided */ 25098334d2aSAlex Elder if (mem->offset) 25198334d2aSAlex Elder dev_warn(dev, "empty region %u has non-zero offset\n", 252*8cc7ebbfSAlex Elder mem->id); 253d39ffb97SAlex Elder } 254d39ffb97SAlex Elder 255d39ffb97SAlex Elder /* Now see if any required regions are not defined */ 2566857b023SAlex Elder for (mem_id = find_first_zero_bit(regions, IPA_MEM_COUNT); 2576857b023SAlex Elder mem_id < IPA_MEM_COUNT; 2586857b023SAlex Elder mem_id = find_next_zero_bit(regions, IPA_MEM_COUNT, mem_id + 1)) { 2596857b023SAlex Elder if (ipa_mem_id_required(ipa, mem_id)) 260d39ffb97SAlex Elder dev_err(dev, "required memory region %u missing\n", 261d39ffb97SAlex Elder mem_id); 2626857b023SAlex Elder } 26398334d2aSAlex Elder 26498334d2aSAlex Elder return true; 26598334d2aSAlex Elder } 26698334d2aSAlex Elder 2675e57c6c5SAlex Elder /* Do all memory regions fit within the IPA local memory? */ 2685e57c6c5SAlex Elder static bool ipa_mem_size_valid(struct ipa *ipa) 2695e57c6c5SAlex Elder { 2705e57c6c5SAlex Elder struct device *dev = &ipa->pdev->dev; 2715e57c6c5SAlex Elder u32 limit = ipa->mem_size; 272*8cc7ebbfSAlex Elder u32 i; 2735e57c6c5SAlex Elder 274*8cc7ebbfSAlex Elder for (i = 0; i < ipa->mem_count; i++) { 275*8cc7ebbfSAlex Elder const struct ipa_mem *mem = &ipa->mem[i]; 2765e57c6c5SAlex Elder 2775e57c6c5SAlex Elder if (mem->offset + mem->size <= limit) 2785e57c6c5SAlex Elder continue; 2795e57c6c5SAlex Elder 2805e57c6c5SAlex Elder dev_err(dev, "region %u ends beyond memory limit (0x%08x)\n", 281*8cc7ebbfSAlex Elder mem->id, limit); 2825e57c6c5SAlex Elder 2835e57c6c5SAlex Elder return false; 2845e57c6c5SAlex Elder } 2855e57c6c5SAlex Elder 2865e57c6c5SAlex Elder return true; 2875e57c6c5SAlex Elder } 2885e57c6c5SAlex Elder 289ba764c4dSAlex Elder /** 290ba764c4dSAlex Elder * ipa_mem_config() - Configure IPA shared memory 291e3eea08eSAlex Elder * @ipa: IPA pointer 292ba764c4dSAlex Elder * 293e3eea08eSAlex Elder * Return: 0 if successful, or a negative error code 294ba764c4dSAlex Elder */ 295ba764c4dSAlex Elder int ipa_mem_config(struct ipa *ipa) 296ba764c4dSAlex Elder { 297ba764c4dSAlex Elder struct device *dev = &ipa->pdev->dev; 298ba764c4dSAlex Elder dma_addr_t addr; 299ba764c4dSAlex Elder u32 mem_size; 300ba764c4dSAlex Elder void *virt; 301ba764c4dSAlex Elder u32 val; 302*8cc7ebbfSAlex Elder u32 i; 303ba764c4dSAlex Elder 304ba764c4dSAlex Elder /* Check the advertised location and size of the shared memory area */ 305ba764c4dSAlex Elder val = ioread32(ipa->reg_virt + IPA_REG_SHARED_MEM_SIZE_OFFSET); 306ba764c4dSAlex Elder 307ba764c4dSAlex Elder /* The fields in the register are in 8 byte units */ 308ba764c4dSAlex Elder ipa->mem_offset = 8 * u32_get_bits(val, SHARED_MEM_BADDR_FMASK); 309ba764c4dSAlex Elder /* Make sure the end is within the region's mapped space */ 310ba764c4dSAlex Elder mem_size = 8 * u32_get_bits(val, SHARED_MEM_SIZE_FMASK); 311ba764c4dSAlex Elder 312ba764c4dSAlex Elder /* If the sizes don't match, issue a warning */ 3132c642c48SAlex Elder if (ipa->mem_offset + mem_size < ipa->mem_size) { 314ba764c4dSAlex Elder dev_warn(dev, "limiting IPA memory size to 0x%08x\n", 315ba764c4dSAlex Elder mem_size); 316ba764c4dSAlex Elder ipa->mem_size = mem_size; 3172c642c48SAlex Elder } else if (ipa->mem_offset + mem_size > ipa->mem_size) { 3182c642c48SAlex Elder dev_dbg(dev, "ignoring larger reported memory size: 0x%08x\n", 3192c642c48SAlex Elder mem_size); 320ba764c4dSAlex Elder } 321ba764c4dSAlex Elder 3225e57c6c5SAlex Elder /* We know our memory size; make sure regions are all in range */ 3235e57c6c5SAlex Elder if (!ipa_mem_size_valid(ipa)) 3245e57c6c5SAlex Elder return -EINVAL; 3255e57c6c5SAlex Elder 326ba764c4dSAlex Elder /* Prealloc DMA memory for zeroing regions */ 327ba764c4dSAlex Elder virt = dma_alloc_coherent(dev, IPA_MEM_MAX, &addr, GFP_KERNEL); 328ba764c4dSAlex Elder if (!virt) 329ba764c4dSAlex Elder return -ENOMEM; 330ba764c4dSAlex Elder ipa->zero_addr = addr; 331ba764c4dSAlex Elder ipa->zero_virt = virt; 332ba764c4dSAlex Elder ipa->zero_size = IPA_MEM_MAX; 333ba764c4dSAlex Elder 334*8cc7ebbfSAlex Elder /* For each defined region, write "canary" values in the 335*8cc7ebbfSAlex Elder * space prior to the region's base address if indicated. 336ba764c4dSAlex Elder */ 337*8cc7ebbfSAlex Elder for (i = 0; i < ipa->mem_count; i++) { 338*8cc7ebbfSAlex Elder const struct ipa_mem *mem = &ipa->mem[i]; 339ba764c4dSAlex Elder u16 canary_count; 340ba764c4dSAlex Elder __le32 *canary; 341ba764c4dSAlex Elder 342ba764c4dSAlex Elder /* Skip over undefined regions */ 343ba764c4dSAlex Elder if (!mem->offset && !mem->size) 344ba764c4dSAlex Elder continue; 345ba764c4dSAlex Elder 346ba764c4dSAlex Elder canary_count = mem->canary_count; 347ba764c4dSAlex Elder if (!canary_count) 348ba764c4dSAlex Elder continue; 349ba764c4dSAlex Elder 350ba764c4dSAlex Elder /* Write canary values in the space before the region */ 351ba764c4dSAlex Elder canary = ipa->mem_virt + ipa->mem_offset + mem->offset; 352ba764c4dSAlex Elder do 353ba764c4dSAlex Elder *--canary = IPA_MEM_CANARY_VAL; 354ba764c4dSAlex Elder while (--canary_count); 355ba764c4dSAlex Elder } 356ba764c4dSAlex Elder 357ba764c4dSAlex Elder /* Make sure filter and route table memory regions are valid */ 358ba764c4dSAlex Elder if (!ipa_table_valid(ipa)) 359ba764c4dSAlex Elder goto err_dma_free; 360ba764c4dSAlex Elder 361ba764c4dSAlex Elder /* Validate memory-related properties relevant to immediate commands */ 362ba764c4dSAlex Elder if (!ipa_cmd_data_valid(ipa)) 363ba764c4dSAlex Elder goto err_dma_free; 364ba764c4dSAlex Elder 365ba764c4dSAlex Elder /* Verify the microcontroller ring alignment (0 is OK too) */ 366ba764c4dSAlex Elder if (ipa->mem[IPA_MEM_UC_EVENT_RING].offset % 1024) { 367ba764c4dSAlex Elder dev_err(dev, "microcontroller ring not 1024-byte aligned\n"); 368ba764c4dSAlex Elder goto err_dma_free; 369ba764c4dSAlex Elder } 370ba764c4dSAlex Elder 371ba764c4dSAlex Elder return 0; 372ba764c4dSAlex Elder 373ba764c4dSAlex Elder err_dma_free: 374ba764c4dSAlex Elder dma_free_coherent(dev, IPA_MEM_MAX, ipa->zero_virt, ipa->zero_addr); 375ba764c4dSAlex Elder 376ba764c4dSAlex Elder return -EINVAL; 377ba764c4dSAlex Elder } 378ba764c4dSAlex Elder 379ba764c4dSAlex Elder /* Inverse of ipa_mem_config() */ 380ba764c4dSAlex Elder void ipa_mem_deconfig(struct ipa *ipa) 381ba764c4dSAlex Elder { 382ba764c4dSAlex Elder struct device *dev = &ipa->pdev->dev; 383ba764c4dSAlex Elder 384ba764c4dSAlex Elder dma_free_coherent(dev, ipa->zero_size, ipa->zero_virt, ipa->zero_addr); 385ba764c4dSAlex Elder ipa->zero_size = 0; 386ba764c4dSAlex Elder ipa->zero_virt = NULL; 387ba764c4dSAlex Elder ipa->zero_addr = 0; 388ba764c4dSAlex Elder } 389ba764c4dSAlex Elder 390ba764c4dSAlex Elder /** 391ba764c4dSAlex Elder * ipa_mem_zero_modem() - Zero IPA-local memory regions owned by the modem 392e3eea08eSAlex Elder * @ipa: IPA pointer 393ba764c4dSAlex Elder * 394ba764c4dSAlex Elder * Zero regions of IPA-local memory used by the modem. These are configured 395ba764c4dSAlex Elder * (and initially zeroed) by ipa_mem_setup(), but if the modem crashes and 396ba764c4dSAlex Elder * restarts via SSR we need to re-initialize them. A QMI message tells the 397ba764c4dSAlex Elder * modem where to find regions of IPA local memory it needs to know about 398ba764c4dSAlex Elder * (these included). 399ba764c4dSAlex Elder */ 400ba764c4dSAlex Elder int ipa_mem_zero_modem(struct ipa *ipa) 401ba764c4dSAlex Elder { 402ba764c4dSAlex Elder struct gsi_trans *trans; 403ba764c4dSAlex Elder 404ba764c4dSAlex Elder /* Get a transaction to zero the modem memory, modem header, 405ba764c4dSAlex Elder * and modem processing context regions. 406ba764c4dSAlex Elder */ 407ba764c4dSAlex Elder trans = ipa_cmd_trans_alloc(ipa, 3); 408ba764c4dSAlex Elder if (!trans) { 409ba764c4dSAlex Elder dev_err(&ipa->pdev->dev, 410ba764c4dSAlex Elder "no transaction to zero modem memory\n"); 411ba764c4dSAlex Elder return -EBUSY; 412ba764c4dSAlex Elder } 413ba764c4dSAlex Elder 414ba764c4dSAlex Elder ipa_mem_zero_region_add(trans, &ipa->mem[IPA_MEM_MODEM_HEADER]); 415ba764c4dSAlex Elder 416ba764c4dSAlex Elder ipa_mem_zero_region_add(trans, &ipa->mem[IPA_MEM_MODEM_PROC_CTX]); 417ba764c4dSAlex Elder 418ba764c4dSAlex Elder ipa_mem_zero_region_add(trans, &ipa->mem[IPA_MEM_MODEM]); 419ba764c4dSAlex Elder 420ba764c4dSAlex Elder gsi_trans_commit_wait(trans); 421ba764c4dSAlex Elder 422ba764c4dSAlex Elder return 0; 423ba764c4dSAlex Elder } 424ba764c4dSAlex Elder 4253e313c3fSAlex Elder /** 4263e313c3fSAlex Elder * ipa_imem_init() - Initialize IMEM memory used by the IPA 4273e313c3fSAlex Elder * @ipa: IPA pointer 4283e313c3fSAlex Elder * @addr: Physical address of the IPA region in IMEM 4293e313c3fSAlex Elder * @size: Size (bytes) of the IPA region in IMEM 4303e313c3fSAlex Elder * 4313e313c3fSAlex Elder * IMEM is a block of shared memory separate from system DRAM, and 4323e313c3fSAlex Elder * a portion of this memory is available for the IPA to use. The 4333e313c3fSAlex Elder * modem accesses this memory directly, but the IPA accesses it 4343e313c3fSAlex Elder * via the IOMMU, using the AP's credentials. 4353e313c3fSAlex Elder * 4363e313c3fSAlex Elder * If this region exists (size > 0) we map it for read/write access 4373e313c3fSAlex Elder * through the IOMMU using the IPA device. 4383e313c3fSAlex Elder * 4393e313c3fSAlex Elder * Note: @addr and @size are not guaranteed to be page-aligned. 4403e313c3fSAlex Elder */ 4413e313c3fSAlex Elder static int ipa_imem_init(struct ipa *ipa, unsigned long addr, size_t size) 4423e313c3fSAlex Elder { 4433e313c3fSAlex Elder struct device *dev = &ipa->pdev->dev; 4443e313c3fSAlex Elder struct iommu_domain *domain; 4453e313c3fSAlex Elder unsigned long iova; 4463e313c3fSAlex Elder phys_addr_t phys; 4473e313c3fSAlex Elder int ret; 4483e313c3fSAlex Elder 4493e313c3fSAlex Elder if (!size) 4503e313c3fSAlex Elder return 0; /* IMEM memory not used */ 4513e313c3fSAlex Elder 4523e313c3fSAlex Elder domain = iommu_get_domain_for_dev(dev); 4533e313c3fSAlex Elder if (!domain) { 4543e313c3fSAlex Elder dev_err(dev, "no IOMMU domain found for IMEM\n"); 4553e313c3fSAlex Elder return -EINVAL; 4563e313c3fSAlex Elder } 4573e313c3fSAlex Elder 4583e313c3fSAlex Elder /* Align the address down and the size up to page boundaries */ 4593e313c3fSAlex Elder phys = addr & PAGE_MASK; 4603e313c3fSAlex Elder size = PAGE_ALIGN(size + addr - phys); 4613e313c3fSAlex Elder iova = phys; /* We just want a direct mapping */ 4623e313c3fSAlex Elder 4633e313c3fSAlex Elder ret = iommu_map(domain, iova, phys, size, IOMMU_READ | IOMMU_WRITE); 4643e313c3fSAlex Elder if (ret) 4653e313c3fSAlex Elder return ret; 4663e313c3fSAlex Elder 4673e313c3fSAlex Elder ipa->imem_iova = iova; 4683e313c3fSAlex Elder ipa->imem_size = size; 4693e313c3fSAlex Elder 4703e313c3fSAlex Elder return 0; 4713e313c3fSAlex Elder } 4723e313c3fSAlex Elder 4733e313c3fSAlex Elder static void ipa_imem_exit(struct ipa *ipa) 4743e313c3fSAlex Elder { 4753e313c3fSAlex Elder struct iommu_domain *domain; 4763e313c3fSAlex Elder struct device *dev; 4773e313c3fSAlex Elder 4783e313c3fSAlex Elder if (!ipa->imem_size) 4793e313c3fSAlex Elder return; 4803e313c3fSAlex Elder 4813e313c3fSAlex Elder dev = &ipa->pdev->dev; 4823e313c3fSAlex Elder domain = iommu_get_domain_for_dev(dev); 4833e313c3fSAlex Elder if (domain) { 4843e313c3fSAlex Elder size_t size; 4853e313c3fSAlex Elder 4863e313c3fSAlex Elder size = iommu_unmap(domain, ipa->imem_iova, ipa->imem_size); 4873e313c3fSAlex Elder if (size != ipa->imem_size) 488113b6ea0SAlex Elder dev_warn(dev, "unmapped %zu IMEM bytes, expected %zu\n", 4893e313c3fSAlex Elder size, ipa->imem_size); 4903e313c3fSAlex Elder } else { 4913e313c3fSAlex Elder dev_err(dev, "couldn't get IPA IOMMU domain for IMEM\n"); 4923e313c3fSAlex Elder } 4933e313c3fSAlex Elder 4943e313c3fSAlex Elder ipa->imem_size = 0; 4953e313c3fSAlex Elder ipa->imem_iova = 0; 4963e313c3fSAlex Elder } 4973e313c3fSAlex Elder 498a0036bb4SAlex Elder /** 499a0036bb4SAlex Elder * ipa_smem_init() - Initialize SMEM memory used by the IPA 500a0036bb4SAlex Elder * @ipa: IPA pointer 501a0036bb4SAlex Elder * @item: Item ID of SMEM memory 502a0036bb4SAlex Elder * @size: Size (bytes) of SMEM memory region 503a0036bb4SAlex Elder * 504a0036bb4SAlex Elder * SMEM is a managed block of shared DRAM, from which numbered "items" 505a0036bb4SAlex Elder * can be allocated. One item is designated for use by the IPA. 506a0036bb4SAlex Elder * 507a0036bb4SAlex Elder * The modem accesses SMEM memory directly, but the IPA accesses it 508a0036bb4SAlex Elder * via the IOMMU, using the AP's credentials. 509a0036bb4SAlex Elder * 510a0036bb4SAlex Elder * If size provided is non-zero, we allocate it and map it for 511a0036bb4SAlex Elder * access through the IOMMU. 512a0036bb4SAlex Elder * 513a0036bb4SAlex Elder * Note: @size and the item address are is not guaranteed to be page-aligned. 514a0036bb4SAlex Elder */ 515a0036bb4SAlex Elder static int ipa_smem_init(struct ipa *ipa, u32 item, size_t size) 516a0036bb4SAlex Elder { 517a0036bb4SAlex Elder struct device *dev = &ipa->pdev->dev; 518a0036bb4SAlex Elder struct iommu_domain *domain; 519a0036bb4SAlex Elder unsigned long iova; 520a0036bb4SAlex Elder phys_addr_t phys; 521a0036bb4SAlex Elder phys_addr_t addr; 522a0036bb4SAlex Elder size_t actual; 523a0036bb4SAlex Elder void *virt; 524a0036bb4SAlex Elder int ret; 525a0036bb4SAlex Elder 526a0036bb4SAlex Elder if (!size) 527a0036bb4SAlex Elder return 0; /* SMEM memory not used */ 528a0036bb4SAlex Elder 529a0036bb4SAlex Elder /* SMEM is memory shared between the AP and another system entity 530a0036bb4SAlex Elder * (in this case, the modem). An allocation from SMEM is persistent 531a0036bb4SAlex Elder * until the AP reboots; there is no way to free an allocated SMEM 532a0036bb4SAlex Elder * region. Allocation only reserves the space; to use it you need 533a0036bb4SAlex Elder * to "get" a pointer it (this implies no reference counting). 534a0036bb4SAlex Elder * The item might have already been allocated, in which case we 535a0036bb4SAlex Elder * use it unless the size isn't what we expect. 536a0036bb4SAlex Elder */ 537a0036bb4SAlex Elder ret = qcom_smem_alloc(QCOM_SMEM_HOST_MODEM, item, size); 538a0036bb4SAlex Elder if (ret && ret != -EEXIST) { 539a0036bb4SAlex Elder dev_err(dev, "error %d allocating size %zu SMEM item %u\n", 540a0036bb4SAlex Elder ret, size, item); 541a0036bb4SAlex Elder return ret; 542a0036bb4SAlex Elder } 543a0036bb4SAlex Elder 544a0036bb4SAlex Elder /* Now get the address of the SMEM memory region */ 545a0036bb4SAlex Elder virt = qcom_smem_get(QCOM_SMEM_HOST_MODEM, item, &actual); 546a0036bb4SAlex Elder if (IS_ERR(virt)) { 547a0036bb4SAlex Elder ret = PTR_ERR(virt); 548a0036bb4SAlex Elder dev_err(dev, "error %d getting SMEM item %u\n", ret, item); 549a0036bb4SAlex Elder return ret; 550a0036bb4SAlex Elder } 551a0036bb4SAlex Elder 552a0036bb4SAlex Elder /* In case the region was already allocated, verify the size */ 553a0036bb4SAlex Elder if (ret && actual != size) { 554a0036bb4SAlex Elder dev_err(dev, "SMEM item %u has size %zu, expected %zu\n", 555a0036bb4SAlex Elder item, actual, size); 556a0036bb4SAlex Elder return -EINVAL; 557a0036bb4SAlex Elder } 558a0036bb4SAlex Elder 559a0036bb4SAlex Elder domain = iommu_get_domain_for_dev(dev); 560a0036bb4SAlex Elder if (!domain) { 561a0036bb4SAlex Elder dev_err(dev, "no IOMMU domain found for SMEM\n"); 562a0036bb4SAlex Elder return -EINVAL; 563a0036bb4SAlex Elder } 564a0036bb4SAlex Elder 565a0036bb4SAlex Elder /* Align the address down and the size up to a page boundary */ 566a0036bb4SAlex Elder addr = qcom_smem_virt_to_phys(virt) & PAGE_MASK; 567a0036bb4SAlex Elder phys = addr & PAGE_MASK; 568a0036bb4SAlex Elder size = PAGE_ALIGN(size + addr - phys); 569a0036bb4SAlex Elder iova = phys; /* We just want a direct mapping */ 570a0036bb4SAlex Elder 571a0036bb4SAlex Elder ret = iommu_map(domain, iova, phys, size, IOMMU_READ | IOMMU_WRITE); 572a0036bb4SAlex Elder if (ret) 573a0036bb4SAlex Elder return ret; 574a0036bb4SAlex Elder 575a0036bb4SAlex Elder ipa->smem_iova = iova; 576a0036bb4SAlex Elder ipa->smem_size = size; 577a0036bb4SAlex Elder 578a0036bb4SAlex Elder return 0; 579a0036bb4SAlex Elder } 580a0036bb4SAlex Elder 581a0036bb4SAlex Elder static void ipa_smem_exit(struct ipa *ipa) 582a0036bb4SAlex Elder { 583a0036bb4SAlex Elder struct device *dev = &ipa->pdev->dev; 584a0036bb4SAlex Elder struct iommu_domain *domain; 585a0036bb4SAlex Elder 586a0036bb4SAlex Elder domain = iommu_get_domain_for_dev(dev); 587a0036bb4SAlex Elder if (domain) { 588a0036bb4SAlex Elder size_t size; 589a0036bb4SAlex Elder 590a0036bb4SAlex Elder size = iommu_unmap(domain, ipa->smem_iova, ipa->smem_size); 591a0036bb4SAlex Elder if (size != ipa->smem_size) 592113b6ea0SAlex Elder dev_warn(dev, "unmapped %zu SMEM bytes, expected %zu\n", 593a0036bb4SAlex Elder size, ipa->smem_size); 594a0036bb4SAlex Elder 595a0036bb4SAlex Elder } else { 596a0036bb4SAlex Elder dev_err(dev, "couldn't get IPA IOMMU domain for SMEM\n"); 597a0036bb4SAlex Elder } 598a0036bb4SAlex Elder 599a0036bb4SAlex Elder ipa->smem_size = 0; 600a0036bb4SAlex Elder ipa->smem_iova = 0; 601a0036bb4SAlex Elder } 602a0036bb4SAlex Elder 603ba764c4dSAlex Elder /* Perform memory region-related initialization */ 6043128aae8SAlex Elder int ipa_mem_init(struct ipa *ipa, const struct ipa_mem_data *mem_data) 605ba764c4dSAlex Elder { 606ba764c4dSAlex Elder struct device *dev = &ipa->pdev->dev; 607ba764c4dSAlex Elder struct resource *res; 608ba764c4dSAlex Elder int ret; 609ba764c4dSAlex Elder 6101eec7677SAlex Elder /* Make sure the set of defined memory regions is valid */ 6111eec7677SAlex Elder if (!ipa_mem_valid(ipa, mem_data)) 612ba764c4dSAlex Elder return -EINVAL; 6131eec7677SAlex Elder 6141eec7677SAlex Elder ipa->mem_count = mem_data->local_count; 6151eec7677SAlex Elder ipa->mem = mem_data->local; 616ba764c4dSAlex Elder 617ba764c4dSAlex Elder ret = dma_set_mask_and_coherent(&ipa->pdev->dev, DMA_BIT_MASK(64)); 618ba764c4dSAlex Elder if (ret) { 619ba764c4dSAlex Elder dev_err(dev, "error %d setting DMA mask\n", ret); 620ba764c4dSAlex Elder return ret; 621ba764c4dSAlex Elder } 622ba764c4dSAlex Elder 623ba764c4dSAlex Elder res = platform_get_resource_byname(ipa->pdev, IORESOURCE_MEM, 624ba764c4dSAlex Elder "ipa-shared"); 625ba764c4dSAlex Elder if (!res) { 626ba764c4dSAlex Elder dev_err(dev, 627ba764c4dSAlex Elder "DT error getting \"ipa-shared\" memory property\n"); 628ba764c4dSAlex Elder return -ENODEV; 629ba764c4dSAlex Elder } 630ba764c4dSAlex Elder 631ba764c4dSAlex Elder ipa->mem_virt = memremap(res->start, resource_size(res), MEMREMAP_WC); 632ba764c4dSAlex Elder if (!ipa->mem_virt) { 633ba764c4dSAlex Elder dev_err(dev, "unable to remap \"ipa-shared\" memory\n"); 634ba764c4dSAlex Elder return -ENOMEM; 635ba764c4dSAlex Elder } 636ba764c4dSAlex Elder 637ba764c4dSAlex Elder ipa->mem_addr = res->start; 638ba764c4dSAlex Elder ipa->mem_size = resource_size(res); 639ba764c4dSAlex Elder 6403e313c3fSAlex Elder ret = ipa_imem_init(ipa, mem_data->imem_addr, mem_data->imem_size); 6413e313c3fSAlex Elder if (ret) 6423e313c3fSAlex Elder goto err_unmap; 6433e313c3fSAlex Elder 644a0036bb4SAlex Elder ret = ipa_smem_init(ipa, mem_data->smem_id, mem_data->smem_size); 645a0036bb4SAlex Elder if (ret) 646a0036bb4SAlex Elder goto err_imem_exit; 647a0036bb4SAlex Elder 648ba764c4dSAlex Elder return 0; 6493e313c3fSAlex Elder 650a0036bb4SAlex Elder err_imem_exit: 651a0036bb4SAlex Elder ipa_imem_exit(ipa); 6523e313c3fSAlex Elder err_unmap: 6533e313c3fSAlex Elder memunmap(ipa->mem_virt); 6543e313c3fSAlex Elder 6553e313c3fSAlex Elder return ret; 656ba764c4dSAlex Elder } 657ba764c4dSAlex Elder 658ba764c4dSAlex Elder /* Inverse of ipa_mem_init() */ 659ba764c4dSAlex Elder void ipa_mem_exit(struct ipa *ipa) 660ba764c4dSAlex Elder { 661a0036bb4SAlex Elder ipa_smem_exit(ipa); 6623e313c3fSAlex Elder ipa_imem_exit(ipa); 663ba764c4dSAlex Elder memunmap(ipa->mem_virt); 664ba764c4dSAlex Elder } 665