1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * The On Chip Memory (OCMEM) allocator allows various clients to allocate 4 * memory from OCMEM based on performance, latency and power requirements. 5 * This is typically used by the GPU, camera/video, and audio components on 6 * some Snapdragon SoCs. 7 * 8 * Copyright (C) 2019 Brian Masney <masneyb@onstation.org> 9 * Copyright (C) 2015 Red Hat. Author: Rob Clark <robdclark@gmail.com> 10 */ 11 12 #include <linux/bitfield.h> 13 #include <linux/clk.h> 14 #include <linux/io.h> 15 #include <linux/kernel.h> 16 #include <linux/module.h> 17 #include <linux/of_device.h> 18 #include <linux/platform_device.h> 19 #include <linux/firmware/qcom/qcom_scm.h> 20 #include <linux/sizes.h> 21 #include <linux/slab.h> 22 #include <linux/types.h> 23 #include <soc/qcom/ocmem.h> 24 25 enum region_mode { 26 WIDE_MODE = 0x0, 27 THIN_MODE, 28 MODE_DEFAULT = WIDE_MODE, 29 }; 30 31 enum ocmem_macro_state { 32 PASSTHROUGH = 0, 33 PERI_ON = 1, 34 CORE_ON = 2, 35 CLK_OFF = 4, 36 }; 37 38 struct ocmem_region { 39 bool interleaved; 40 enum region_mode mode; 41 unsigned int num_macros; 42 enum ocmem_macro_state macro_state[4]; 43 unsigned long macro_size; 44 unsigned long region_size; 45 }; 46 47 struct ocmem_config { 48 uint8_t num_regions; 49 unsigned long macro_size; 50 }; 51 52 struct ocmem { 53 struct device *dev; 54 const struct ocmem_config *config; 55 struct resource *memory; 56 void __iomem *mmio; 57 unsigned int num_ports; 58 unsigned int num_macros; 59 bool interleaved; 60 struct ocmem_region *regions; 61 unsigned long active_allocations; 62 }; 63 64 #define OCMEM_MIN_ALIGN SZ_64K 65 #define OCMEM_MIN_ALLOC SZ_64K 66 67 #define OCMEM_REG_HW_VERSION 0x00000000 68 #define OCMEM_REG_HW_PROFILE 0x00000004 69 70 #define OCMEM_REG_REGION_MODE_CTL 0x00001000 71 #define OCMEM_REGION_MODE_CTL_REG0_THIN 0x00000001 72 #define OCMEM_REGION_MODE_CTL_REG1_THIN 0x00000002 73 #define OCMEM_REGION_MODE_CTL_REG2_THIN 0x00000004 74 #define OCMEM_REGION_MODE_CTL_REG3_THIN 0x00000008 75 76 #define OCMEM_REG_GFX_MPU_START 0x00001004 77 #define OCMEM_REG_GFX_MPU_END 0x00001008 78 79 #define OCMEM_HW_VERSION_MAJOR(val) FIELD_GET(GENMASK(31, 28), val) 80 #define OCMEM_HW_VERSION_MINOR(val) FIELD_GET(GENMASK(27, 16), val) 81 #define OCMEM_HW_VERSION_STEP(val) FIELD_GET(GENMASK(15, 0), val) 82 83 #define OCMEM_HW_PROFILE_NUM_PORTS(val) FIELD_PREP(0x0000000f, (val)) 84 #define OCMEM_HW_PROFILE_NUM_MACROS(val) FIELD_PREP(0x00003f00, (val)) 85 86 #define OCMEM_HW_PROFILE_LAST_REGN_HALFSIZE 0x00010000 87 #define OCMEM_HW_PROFILE_INTERLEAVING 0x00020000 88 #define OCMEM_REG_GEN_STATUS 0x0000000c 89 90 #define OCMEM_REG_PSGSC_STATUS 0x00000038 91 #define OCMEM_REG_PSGSC_CTL(i0) (0x0000003c + 0x1*(i0)) 92 93 #define OCMEM_PSGSC_CTL_MACRO0_MODE(val) FIELD_PREP(0x00000007, (val)) 94 #define OCMEM_PSGSC_CTL_MACRO1_MODE(val) FIELD_PREP(0x00000070, (val)) 95 #define OCMEM_PSGSC_CTL_MACRO2_MODE(val) FIELD_PREP(0x00000700, (val)) 96 #define OCMEM_PSGSC_CTL_MACRO3_MODE(val) FIELD_PREP(0x00007000, (val)) 97 98 #define OCMEM_CLK_CORE_IDX 0 99 static struct clk_bulk_data ocmem_clks[] = { 100 { 101 .id = "core", 102 }, 103 { 104 .id = "iface", 105 }, 106 }; 107 108 static inline void ocmem_write(struct ocmem *ocmem, u32 reg, u32 data) 109 { 110 writel(data, ocmem->mmio + reg); 111 } 112 113 static inline u32 ocmem_read(struct ocmem *ocmem, u32 reg) 114 { 115 return readl(ocmem->mmio + reg); 116 } 117 118 static void update_ocmem(struct ocmem *ocmem) 119 { 120 uint32_t region_mode_ctrl = 0x0; 121 int i; 122 123 if (!qcom_scm_ocmem_lock_available()) { 124 for (i = 0; i < ocmem->config->num_regions; i++) { 125 struct ocmem_region *region = &ocmem->regions[i]; 126 127 if (region->mode == THIN_MODE) 128 region_mode_ctrl |= BIT(i); 129 } 130 131 dev_dbg(ocmem->dev, "ocmem_region_mode_control %x\n", 132 region_mode_ctrl); 133 ocmem_write(ocmem, OCMEM_REG_REGION_MODE_CTL, region_mode_ctrl); 134 } 135 136 for (i = 0; i < ocmem->config->num_regions; i++) { 137 struct ocmem_region *region = &ocmem->regions[i]; 138 u32 data; 139 140 data = OCMEM_PSGSC_CTL_MACRO0_MODE(region->macro_state[0]) | 141 OCMEM_PSGSC_CTL_MACRO1_MODE(region->macro_state[1]) | 142 OCMEM_PSGSC_CTL_MACRO2_MODE(region->macro_state[2]) | 143 OCMEM_PSGSC_CTL_MACRO3_MODE(region->macro_state[3]); 144 145 ocmem_write(ocmem, OCMEM_REG_PSGSC_CTL(i), data); 146 } 147 } 148 149 static unsigned long phys_to_offset(struct ocmem *ocmem, 150 unsigned long addr) 151 { 152 if (addr < ocmem->memory->start || addr >= ocmem->memory->end) 153 return 0; 154 155 return addr - ocmem->memory->start; 156 } 157 158 static unsigned long device_address(struct ocmem *ocmem, 159 enum ocmem_client client, 160 unsigned long addr) 161 { 162 WARN_ON(client != OCMEM_GRAPHICS); 163 164 /* TODO: gpu uses phys_to_offset, but others do not.. */ 165 return phys_to_offset(ocmem, addr); 166 } 167 168 static void update_range(struct ocmem *ocmem, struct ocmem_buf *buf, 169 enum ocmem_macro_state mstate, enum region_mode rmode) 170 { 171 unsigned long offset = 0; 172 int i, j; 173 174 for (i = 0; i < ocmem->config->num_regions; i++) { 175 struct ocmem_region *region = &ocmem->regions[i]; 176 177 if (buf->offset <= offset && offset < buf->offset + buf->len) 178 region->mode = rmode; 179 180 for (j = 0; j < region->num_macros; j++) { 181 if (buf->offset <= offset && 182 offset < buf->offset + buf->len) 183 region->macro_state[j] = mstate; 184 185 offset += region->macro_size; 186 } 187 } 188 189 update_ocmem(ocmem); 190 } 191 192 struct ocmem *of_get_ocmem(struct device *dev) 193 { 194 struct platform_device *pdev; 195 struct device_node *devnode; 196 struct ocmem *ocmem; 197 198 devnode = of_parse_phandle(dev->of_node, "sram", 0); 199 if (!devnode || !devnode->parent) { 200 dev_err(dev, "Cannot look up sram phandle\n"); 201 of_node_put(devnode); 202 return ERR_PTR(-ENODEV); 203 } 204 205 pdev = of_find_device_by_node(devnode->parent); 206 if (!pdev) { 207 dev_err(dev, "Cannot find device node %s\n", devnode->name); 208 of_node_put(devnode); 209 return ERR_PTR(-EPROBE_DEFER); 210 } 211 of_node_put(devnode); 212 213 ocmem = platform_get_drvdata(pdev); 214 if (!ocmem) { 215 dev_err(dev, "Cannot get ocmem\n"); 216 put_device(&pdev->dev); 217 return ERR_PTR(-ENODEV); 218 } 219 return ocmem; 220 } 221 EXPORT_SYMBOL(of_get_ocmem); 222 223 struct ocmem_buf *ocmem_allocate(struct ocmem *ocmem, enum ocmem_client client, 224 unsigned long size) 225 { 226 struct ocmem_buf *buf; 227 int ret; 228 229 /* TODO: add support for other clients... */ 230 if (WARN_ON(client != OCMEM_GRAPHICS)) 231 return ERR_PTR(-ENODEV); 232 233 if (size < OCMEM_MIN_ALLOC || !IS_ALIGNED(size, OCMEM_MIN_ALIGN)) 234 return ERR_PTR(-EINVAL); 235 236 if (test_and_set_bit_lock(BIT(client), &ocmem->active_allocations)) 237 return ERR_PTR(-EBUSY); 238 239 buf = kzalloc(sizeof(*buf), GFP_KERNEL); 240 if (!buf) { 241 ret = -ENOMEM; 242 goto err_unlock; 243 } 244 245 buf->offset = 0; 246 buf->addr = device_address(ocmem, client, buf->offset); 247 buf->len = size; 248 249 update_range(ocmem, buf, CORE_ON, WIDE_MODE); 250 251 if (qcom_scm_ocmem_lock_available()) { 252 ret = qcom_scm_ocmem_lock(QCOM_SCM_OCMEM_GRAPHICS_ID, 253 buf->offset, buf->len, WIDE_MODE); 254 if (ret) { 255 dev_err(ocmem->dev, "could not lock: %d\n", ret); 256 ret = -EINVAL; 257 goto err_kfree; 258 } 259 } else { 260 ocmem_write(ocmem, OCMEM_REG_GFX_MPU_START, buf->offset); 261 ocmem_write(ocmem, OCMEM_REG_GFX_MPU_END, 262 buf->offset + buf->len); 263 } 264 265 dev_dbg(ocmem->dev, "using %ldK of OCMEM at 0x%08lx for client %d\n", 266 size / 1024, buf->addr, client); 267 268 return buf; 269 270 err_kfree: 271 kfree(buf); 272 err_unlock: 273 clear_bit_unlock(BIT(client), &ocmem->active_allocations); 274 275 return ERR_PTR(ret); 276 } 277 EXPORT_SYMBOL(ocmem_allocate); 278 279 void ocmem_free(struct ocmem *ocmem, enum ocmem_client client, 280 struct ocmem_buf *buf) 281 { 282 /* TODO: add support for other clients... */ 283 if (WARN_ON(client != OCMEM_GRAPHICS)) 284 return; 285 286 update_range(ocmem, buf, CLK_OFF, MODE_DEFAULT); 287 288 if (qcom_scm_ocmem_lock_available()) { 289 int ret; 290 291 ret = qcom_scm_ocmem_unlock(QCOM_SCM_OCMEM_GRAPHICS_ID, 292 buf->offset, buf->len); 293 if (ret) 294 dev_err(ocmem->dev, "could not unlock: %d\n", ret); 295 } else { 296 ocmem_write(ocmem, OCMEM_REG_GFX_MPU_START, 0x0); 297 ocmem_write(ocmem, OCMEM_REG_GFX_MPU_END, 0x0); 298 } 299 300 kfree(buf); 301 302 clear_bit_unlock(BIT(client), &ocmem->active_allocations); 303 } 304 EXPORT_SYMBOL(ocmem_free); 305 306 static int ocmem_dev_probe(struct platform_device *pdev) 307 { 308 struct device *dev = &pdev->dev; 309 unsigned long reg, region_size; 310 int i, j, ret, num_banks; 311 struct ocmem *ocmem; 312 313 if (!qcom_scm_is_available()) 314 return -EPROBE_DEFER; 315 316 ocmem = devm_kzalloc(dev, sizeof(*ocmem), GFP_KERNEL); 317 if (!ocmem) 318 return -ENOMEM; 319 320 ocmem->dev = dev; 321 ocmem->config = device_get_match_data(dev); 322 323 ret = devm_clk_bulk_get(dev, ARRAY_SIZE(ocmem_clks), ocmem_clks); 324 if (ret) { 325 if (ret != -EPROBE_DEFER) 326 dev_err(dev, "Unable to get clocks\n"); 327 328 return ret; 329 } 330 331 ocmem->mmio = devm_platform_ioremap_resource_byname(pdev, "ctrl"); 332 if (IS_ERR(ocmem->mmio)) { 333 dev_err(&pdev->dev, "Failed to ioremap ocmem_ctrl resource\n"); 334 return PTR_ERR(ocmem->mmio); 335 } 336 337 ocmem->memory = platform_get_resource_byname(pdev, IORESOURCE_MEM, 338 "mem"); 339 if (!ocmem->memory) { 340 dev_err(dev, "Could not get mem region\n"); 341 return -ENXIO; 342 } 343 344 /* The core clock is synchronous with graphics */ 345 WARN_ON(clk_set_rate(ocmem_clks[OCMEM_CLK_CORE_IDX].clk, 1000) < 0); 346 347 ret = clk_bulk_prepare_enable(ARRAY_SIZE(ocmem_clks), ocmem_clks); 348 if (ret) { 349 dev_info(ocmem->dev, "Failed to enable clocks\n"); 350 return ret; 351 } 352 353 if (qcom_scm_restore_sec_cfg_available()) { 354 dev_dbg(dev, "configuring scm\n"); 355 ret = qcom_scm_restore_sec_cfg(QCOM_SCM_OCMEM_DEV_ID, 0); 356 if (ret) { 357 dev_err(dev, "Could not enable secure configuration\n"); 358 goto err_clk_disable; 359 } 360 } 361 362 reg = ocmem_read(ocmem, OCMEM_REG_HW_VERSION); 363 dev_dbg(dev, "OCMEM hardware version: %lu.%lu.%lu\n", 364 OCMEM_HW_VERSION_MAJOR(reg), 365 OCMEM_HW_VERSION_MINOR(reg), 366 OCMEM_HW_VERSION_STEP(reg)); 367 368 reg = ocmem_read(ocmem, OCMEM_REG_HW_PROFILE); 369 ocmem->num_ports = OCMEM_HW_PROFILE_NUM_PORTS(reg); 370 ocmem->num_macros = OCMEM_HW_PROFILE_NUM_MACROS(reg); 371 ocmem->interleaved = !!(reg & OCMEM_HW_PROFILE_INTERLEAVING); 372 373 num_banks = ocmem->num_ports / 2; 374 region_size = ocmem->config->macro_size * num_banks; 375 376 dev_info(dev, "%u ports, %u regions, %u macros, %sinterleaved\n", 377 ocmem->num_ports, ocmem->config->num_regions, 378 ocmem->num_macros, ocmem->interleaved ? "" : "not "); 379 380 ocmem->regions = devm_kcalloc(dev, ocmem->config->num_regions, 381 sizeof(struct ocmem_region), GFP_KERNEL); 382 if (!ocmem->regions) { 383 ret = -ENOMEM; 384 goto err_clk_disable; 385 } 386 387 for (i = 0; i < ocmem->config->num_regions; i++) { 388 struct ocmem_region *region = &ocmem->regions[i]; 389 390 if (WARN_ON(num_banks > ARRAY_SIZE(region->macro_state))) { 391 ret = -EINVAL; 392 goto err_clk_disable; 393 } 394 395 region->mode = MODE_DEFAULT; 396 region->num_macros = num_banks; 397 398 if (i == (ocmem->config->num_regions - 1) && 399 reg & OCMEM_HW_PROFILE_LAST_REGN_HALFSIZE) { 400 region->macro_size = ocmem->config->macro_size / 2; 401 region->region_size = region_size / 2; 402 } else { 403 region->macro_size = ocmem->config->macro_size; 404 region->region_size = region_size; 405 } 406 407 for (j = 0; j < ARRAY_SIZE(region->macro_state); j++) 408 region->macro_state[j] = CLK_OFF; 409 } 410 411 platform_set_drvdata(pdev, ocmem); 412 413 return 0; 414 415 err_clk_disable: 416 clk_bulk_disable_unprepare(ARRAY_SIZE(ocmem_clks), ocmem_clks); 417 return ret; 418 } 419 420 static int ocmem_dev_remove(struct platform_device *pdev) 421 { 422 clk_bulk_disable_unprepare(ARRAY_SIZE(ocmem_clks), ocmem_clks); 423 424 return 0; 425 } 426 427 static const struct ocmem_config ocmem_8974_config = { 428 .num_regions = 3, 429 .macro_size = SZ_128K, 430 }; 431 432 static const struct of_device_id ocmem_of_match[] = { 433 { .compatible = "qcom,msm8974-ocmem", .data = &ocmem_8974_config }, 434 { } 435 }; 436 437 MODULE_DEVICE_TABLE(of, ocmem_of_match); 438 439 static struct platform_driver ocmem_driver = { 440 .probe = ocmem_dev_probe, 441 .remove = ocmem_dev_remove, 442 .driver = { 443 .name = "ocmem", 444 .of_match_table = ocmem_of_match, 445 }, 446 }; 447 448 module_platform_driver(ocmem_driver); 449 450 MODULE_DESCRIPTION("On Chip Memory (OCMEM) allocator for some Snapdragon SoCs"); 451 MODULE_LICENSE("GPL v2"); 452