1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * The On Chip Memory (OCMEM) allocator allows various clients to allocate
4 * memory from OCMEM based on performance, latency and power requirements.
5 * This is typically used by the GPU, camera/video, and audio components on
6 * some Snapdragon SoCs.
7 *
8 * Copyright (C) 2019 Brian Masney <masneyb@onstation.org>
9 * Copyright (C) 2015 Red Hat. Author: Rob Clark <robdclark@gmail.com>
10 */
11
12 #include <linux/bitfield.h>
13 #include <linux/clk.h>
14 #include <linux/io.h>
15 #include <linux/kernel.h>
16 #include <linux/module.h>
17 #include <linux/of.h>
18 #include <linux/of_platform.h>
19 #include <linux/platform_device.h>
20 #include <linux/firmware/qcom/qcom_scm.h>
21 #include <linux/sizes.h>
22 #include <linux/slab.h>
23 #include <linux/types.h>
24 #include <soc/qcom/ocmem.h>
25
26 enum region_mode {
27 WIDE_MODE = 0x0,
28 THIN_MODE,
29 MODE_DEFAULT = WIDE_MODE,
30 };
31
32 enum ocmem_macro_state {
33 PASSTHROUGH = 0,
34 PERI_ON = 1,
35 CORE_ON = 2,
36 CLK_OFF = 4,
37 };
38
39 struct ocmem_region {
40 bool interleaved;
41 enum region_mode mode;
42 unsigned int num_macros;
43 enum ocmem_macro_state macro_state[4];
44 unsigned long macro_size;
45 unsigned long region_size;
46 };
47
48 struct ocmem_config {
49 uint8_t num_regions;
50 unsigned long macro_size;
51 };
52
53 struct ocmem {
54 struct device *dev;
55 const struct ocmem_config *config;
56 struct resource *memory;
57 void __iomem *mmio;
58 struct clk *core_clk;
59 struct clk *iface_clk;
60 unsigned int num_ports;
61 unsigned int num_macros;
62 bool interleaved;
63 struct ocmem_region *regions;
64 unsigned long active_allocations;
65 };
66
67 #define OCMEM_MIN_ALIGN SZ_64K
68 #define OCMEM_MIN_ALLOC SZ_64K
69
70 #define OCMEM_REG_HW_VERSION 0x00000000
71 #define OCMEM_REG_HW_PROFILE 0x00000004
72
73 #define OCMEM_REG_REGION_MODE_CTL 0x00001000
74 #define OCMEM_REGION_MODE_CTL_REG0_THIN 0x00000001
75 #define OCMEM_REGION_MODE_CTL_REG1_THIN 0x00000002
76 #define OCMEM_REGION_MODE_CTL_REG2_THIN 0x00000004
77 #define OCMEM_REGION_MODE_CTL_REG3_THIN 0x00000008
78
79 #define OCMEM_REG_GFX_MPU_START 0x00001004
80 #define OCMEM_REG_GFX_MPU_END 0x00001008
81
82 #define OCMEM_HW_VERSION_MAJOR(val) FIELD_GET(GENMASK(31, 28), val)
83 #define OCMEM_HW_VERSION_MINOR(val) FIELD_GET(GENMASK(27, 16), val)
84 #define OCMEM_HW_VERSION_STEP(val) FIELD_GET(GENMASK(15, 0), val)
85
86 #define OCMEM_HW_PROFILE_NUM_PORTS(val) FIELD_GET(0x0000000f, (val))
87 #define OCMEM_HW_PROFILE_NUM_MACROS(val) FIELD_GET(0x00003f00, (val))
88
89 #define OCMEM_HW_PROFILE_LAST_REGN_HALFSIZE 0x00010000
90 #define OCMEM_HW_PROFILE_INTERLEAVING 0x00020000
91 #define OCMEM_REG_GEN_STATUS 0x0000000c
92
93 #define OCMEM_REG_PSGSC_STATUS 0x00000038
94 #define OCMEM_REG_PSGSC_CTL(i0) (0x0000003c + 0x1*(i0))
95
96 #define OCMEM_PSGSC_CTL_MACRO0_MODE(val) FIELD_PREP(0x00000007, (val))
97 #define OCMEM_PSGSC_CTL_MACRO1_MODE(val) FIELD_PREP(0x00000070, (val))
98 #define OCMEM_PSGSC_CTL_MACRO2_MODE(val) FIELD_PREP(0x00000700, (val))
99 #define OCMEM_PSGSC_CTL_MACRO3_MODE(val) FIELD_PREP(0x00007000, (val))
100
ocmem_write(struct ocmem * ocmem,u32 reg,u32 data)101 static inline void ocmem_write(struct ocmem *ocmem, u32 reg, u32 data)
102 {
103 writel(data, ocmem->mmio + reg);
104 }
105
ocmem_read(struct ocmem * ocmem,u32 reg)106 static inline u32 ocmem_read(struct ocmem *ocmem, u32 reg)
107 {
108 return readl(ocmem->mmio + reg);
109 }
110
update_ocmem(struct ocmem * ocmem)111 static void update_ocmem(struct ocmem *ocmem)
112 {
113 uint32_t region_mode_ctrl = 0x0;
114 int i;
115
116 if (!qcom_scm_ocmem_lock_available()) {
117 for (i = 0; i < ocmem->config->num_regions; i++) {
118 struct ocmem_region *region = &ocmem->regions[i];
119
120 if (region->mode == THIN_MODE)
121 region_mode_ctrl |= BIT(i);
122 }
123
124 dev_dbg(ocmem->dev, "ocmem_region_mode_control %x\n",
125 region_mode_ctrl);
126 ocmem_write(ocmem, OCMEM_REG_REGION_MODE_CTL, region_mode_ctrl);
127 }
128
129 for (i = 0; i < ocmem->config->num_regions; i++) {
130 struct ocmem_region *region = &ocmem->regions[i];
131 u32 data;
132
133 data = OCMEM_PSGSC_CTL_MACRO0_MODE(region->macro_state[0]) |
134 OCMEM_PSGSC_CTL_MACRO1_MODE(region->macro_state[1]) |
135 OCMEM_PSGSC_CTL_MACRO2_MODE(region->macro_state[2]) |
136 OCMEM_PSGSC_CTL_MACRO3_MODE(region->macro_state[3]);
137
138 ocmem_write(ocmem, OCMEM_REG_PSGSC_CTL(i), data);
139 }
140 }
141
phys_to_offset(struct ocmem * ocmem,unsigned long addr)142 static unsigned long phys_to_offset(struct ocmem *ocmem,
143 unsigned long addr)
144 {
145 if (addr < ocmem->memory->start || addr >= ocmem->memory->end)
146 return 0;
147
148 return addr - ocmem->memory->start;
149 }
150
device_address(struct ocmem * ocmem,enum ocmem_client client,unsigned long addr)151 static unsigned long device_address(struct ocmem *ocmem,
152 enum ocmem_client client,
153 unsigned long addr)
154 {
155 WARN_ON(client != OCMEM_GRAPHICS);
156
157 /* TODO: gpu uses phys_to_offset, but others do not.. */
158 return phys_to_offset(ocmem, addr);
159 }
160
update_range(struct ocmem * ocmem,struct ocmem_buf * buf,enum ocmem_macro_state mstate,enum region_mode rmode)161 static void update_range(struct ocmem *ocmem, struct ocmem_buf *buf,
162 enum ocmem_macro_state mstate, enum region_mode rmode)
163 {
164 unsigned long offset = 0;
165 int i, j;
166
167 for (i = 0; i < ocmem->config->num_regions; i++) {
168 struct ocmem_region *region = &ocmem->regions[i];
169
170 if (buf->offset <= offset && offset < buf->offset + buf->len)
171 region->mode = rmode;
172
173 for (j = 0; j < region->num_macros; j++) {
174 if (buf->offset <= offset &&
175 offset < buf->offset + buf->len)
176 region->macro_state[j] = mstate;
177
178 offset += region->macro_size;
179 }
180 }
181
182 update_ocmem(ocmem);
183 }
184
of_get_ocmem(struct device * dev)185 struct ocmem *of_get_ocmem(struct device *dev)
186 {
187 struct platform_device *pdev;
188 struct device_node *devnode;
189 struct ocmem *ocmem;
190
191 devnode = of_parse_phandle(dev->of_node, "sram", 0);
192 if (!devnode || !devnode->parent) {
193 dev_err(dev, "Cannot look up sram phandle\n");
194 of_node_put(devnode);
195 return ERR_PTR(-ENODEV);
196 }
197
198 pdev = of_find_device_by_node(devnode->parent);
199 if (!pdev) {
200 dev_err(dev, "Cannot find device node %s\n", devnode->name);
201 of_node_put(devnode);
202 return ERR_PTR(-EPROBE_DEFER);
203 }
204 of_node_put(devnode);
205
206 ocmem = platform_get_drvdata(pdev);
207 if (!ocmem) {
208 dev_err(dev, "Cannot get ocmem\n");
209 put_device(&pdev->dev);
210 return ERR_PTR(-ENODEV);
211 }
212 return ocmem;
213 }
214 EXPORT_SYMBOL(of_get_ocmem);
215
ocmem_allocate(struct ocmem * ocmem,enum ocmem_client client,unsigned long size)216 struct ocmem_buf *ocmem_allocate(struct ocmem *ocmem, enum ocmem_client client,
217 unsigned long size)
218 {
219 struct ocmem_buf *buf;
220 int ret;
221
222 /* TODO: add support for other clients... */
223 if (WARN_ON(client != OCMEM_GRAPHICS))
224 return ERR_PTR(-ENODEV);
225
226 if (size < OCMEM_MIN_ALLOC || !IS_ALIGNED(size, OCMEM_MIN_ALIGN))
227 return ERR_PTR(-EINVAL);
228
229 if (test_and_set_bit_lock(BIT(client), &ocmem->active_allocations))
230 return ERR_PTR(-EBUSY);
231
232 buf = kzalloc(sizeof(*buf), GFP_KERNEL);
233 if (!buf) {
234 ret = -ENOMEM;
235 goto err_unlock;
236 }
237
238 buf->offset = 0;
239 buf->addr = device_address(ocmem, client, buf->offset);
240 buf->len = size;
241
242 update_range(ocmem, buf, CORE_ON, WIDE_MODE);
243
244 if (qcom_scm_ocmem_lock_available()) {
245 ret = qcom_scm_ocmem_lock(QCOM_SCM_OCMEM_GRAPHICS_ID,
246 buf->offset, buf->len, WIDE_MODE);
247 if (ret) {
248 dev_err(ocmem->dev, "could not lock: %d\n", ret);
249 ret = -EINVAL;
250 goto err_kfree;
251 }
252 } else {
253 ocmem_write(ocmem, OCMEM_REG_GFX_MPU_START, buf->offset);
254 ocmem_write(ocmem, OCMEM_REG_GFX_MPU_END,
255 buf->offset + buf->len);
256 }
257
258 dev_dbg(ocmem->dev, "using %ldK of OCMEM at 0x%08lx for client %d\n",
259 size / 1024, buf->addr, client);
260
261 return buf;
262
263 err_kfree:
264 kfree(buf);
265 err_unlock:
266 clear_bit_unlock(BIT(client), &ocmem->active_allocations);
267
268 return ERR_PTR(ret);
269 }
270 EXPORT_SYMBOL(ocmem_allocate);
271
ocmem_free(struct ocmem * ocmem,enum ocmem_client client,struct ocmem_buf * buf)272 void ocmem_free(struct ocmem *ocmem, enum ocmem_client client,
273 struct ocmem_buf *buf)
274 {
275 /* TODO: add support for other clients... */
276 if (WARN_ON(client != OCMEM_GRAPHICS))
277 return;
278
279 update_range(ocmem, buf, CLK_OFF, MODE_DEFAULT);
280
281 if (qcom_scm_ocmem_lock_available()) {
282 int ret;
283
284 ret = qcom_scm_ocmem_unlock(QCOM_SCM_OCMEM_GRAPHICS_ID,
285 buf->offset, buf->len);
286 if (ret)
287 dev_err(ocmem->dev, "could not unlock: %d\n", ret);
288 } else {
289 ocmem_write(ocmem, OCMEM_REG_GFX_MPU_START, 0x0);
290 ocmem_write(ocmem, OCMEM_REG_GFX_MPU_END, 0x0);
291 }
292
293 kfree(buf);
294
295 clear_bit_unlock(BIT(client), &ocmem->active_allocations);
296 }
297 EXPORT_SYMBOL(ocmem_free);
298
ocmem_dev_probe(struct platform_device * pdev)299 static int ocmem_dev_probe(struct platform_device *pdev)
300 {
301 struct device *dev = &pdev->dev;
302 unsigned long reg, region_size;
303 int i, j, ret, num_banks;
304 struct ocmem *ocmem;
305
306 if (!qcom_scm_is_available())
307 return -EPROBE_DEFER;
308
309 ocmem = devm_kzalloc(dev, sizeof(*ocmem), GFP_KERNEL);
310 if (!ocmem)
311 return -ENOMEM;
312
313 ocmem->dev = dev;
314 ocmem->config = device_get_match_data(dev);
315
316 ocmem->core_clk = devm_clk_get(dev, "core");
317 if (IS_ERR(ocmem->core_clk))
318 return dev_err_probe(dev, PTR_ERR(ocmem->core_clk),
319 "Unable to get core clock\n");
320
321 ocmem->iface_clk = devm_clk_get_optional(dev, "iface");
322 if (IS_ERR(ocmem->iface_clk))
323 return dev_err_probe(dev, PTR_ERR(ocmem->iface_clk),
324 "Unable to get iface clock\n");
325
326 ocmem->mmio = devm_platform_ioremap_resource_byname(pdev, "ctrl");
327 if (IS_ERR(ocmem->mmio))
328 return dev_err_probe(&pdev->dev, PTR_ERR(ocmem->mmio),
329 "Failed to ioremap ocmem_ctrl resource\n");
330
331 ocmem->memory = platform_get_resource_byname(pdev, IORESOURCE_MEM,
332 "mem");
333 if (!ocmem->memory) {
334 dev_err(dev, "Could not get mem region\n");
335 return -ENXIO;
336 }
337
338 /* The core clock is synchronous with graphics */
339 WARN_ON(clk_set_rate(ocmem->core_clk, 1000) < 0);
340
341 ret = clk_prepare_enable(ocmem->core_clk);
342 if (ret)
343 return dev_err_probe(ocmem->dev, ret, "Failed to enable core clock\n");
344
345 ret = clk_prepare_enable(ocmem->iface_clk);
346 if (ret) {
347 clk_disable_unprepare(ocmem->core_clk);
348 return dev_err_probe(ocmem->dev, ret, "Failed to enable iface clock\n");
349 }
350
351 if (qcom_scm_restore_sec_cfg_available()) {
352 dev_dbg(dev, "configuring scm\n");
353 ret = qcom_scm_restore_sec_cfg(QCOM_SCM_OCMEM_DEV_ID, 0);
354 if (ret) {
355 dev_err_probe(dev, ret, "Could not enable secure configuration\n");
356 goto err_clk_disable;
357 }
358 }
359
360 reg = ocmem_read(ocmem, OCMEM_REG_HW_VERSION);
361 dev_dbg(dev, "OCMEM hardware version: %lu.%lu.%lu\n",
362 OCMEM_HW_VERSION_MAJOR(reg),
363 OCMEM_HW_VERSION_MINOR(reg),
364 OCMEM_HW_VERSION_STEP(reg));
365
366 reg = ocmem_read(ocmem, OCMEM_REG_HW_PROFILE);
367 ocmem->num_ports = OCMEM_HW_PROFILE_NUM_PORTS(reg);
368 ocmem->num_macros = OCMEM_HW_PROFILE_NUM_MACROS(reg);
369 ocmem->interleaved = !!(reg & OCMEM_HW_PROFILE_INTERLEAVING);
370
371 num_banks = ocmem->num_ports / 2;
372 region_size = ocmem->config->macro_size * num_banks;
373
374 dev_info(dev, "%u ports, %u regions, %u macros, %sinterleaved\n",
375 ocmem->num_ports, ocmem->config->num_regions,
376 ocmem->num_macros, ocmem->interleaved ? "" : "not ");
377
378 ocmem->regions = devm_kcalloc(dev, ocmem->config->num_regions,
379 sizeof(struct ocmem_region), GFP_KERNEL);
380 if (!ocmem->regions) {
381 ret = -ENOMEM;
382 goto err_clk_disable;
383 }
384
385 for (i = 0; i < ocmem->config->num_regions; i++) {
386 struct ocmem_region *region = &ocmem->regions[i];
387
388 if (WARN_ON(num_banks > ARRAY_SIZE(region->macro_state))) {
389 ret = -EINVAL;
390 goto err_clk_disable;
391 }
392
393 region->mode = MODE_DEFAULT;
394 region->num_macros = num_banks;
395
396 if (i == (ocmem->config->num_regions - 1) &&
397 reg & OCMEM_HW_PROFILE_LAST_REGN_HALFSIZE) {
398 region->macro_size = ocmem->config->macro_size / 2;
399 region->region_size = region_size / 2;
400 } else {
401 region->macro_size = ocmem->config->macro_size;
402 region->region_size = region_size;
403 }
404
405 for (j = 0; j < ARRAY_SIZE(region->macro_state); j++)
406 region->macro_state[j] = CLK_OFF;
407 }
408
409 platform_set_drvdata(pdev, ocmem);
410
411 return 0;
412
413 err_clk_disable:
414 clk_disable_unprepare(ocmem->core_clk);
415 clk_disable_unprepare(ocmem->iface_clk);
416 return ret;
417 }
418
ocmem_dev_remove(struct platform_device * pdev)419 static int ocmem_dev_remove(struct platform_device *pdev)
420 {
421 struct ocmem *ocmem = platform_get_drvdata(pdev);
422
423 clk_disable_unprepare(ocmem->core_clk);
424 clk_disable_unprepare(ocmem->iface_clk);
425
426 return 0;
427 }
428
429 static const struct ocmem_config ocmem_8226_config = {
430 .num_regions = 1,
431 .macro_size = SZ_128K,
432 };
433
434 static const struct ocmem_config ocmem_8974_config = {
435 .num_regions = 3,
436 .macro_size = SZ_128K,
437 };
438
439 static const struct of_device_id ocmem_of_match[] = {
440 { .compatible = "qcom,msm8226-ocmem", .data = &ocmem_8226_config },
441 { .compatible = "qcom,msm8974-ocmem", .data = &ocmem_8974_config },
442 { }
443 };
444
445 MODULE_DEVICE_TABLE(of, ocmem_of_match);
446
447 static struct platform_driver ocmem_driver = {
448 .probe = ocmem_dev_probe,
449 .remove = ocmem_dev_remove,
450 .driver = {
451 .name = "ocmem",
452 .of_match_table = ocmem_of_match,
453 },
454 };
455
456 module_platform_driver(ocmem_driver);
457
458 MODULE_DESCRIPTION("On Chip Memory (OCMEM) allocator for some Snapdragon SoCs");
459 MODULE_LICENSE("GPL v2");
460