1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Support for Medifield PNW Camera Imaging ISP subsystem.
4 *
5 * Copyright (c) 2010-2017 Intel Corporation. All Rights Reserved.
6 *
7 * Copyright (c) 2010 Silicon Hive www.siliconhive.com.
8 *
9 * This program is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU General Public License version
11 * 2 as published by the Free Software Foundation.
12 *
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
17 *
18 *
19 */
20 /*
21 * This file contains entry functions for memory management of ISP driver
22 */
23 #include <linux/kernel.h>
24 #include <linux/types.h>
25 #include <linux/mm.h>
26 #include <linux/highmem.h> /* for kmap */
27 #include <linux/io.h> /* for page_to_phys */
28 #include <linux/sysfs.h>
29
30 #include "hmm/hmm.h"
31 #include "hmm/hmm_bo.h"
32
33 #include "atomisp_internal.h"
34 #include "asm/cacheflush.h"
35 #include "mmu/isp_mmu.h"
36 #include "mmu/sh_mmu_mrfld.h"
37
38 struct hmm_bo_device bo_device;
39 static ia_css_ptr dummy_ptr = mmgr_EXCEPTION;
40 static bool hmm_initialized;
41
42 /*
43 * p: private
44 * v: vmalloc
45 */
46 static const char hmm_bo_type_string[] = "pv";
47
bo_show(struct device * dev,struct device_attribute * attr,char * buf,struct list_head * bo_list,bool active)48 static ssize_t bo_show(struct device *dev, struct device_attribute *attr,
49 char *buf, struct list_head *bo_list, bool active)
50 {
51 ssize_t ret = 0;
52 struct hmm_buffer_object *bo;
53 unsigned long flags;
54 int i;
55 long total[HMM_BO_LAST] = { 0 };
56 long count[HMM_BO_LAST] = { 0 };
57 int index1 = 0;
58 int index2 = 0;
59
60 ret = scnprintf(buf, PAGE_SIZE, "type pgnr\n");
61 if (ret <= 0)
62 return 0;
63
64 index1 += ret;
65
66 spin_lock_irqsave(&bo_device.list_lock, flags);
67 list_for_each_entry(bo, bo_list, list) {
68 if ((active && (bo->status & HMM_BO_ALLOCED)) ||
69 (!active && !(bo->status & HMM_BO_ALLOCED))) {
70 ret = scnprintf(buf + index1, PAGE_SIZE - index1,
71 "%c %d\n",
72 hmm_bo_type_string[bo->type], bo->pgnr);
73
74 total[bo->type] += bo->pgnr;
75 count[bo->type]++;
76 if (ret > 0)
77 index1 += ret;
78 }
79 }
80 spin_unlock_irqrestore(&bo_device.list_lock, flags);
81
82 for (i = 0; i < HMM_BO_LAST; i++) {
83 if (count[i]) {
84 ret = scnprintf(buf + index1 + index2,
85 PAGE_SIZE - index1 - index2,
86 "%ld %c buffer objects: %ld KB\n",
87 count[i], hmm_bo_type_string[i],
88 total[i] * 4);
89 if (ret > 0)
90 index2 += ret;
91 }
92 }
93
94 /* Add trailing zero, not included by scnprintf */
95 return index1 + index2 + 1;
96 }
97
active_bo_show(struct device * dev,struct device_attribute * attr,char * buf)98 static ssize_t active_bo_show(struct device *dev, struct device_attribute *attr,
99 char *buf)
100 {
101 return bo_show(dev, attr, buf, &bo_device.entire_bo_list, true);
102 }
103
free_bo_show(struct device * dev,struct device_attribute * attr,char * buf)104 static ssize_t free_bo_show(struct device *dev, struct device_attribute *attr,
105 char *buf)
106 {
107 return bo_show(dev, attr, buf, &bo_device.entire_bo_list, false);
108 }
109
110
111 static DEVICE_ATTR_RO(active_bo);
112 static DEVICE_ATTR_RO(free_bo);
113
114 static struct attribute *sysfs_attrs_ctrl[] = {
115 &dev_attr_active_bo.attr,
116 &dev_attr_free_bo.attr,
117 NULL
118 };
119
120 static struct attribute_group atomisp_attribute_group[] = {
121 {.attrs = sysfs_attrs_ctrl },
122 };
123
hmm_init(void)124 int hmm_init(void)
125 {
126 int ret;
127
128 ret = hmm_bo_device_init(&bo_device, &sh_mmu_mrfld,
129 ISP_VM_START, ISP_VM_SIZE);
130 if (ret)
131 dev_err(atomisp_dev, "hmm_bo_device_init failed.\n");
132
133 hmm_initialized = true;
134
135 /*
136 * As hmm use NULL to indicate invalid ISP virtual address,
137 * and ISP_VM_START is defined to 0 too, so we allocate
138 * one piece of dummy memory, which should return value 0,
139 * at the beginning, to avoid hmm_alloc return 0 in the
140 * further allocation.
141 */
142 dummy_ptr = hmm_alloc(1);
143
144 if (!ret) {
145 ret = sysfs_create_group(&atomisp_dev->kobj,
146 atomisp_attribute_group);
147 if (ret)
148 dev_err(atomisp_dev,
149 "%s Failed to create sysfs\n", __func__);
150 }
151
152 return ret;
153 }
154
hmm_cleanup(void)155 void hmm_cleanup(void)
156 {
157 if (dummy_ptr == mmgr_EXCEPTION)
158 return;
159 sysfs_remove_group(&atomisp_dev->kobj, atomisp_attribute_group);
160
161 /* free dummy memory first */
162 hmm_free(dummy_ptr);
163 dummy_ptr = 0;
164
165 hmm_bo_device_exit(&bo_device);
166 hmm_initialized = false;
167 }
168
__hmm_alloc(size_t bytes,enum hmm_bo_type type,void * vmalloc_addr)169 static ia_css_ptr __hmm_alloc(size_t bytes, enum hmm_bo_type type,
170 void *vmalloc_addr)
171 {
172 unsigned int pgnr;
173 struct hmm_buffer_object *bo;
174 int ret;
175
176 /*
177 * Check if we are initialized. In the ideal world we wouldn't need
178 * this but we can tackle it once the driver is a lot cleaner
179 */
180
181 if (!hmm_initialized)
182 hmm_init();
183 /* Get page number from size */
184 pgnr = size_to_pgnr_ceil(bytes);
185
186 /* Buffer object structure init */
187 bo = hmm_bo_alloc(&bo_device, pgnr);
188 if (!bo) {
189 dev_err(atomisp_dev, "hmm_bo_create failed.\n");
190 goto create_bo_err;
191 }
192
193 /* Allocate pages for memory */
194 ret = hmm_bo_alloc_pages(bo, type, vmalloc_addr);
195 if (ret) {
196 dev_err(atomisp_dev, "hmm_bo_alloc_pages failed.\n");
197 goto alloc_page_err;
198 }
199
200 /* Combine the virtual address and pages together */
201 ret = hmm_bo_bind(bo);
202 if (ret) {
203 dev_err(atomisp_dev, "hmm_bo_bind failed.\n");
204 goto bind_err;
205 }
206
207 dev_dbg(atomisp_dev, "pages: 0x%08x (%zu bytes), type: %d, vmalloc %p\n",
208 bo->start, bytes, type, vmalloc);
209
210 return bo->start;
211
212 bind_err:
213 hmm_bo_free_pages(bo);
214 alloc_page_err:
215 hmm_bo_unref(bo);
216 create_bo_err:
217 return 0;
218 }
219
hmm_alloc(size_t bytes)220 ia_css_ptr hmm_alloc(size_t bytes)
221 {
222 return __hmm_alloc(bytes, HMM_BO_PRIVATE, NULL);
223 }
224
hmm_create_from_vmalloc_buf(size_t bytes,void * vmalloc_addr)225 ia_css_ptr hmm_create_from_vmalloc_buf(size_t bytes, void *vmalloc_addr)
226 {
227 return __hmm_alloc(bytes, HMM_BO_VMALLOC, vmalloc_addr);
228 }
229
hmm_free(ia_css_ptr virt)230 void hmm_free(ia_css_ptr virt)
231 {
232 struct hmm_buffer_object *bo;
233
234 dev_dbg(atomisp_dev, "%s: free 0x%08x\n", __func__, virt);
235
236 if (WARN_ON(virt == mmgr_EXCEPTION))
237 return;
238
239 bo = hmm_bo_device_search_start(&bo_device, (unsigned int)virt);
240
241 if (!bo) {
242 dev_err(atomisp_dev,
243 "can not find buffer object start with address 0x%x\n",
244 (unsigned int)virt);
245 return;
246 }
247
248 hmm_bo_unbind(bo);
249 hmm_bo_free_pages(bo);
250 hmm_bo_unref(bo);
251 }
252
hmm_check_bo(struct hmm_buffer_object * bo,unsigned int ptr)253 static inline int hmm_check_bo(struct hmm_buffer_object *bo, unsigned int ptr)
254 {
255 if (!bo) {
256 dev_err(atomisp_dev,
257 "can not find buffer object contains address 0x%x\n",
258 ptr);
259 return -EINVAL;
260 }
261
262 if (!hmm_bo_page_allocated(bo)) {
263 dev_err(atomisp_dev,
264 "buffer object has no page allocated.\n");
265 return -EINVAL;
266 }
267
268 if (!hmm_bo_allocated(bo)) {
269 dev_err(atomisp_dev,
270 "buffer object has no virtual address space allocated.\n");
271 return -EINVAL;
272 }
273
274 return 0;
275 }
276
277 /* Read function in ISP memory management */
load_and_flush_by_kmap(ia_css_ptr virt,void * data,unsigned int bytes)278 static int load_and_flush_by_kmap(ia_css_ptr virt, void *data,
279 unsigned int bytes)
280 {
281 struct hmm_buffer_object *bo;
282 unsigned int idx, offset, len;
283 char *src, *des;
284 int ret;
285
286 bo = hmm_bo_device_search_in_range(&bo_device, virt);
287 ret = hmm_check_bo(bo, virt);
288 if (ret)
289 return ret;
290
291 des = (char *)data;
292 while (bytes) {
293 idx = (virt - bo->start) >> PAGE_SHIFT;
294 offset = (virt - bo->start) - (idx << PAGE_SHIFT);
295
296 src = (char *)kmap_local_page(bo->pages[idx]) + offset;
297
298 if ((bytes + offset) >= PAGE_SIZE) {
299 len = PAGE_SIZE - offset;
300 bytes -= len;
301 } else {
302 len = bytes;
303 bytes = 0;
304 }
305
306 virt += len; /* update virt for next loop */
307
308 if (des) {
309 memcpy(des, src, len);
310 des += len;
311 }
312
313 clflush_cache_range(src, len);
314
315 kunmap_local(src);
316 }
317
318 return 0;
319 }
320
321 /* Read function in ISP memory management */
load_and_flush(ia_css_ptr virt,void * data,unsigned int bytes)322 static int load_and_flush(ia_css_ptr virt, void *data, unsigned int bytes)
323 {
324 struct hmm_buffer_object *bo;
325 int ret;
326
327 bo = hmm_bo_device_search_in_range(&bo_device, virt);
328 ret = hmm_check_bo(bo, virt);
329 if (ret)
330 return ret;
331
332 if (bo->status & HMM_BO_VMAPED || bo->status & HMM_BO_VMAPED_CACHED) {
333 void *src = bo->vmap_addr;
334
335 src += (virt - bo->start);
336 memcpy(data, src, bytes);
337 if (bo->status & HMM_BO_VMAPED_CACHED)
338 clflush_cache_range(src, bytes);
339 } else {
340 void *vptr;
341
342 vptr = hmm_bo_vmap(bo, true);
343 if (!vptr)
344 return load_and_flush_by_kmap(virt, data, bytes);
345 else
346 vptr = vptr + (virt - bo->start);
347
348 memcpy(data, vptr, bytes);
349 clflush_cache_range(vptr, bytes);
350 hmm_bo_vunmap(bo);
351 }
352
353 return 0;
354 }
355
356 /* Read function in ISP memory management */
hmm_load(ia_css_ptr virt,void * data,unsigned int bytes)357 int hmm_load(ia_css_ptr virt, void *data, unsigned int bytes)
358 {
359 if (!virt) {
360 dev_warn(atomisp_dev,
361 "hmm_store: address is NULL\n");
362 return -EINVAL;
363 }
364 if (!data) {
365 dev_err(atomisp_dev,
366 "hmm_store: data is a NULL argument\n");
367 return -EINVAL;
368 }
369 return load_and_flush(virt, data, bytes);
370 }
371
372 /* Flush hmm data from the data cache */
hmm_flush(ia_css_ptr virt,unsigned int bytes)373 int hmm_flush(ia_css_ptr virt, unsigned int bytes)
374 {
375 return load_and_flush(virt, NULL, bytes);
376 }
377
378 /* Write function in ISP memory management */
hmm_store(ia_css_ptr virt,const void * data,unsigned int bytes)379 int hmm_store(ia_css_ptr virt, const void *data, unsigned int bytes)
380 {
381 struct hmm_buffer_object *bo;
382 unsigned int idx, offset, len;
383 char *src, *des;
384 int ret;
385
386 if (!virt) {
387 dev_warn(atomisp_dev,
388 "hmm_store: address is NULL\n");
389 return -EINVAL;
390 }
391 if (!data) {
392 dev_err(atomisp_dev,
393 "hmm_store: data is a NULL argument\n");
394 return -EINVAL;
395 }
396
397 bo = hmm_bo_device_search_in_range(&bo_device, virt);
398 ret = hmm_check_bo(bo, virt);
399 if (ret)
400 return ret;
401
402 if (bo->status & HMM_BO_VMAPED || bo->status & HMM_BO_VMAPED_CACHED) {
403 void *dst = bo->vmap_addr;
404
405 dst += (virt - bo->start);
406 memcpy(dst, data, bytes);
407 if (bo->status & HMM_BO_VMAPED_CACHED)
408 clflush_cache_range(dst, bytes);
409 } else {
410 void *vptr;
411
412 vptr = hmm_bo_vmap(bo, true);
413 if (vptr) {
414 vptr = vptr + (virt - bo->start);
415
416 memcpy(vptr, data, bytes);
417 clflush_cache_range(vptr, bytes);
418 hmm_bo_vunmap(bo);
419 return 0;
420 }
421 }
422
423 src = (char *)data;
424 while (bytes) {
425 idx = (virt - bo->start) >> PAGE_SHIFT;
426 offset = (virt - bo->start) - (idx << PAGE_SHIFT);
427
428 des = (char *)kmap_local_page(bo->pages[idx]);
429
430 if (!des) {
431 dev_err(atomisp_dev,
432 "kmap buffer object page failed: pg_idx = %d\n",
433 idx);
434 return -EINVAL;
435 }
436
437 des += offset;
438
439 if ((bytes + offset) >= PAGE_SIZE) {
440 len = PAGE_SIZE - offset;
441 bytes -= len;
442 } else {
443 len = bytes;
444 bytes = 0;
445 }
446
447 virt += len;
448
449 memcpy(des, src, len);
450
451 src += len;
452
453 clflush_cache_range(des, len);
454
455 kunmap_local(des);
456 }
457
458 return 0;
459 }
460
461 /* memset function in ISP memory management */
hmm_set(ia_css_ptr virt,int c,unsigned int bytes)462 int hmm_set(ia_css_ptr virt, int c, unsigned int bytes)
463 {
464 struct hmm_buffer_object *bo;
465 unsigned int idx, offset, len;
466 char *des;
467 int ret;
468
469 bo = hmm_bo_device_search_in_range(&bo_device, virt);
470 ret = hmm_check_bo(bo, virt);
471 if (ret)
472 return ret;
473
474 if (bo->status & HMM_BO_VMAPED || bo->status & HMM_BO_VMAPED_CACHED) {
475 void *dst = bo->vmap_addr;
476
477 dst += (virt - bo->start);
478 memset(dst, c, bytes);
479
480 if (bo->status & HMM_BO_VMAPED_CACHED)
481 clflush_cache_range(dst, bytes);
482 } else {
483 void *vptr;
484
485 vptr = hmm_bo_vmap(bo, true);
486 if (vptr) {
487 vptr = vptr + (virt - bo->start);
488 memset(vptr, c, bytes);
489 clflush_cache_range(vptr, bytes);
490 hmm_bo_vunmap(bo);
491 return 0;
492 }
493 }
494
495 while (bytes) {
496 idx = (virt - bo->start) >> PAGE_SHIFT;
497 offset = (virt - bo->start) - (idx << PAGE_SHIFT);
498
499 des = (char *)kmap_local_page(bo->pages[idx]) + offset;
500
501 if ((bytes + offset) >= PAGE_SIZE) {
502 len = PAGE_SIZE - offset;
503 bytes -= len;
504 } else {
505 len = bytes;
506 bytes = 0;
507 }
508
509 virt += len;
510
511 memset(des, c, len);
512
513 clflush_cache_range(des, len);
514
515 kunmap_local(des);
516 }
517
518 return 0;
519 }
520
521 /* Virtual address to physical address convert */
hmm_virt_to_phys(ia_css_ptr virt)522 phys_addr_t hmm_virt_to_phys(ia_css_ptr virt)
523 {
524 unsigned int idx, offset;
525 struct hmm_buffer_object *bo;
526
527 bo = hmm_bo_device_search_in_range(&bo_device, virt);
528 if (!bo) {
529 dev_err(atomisp_dev,
530 "can not find buffer object contains address 0x%x\n",
531 virt);
532 return -1;
533 }
534
535 idx = (virt - bo->start) >> PAGE_SHIFT;
536 offset = (virt - bo->start) - (idx << PAGE_SHIFT);
537
538 return page_to_phys(bo->pages[idx]) + offset;
539 }
540
hmm_mmap(struct vm_area_struct * vma,ia_css_ptr virt)541 int hmm_mmap(struct vm_area_struct *vma, ia_css_ptr virt)
542 {
543 struct hmm_buffer_object *bo;
544
545 bo = hmm_bo_device_search_start(&bo_device, virt);
546 if (!bo) {
547 dev_err(atomisp_dev,
548 "can not find buffer object start with address 0x%x\n",
549 virt);
550 return -EINVAL;
551 }
552
553 return hmm_bo_mmap(vma, bo);
554 }
555
556 /* Map ISP virtual address into IA virtual address */
hmm_vmap(ia_css_ptr virt,bool cached)557 void *hmm_vmap(ia_css_ptr virt, bool cached)
558 {
559 struct hmm_buffer_object *bo;
560 void *ptr;
561
562 bo = hmm_bo_device_search_in_range(&bo_device, virt);
563 if (!bo) {
564 dev_err(atomisp_dev,
565 "can not find buffer object contains address 0x%x\n",
566 virt);
567 return NULL;
568 }
569
570 ptr = hmm_bo_vmap(bo, cached);
571 if (ptr)
572 return ptr + (virt - bo->start);
573 else
574 return NULL;
575 }
576
577 /* Flush the memory which is mapped as cached memory through hmm_vmap */
hmm_flush_vmap(ia_css_ptr virt)578 void hmm_flush_vmap(ia_css_ptr virt)
579 {
580 struct hmm_buffer_object *bo;
581
582 bo = hmm_bo_device_search_in_range(&bo_device, virt);
583 if (!bo) {
584 dev_warn(atomisp_dev,
585 "can not find buffer object contains address 0x%x\n",
586 virt);
587 return;
588 }
589
590 hmm_bo_flush_vmap(bo);
591 }
592
hmm_vunmap(ia_css_ptr virt)593 void hmm_vunmap(ia_css_ptr virt)
594 {
595 struct hmm_buffer_object *bo;
596
597 bo = hmm_bo_device_search_in_range(&bo_device, virt);
598 if (!bo) {
599 dev_warn(atomisp_dev,
600 "can not find buffer object contains address 0x%x\n",
601 virt);
602 return;
603 }
604
605 hmm_bo_vunmap(bo);
606 }
607