1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright(c) 2017 IBM Corporation. All rights reserved.
4 */
5
6 #include <linux/string.h>
7 #include <linux/export.h>
8 #include <linux/uaccess.h>
9 #include <linux/libnvdimm.h>
10
11 #include <asm/cacheflush.h>
12
__clean_pmem_range(unsigned long start,unsigned long stop)13 static inline void __clean_pmem_range(unsigned long start, unsigned long stop)
14 {
15 unsigned long shift = l1_dcache_shift();
16 unsigned long bytes = l1_dcache_bytes();
17 void *addr = (void *)(start & ~(bytes - 1));
18 unsigned long size = stop - (unsigned long)addr + (bytes - 1);
19 unsigned long i;
20
21 for (i = 0; i < size >> shift; i++, addr += bytes)
22 asm volatile(PPC_DCBSTPS(%0, %1): :"i"(0), "r"(addr): "memory");
23 }
24
__flush_pmem_range(unsigned long start,unsigned long stop)25 static inline void __flush_pmem_range(unsigned long start, unsigned long stop)
26 {
27 unsigned long shift = l1_dcache_shift();
28 unsigned long bytes = l1_dcache_bytes();
29 void *addr = (void *)(start & ~(bytes - 1));
30 unsigned long size = stop - (unsigned long)addr + (bytes - 1);
31 unsigned long i;
32
33 for (i = 0; i < size >> shift; i++, addr += bytes)
34 asm volatile(PPC_DCBFPS(%0, %1): :"i"(0), "r"(addr): "memory");
35 }
36
clean_pmem_range(unsigned long start,unsigned long stop)37 static inline void clean_pmem_range(unsigned long start, unsigned long stop)
38 {
39 if (cpu_has_feature(CPU_FTR_ARCH_207S))
40 return __clean_pmem_range(start, stop);
41 }
42
flush_pmem_range(unsigned long start,unsigned long stop)43 static inline void flush_pmem_range(unsigned long start, unsigned long stop)
44 {
45 if (cpu_has_feature(CPU_FTR_ARCH_207S))
46 return __flush_pmem_range(start, stop);
47 }
48
49 /*
50 * CONFIG_ARCH_HAS_PMEM_API symbols
51 */
arch_wb_cache_pmem(void * addr,size_t size)52 void arch_wb_cache_pmem(void *addr, size_t size)
53 {
54 unsigned long start = (unsigned long) addr;
55 clean_pmem_range(start, start + size);
56 }
57 EXPORT_SYMBOL_GPL(arch_wb_cache_pmem);
58
arch_invalidate_pmem(void * addr,size_t size)59 void arch_invalidate_pmem(void *addr, size_t size)
60 {
61 unsigned long start = (unsigned long) addr;
62 flush_pmem_range(start, start + size);
63 }
64 EXPORT_SYMBOL_GPL(arch_invalidate_pmem);
65
66 /*
67 * CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE symbols
68 */
__copy_from_user_flushcache(void * dest,const void __user * src,unsigned size)69 long __copy_from_user_flushcache(void *dest, const void __user *src,
70 unsigned size)
71 {
72 unsigned long copied, start = (unsigned long) dest;
73
74 copied = __copy_from_user(dest, src, size);
75 clean_pmem_range(start, start + size);
76
77 return copied;
78 }
79
memcpy_flushcache(void * dest,const void * src,size_t size)80 void memcpy_flushcache(void *dest, const void *src, size_t size)
81 {
82 unsigned long start = (unsigned long) dest;
83
84 memcpy(dest, src, size);
85 clean_pmem_range(start, start + size);
86 }
87 EXPORT_SYMBOL(memcpy_flushcache);
88