1 // SPDX-License-Identifier: GPL-2.0-or-later 2 3 #include <linux/highmem.h> 4 #include <linux/kprobes.h> 5 6 /** 7 * flush_coherent_icache() - if a CPU has a coherent icache, flush it 8 * Return true if the cache was flushed, false otherwise 9 */ 10 static inline bool flush_coherent_icache(void) 11 { 12 /* 13 * For a snooping icache, we still need a dummy icbi to purge all the 14 * prefetched instructions from the ifetch buffers. We also need a sync 15 * before the icbi to order the actual stores to memory that might 16 * have modified instructions with the icbi. 17 */ 18 if (cpu_has_feature(CPU_FTR_COHERENT_ICACHE)) { 19 mb(); /* sync */ 20 icbi((void *)PAGE_OFFSET); 21 mb(); /* sync */ 22 isync(); 23 return true; 24 } 25 26 return false; 27 } 28 29 /** 30 * invalidate_icache_range() - Flush the icache by issuing icbi across an address range 31 * @start: the start address 32 * @stop: the stop address (exclusive) 33 */ 34 static void invalidate_icache_range(unsigned long start, unsigned long stop) 35 { 36 unsigned long shift = l1_icache_shift(); 37 unsigned long bytes = l1_icache_bytes(); 38 char *addr = (char *)(start & ~(bytes - 1)); 39 unsigned long size = stop - (unsigned long)addr + (bytes - 1); 40 unsigned long i; 41 42 for (i = 0; i < size >> shift; i++, addr += bytes) 43 icbi(addr); 44 45 mb(); /* sync */ 46 isync(); 47 } 48 49 /** 50 * flush_icache_range: Write any modified data cache blocks out to memory 51 * and invalidate the corresponding blocks in the instruction cache 52 * 53 * Generic code will call this after writing memory, before executing from it. 54 * 55 * @start: the start address 56 * @stop: the stop address (exclusive) 57 */ 58 void flush_icache_range(unsigned long start, unsigned long stop) 59 { 60 if (flush_coherent_icache()) 61 return; 62 63 clean_dcache_range(start, stop); 64 65 if (IS_ENABLED(CONFIG_44x)) { 66 /* 67 * Flash invalidate on 44x because we are passed kmapped 68 * addresses and this doesn't work for userspace pages due to 69 * the virtually tagged icache. 70 */ 71 iccci((void *)start); 72 mb(); /* sync */ 73 isync(); 74 } else 75 invalidate_icache_range(start, stop); 76 } 77 EXPORT_SYMBOL(flush_icache_range); 78 79 #ifdef CONFIG_HIGHMEM 80 /** 81 * flush_dcache_icache_phys() - Flush a page by it's physical address 82 * @physaddr: the physical address of the page 83 */ 84 static void flush_dcache_icache_phys(unsigned long physaddr) 85 { 86 unsigned long bytes = l1_dcache_bytes(); 87 unsigned long nb = PAGE_SIZE / bytes; 88 unsigned long addr = physaddr & PAGE_MASK; 89 unsigned long msr, msr0; 90 unsigned long loop1 = addr, loop2 = addr; 91 92 msr0 = mfmsr(); 93 msr = msr0 & ~MSR_DR; 94 /* 95 * This must remain as ASM to prevent potential memory accesses 96 * while the data MMU is disabled 97 */ 98 asm volatile( 99 " mtctr %2;\n" 100 " mtmsr %3;\n" 101 " isync;\n" 102 "0: dcbst 0, %0;\n" 103 " addi %0, %0, %4;\n" 104 " bdnz 0b;\n" 105 " sync;\n" 106 " mtctr %2;\n" 107 "1: icbi 0, %1;\n" 108 " addi %1, %1, %4;\n" 109 " bdnz 1b;\n" 110 " sync;\n" 111 " mtmsr %5;\n" 112 " isync;\n" 113 : "+&r" (loop1), "+&r" (loop2) 114 : "r" (nb), "r" (msr), "i" (bytes), "r" (msr0) 115 : "ctr", "memory"); 116 } 117 NOKPROBE_SYMBOL(flush_dcache_icache_phys) 118 #else 119 static void flush_dcache_icache_phys(unsigned long physaddr) 120 { 121 } 122 #endif 123 124 /** 125 * __flush_dcache_icache(): Flush a particular page from the data cache to RAM. 126 * Note: this is necessary because the instruction cache does *not* 127 * snoop from the data cache. 128 * 129 * @p: the address of the page to flush 130 */ 131 static void __flush_dcache_icache(void *p) 132 { 133 unsigned long addr = (unsigned long)p & PAGE_MASK; 134 135 clean_dcache_range(addr, addr + PAGE_SIZE); 136 137 /* 138 * We don't flush the icache on 44x. Those have a virtual icache and we 139 * don't have access to the virtual address here (it's not the page 140 * vaddr but where it's mapped in user space). The flushing of the 141 * icache on these is handled elsewhere, when a change in the address 142 * space occurs, before returning to user space. 143 */ 144 145 if (mmu_has_feature(MMU_FTR_TYPE_44x)) 146 return; 147 148 invalidate_icache_range(addr, addr + PAGE_SIZE); 149 } 150 151 static void flush_dcache_icache_hugepage(struct page *page) 152 { 153 int i; 154 int nr = compound_nr(page); 155 156 if (!PageHighMem(page)) { 157 for (i = 0; i < nr; i++) 158 __flush_dcache_icache(lowmem_page_address(page + i)); 159 } else { 160 for (i = 0; i < nr; i++) { 161 void *start = kmap_local_page(page + i); 162 163 __flush_dcache_icache(start); 164 kunmap_local(start); 165 } 166 } 167 } 168 169 void flush_dcache_icache_page(struct page *page) 170 { 171 if (flush_coherent_icache()) 172 return; 173 174 if (PageCompound(page)) 175 return flush_dcache_icache_hugepage(page); 176 177 if (!PageHighMem(page)) { 178 __flush_dcache_icache(lowmem_page_address(page)); 179 } else if (IS_ENABLED(CONFIG_BOOKE) || sizeof(phys_addr_t) > sizeof(void *)) { 180 void *start = kmap_local_page(page); 181 182 __flush_dcache_icache(start); 183 kunmap_local(start); 184 } else { 185 flush_dcache_icache_phys(page_to_phys(page)); 186 } 187 } 188 EXPORT_SYMBOL(flush_dcache_icache_page); 189 190 void clear_user_page(void *page, unsigned long vaddr, struct page *pg) 191 { 192 clear_page(page); 193 194 /* 195 * We shouldn't have to do this, but some versions of glibc 196 * require it (ld.so assumes zero filled pages are icache clean) 197 * - Anton 198 */ 199 flush_dcache_page(pg); 200 } 201 EXPORT_SYMBOL(clear_user_page); 202 203 void copy_user_page(void *vto, void *vfrom, unsigned long vaddr, 204 struct page *pg) 205 { 206 copy_page(vto, vfrom); 207 208 /* 209 * We should be able to use the following optimisation, however 210 * there are two problems. 211 * Firstly a bug in some versions of binutils meant PLT sections 212 * were not marked executable. 213 * Secondly the first word in the GOT section is blrl, used 214 * to establish the GOT address. Until recently the GOT was 215 * not marked executable. 216 * - Anton 217 */ 218 #if 0 219 if (!vma->vm_file && ((vma->vm_flags & VM_EXEC) == 0)) 220 return; 221 #endif 222 223 flush_dcache_page(pg); 224 } 225 226 void flush_icache_user_page(struct vm_area_struct *vma, struct page *page, 227 unsigned long addr, int len) 228 { 229 void *maddr; 230 231 maddr = kmap_local_page(page) + (addr & ~PAGE_MASK); 232 flush_icache_range((unsigned long)maddr, (unsigned long)maddr + len); 233 kunmap_local(maddr); 234 } 235