xref: /openbmc/linux/arch/sh/mm/cache-sh4.c (revision 22246614)
1 /*
2  * arch/sh/mm/cache-sh4.c
3  *
4  * Copyright (C) 1999, 2000, 2002  Niibe Yutaka
5  * Copyright (C) 2001 - 2007  Paul Mundt
6  * Copyright (C) 2003  Richard Curnow
7  *
8  * This file is subject to the terms and conditions of the GNU General Public
9  * License.  See the file "COPYING" in the main directory of this archive
10  * for more details.
11  */
12 #include <linux/init.h>
13 #include <linux/mm.h>
14 #include <linux/io.h>
15 #include <linux/mutex.h>
16 #include <asm/mmu_context.h>
17 #include <asm/cacheflush.h>
18 
19 /*
20  * The maximum number of pages we support up to when doing ranged dcache
21  * flushing. Anything exceeding this will simply flush the dcache in its
22  * entirety.
23  */
24 #define MAX_DCACHE_PAGES	64	/* XXX: Tune for ways */
25 
26 static void __flush_dcache_segment_1way(unsigned long start,
27 					unsigned long extent);
28 static void __flush_dcache_segment_2way(unsigned long start,
29 					unsigned long extent);
30 static void __flush_dcache_segment_4way(unsigned long start,
31 					unsigned long extent);
32 
33 static void __flush_cache_4096(unsigned long addr, unsigned long phys,
34 			       unsigned long exec_offset);
35 
36 /*
37  * This is initialised here to ensure that it is not placed in the BSS.  If
38  * that were to happen, note that cache_init gets called before the BSS is
39  * cleared, so this would get nulled out which would be hopeless.
40  */
41 static void (*__flush_dcache_segment_fn)(unsigned long, unsigned long) =
42 	(void (*)(unsigned long, unsigned long))0xdeadbeef;
43 
44 static void compute_alias(struct cache_info *c)
45 {
46 	c->alias_mask = ((c->sets - 1) << c->entry_shift) & ~(PAGE_SIZE - 1);
47 	c->n_aliases = c->alias_mask ? (c->alias_mask >> PAGE_SHIFT) + 1 : 0;
48 }
49 
50 static void __init emit_cache_params(void)
51 {
52 	printk("PVR=%08x CVR=%08x PRR=%08x\n",
53 		ctrl_inl(CCN_PVR),
54 		ctrl_inl(CCN_CVR),
55 		ctrl_inl(CCN_PRR));
56 	printk("I-cache : n_ways=%d n_sets=%d way_incr=%d\n",
57 		boot_cpu_data.icache.ways,
58 		boot_cpu_data.icache.sets,
59 		boot_cpu_data.icache.way_incr);
60 	printk("I-cache : entry_mask=0x%08x alias_mask=0x%08x n_aliases=%d\n",
61 		boot_cpu_data.icache.entry_mask,
62 		boot_cpu_data.icache.alias_mask,
63 		boot_cpu_data.icache.n_aliases);
64 	printk("D-cache : n_ways=%d n_sets=%d way_incr=%d\n",
65 		boot_cpu_data.dcache.ways,
66 		boot_cpu_data.dcache.sets,
67 		boot_cpu_data.dcache.way_incr);
68 	printk("D-cache : entry_mask=0x%08x alias_mask=0x%08x n_aliases=%d\n",
69 		boot_cpu_data.dcache.entry_mask,
70 		boot_cpu_data.dcache.alias_mask,
71 		boot_cpu_data.dcache.n_aliases);
72 
73 	/*
74 	 * Emit Secondary Cache parameters if the CPU has a probed L2.
75 	 */
76 	if (boot_cpu_data.flags & CPU_HAS_L2_CACHE) {
77 		printk("S-cache : n_ways=%d n_sets=%d way_incr=%d\n",
78 			boot_cpu_data.scache.ways,
79 			boot_cpu_data.scache.sets,
80 			boot_cpu_data.scache.way_incr);
81 		printk("S-cache : entry_mask=0x%08x alias_mask=0x%08x n_aliases=%d\n",
82 			boot_cpu_data.scache.entry_mask,
83 			boot_cpu_data.scache.alias_mask,
84 			boot_cpu_data.scache.n_aliases);
85 	}
86 
87 	if (!__flush_dcache_segment_fn)
88 		panic("unknown number of cache ways\n");
89 }
90 
91 /*
92  * SH-4 has virtually indexed and physically tagged cache.
93  */
94 void __init p3_cache_init(void)
95 {
96 	compute_alias(&boot_cpu_data.icache);
97 	compute_alias(&boot_cpu_data.dcache);
98 	compute_alias(&boot_cpu_data.scache);
99 
100 	switch (boot_cpu_data.dcache.ways) {
101 	case 1:
102 		__flush_dcache_segment_fn = __flush_dcache_segment_1way;
103 		break;
104 	case 2:
105 		__flush_dcache_segment_fn = __flush_dcache_segment_2way;
106 		break;
107 	case 4:
108 		__flush_dcache_segment_fn = __flush_dcache_segment_4way;
109 		break;
110 	default:
111 		__flush_dcache_segment_fn = NULL;
112 		break;
113 	}
114 
115 	emit_cache_params();
116 }
117 
118 /*
119  * Write back the dirty D-caches, but not invalidate them.
120  *
121  * START: Virtual Address (U0, P1, or P3)
122  * SIZE: Size of the region.
123  */
124 void __flush_wback_region(void *start, int size)
125 {
126 	unsigned long v;
127 	unsigned long begin, end;
128 
129 	begin = (unsigned long)start & ~(L1_CACHE_BYTES-1);
130 	end = ((unsigned long)start + size + L1_CACHE_BYTES-1)
131 		& ~(L1_CACHE_BYTES-1);
132 	for (v = begin; v < end; v+=L1_CACHE_BYTES) {
133 		asm volatile("ocbwb	%0"
134 			     : /* no output */
135 			     : "m" (__m(v)));
136 	}
137 }
138 
139 /*
140  * Write back the dirty D-caches and invalidate them.
141  *
142  * START: Virtual Address (U0, P1, or P3)
143  * SIZE: Size of the region.
144  */
145 void __flush_purge_region(void *start, int size)
146 {
147 	unsigned long v;
148 	unsigned long begin, end;
149 
150 	begin = (unsigned long)start & ~(L1_CACHE_BYTES-1);
151 	end = ((unsigned long)start + size + L1_CACHE_BYTES-1)
152 		& ~(L1_CACHE_BYTES-1);
153 	for (v = begin; v < end; v+=L1_CACHE_BYTES) {
154 		asm volatile("ocbp	%0"
155 			     : /* no output */
156 			     : "m" (__m(v)));
157 	}
158 }
159 
160 /*
161  * No write back please
162  */
163 void __flush_invalidate_region(void *start, int size)
164 {
165 	unsigned long v;
166 	unsigned long begin, end;
167 
168 	begin = (unsigned long)start & ~(L1_CACHE_BYTES-1);
169 	end = ((unsigned long)start + size + L1_CACHE_BYTES-1)
170 		& ~(L1_CACHE_BYTES-1);
171 	for (v = begin; v < end; v+=L1_CACHE_BYTES) {
172 		asm volatile("ocbi	%0"
173 			     : /* no output */
174 			     : "m" (__m(v)));
175 	}
176 }
177 
178 /*
179  * Write back the range of D-cache, and purge the I-cache.
180  *
181  * Called from kernel/module.c:sys_init_module and routine for a.out format.
182  */
183 void flush_icache_range(unsigned long start, unsigned long end)
184 {
185 	flush_cache_all();
186 }
187 
188 /*
189  * Write back the D-cache and purge the I-cache for signal trampoline.
190  * .. which happens to be the same behavior as flush_icache_range().
191  * So, we simply flush out a line.
192  */
193 void __uses_jump_to_uncached flush_cache_sigtramp(unsigned long addr)
194 {
195 	unsigned long v, index;
196 	unsigned long flags;
197 	int i;
198 
199 	v = addr & ~(L1_CACHE_BYTES-1);
200 	asm volatile("ocbwb	%0"
201 		     : /* no output */
202 		     : "m" (__m(v)));
203 
204 	index = CACHE_IC_ADDRESS_ARRAY |
205 			(v & boot_cpu_data.icache.entry_mask);
206 
207 	local_irq_save(flags);
208 	jump_to_uncached();
209 
210 	for (i = 0; i < boot_cpu_data.icache.ways;
211 	     i++, index += boot_cpu_data.icache.way_incr)
212 		ctrl_outl(0, index);	/* Clear out Valid-bit */
213 
214 	back_to_cached();
215 	wmb();
216 	local_irq_restore(flags);
217 }
218 
219 static inline void flush_cache_4096(unsigned long start,
220 				    unsigned long phys)
221 {
222 	unsigned long flags, exec_offset = 0;
223 
224 	/*
225 	 * All types of SH-4 require PC to be in P2 to operate on the I-cache.
226 	 * Some types of SH-4 require PC to be in P2 to operate on the D-cache.
227 	 */
228 	if ((boot_cpu_data.flags & CPU_HAS_P2_FLUSH_BUG) ||
229 	    (start < CACHE_OC_ADDRESS_ARRAY))
230 		exec_offset = 0x20000000;
231 
232 	local_irq_save(flags);
233 	__flush_cache_4096(start | SH_CACHE_ASSOC,
234 			   P1SEGADDR(phys), exec_offset);
235 	local_irq_restore(flags);
236 }
237 
238 /*
239  * Write back & invalidate the D-cache of the page.
240  * (To avoid "alias" issues)
241  */
242 void flush_dcache_page(struct page *page)
243 {
244 	if (test_bit(PG_mapped, &page->flags)) {
245 		unsigned long phys = PHYSADDR(page_address(page));
246 		unsigned long addr = CACHE_OC_ADDRESS_ARRAY;
247 		int i, n;
248 
249 		/* Loop all the D-cache */
250 		n = boot_cpu_data.dcache.n_aliases;
251 		for (i = 0; i < n; i++, addr += 4096)
252 			flush_cache_4096(addr, phys);
253 	}
254 
255 	wmb();
256 }
257 
258 /* TODO: Selective icache invalidation through IC address array.. */
259 static inline void __uses_jump_to_uncached flush_icache_all(void)
260 {
261 	unsigned long flags, ccr;
262 
263 	local_irq_save(flags);
264 	jump_to_uncached();
265 
266 	/* Flush I-cache */
267 	ccr = ctrl_inl(CCR);
268 	ccr |= CCR_CACHE_ICI;
269 	ctrl_outl(ccr, CCR);
270 
271 	/*
272 	 * back_to_cached() will take care of the barrier for us, don't add
273 	 * another one!
274 	 */
275 
276 	back_to_cached();
277 	local_irq_restore(flags);
278 }
279 
280 void flush_dcache_all(void)
281 {
282 	(*__flush_dcache_segment_fn)(0UL, boot_cpu_data.dcache.way_size);
283 	wmb();
284 }
285 
286 void flush_cache_all(void)
287 {
288 	flush_dcache_all();
289 	flush_icache_all();
290 }
291 
292 static void __flush_cache_mm(struct mm_struct *mm, unsigned long start,
293 			     unsigned long end)
294 {
295 	unsigned long d = 0, p = start & PAGE_MASK;
296 	unsigned long alias_mask = boot_cpu_data.dcache.alias_mask;
297 	unsigned long n_aliases = boot_cpu_data.dcache.n_aliases;
298 	unsigned long select_bit;
299 	unsigned long all_aliases_mask;
300 	unsigned long addr_offset;
301 	pgd_t *dir;
302 	pmd_t *pmd;
303 	pud_t *pud;
304 	pte_t *pte;
305 	int i;
306 
307 	dir = pgd_offset(mm, p);
308 	pud = pud_offset(dir, p);
309 	pmd = pmd_offset(pud, p);
310 	end = PAGE_ALIGN(end);
311 
312 	all_aliases_mask = (1 << n_aliases) - 1;
313 
314 	do {
315 		if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd))) {
316 			p &= PMD_MASK;
317 			p += PMD_SIZE;
318 			pmd++;
319 
320 			continue;
321 		}
322 
323 		pte = pte_offset_kernel(pmd, p);
324 
325 		do {
326 			unsigned long phys;
327 			pte_t entry = *pte;
328 
329 			if (!(pte_val(entry) & _PAGE_PRESENT)) {
330 				pte++;
331 				p += PAGE_SIZE;
332 				continue;
333 			}
334 
335 			phys = pte_val(entry) & PTE_PHYS_MASK;
336 
337 			if ((p ^ phys) & alias_mask) {
338 				d |= 1 << ((p & alias_mask) >> PAGE_SHIFT);
339 				d |= 1 << ((phys & alias_mask) >> PAGE_SHIFT);
340 
341 				if (d == all_aliases_mask)
342 					goto loop_exit;
343 			}
344 
345 			pte++;
346 			p += PAGE_SIZE;
347 		} while (p < end && ((unsigned long)pte & ~PAGE_MASK));
348 		pmd++;
349 	} while (p < end);
350 
351 loop_exit:
352 	addr_offset = 0;
353 	select_bit = 1;
354 
355 	for (i = 0; i < n_aliases; i++) {
356 		if (d & select_bit) {
357 			(*__flush_dcache_segment_fn)(addr_offset, PAGE_SIZE);
358 			wmb();
359 		}
360 
361 		select_bit <<= 1;
362 		addr_offset += PAGE_SIZE;
363 	}
364 }
365 
366 /*
367  * Note : (RPC) since the caches are physically tagged, the only point
368  * of flush_cache_mm for SH-4 is to get rid of aliases from the
369  * D-cache.  The assumption elsewhere, e.g. flush_cache_range, is that
370  * lines can stay resident so long as the virtual address they were
371  * accessed with (hence cache set) is in accord with the physical
372  * address (i.e. tag).  It's no different here.  So I reckon we don't
373  * need to flush the I-cache, since aliases don't matter for that.  We
374  * should try that.
375  *
376  * Caller takes mm->mmap_sem.
377  */
378 void flush_cache_mm(struct mm_struct *mm)
379 {
380 	/*
381 	 * If cache is only 4k-per-way, there are never any 'aliases'.  Since
382 	 * the cache is physically tagged, the data can just be left in there.
383 	 */
384 	if (boot_cpu_data.dcache.n_aliases == 0)
385 		return;
386 
387 	/*
388 	 * Don't bother groveling around the dcache for the VMA ranges
389 	 * if there are too many PTEs to make it worthwhile.
390 	 */
391 	if (mm->nr_ptes >= MAX_DCACHE_PAGES)
392 		flush_dcache_all();
393 	else {
394 		struct vm_area_struct *vma;
395 
396 		/*
397 		 * In this case there are reasonably sized ranges to flush,
398 		 * iterate through the VMA list and take care of any aliases.
399 		 */
400 		for (vma = mm->mmap; vma; vma = vma->vm_next)
401 			__flush_cache_mm(mm, vma->vm_start, vma->vm_end);
402 	}
403 
404 	/* Only touch the icache if one of the VMAs has VM_EXEC set. */
405 	if (mm->exec_vm)
406 		flush_icache_all();
407 }
408 
409 /*
410  * Write back and invalidate I/D-caches for the page.
411  *
412  * ADDR: Virtual Address (U0 address)
413  * PFN: Physical page number
414  */
415 void flush_cache_page(struct vm_area_struct *vma, unsigned long address,
416 		      unsigned long pfn)
417 {
418 	unsigned long phys = pfn << PAGE_SHIFT;
419 	unsigned int alias_mask;
420 
421 	alias_mask = boot_cpu_data.dcache.alias_mask;
422 
423 	/* We only need to flush D-cache when we have alias */
424 	if ((address^phys) & alias_mask) {
425 		/* Loop 4K of the D-cache */
426 		flush_cache_4096(
427 			CACHE_OC_ADDRESS_ARRAY | (address & alias_mask),
428 			phys);
429 		/* Loop another 4K of the D-cache */
430 		flush_cache_4096(
431 			CACHE_OC_ADDRESS_ARRAY | (phys & alias_mask),
432 			phys);
433 	}
434 
435 	alias_mask = boot_cpu_data.icache.alias_mask;
436 	if (vma->vm_flags & VM_EXEC) {
437 		/*
438 		 * Evict entries from the portion of the cache from which code
439 		 * may have been executed at this address (virtual).  There's
440 		 * no need to evict from the portion corresponding to the
441 		 * physical address as for the D-cache, because we know the
442 		 * kernel has never executed the code through its identity
443 		 * translation.
444 		 */
445 		flush_cache_4096(
446 			CACHE_IC_ADDRESS_ARRAY | (address & alias_mask),
447 			phys);
448 	}
449 }
450 
451 /*
452  * Write back and invalidate D-caches.
453  *
454  * START, END: Virtual Address (U0 address)
455  *
456  * NOTE: We need to flush the _physical_ page entry.
457  * Flushing the cache lines for U0 only isn't enough.
458  * We need to flush for P1 too, which may contain aliases.
459  */
460 void flush_cache_range(struct vm_area_struct *vma, unsigned long start,
461 		       unsigned long end)
462 {
463 	/*
464 	 * If cache is only 4k-per-way, there are never any 'aliases'.  Since
465 	 * the cache is physically tagged, the data can just be left in there.
466 	 */
467 	if (boot_cpu_data.dcache.n_aliases == 0)
468 		return;
469 
470 	/*
471 	 * Don't bother with the lookup and alias check if we have a
472 	 * wide range to cover, just blow away the dcache in its
473 	 * entirety instead. -- PFM.
474 	 */
475 	if (((end - start) >> PAGE_SHIFT) >= MAX_DCACHE_PAGES)
476 		flush_dcache_all();
477 	else
478 		__flush_cache_mm(vma->vm_mm, start, end);
479 
480 	if (vma->vm_flags & VM_EXEC) {
481 		/*
482 		 * TODO: Is this required???  Need to look at how I-cache
483 		 * coherency is assured when new programs are loaded to see if
484 		 * this matters.
485 		 */
486 		flush_icache_all();
487 	}
488 }
489 
490 /*
491  * flush_icache_user_range
492  * @vma: VMA of the process
493  * @page: page
494  * @addr: U0 address
495  * @len: length of the range (< page size)
496  */
497 void flush_icache_user_range(struct vm_area_struct *vma,
498 			     struct page *page, unsigned long addr, int len)
499 {
500 	flush_cache_page(vma, addr, page_to_pfn(page));
501 	mb();
502 }
503 
504 /**
505  * __flush_cache_4096
506  *
507  * @addr:  address in memory mapped cache array
508  * @phys:  P1 address to flush (has to match tags if addr has 'A' bit
509  *         set i.e. associative write)
510  * @exec_offset: set to 0x20000000 if flush has to be executed from P2
511  *               region else 0x0
512  *
513  * The offset into the cache array implied by 'addr' selects the
514  * 'colour' of the virtual address range that will be flushed.  The
515  * operation (purge/write-back) is selected by the lower 2 bits of
516  * 'phys'.
517  */
518 static void __flush_cache_4096(unsigned long addr, unsigned long phys,
519 			       unsigned long exec_offset)
520 {
521 	int way_count;
522 	unsigned long base_addr = addr;
523 	struct cache_info *dcache;
524 	unsigned long way_incr;
525 	unsigned long a, ea, p;
526 	unsigned long temp_pc;
527 
528 	dcache = &boot_cpu_data.dcache;
529 	/* Write this way for better assembly. */
530 	way_count = dcache->ways;
531 	way_incr = dcache->way_incr;
532 
533 	/*
534 	 * Apply exec_offset (i.e. branch to P2 if required.).
535 	 *
536 	 * FIXME:
537 	 *
538 	 *	If I write "=r" for the (temp_pc), it puts this in r6 hence
539 	 *	trashing exec_offset before it's been added on - why?  Hence
540 	 *	"=&r" as a 'workaround'
541 	 */
542 	asm volatile("mov.l 1f, %0\n\t"
543 		     "add   %1, %0\n\t"
544 		     "jmp   @%0\n\t"
545 		     "nop\n\t"
546 		     ".balign 4\n\t"
547 		     "1:  .long 2f\n\t"
548 		     "2:\n" : "=&r" (temp_pc) : "r" (exec_offset));
549 
550 	/*
551 	 * We know there will be >=1 iteration, so write as do-while to avoid
552 	 * pointless nead-of-loop check for 0 iterations.
553 	 */
554 	do {
555 		ea = base_addr + PAGE_SIZE;
556 		a = base_addr;
557 		p = phys;
558 
559 		do {
560 			*(volatile unsigned long *)a = p;
561 			/*
562 			 * Next line: intentionally not p+32, saves an add, p
563 			 * will do since only the cache tag bits need to
564 			 * match.
565 			 */
566 			*(volatile unsigned long *)(a+32) = p;
567 			a += 64;
568 			p += 64;
569 		} while (a < ea);
570 
571 		base_addr += way_incr;
572 	} while (--way_count != 0);
573 }
574 
575 /*
576  * Break the 1, 2 and 4 way variants of this out into separate functions to
577  * avoid nearly all the overhead of having the conditional stuff in the function
578  * bodies (+ the 1 and 2 way cases avoid saving any registers too).
579  */
580 static void __flush_dcache_segment_1way(unsigned long start,
581 					unsigned long extent_per_way)
582 {
583 	unsigned long orig_sr, sr_with_bl;
584 	unsigned long base_addr;
585 	unsigned long way_incr, linesz, way_size;
586 	struct cache_info *dcache;
587 	register unsigned long a0, a0e;
588 
589 	asm volatile("stc sr, %0" : "=r" (orig_sr));
590 	sr_with_bl = orig_sr | (1<<28);
591 	base_addr = ((unsigned long)&empty_zero_page[0]);
592 
593 	/*
594 	 * The previous code aligned base_addr to 16k, i.e. the way_size of all
595 	 * existing SH-4 D-caches.  Whilst I don't see a need to have this
596 	 * aligned to any better than the cache line size (which it will be
597 	 * anyway by construction), let's align it to at least the way_size of
598 	 * any existing or conceivable SH-4 D-cache.  -- RPC
599 	 */
600 	base_addr = ((base_addr >> 16) << 16);
601 	base_addr |= start;
602 
603 	dcache = &boot_cpu_data.dcache;
604 	linesz = dcache->linesz;
605 	way_incr = dcache->way_incr;
606 	way_size = dcache->way_size;
607 
608 	a0 = base_addr;
609 	a0e = base_addr + extent_per_way;
610 	do {
611 		asm volatile("ldc %0, sr" : : "r" (sr_with_bl));
612 		asm volatile("movca.l r0, @%0\n\t"
613 			     "ocbi @%0" : : "r" (a0));
614 		a0 += linesz;
615 		asm volatile("movca.l r0, @%0\n\t"
616 			     "ocbi @%0" : : "r" (a0));
617 		a0 += linesz;
618 		asm volatile("movca.l r0, @%0\n\t"
619 			     "ocbi @%0" : : "r" (a0));
620 		a0 += linesz;
621 		asm volatile("movca.l r0, @%0\n\t"
622 			     "ocbi @%0" : : "r" (a0));
623 		asm volatile("ldc %0, sr" : : "r" (orig_sr));
624 		a0 += linesz;
625 	} while (a0 < a0e);
626 }
627 
628 static void __flush_dcache_segment_2way(unsigned long start,
629 					unsigned long extent_per_way)
630 {
631 	unsigned long orig_sr, sr_with_bl;
632 	unsigned long base_addr;
633 	unsigned long way_incr, linesz, way_size;
634 	struct cache_info *dcache;
635 	register unsigned long a0, a1, a0e;
636 
637 	asm volatile("stc sr, %0" : "=r" (orig_sr));
638 	sr_with_bl = orig_sr | (1<<28);
639 	base_addr = ((unsigned long)&empty_zero_page[0]);
640 
641 	/* See comment under 1-way above */
642 	base_addr = ((base_addr >> 16) << 16);
643 	base_addr |= start;
644 
645 	dcache = &boot_cpu_data.dcache;
646 	linesz = dcache->linesz;
647 	way_incr = dcache->way_incr;
648 	way_size = dcache->way_size;
649 
650 	a0 = base_addr;
651 	a1 = a0 + way_incr;
652 	a0e = base_addr + extent_per_way;
653 	do {
654 		asm volatile("ldc %0, sr" : : "r" (sr_with_bl));
655 		asm volatile("movca.l r0, @%0\n\t"
656 			     "movca.l r0, @%1\n\t"
657 			     "ocbi @%0\n\t"
658 			     "ocbi @%1" : :
659 			     "r" (a0), "r" (a1));
660 		a0 += linesz;
661 		a1 += linesz;
662 		asm volatile("movca.l r0, @%0\n\t"
663 			     "movca.l r0, @%1\n\t"
664 			     "ocbi @%0\n\t"
665 			     "ocbi @%1" : :
666 			     "r" (a0), "r" (a1));
667 		a0 += linesz;
668 		a1 += linesz;
669 		asm volatile("movca.l r0, @%0\n\t"
670 			     "movca.l r0, @%1\n\t"
671 			     "ocbi @%0\n\t"
672 			     "ocbi @%1" : :
673 			     "r" (a0), "r" (a1));
674 		a0 += linesz;
675 		a1 += linesz;
676 		asm volatile("movca.l r0, @%0\n\t"
677 			     "movca.l r0, @%1\n\t"
678 			     "ocbi @%0\n\t"
679 			     "ocbi @%1" : :
680 			     "r" (a0), "r" (a1));
681 		asm volatile("ldc %0, sr" : : "r" (orig_sr));
682 		a0 += linesz;
683 		a1 += linesz;
684 	} while (a0 < a0e);
685 }
686 
687 static void __flush_dcache_segment_4way(unsigned long start,
688 					unsigned long extent_per_way)
689 {
690 	unsigned long orig_sr, sr_with_bl;
691 	unsigned long base_addr;
692 	unsigned long way_incr, linesz, way_size;
693 	struct cache_info *dcache;
694 	register unsigned long a0, a1, a2, a3, a0e;
695 
696 	asm volatile("stc sr, %0" : "=r" (orig_sr));
697 	sr_with_bl = orig_sr | (1<<28);
698 	base_addr = ((unsigned long)&empty_zero_page[0]);
699 
700 	/* See comment under 1-way above */
701 	base_addr = ((base_addr >> 16) << 16);
702 	base_addr |= start;
703 
704 	dcache = &boot_cpu_data.dcache;
705 	linesz = dcache->linesz;
706 	way_incr = dcache->way_incr;
707 	way_size = dcache->way_size;
708 
709 	a0 = base_addr;
710 	a1 = a0 + way_incr;
711 	a2 = a1 + way_incr;
712 	a3 = a2 + way_incr;
713 	a0e = base_addr + extent_per_way;
714 	do {
715 		asm volatile("ldc %0, sr" : : "r" (sr_with_bl));
716 		asm volatile("movca.l r0, @%0\n\t"
717 			     "movca.l r0, @%1\n\t"
718 			     "movca.l r0, @%2\n\t"
719 			     "movca.l r0, @%3\n\t"
720 			     "ocbi @%0\n\t"
721 			     "ocbi @%1\n\t"
722 			     "ocbi @%2\n\t"
723 			     "ocbi @%3\n\t" : :
724 			     "r" (a0), "r" (a1), "r" (a2), "r" (a3));
725 		a0 += linesz;
726 		a1 += linesz;
727 		a2 += linesz;
728 		a3 += linesz;
729 		asm volatile("movca.l r0, @%0\n\t"
730 			     "movca.l r0, @%1\n\t"
731 			     "movca.l r0, @%2\n\t"
732 			     "movca.l r0, @%3\n\t"
733 			     "ocbi @%0\n\t"
734 			     "ocbi @%1\n\t"
735 			     "ocbi @%2\n\t"
736 			     "ocbi @%3\n\t" : :
737 			     "r" (a0), "r" (a1), "r" (a2), "r" (a3));
738 		a0 += linesz;
739 		a1 += linesz;
740 		a2 += linesz;
741 		a3 += linesz;
742 		asm volatile("movca.l r0, @%0\n\t"
743 			     "movca.l r0, @%1\n\t"
744 			     "movca.l r0, @%2\n\t"
745 			     "movca.l r0, @%3\n\t"
746 			     "ocbi @%0\n\t"
747 			     "ocbi @%1\n\t"
748 			     "ocbi @%2\n\t"
749 			     "ocbi @%3\n\t" : :
750 			     "r" (a0), "r" (a1), "r" (a2), "r" (a3));
751 		a0 += linesz;
752 		a1 += linesz;
753 		a2 += linesz;
754 		a3 += linesz;
755 		asm volatile("movca.l r0, @%0\n\t"
756 			     "movca.l r0, @%1\n\t"
757 			     "movca.l r0, @%2\n\t"
758 			     "movca.l r0, @%3\n\t"
759 			     "ocbi @%0\n\t"
760 			     "ocbi @%1\n\t"
761 			     "ocbi @%2\n\t"
762 			     "ocbi @%3\n\t" : :
763 			     "r" (a0), "r" (a1), "r" (a2), "r" (a3));
764 		asm volatile("ldc %0, sr" : : "r" (orig_sr));
765 		a0 += linesz;
766 		a1 += linesz;
767 		a2 += linesz;
768 		a3 += linesz;
769 	} while (a0 < a0e);
770 }
771