56 static inline void psb_mmu_clflush(struct psb_mmu_driver *driver, void *addr)  in psb_mmu_clflush()  argument
58 	if (!driver->has_clflush)  in psb_mmu_clflush()
66 static void psb_mmu_flush_pd_locked(struct psb_mmu_driver *driver, int force)  in psb_mmu_flush_pd_locked()  argument
68 	struct drm_device *dev = driver->dev;  in psb_mmu_flush_pd_locked()
71 	if (atomic_read(&driver->needs_tlbflush) || force) {  in psb_mmu_flush_pd_locked()
79 		if (driver->msvdx_mmu_invaldc)  in psb_mmu_flush_pd_locked()
80 			atomic_set(driver->msvdx_mmu_invaldc, 1);  in psb_mmu_flush_pd_locked()
82 	atomic_set(&driver->needs_tlbflush, 0);  in psb_mmu_flush_pd_locked()
86 static void psb_mmu_flush_pd(struct psb_mmu_driver *driver, int force)
88 	down_write(&driver->sem);
89 	psb_mmu_flush_pd_locked(driver, force);
90 	up_write(&driver->sem);
94 void psb_mmu_flush(struct psb_mmu_driver *driver)  in psb_mmu_flush()  argument
96 	struct drm_device *dev = driver->dev;  in psb_mmu_flush()
100 	down_write(&driver->sem);  in psb_mmu_flush()
102 	if (atomic_read(&driver->needs_tlbflush))  in psb_mmu_flush()
114 	atomic_set(&driver->needs_tlbflush, 0);  in psb_mmu_flush()
115 	if (driver->msvdx_mmu_invaldc)  in psb_mmu_flush()
116 		atomic_set(driver->msvdx_mmu_invaldc, 1);  in psb_mmu_flush()
117 	up_write(&driver->sem);  in psb_mmu_flush()
122 	struct drm_device *dev = pd->driver->dev;  in psb_mmu_set_pd_context()
127 	down_write(&pd->driver->sem);  in psb_mmu_set_pd_context()
130 	psb_mmu_flush_pd_locked(pd->driver, 1);  in psb_mmu_set_pd_context()
132 	up_write(&pd->driver->sem);  in psb_mmu_set_pd_context()
157 struct psb_mmu_pd *psb_mmu_alloc_pd(struct psb_mmu_driver *driver,  in psb_mmu_alloc_pd()  argument
208 	pd->driver = driver;  in psb_mmu_alloc_pd()
231 	struct psb_mmu_driver *driver = pd->driver;  in psb_mmu_free_pagedir()  local
232 	struct drm_device *dev = driver->dev;  in psb_mmu_free_pagedir()
237 	down_write(&driver->sem);  in psb_mmu_free_pagedir()
240 		psb_mmu_flush_pd_locked(driver, 1);  in psb_mmu_free_pagedir()
257 	up_write(&driver->sem);  in psb_mmu_free_pagedir()
264 	uint32_t clflush_add = pd->driver->clflush_add >> PAGE_SHIFT;  in psb_mmu_alloc_pt()
266 	spinlock_t *lock = &pd->driver->lock;  in psb_mmu_alloc_pt()
288 	if (pd->driver->has_clflush && pd->hw_context != -1) {  in psb_mmu_alloc_pt()
312 	spinlock_t *lock = &pd->driver->lock;  in psb_mmu_pt_alloc_map_lock()
338 			psb_mmu_clflush(pd->driver, (void *)&v[index]);  in psb_mmu_pt_alloc_map_lock()
339 			atomic_set(&pd->driver->needs_tlbflush, 1);  in psb_mmu_pt_alloc_map_lock()
351 	spinlock_t *lock = &pd->driver->lock;  in psb_mmu_pt_map_lock()
375 			psb_mmu_clflush(pd->driver, (void *)&v[pt->index]);  in psb_mmu_pt_unmap_unlock()
376 			atomic_set(&pd->driver->needs_tlbflush, 1);  in psb_mmu_pt_unmap_unlock()
379 		spin_unlock(&pd->driver->lock);  in psb_mmu_pt_unmap_unlock()
383 	spin_unlock(&pd->driver->lock);  in psb_mmu_pt_unmap_unlock()
398 struct psb_mmu_pd *psb_mmu_get_default_pd(struct psb_mmu_driver *driver)  in psb_mmu_get_default_pd()  argument
402 	down_read(&driver->sem);  in psb_mmu_get_default_pd()
403 	pd = driver->default_pd;  in psb_mmu_get_default_pd()
404 	up_read(&driver->sem);  in psb_mmu_get_default_pd()
409 void psb_mmu_driver_takedown(struct psb_mmu_driver *driver)  in psb_mmu_driver_takedown()  argument
411 	struct drm_device *dev = driver->dev;  in psb_mmu_driver_takedown()
414 	PSB_WSGX32(driver->bif_ctrl, PSB_CR_BIF_CTRL);  in psb_mmu_driver_takedown()
415 	psb_mmu_free_pagedir(driver->default_pd);  in psb_mmu_driver_takedown()
416 	kfree(driver);  in psb_mmu_driver_takedown()
424 	struct psb_mmu_driver *driver;  in psb_mmu_driver_init()  local
427 	driver = kmalloc(sizeof(*driver), GFP_KERNEL);  in psb_mmu_driver_init()
429 	if (!driver)  in psb_mmu_driver_init()
432 	driver->dev = dev;  in psb_mmu_driver_init()
433 	driver->default_pd = psb_mmu_alloc_pd(driver, trap_pagefaults,  in psb_mmu_driver_init()
435 	if (!driver->default_pd)  in psb_mmu_driver_init()
438 	spin_lock_init(&driver->lock);  in psb_mmu_driver_init()
439 	init_rwsem(&driver->sem);  in psb_mmu_driver_init()
440 	down_write(&driver->sem);  in psb_mmu_driver_init()
441 	atomic_set(&driver->needs_tlbflush, 1);  in psb_mmu_driver_init()
442 	driver->msvdx_mmu_invaldc = msvdx_mmu_invaldc;  in psb_mmu_driver_init()
444 	driver->bif_ctrl = PSB_RSGX32(PSB_CR_BIF_CTRL);  in psb_mmu_driver_init()
445 	PSB_WSGX32(driver->bif_ctrl | _PSB_CB_CTRL_CLEAR_FAULT,  in psb_mmu_driver_init()
447 	PSB_WSGX32(driver->bif_ctrl & ~_PSB_CB_CTRL_CLEAR_FAULT,  in psb_mmu_driver_init()
450 	driver->has_clflush = 0;  in psb_mmu_driver_init()
462 		driver->has_clflush = 1;  in psb_mmu_driver_init()
463 		driver->clflush_add =  in psb_mmu_driver_init()
465 		driver->clflush_mask = driver->clflush_add - 1;  in psb_mmu_driver_init()
466 		driver->clflush_mask = ~driver->clflush_mask;  in psb_mmu_driver_init()
469 	up_write(&driver->sem);  in psb_mmu_driver_init()
470 	return driver;  in psb_mmu_driver_init()
473 	kfree(driver);  in psb_mmu_driver_init()
489 	unsigned long clflush_add = pd->driver->clflush_add;  in psb_mmu_flush_ptes()
490 	unsigned long clflush_mask = pd->driver->clflush_mask;  in psb_mmu_flush_ptes()
492 	if (!pd->driver->has_clflush)  in psb_mmu_flush_ptes()
534 	down_read(&pd->driver->sem);  in psb_mmu_remove_pfn_sequence()
556 	up_read(&pd->driver->sem);  in psb_mmu_remove_pfn_sequence()
559 		psb_mmu_flush(pd->driver);  in psb_mmu_remove_pfn_sequence()
586 	down_read(&pd->driver->sem);  in psb_mmu_remove_pages()
614 	up_read(&pd->driver->sem);  in psb_mmu_remove_pages()
617 		psb_mmu_flush(pd->driver);  in psb_mmu_remove_pages()
632 	down_read(&pd->driver->sem);  in psb_mmu_insert_pfn_sequence()
658 	up_read(&pd->driver->sem);  in psb_mmu_insert_pfn_sequence()
661 		psb_mmu_flush(pd->driver);  in psb_mmu_insert_pfn_sequence()
694 	down_read(&pd->driver->sem);  in psb_mmu_insert_pages()
725 	up_read(&pd->driver->sem);  in psb_mmu_insert_pages()
728 		psb_mmu_flush(pd->driver);  in psb_mmu_insert_pages()
739 	spinlock_t *lock = &pd->driver->lock;  in psb_mmu_virtual_to_pfn()
741 	down_read(&pd->driver->sem);  in psb_mmu_virtual_to_pfn()
770 	up_read(&pd->driver->sem);  in psb_mmu_virtual_to_pfn()