62 int dax_add_host(struct dax_device *dax_dev, struct gendisk *disk)  in dax_add_host()  argument
64 	return xa_insert(&dax_hosts, (unsigned long)disk, dax_dev, GFP_KERNEL);  in dax_add_host()
84 	struct dax_device *dax_dev;  in fs_dax_get_by_bdev()  local
99 	dax_dev = xa_load(&dax_hosts, (unsigned long)bdev->bd_disk);  in fs_dax_get_by_bdev()
100 	if (!dax_dev || !dax_alive(dax_dev) || !igrab(&dax_dev->inode))  in fs_dax_get_by_bdev()
101 		dax_dev = NULL;  in fs_dax_get_by_bdev()
103 		if (!cmpxchg(&dax_dev->holder_data, NULL, holder))  in fs_dax_get_by_bdev()
104 			dax_dev->holder_ops = ops;  in fs_dax_get_by_bdev()
106 			dax_dev = NULL;  in fs_dax_get_by_bdev()
110 	return dax_dev;  in fs_dax_get_by_bdev()
114 void fs_put_dax(struct dax_device *dax_dev, void *holder)  in fs_put_dax()  argument
116 	if (dax_dev && holder &&  in fs_put_dax()
117 	    cmpxchg(&dax_dev->holder_data, holder, NULL) == holder)  in fs_put_dax()
118 		dax_dev->holder_ops = NULL;  in fs_put_dax()
119 	put_dax(dax_dev);  in fs_put_dax()
149 long dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, long nr_pages,  in dax_direct_access()  argument
154 	if (!dax_dev)  in dax_direct_access()
157 	if (!dax_alive(dax_dev))  in dax_direct_access()
163 	avail = dax_dev->ops->direct_access(dax_dev, pgoff, nr_pages,  in dax_direct_access()
171 size_t dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr,  in dax_copy_from_iter()  argument
174 	if (!dax_alive(dax_dev))  in dax_copy_from_iter()
182 	if (test_bit(DAXDEV_NOCACHE, &dax_dev->flags))  in dax_copy_from_iter()
187 size_t dax_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr,  in dax_copy_to_iter()  argument
190 	if (!dax_alive(dax_dev))  in dax_copy_to_iter()
198 	if (test_bit(DAXDEV_NOMC, &dax_dev->flags))  in dax_copy_to_iter()
203 int dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff,  in dax_zero_page_range()  argument
208 	if (!dax_alive(dax_dev))  in dax_zero_page_range()
218 	ret = dax_dev->ops->zero_page_range(dax_dev, pgoff, nr_pages);  in dax_zero_page_range()
223 size_t dax_recovery_write(struct dax_device *dax_dev, pgoff_t pgoff,  in dax_recovery_write()  argument
226 	if (!dax_dev->ops->recovery_write)  in dax_recovery_write()
228 	return dax_dev->ops->recovery_write(dax_dev, pgoff, addr, bytes, iter);  in dax_recovery_write()
232 int dax_holder_notify_failure(struct dax_device *dax_dev, u64 off,  in dax_holder_notify_failure()  argument
238 	if (!dax_alive(dax_dev)) {  in dax_holder_notify_failure()
243 	if (!dax_dev->holder_ops) {  in dax_holder_notify_failure()
248 	rc = dax_dev->holder_ops->notify_failure(dax_dev, off, len, mf_flags);  in dax_holder_notify_failure()
257 void dax_flush(struct dax_device *dax_dev, void *addr, size_t size)  in dax_flush()  argument
259 	if (unlikely(!dax_write_cache_enabled(dax_dev)))  in dax_flush()
265 void dax_flush(struct dax_device *dax_dev, void *addr, size_t size)  in dax_flush()  argument
271 void dax_write_cache(struct dax_device *dax_dev, bool wc)  in dax_write_cache()  argument
274 		set_bit(DAXDEV_WRITE_CACHE, &dax_dev->flags);  in dax_write_cache()
276 		clear_bit(DAXDEV_WRITE_CACHE, &dax_dev->flags);  in dax_write_cache()
280 bool dax_write_cache_enabled(struct dax_device *dax_dev)  in dax_write_cache_enabled()  argument
282 	return test_bit(DAXDEV_WRITE_CACHE, &dax_dev->flags);  in dax_write_cache_enabled()
286 bool dax_synchronous(struct dax_device *dax_dev)  in dax_synchronous()  argument
288 	return test_bit(DAXDEV_SYNC, &dax_dev->flags);  in dax_synchronous()
292 void set_dax_synchronous(struct dax_device *dax_dev)  in set_dax_synchronous()  argument
294 	set_bit(DAXDEV_SYNC, &dax_dev->flags);  in set_dax_synchronous()
298 void set_dax_nocache(struct dax_device *dax_dev)  in set_dax_nocache()  argument
300 	set_bit(DAXDEV_NOCACHE, &dax_dev->flags);  in set_dax_nocache()
304 void set_dax_nomc(struct dax_device *dax_dev)  in set_dax_nomc()  argument
306 	set_bit(DAXDEV_NOMC, &dax_dev->flags);  in set_dax_nomc()
310 bool dax_alive(struct dax_device *dax_dev)  in dax_alive()  argument
313 	return test_bit(DAXDEV_ALIVE, &dax_dev->flags);  in dax_alive()
323 void kill_dax(struct dax_device *dax_dev)  in kill_dax()  argument
325 	if (!dax_dev)  in kill_dax()
328 	if (dax_dev->holder_data != NULL)  in kill_dax()
329 		dax_holder_notify_failure(dax_dev, 0, U64_MAX, 0);  in kill_dax()
331 	clear_bit(DAXDEV_ALIVE, &dax_dev->flags);  in kill_dax()
335 	dax_dev->holder_ops = NULL;  in kill_dax()
336 	dax_dev->holder_data = NULL;  in kill_dax()
340 void run_dax(struct dax_device *dax_dev)  in run_dax()  argument
342 	set_bit(DAXDEV_ALIVE, &dax_dev->flags);  in run_dax()
348 	struct dax_device *dax_dev;  in dax_alloc_inode()  local
351 	dax_dev = alloc_inode_sb(sb, dax_cache, GFP_KERNEL);  in dax_alloc_inode()
352 	if (!dax_dev)  in dax_alloc_inode()
355 	inode = &dax_dev->inode;  in dax_alloc_inode()
367 	struct dax_device *dax_dev = to_dax_dev(inode);  in dax_free_inode()  local
370 	kmem_cache_free(dax_cache, dax_dev);  in dax_free_inode()
375 	struct dax_device *dax_dev = to_dax_dev(inode);  in dax_destroy_inode()  local
376 	WARN_ONCE(test_bit(DAXDEV_ALIVE, &dax_dev->flags),  in dax_destroy_inode()
420 	struct dax_device *dax_dev;  in dax_dev_get()  local
429 	dax_dev = to_dax_dev(inode);  in dax_dev_get()
431 		set_bit(DAXDEV_ALIVE, &dax_dev->flags);  in dax_dev_get()
432 		inode->i_cdev = &dax_dev->cdev;  in dax_dev_get()
439 	return dax_dev;  in dax_dev_get()
444 	struct dax_device *dax_dev;  in alloc_dax()  local
456 	dax_dev = dax_dev_get(devt);  in alloc_dax()
457 	if (!dax_dev)  in alloc_dax()
460 	dax_dev->ops = ops;  in alloc_dax()
461 	dax_dev->private = private;  in alloc_dax()
462 	return dax_dev;  in alloc_dax()
470 void put_dax(struct dax_device *dax_dev)  in put_dax()  argument
472 	if (!dax_dev)  in put_dax()
474 	iput(&dax_dev->inode);  in put_dax()
485 void *dax_holder(struct dax_device *dax_dev)  in dax_holder()  argument
487 	return dax_dev->holder_data;  in dax_holder()
506 struct inode *dax_inode(struct dax_device *dax_dev)  in dax_inode()  argument
508 	return &dax_dev->inode;  in dax_inode()
512 void *dax_get_private(struct dax_device *dax_dev)  in dax_get_private()  argument
514 	if (!test_bit(DAXDEV_ALIVE, &dax_dev->flags))  in dax_get_private()
516 	return dax_dev->private;  in dax_get_private()
522 	struct dax_device *dax_dev = _dax_dev;  in init_once()  local
523 	struct inode *inode = &dax_dev->inode;  in init_once()
525 	memset(dax_dev, 0, sizeof(*dax_dev));  in init_once()