xref: /openbmc/linux/drivers/dax/super.c (revision 273752c9)
17b6be844SDan Williams /*
27b6be844SDan Williams  * Copyright(c) 2017 Intel Corporation. All rights reserved.
37b6be844SDan Williams  *
47b6be844SDan Williams  * This program is free software; you can redistribute it and/or modify
57b6be844SDan Williams  * it under the terms of version 2 of the GNU General Public License as
67b6be844SDan Williams  * published by the Free Software Foundation.
77b6be844SDan Williams  *
87b6be844SDan Williams  * This program is distributed in the hope that it will be useful, but
97b6be844SDan Williams  * WITHOUT ANY WARRANTY; without even the implied warranty of
107b6be844SDan Williams  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
117b6be844SDan Williams  * General Public License for more details.
127b6be844SDan Williams  */
137b6be844SDan Williams #include <linux/pagemap.h>
147b6be844SDan Williams #include <linux/module.h>
157b6be844SDan Williams #include <linux/mount.h>
167b6be844SDan Williams #include <linux/magic.h>
17ef510424SDan Williams #include <linux/genhd.h>
187b6be844SDan Williams #include <linux/cdev.h>
197b6be844SDan Williams #include <linux/hash.h>
207b6be844SDan Williams #include <linux/slab.h>
217e026c8cSDan Williams #include <linux/uio.h>
226568b08bSDan Williams #include <linux/dax.h>
237b6be844SDan Williams #include <linux/fs.h>
247b6be844SDan Williams 
257b6be844SDan Williams static dev_t dax_devt;
267b6be844SDan Williams DEFINE_STATIC_SRCU(dax_srcu);
277b6be844SDan Williams static struct vfsmount *dax_mnt;
287b6be844SDan Williams static DEFINE_IDA(dax_minor_ida);
297b6be844SDan Williams static struct kmem_cache *dax_cache __read_mostly;
307b6be844SDan Williams static struct super_block *dax_superblock __read_mostly;
317b6be844SDan Williams 
3272058005SDan Williams #define DAX_HASH_SIZE (PAGE_SIZE / sizeof(struct hlist_head))
3372058005SDan Williams static struct hlist_head dax_host_list[DAX_HASH_SIZE];
3472058005SDan Williams static DEFINE_SPINLOCK(dax_host_lock);
3572058005SDan Williams 
367b6be844SDan Williams int dax_read_lock(void)
377b6be844SDan Williams {
387b6be844SDan Williams 	return srcu_read_lock(&dax_srcu);
397b6be844SDan Williams }
407b6be844SDan Williams EXPORT_SYMBOL_GPL(dax_read_lock);
417b6be844SDan Williams 
427b6be844SDan Williams void dax_read_unlock(int id)
437b6be844SDan Williams {
447b6be844SDan Williams 	srcu_read_unlock(&dax_srcu, id);
457b6be844SDan Williams }
467b6be844SDan Williams EXPORT_SYMBOL_GPL(dax_read_unlock);
477b6be844SDan Williams 
489d109081SDan Williams #ifdef CONFIG_BLOCK
49ef510424SDan Williams int bdev_dax_pgoff(struct block_device *bdev, sector_t sector, size_t size,
50ef510424SDan Williams 		pgoff_t *pgoff)
51ef510424SDan Williams {
52ef510424SDan Williams 	phys_addr_t phys_off = (get_start_sect(bdev) + sector) * 512;
53ef510424SDan Williams 
54ef510424SDan Williams 	if (pgoff)
55ef510424SDan Williams 		*pgoff = PHYS_PFN(phys_off);
56ef510424SDan Williams 	if (phys_off % PAGE_SIZE || size % PAGE_SIZE)
57ef510424SDan Williams 		return -EINVAL;
58ef510424SDan Williams 	return 0;
59ef510424SDan Williams }
60ef510424SDan Williams EXPORT_SYMBOL(bdev_dax_pgoff);
61ef510424SDan Williams 
62ef510424SDan Williams /**
63ef510424SDan Williams  * __bdev_dax_supported() - Check if the device supports dax for filesystem
64ef510424SDan Williams  * @sb: The superblock of the device
65ef510424SDan Williams  * @blocksize: The block size of the device
66ef510424SDan Williams  *
67ef510424SDan Williams  * This is a library function for filesystems to check if the block device
68ef510424SDan Williams  * can be mounted with dax option.
69ef510424SDan Williams  *
70ef510424SDan Williams  * Return: negative errno if unsupported, 0 if supported.
71ef510424SDan Williams  */
72ef510424SDan Williams int __bdev_dax_supported(struct super_block *sb, int blocksize)
73ef510424SDan Williams {
74ef510424SDan Williams 	struct block_device *bdev = sb->s_bdev;
75ef510424SDan Williams 	struct dax_device *dax_dev;
76ef510424SDan Williams 	pgoff_t pgoff;
77ef510424SDan Williams 	int err, id;
78ef510424SDan Williams 	void *kaddr;
79ef510424SDan Williams 	pfn_t pfn;
80ef510424SDan Williams 	long len;
81ef510424SDan Williams 
82ef510424SDan Williams 	if (blocksize != PAGE_SIZE) {
83ef510424SDan Williams 		pr_err("VFS (%s): error: unsupported blocksize for dax\n",
84ef510424SDan Williams 				sb->s_id);
85ef510424SDan Williams 		return -EINVAL;
86ef510424SDan Williams 	}
87ef510424SDan Williams 
88ef510424SDan Williams 	err = bdev_dax_pgoff(bdev, 0, PAGE_SIZE, &pgoff);
89ef510424SDan Williams 	if (err) {
90ef510424SDan Williams 		pr_err("VFS (%s): error: unaligned partition for dax\n",
91ef510424SDan Williams 				sb->s_id);
92ef510424SDan Williams 		return err;
93ef510424SDan Williams 	}
94ef510424SDan Williams 
95ef510424SDan Williams 	dax_dev = dax_get_by_host(bdev->bd_disk->disk_name);
96ef510424SDan Williams 	if (!dax_dev) {
97ef510424SDan Williams 		pr_err("VFS (%s): error: device does not support dax\n",
98ef510424SDan Williams 				sb->s_id);
99ef510424SDan Williams 		return -EOPNOTSUPP;
100ef510424SDan Williams 	}
101ef510424SDan Williams 
102ef510424SDan Williams 	id = dax_read_lock();
103ef510424SDan Williams 	len = dax_direct_access(dax_dev, pgoff, 1, &kaddr, &pfn);
104ef510424SDan Williams 	dax_read_unlock(id);
105ef510424SDan Williams 
106ef510424SDan Williams 	put_dax(dax_dev);
107ef510424SDan Williams 
108ef510424SDan Williams 	if (len < 1) {
109ef510424SDan Williams 		pr_err("VFS (%s): error: dax access failed (%ld)",
110ef510424SDan Williams 				sb->s_id, len);
111ef510424SDan Williams 		return len < 0 ? len : -EIO;
112ef510424SDan Williams 	}
113ef510424SDan Williams 
114ef510424SDan Williams 	return 0;
115ef510424SDan Williams }
116ef510424SDan Williams EXPORT_SYMBOL_GPL(__bdev_dax_supported);
1179d109081SDan Williams #endif
118ef510424SDan Williams 
1199a60c3efSDan Williams enum dax_device_flags {
1209a60c3efSDan Williams 	/* !alive + rcu grace period == no new operations / mappings */
1219a60c3efSDan Williams 	DAXDEV_ALIVE,
1226e0c90d6SDan Williams 	/* gate whether dax_flush() calls the low level flush routine */
1236e0c90d6SDan Williams 	DAXDEV_WRITE_CACHE,
1249a60c3efSDan Williams };
1259a60c3efSDan Williams 
1267b6be844SDan Williams /**
1277b6be844SDan Williams  * struct dax_device - anchor object for dax services
1287b6be844SDan Williams  * @inode: core vfs
1297b6be844SDan Williams  * @cdev: optional character interface for "device dax"
13072058005SDan Williams  * @host: optional name for lookups where the device path is not available
1317b6be844SDan Williams  * @private: dax driver private data
1329a60c3efSDan Williams  * @flags: state and boolean properties
1337b6be844SDan Williams  */
1347b6be844SDan Williams struct dax_device {
13572058005SDan Williams 	struct hlist_node list;
1367b6be844SDan Williams 	struct inode inode;
1377b6be844SDan Williams 	struct cdev cdev;
13872058005SDan Williams 	const char *host;
1397b6be844SDan Williams 	void *private;
1409a60c3efSDan Williams 	unsigned long flags;
1416568b08bSDan Williams 	const struct dax_operations *ops;
1427b6be844SDan Williams };
1437b6be844SDan Williams 
1446e0c90d6SDan Williams static ssize_t write_cache_show(struct device *dev,
1456e0c90d6SDan Williams 		struct device_attribute *attr, char *buf)
1466e0c90d6SDan Williams {
1476e0c90d6SDan Williams 	struct dax_device *dax_dev = dax_get_by_host(dev_name(dev));
1486e0c90d6SDan Williams 	ssize_t rc;
1496e0c90d6SDan Williams 
1506e0c90d6SDan Williams 	WARN_ON_ONCE(!dax_dev);
1516e0c90d6SDan Williams 	if (!dax_dev)
1526e0c90d6SDan Williams 		return -ENXIO;
1536e0c90d6SDan Williams 
1546e0c90d6SDan Williams 	rc = sprintf(buf, "%d\n", !!test_bit(DAXDEV_WRITE_CACHE,
1556e0c90d6SDan Williams 				&dax_dev->flags));
1566e0c90d6SDan Williams 	put_dax(dax_dev);
1576e0c90d6SDan Williams 	return rc;
1586e0c90d6SDan Williams }
1596e0c90d6SDan Williams 
1606e0c90d6SDan Williams static ssize_t write_cache_store(struct device *dev,
1616e0c90d6SDan Williams 		struct device_attribute *attr, const char *buf, size_t len)
1626e0c90d6SDan Williams {
1636e0c90d6SDan Williams 	bool write_cache;
1646e0c90d6SDan Williams 	int rc = strtobool(buf, &write_cache);
1656e0c90d6SDan Williams 	struct dax_device *dax_dev = dax_get_by_host(dev_name(dev));
1666e0c90d6SDan Williams 
1676e0c90d6SDan Williams 	WARN_ON_ONCE(!dax_dev);
1686e0c90d6SDan Williams 	if (!dax_dev)
1696e0c90d6SDan Williams 		return -ENXIO;
1706e0c90d6SDan Williams 
1716e0c90d6SDan Williams 	if (rc)
1726e0c90d6SDan Williams 		len = rc;
1736e0c90d6SDan Williams 	else if (write_cache)
1746e0c90d6SDan Williams 		set_bit(DAXDEV_WRITE_CACHE, &dax_dev->flags);
1756e0c90d6SDan Williams 	else
1766e0c90d6SDan Williams 		clear_bit(DAXDEV_WRITE_CACHE, &dax_dev->flags);
1776e0c90d6SDan Williams 
1786e0c90d6SDan Williams 	put_dax(dax_dev);
1796e0c90d6SDan Williams 	return len;
1806e0c90d6SDan Williams }
1816e0c90d6SDan Williams static DEVICE_ATTR_RW(write_cache);
1826e0c90d6SDan Williams 
1836e0c90d6SDan Williams static umode_t dax_visible(struct kobject *kobj, struct attribute *a, int n)
1846e0c90d6SDan Williams {
1856e0c90d6SDan Williams 	struct device *dev = container_of(kobj, typeof(*dev), kobj);
1866e0c90d6SDan Williams 	struct dax_device *dax_dev = dax_get_by_host(dev_name(dev));
1876e0c90d6SDan Williams 
1886e0c90d6SDan Williams 	WARN_ON_ONCE(!dax_dev);
1896e0c90d6SDan Williams 	if (!dax_dev)
1906e0c90d6SDan Williams 		return 0;
1916e0c90d6SDan Williams 
1926e0c90d6SDan Williams 	if (a == &dev_attr_write_cache.attr && !dax_dev->ops->flush)
1936e0c90d6SDan Williams 		return 0;
1946e0c90d6SDan Williams 	return a->mode;
1956e0c90d6SDan Williams }
1966e0c90d6SDan Williams 
1976e0c90d6SDan Williams static struct attribute *dax_attributes[] = {
1986e0c90d6SDan Williams 	&dev_attr_write_cache.attr,
1996e0c90d6SDan Williams 	NULL,
2006e0c90d6SDan Williams };
2016e0c90d6SDan Williams 
2026e0c90d6SDan Williams struct attribute_group dax_attribute_group = {
2036e0c90d6SDan Williams 	.name = "dax",
2046e0c90d6SDan Williams 	.attrs = dax_attributes,
2056e0c90d6SDan Williams 	.is_visible = dax_visible,
2066e0c90d6SDan Williams };
2076e0c90d6SDan Williams EXPORT_SYMBOL_GPL(dax_attribute_group);
2086e0c90d6SDan Williams 
209b0686260SDan Williams /**
210b0686260SDan Williams  * dax_direct_access() - translate a device pgoff to an absolute pfn
211b0686260SDan Williams  * @dax_dev: a dax_device instance representing the logical memory range
212b0686260SDan Williams  * @pgoff: offset in pages from the start of the device to translate
213b0686260SDan Williams  * @nr_pages: number of consecutive pages caller can handle relative to @pfn
214b0686260SDan Williams  * @kaddr: output parameter that returns a virtual address mapping of pfn
215b0686260SDan Williams  * @pfn: output parameter that returns an absolute pfn translation of @pgoff
216b0686260SDan Williams  *
217b0686260SDan Williams  * Return: negative errno if an error occurs, otherwise the number of
218b0686260SDan Williams  * pages accessible at the device relative @pgoff.
219b0686260SDan Williams  */
220b0686260SDan Williams long dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, long nr_pages,
221b0686260SDan Williams 		void **kaddr, pfn_t *pfn)
222b0686260SDan Williams {
223b0686260SDan Williams 	long avail;
224b0686260SDan Williams 
225b0686260SDan Williams 	/*
226b0686260SDan Williams 	 * The device driver is allowed to sleep, in order to make the
227b0686260SDan Williams 	 * memory directly accessible.
228b0686260SDan Williams 	 */
229b0686260SDan Williams 	might_sleep();
230b0686260SDan Williams 
231b0686260SDan Williams 	if (!dax_dev)
232b0686260SDan Williams 		return -EOPNOTSUPP;
233b0686260SDan Williams 
234b0686260SDan Williams 	if (!dax_alive(dax_dev))
235b0686260SDan Williams 		return -ENXIO;
236b0686260SDan Williams 
237b0686260SDan Williams 	if (nr_pages < 0)
238b0686260SDan Williams 		return nr_pages;
239b0686260SDan Williams 
240b0686260SDan Williams 	avail = dax_dev->ops->direct_access(dax_dev, pgoff, nr_pages,
241b0686260SDan Williams 			kaddr, pfn);
242b0686260SDan Williams 	if (!avail)
243b0686260SDan Williams 		return -ERANGE;
244b0686260SDan Williams 	return min(avail, nr_pages);
245b0686260SDan Williams }
246b0686260SDan Williams EXPORT_SYMBOL_GPL(dax_direct_access);
247b0686260SDan Williams 
2487e026c8cSDan Williams size_t dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr,
2497e026c8cSDan Williams 		size_t bytes, struct iov_iter *i)
2507e026c8cSDan Williams {
2517e026c8cSDan Williams 	if (!dax_alive(dax_dev))
2527e026c8cSDan Williams 		return 0;
2537e026c8cSDan Williams 
2547e026c8cSDan Williams 	return dax_dev->ops->copy_from_iter(dax_dev, pgoff, addr, bytes, i);
2557e026c8cSDan Williams }
2567e026c8cSDan Williams EXPORT_SYMBOL_GPL(dax_copy_from_iter);
2577e026c8cSDan Williams 
258abebfbe2SDan Williams void dax_flush(struct dax_device *dax_dev, pgoff_t pgoff, void *addr,
259abebfbe2SDan Williams 		size_t size)
260abebfbe2SDan Williams {
261abebfbe2SDan Williams 	if (!dax_alive(dax_dev))
262abebfbe2SDan Williams 		return;
263abebfbe2SDan Williams 
2646e0c90d6SDan Williams 	if (!test_bit(DAXDEV_WRITE_CACHE, &dax_dev->flags))
2656e0c90d6SDan Williams 		return;
2666e0c90d6SDan Williams 
267abebfbe2SDan Williams 	if (dax_dev->ops->flush)
268abebfbe2SDan Williams 		dax_dev->ops->flush(dax_dev, pgoff, addr, size);
269abebfbe2SDan Williams }
270abebfbe2SDan Williams EXPORT_SYMBOL_GPL(dax_flush);
271abebfbe2SDan Williams 
2726e0c90d6SDan Williams void dax_write_cache(struct dax_device *dax_dev, bool wc)
2736e0c90d6SDan Williams {
2746e0c90d6SDan Williams 	if (wc)
2756e0c90d6SDan Williams 		set_bit(DAXDEV_WRITE_CACHE, &dax_dev->flags);
2766e0c90d6SDan Williams 	else
2776e0c90d6SDan Williams 		clear_bit(DAXDEV_WRITE_CACHE, &dax_dev->flags);
2786e0c90d6SDan Williams }
2796e0c90d6SDan Williams EXPORT_SYMBOL_GPL(dax_write_cache);
2806e0c90d6SDan Williams 
281273752c9SVivek Goyal bool dax_write_cache_enabled(struct dax_device *dax_dev)
282273752c9SVivek Goyal {
283273752c9SVivek Goyal 	return test_bit(DAXDEV_WRITE_CACHE, &dax_dev->flags);
284273752c9SVivek Goyal }
285273752c9SVivek Goyal EXPORT_SYMBOL_GPL(dax_write_cache_enabled);
286273752c9SVivek Goyal 
2877b6be844SDan Williams bool dax_alive(struct dax_device *dax_dev)
2887b6be844SDan Williams {
2897b6be844SDan Williams 	lockdep_assert_held(&dax_srcu);
2909a60c3efSDan Williams 	return test_bit(DAXDEV_ALIVE, &dax_dev->flags);
2917b6be844SDan Williams }
2927b6be844SDan Williams EXPORT_SYMBOL_GPL(dax_alive);
2937b6be844SDan Williams 
29472058005SDan Williams static int dax_host_hash(const char *host)
29572058005SDan Williams {
29672058005SDan Williams 	return hashlen_hash(hashlen_string("DAX", host)) % DAX_HASH_SIZE;
29772058005SDan Williams }
29872058005SDan Williams 
2997b6be844SDan Williams /*
3007b6be844SDan Williams  * Note, rcu is not protecting the liveness of dax_dev, rcu is ensuring
3017b6be844SDan Williams  * that any fault handlers or operations that might have seen
3027b6be844SDan Williams  * dax_alive(), have completed.  Any operations that start after
3037b6be844SDan Williams  * synchronize_srcu() has run will abort upon seeing !dax_alive().
3047b6be844SDan Williams  */
3057b6be844SDan Williams void kill_dax(struct dax_device *dax_dev)
3067b6be844SDan Williams {
3077b6be844SDan Williams 	if (!dax_dev)
3087b6be844SDan Williams 		return;
3097b6be844SDan Williams 
3109a60c3efSDan Williams 	clear_bit(DAXDEV_ALIVE, &dax_dev->flags);
31172058005SDan Williams 
3127b6be844SDan Williams 	synchronize_srcu(&dax_srcu);
31372058005SDan Williams 
31472058005SDan Williams 	spin_lock(&dax_host_lock);
31572058005SDan Williams 	hlist_del_init(&dax_dev->list);
31672058005SDan Williams 	spin_unlock(&dax_host_lock);
31772058005SDan Williams 
3187b6be844SDan Williams 	dax_dev->private = NULL;
3197b6be844SDan Williams }
3207b6be844SDan Williams EXPORT_SYMBOL_GPL(kill_dax);
3217b6be844SDan Williams 
3227b6be844SDan Williams static struct inode *dax_alloc_inode(struct super_block *sb)
3237b6be844SDan Williams {
3247b6be844SDan Williams 	struct dax_device *dax_dev;
325b9d39d17SDan Williams 	struct inode *inode;
3267b6be844SDan Williams 
3277b6be844SDan Williams 	dax_dev = kmem_cache_alloc(dax_cache, GFP_KERNEL);
328b9d39d17SDan Williams 	inode = &dax_dev->inode;
329b9d39d17SDan Williams 	inode->i_rdev = 0;
330b9d39d17SDan Williams 	return inode;
3317b6be844SDan Williams }
3327b6be844SDan Williams 
3337b6be844SDan Williams static struct dax_device *to_dax_dev(struct inode *inode)
3347b6be844SDan Williams {
3357b6be844SDan Williams 	return container_of(inode, struct dax_device, inode);
3367b6be844SDan Williams }
3377b6be844SDan Williams 
3387b6be844SDan Williams static void dax_i_callback(struct rcu_head *head)
3397b6be844SDan Williams {
3407b6be844SDan Williams 	struct inode *inode = container_of(head, struct inode, i_rcu);
3417b6be844SDan Williams 	struct dax_device *dax_dev = to_dax_dev(inode);
3427b6be844SDan Williams 
34372058005SDan Williams 	kfree(dax_dev->host);
34472058005SDan Williams 	dax_dev->host = NULL;
345b9d39d17SDan Williams 	if (inode->i_rdev)
3467b6be844SDan Williams 		ida_simple_remove(&dax_minor_ida, MINOR(inode->i_rdev));
3477b6be844SDan Williams 	kmem_cache_free(dax_cache, dax_dev);
3487b6be844SDan Williams }
3497b6be844SDan Williams 
3507b6be844SDan Williams static void dax_destroy_inode(struct inode *inode)
3517b6be844SDan Williams {
3527b6be844SDan Williams 	struct dax_device *dax_dev = to_dax_dev(inode);
3537b6be844SDan Williams 
3549a60c3efSDan Williams 	WARN_ONCE(test_bit(DAXDEV_ALIVE, &dax_dev->flags),
3557b6be844SDan Williams 			"kill_dax() must be called before final iput()\n");
3567b6be844SDan Williams 	call_rcu(&inode->i_rcu, dax_i_callback);
3577b6be844SDan Williams }
3587b6be844SDan Williams 
3597b6be844SDan Williams static const struct super_operations dax_sops = {
3607b6be844SDan Williams 	.statfs = simple_statfs,
3617b6be844SDan Williams 	.alloc_inode = dax_alloc_inode,
3627b6be844SDan Williams 	.destroy_inode = dax_destroy_inode,
3637b6be844SDan Williams 	.drop_inode = generic_delete_inode,
3647b6be844SDan Williams };
3657b6be844SDan Williams 
3667b6be844SDan Williams static struct dentry *dax_mount(struct file_system_type *fs_type,
3677b6be844SDan Williams 		int flags, const char *dev_name, void *data)
3687b6be844SDan Williams {
3697b6be844SDan Williams 	return mount_pseudo(fs_type, "dax:", &dax_sops, NULL, DAXFS_MAGIC);
3707b6be844SDan Williams }
3717b6be844SDan Williams 
3727b6be844SDan Williams static struct file_system_type dax_fs_type = {
3737b6be844SDan Williams 	.name = "dax",
3747b6be844SDan Williams 	.mount = dax_mount,
3757b6be844SDan Williams 	.kill_sb = kill_anon_super,
3767b6be844SDan Williams };
3777b6be844SDan Williams 
3787b6be844SDan Williams static int dax_test(struct inode *inode, void *data)
3797b6be844SDan Williams {
3807b6be844SDan Williams 	dev_t devt = *(dev_t *) data;
3817b6be844SDan Williams 
3827b6be844SDan Williams 	return inode->i_rdev == devt;
3837b6be844SDan Williams }
3847b6be844SDan Williams 
3857b6be844SDan Williams static int dax_set(struct inode *inode, void *data)
3867b6be844SDan Williams {
3877b6be844SDan Williams 	dev_t devt = *(dev_t *) data;
3887b6be844SDan Williams 
3897b6be844SDan Williams 	inode->i_rdev = devt;
3907b6be844SDan Williams 	return 0;
3917b6be844SDan Williams }
3927b6be844SDan Williams 
3937b6be844SDan Williams static struct dax_device *dax_dev_get(dev_t devt)
3947b6be844SDan Williams {
3957b6be844SDan Williams 	struct dax_device *dax_dev;
3967b6be844SDan Williams 	struct inode *inode;
3977b6be844SDan Williams 
3987b6be844SDan Williams 	inode = iget5_locked(dax_superblock, hash_32(devt + DAXFS_MAGIC, 31),
3997b6be844SDan Williams 			dax_test, dax_set, &devt);
4007b6be844SDan Williams 
4017b6be844SDan Williams 	if (!inode)
4027b6be844SDan Williams 		return NULL;
4037b6be844SDan Williams 
4047b6be844SDan Williams 	dax_dev = to_dax_dev(inode);
4057b6be844SDan Williams 	if (inode->i_state & I_NEW) {
4069a60c3efSDan Williams 		set_bit(DAXDEV_ALIVE, &dax_dev->flags);
4077b6be844SDan Williams 		inode->i_cdev = &dax_dev->cdev;
4087b6be844SDan Williams 		inode->i_mode = S_IFCHR;
4097b6be844SDan Williams 		inode->i_flags = S_DAX;
4107b6be844SDan Williams 		mapping_set_gfp_mask(&inode->i_data, GFP_USER);
4117b6be844SDan Williams 		unlock_new_inode(inode);
4127b6be844SDan Williams 	}
4137b6be844SDan Williams 
4147b6be844SDan Williams 	return dax_dev;
4157b6be844SDan Williams }
4167b6be844SDan Williams 
41772058005SDan Williams static void dax_add_host(struct dax_device *dax_dev, const char *host)
41872058005SDan Williams {
41972058005SDan Williams 	int hash;
42072058005SDan Williams 
42172058005SDan Williams 	/*
42272058005SDan Williams 	 * Unconditionally init dax_dev since it's coming from a
42372058005SDan Williams 	 * non-zeroed slab cache
42472058005SDan Williams 	 */
42572058005SDan Williams 	INIT_HLIST_NODE(&dax_dev->list);
42672058005SDan Williams 	dax_dev->host = host;
42772058005SDan Williams 	if (!host)
42872058005SDan Williams 		return;
42972058005SDan Williams 
43072058005SDan Williams 	hash = dax_host_hash(host);
43172058005SDan Williams 	spin_lock(&dax_host_lock);
43272058005SDan Williams 	hlist_add_head(&dax_dev->list, &dax_host_list[hash]);
43372058005SDan Williams 	spin_unlock(&dax_host_lock);
43472058005SDan Williams }
43572058005SDan Williams 
4366568b08bSDan Williams struct dax_device *alloc_dax(void *private, const char *__host,
4376568b08bSDan Williams 		const struct dax_operations *ops)
4387b6be844SDan Williams {
4397b6be844SDan Williams 	struct dax_device *dax_dev;
44072058005SDan Williams 	const char *host;
4417b6be844SDan Williams 	dev_t devt;
4427b6be844SDan Williams 	int minor;
4437b6be844SDan Williams 
44472058005SDan Williams 	host = kstrdup(__host, GFP_KERNEL);
44572058005SDan Williams 	if (__host && !host)
44672058005SDan Williams 		return NULL;
44772058005SDan Williams 
448cf1e2289SDan Williams 	minor = ida_simple_get(&dax_minor_ida, 0, MINORMASK+1, GFP_KERNEL);
4497b6be844SDan Williams 	if (minor < 0)
45072058005SDan Williams 		goto err_minor;
4517b6be844SDan Williams 
4527b6be844SDan Williams 	devt = MKDEV(MAJOR(dax_devt), minor);
4537b6be844SDan Williams 	dax_dev = dax_dev_get(devt);
4547b6be844SDan Williams 	if (!dax_dev)
45572058005SDan Williams 		goto err_dev;
4567b6be844SDan Williams 
45772058005SDan Williams 	dax_add_host(dax_dev, host);
4586568b08bSDan Williams 	dax_dev->ops = ops;
4597b6be844SDan Williams 	dax_dev->private = private;
4607b6be844SDan Williams 	return dax_dev;
4617b6be844SDan Williams 
46272058005SDan Williams  err_dev:
4637b6be844SDan Williams 	ida_simple_remove(&dax_minor_ida, minor);
46472058005SDan Williams  err_minor:
46572058005SDan Williams 	kfree(host);
4667b6be844SDan Williams 	return NULL;
4677b6be844SDan Williams }
4687b6be844SDan Williams EXPORT_SYMBOL_GPL(alloc_dax);
4697b6be844SDan Williams 
4707b6be844SDan Williams void put_dax(struct dax_device *dax_dev)
4717b6be844SDan Williams {
4727b6be844SDan Williams 	if (!dax_dev)
4737b6be844SDan Williams 		return;
4747b6be844SDan Williams 	iput(&dax_dev->inode);
4757b6be844SDan Williams }
4767b6be844SDan Williams EXPORT_SYMBOL_GPL(put_dax);
4777b6be844SDan Williams 
4787b6be844SDan Williams /**
47972058005SDan Williams  * dax_get_by_host() - temporary lookup mechanism for filesystem-dax
48072058005SDan Williams  * @host: alternate name for the device registered by a dax driver
48172058005SDan Williams  */
48272058005SDan Williams struct dax_device *dax_get_by_host(const char *host)
48372058005SDan Williams {
48472058005SDan Williams 	struct dax_device *dax_dev, *found = NULL;
48572058005SDan Williams 	int hash, id;
48672058005SDan Williams 
48772058005SDan Williams 	if (!host)
48872058005SDan Williams 		return NULL;
48972058005SDan Williams 
49072058005SDan Williams 	hash = dax_host_hash(host);
49172058005SDan Williams 
49272058005SDan Williams 	id = dax_read_lock();
49372058005SDan Williams 	spin_lock(&dax_host_lock);
49472058005SDan Williams 	hlist_for_each_entry(dax_dev, &dax_host_list[hash], list) {
49572058005SDan Williams 		if (!dax_alive(dax_dev)
49672058005SDan Williams 				|| strcmp(host, dax_dev->host) != 0)
49772058005SDan Williams 			continue;
49872058005SDan Williams 
49972058005SDan Williams 		if (igrab(&dax_dev->inode))
50072058005SDan Williams 			found = dax_dev;
50172058005SDan Williams 		break;
50272058005SDan Williams 	}
50372058005SDan Williams 	spin_unlock(&dax_host_lock);
50472058005SDan Williams 	dax_read_unlock(id);
50572058005SDan Williams 
50672058005SDan Williams 	return found;
50772058005SDan Williams }
50872058005SDan Williams EXPORT_SYMBOL_GPL(dax_get_by_host);
50972058005SDan Williams 
51072058005SDan Williams /**
5117b6be844SDan Williams  * inode_dax: convert a public inode into its dax_dev
5127b6be844SDan Williams  * @inode: An inode with i_cdev pointing to a dax_dev
5137b6be844SDan Williams  *
5147b6be844SDan Williams  * Note this is not equivalent to to_dax_dev() which is for private
5157b6be844SDan Williams  * internal use where we know the inode filesystem type == dax_fs_type.
5167b6be844SDan Williams  */
5177b6be844SDan Williams struct dax_device *inode_dax(struct inode *inode)
5187b6be844SDan Williams {
5197b6be844SDan Williams 	struct cdev *cdev = inode->i_cdev;
5207b6be844SDan Williams 
5217b6be844SDan Williams 	return container_of(cdev, struct dax_device, cdev);
5227b6be844SDan Williams }
5237b6be844SDan Williams EXPORT_SYMBOL_GPL(inode_dax);
5247b6be844SDan Williams 
5257b6be844SDan Williams struct inode *dax_inode(struct dax_device *dax_dev)
5267b6be844SDan Williams {
5277b6be844SDan Williams 	return &dax_dev->inode;
5287b6be844SDan Williams }
5297b6be844SDan Williams EXPORT_SYMBOL_GPL(dax_inode);
5307b6be844SDan Williams 
5317b6be844SDan Williams void *dax_get_private(struct dax_device *dax_dev)
5327b6be844SDan Williams {
5337b6be844SDan Williams 	return dax_dev->private;
5347b6be844SDan Williams }
5357b6be844SDan Williams EXPORT_SYMBOL_GPL(dax_get_private);
5367b6be844SDan Williams 
5377b6be844SDan Williams static void init_once(void *_dax_dev)
5387b6be844SDan Williams {
5397b6be844SDan Williams 	struct dax_device *dax_dev = _dax_dev;
5407b6be844SDan Williams 	struct inode *inode = &dax_dev->inode;
5417b6be844SDan Williams 
542b9d39d17SDan Williams 	memset(dax_dev, 0, sizeof(*dax_dev));
5437b6be844SDan Williams 	inode_init_once(inode);
5447b6be844SDan Williams }
5457b6be844SDan Williams 
5467b6be844SDan Williams static int __dax_fs_init(void)
5477b6be844SDan Williams {
5487b6be844SDan Williams 	int rc;
5497b6be844SDan Williams 
5507b6be844SDan Williams 	dax_cache = kmem_cache_create("dax_cache", sizeof(struct dax_device), 0,
5517b6be844SDan Williams 			(SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT|
5527b6be844SDan Williams 			 SLAB_MEM_SPREAD|SLAB_ACCOUNT),
5537b6be844SDan Williams 			init_once);
5547b6be844SDan Williams 	if (!dax_cache)
5557b6be844SDan Williams 		return -ENOMEM;
5567b6be844SDan Williams 
5577b6be844SDan Williams 	rc = register_filesystem(&dax_fs_type);
5587b6be844SDan Williams 	if (rc)
5597b6be844SDan Williams 		goto err_register_fs;
5607b6be844SDan Williams 
5617b6be844SDan Williams 	dax_mnt = kern_mount(&dax_fs_type);
5627b6be844SDan Williams 	if (IS_ERR(dax_mnt)) {
5637b6be844SDan Williams 		rc = PTR_ERR(dax_mnt);
5647b6be844SDan Williams 		goto err_mount;
5657b6be844SDan Williams 	}
5667b6be844SDan Williams 	dax_superblock = dax_mnt->mnt_sb;
5677b6be844SDan Williams 
5687b6be844SDan Williams 	return 0;
5697b6be844SDan Williams 
5707b6be844SDan Williams  err_mount:
5717b6be844SDan Williams 	unregister_filesystem(&dax_fs_type);
5727b6be844SDan Williams  err_register_fs:
5737b6be844SDan Williams 	kmem_cache_destroy(dax_cache);
5747b6be844SDan Williams 
5757b6be844SDan Williams 	return rc;
5767b6be844SDan Williams }
5777b6be844SDan Williams 
5787b6be844SDan Williams static void __dax_fs_exit(void)
5797b6be844SDan Williams {
5807b6be844SDan Williams 	kern_unmount(dax_mnt);
5817b6be844SDan Williams 	unregister_filesystem(&dax_fs_type);
5827b6be844SDan Williams 	kmem_cache_destroy(dax_cache);
5837b6be844SDan Williams }
5847b6be844SDan Williams 
5857b6be844SDan Williams static int __init dax_fs_init(void)
5867b6be844SDan Williams {
5877b6be844SDan Williams 	int rc;
5887b6be844SDan Williams 
5897b6be844SDan Williams 	rc = __dax_fs_init();
5907b6be844SDan Williams 	if (rc)
5917b6be844SDan Williams 		return rc;
5927b6be844SDan Williams 
593cf1e2289SDan Williams 	rc = alloc_chrdev_region(&dax_devt, 0, MINORMASK+1, "dax");
5947b6be844SDan Williams 	if (rc)
5957b6be844SDan Williams 		__dax_fs_exit();
5967b6be844SDan Williams 	return rc;
5977b6be844SDan Williams }
5987b6be844SDan Williams 
5997b6be844SDan Williams static void __exit dax_fs_exit(void)
6007b6be844SDan Williams {
601cf1e2289SDan Williams 	unregister_chrdev_region(dax_devt, MINORMASK+1);
6027b6be844SDan Williams 	ida_destroy(&dax_minor_ida);
6037b6be844SDan Williams 	__dax_fs_exit();
6047b6be844SDan Williams }
6057b6be844SDan Williams 
6067b6be844SDan Williams MODULE_AUTHOR("Intel Corporation");
6077b6be844SDan Williams MODULE_LICENSE("GPL v2");
6087b6be844SDan Williams subsys_initcall(dax_fs_init);
6097b6be844SDan Williams module_exit(dax_fs_exit);
610