1b2441318SGreg Kroah-Hartman /* SPDX-License-Identifier: GPL-2.0 */
2c94c2acfSMatthew Wilcox #ifndef _LINUX_DAX_H
3c94c2acfSMatthew Wilcox #define _LINUX_DAX_H
4c94c2acfSMatthew Wilcox
5c94c2acfSMatthew Wilcox #include <linux/fs.h>
6c94c2acfSMatthew Wilcox #include <linux/mm.h>
74f622938SJan Kara #include <linux/radix-tree.h>
8c94c2acfSMatthew Wilcox
927359fd6SMatthew Wilcox typedef unsigned long dax_entry_t;
1027359fd6SMatthew Wilcox
11fb08a190SChristoph Hellwig struct dax_device;
12fb08a190SChristoph Hellwig struct gendisk;
13a254e568SChristoph Hellwig struct iomap_ops;
14c6f40468SChristoph Hellwig struct iomap_iter;
154f3b4f16SVivek Goyal struct iomap;
16fb08a190SChristoph Hellwig
17e511c4a3SJane Chu enum dax_access_mode {
18e511c4a3SJane Chu DAX_ACCESS,
19e511c4a3SJane Chu DAX_RECOVERY_WRITE,
20e511c4a3SJane Chu };
21e511c4a3SJane Chu
226568b08bSDan Williams struct dax_operations {
236568b08bSDan Williams /*
246568b08bSDan Williams * direct_access: translate a device-relative
256568b08bSDan Williams * logical-page-offset into an absolute physical pfn. Return the
266568b08bSDan Williams * number of pages available for DAX at that pfn.
276568b08bSDan Williams */
286568b08bSDan Williams long (*direct_access)(struct dax_device *, pgoff_t, long,
29e511c4a3SJane Chu enum dax_access_mode, void **, pfn_t *);
307bf7eac8SDan Williams /*
317bf7eac8SDan Williams * Validate whether this device is usable as an fsdax backing
327bf7eac8SDan Williams * device.
337bf7eac8SDan Williams */
347bf7eac8SDan Williams bool (*dax_supported)(struct dax_device *, struct block_device *, int,
357bf7eac8SDan Williams sector_t, sector_t);
36f605a263SVivek Goyal /* zero_page_range: required operation. Zero page range */
37f605a263SVivek Goyal int (*zero_page_range)(struct dax_device *, pgoff_t, size_t);
38047218ecSJane Chu /*
39047218ecSJane Chu * recovery_write: recover a poisoned range by DAX device driver
40047218ecSJane Chu * capable of clearing poison.
41047218ecSJane Chu */
42047218ecSJane Chu size_t (*recovery_write)(struct dax_device *dax_dev, pgoff_t pgoff,
43047218ecSJane Chu void *addr, size_t bytes, struct iov_iter *iter);
446568b08bSDan Williams };
45a254e568SChristoph Hellwig
468012b866SShiyang Ruan struct dax_holder_operations {
478012b866SShiyang Ruan /*
488012b866SShiyang Ruan * notify_failure - notify memory failure into inner holder device
498012b866SShiyang Ruan * @dax_dev: the dax device which contains the holder
508012b866SShiyang Ruan * @offset: offset on this dax device where memory failure occurs
518012b866SShiyang Ruan * @len: length of this memory failure event
528012b866SShiyang Ruan * @flags: action flags for memory failure handler
538012b866SShiyang Ruan */
548012b866SShiyang Ruan int (*notify_failure)(struct dax_device *dax_dev, u64 offset,
558012b866SShiyang Ruan u64 len, int mf_flags);
568012b866SShiyang Ruan };
578012b866SShiyang Ruan
58ef510424SDan Williams #if IS_ENABLED(CONFIG_DAX)
5930c6828aSChristoph Hellwig struct dax_device *alloc_dax(void *private, const struct dax_operations *ops);
608012b866SShiyang Ruan void *dax_holder(struct dax_device *dax_dev);
61ef510424SDan Williams void put_dax(struct dax_device *dax_dev);
62976431b0SDan Williams void kill_dax(struct dax_device *dax_dev);
63976431b0SDan Williams void dax_write_cache(struct dax_device *dax_dev, bool wc);
64976431b0SDan Williams bool dax_write_cache_enabled(struct dax_device *dax_dev);
65fd1d00ecSChristoph Hellwig bool dax_synchronous(struct dax_device *dax_dev);
66fd1d00ecSChristoph Hellwig void set_dax_synchronous(struct dax_device *dax_dev);
67047218ecSJane Chu size_t dax_recovery_write(struct dax_device *dax_dev, pgoff_t pgoff,
68047218ecSJane Chu void *addr, size_t bytes, struct iov_iter *i);
6932de1484SPankaj Gupta /*
7032de1484SPankaj Gupta * Check if given mapping is supported by the file / underlying device.
7132de1484SPankaj Gupta */
daxdev_mapping_supported(struct vm_area_struct * vma,struct dax_device * dax_dev)7232de1484SPankaj Gupta static inline bool daxdev_mapping_supported(struct vm_area_struct *vma,
7332de1484SPankaj Gupta struct dax_device *dax_dev)
7432de1484SPankaj Gupta {
7532de1484SPankaj Gupta if (!(vma->vm_flags & VM_SYNC))
7632de1484SPankaj Gupta return true;
7732de1484SPankaj Gupta if (!IS_DAX(file_inode(vma->vm_file)))
7832de1484SPankaj Gupta return false;
7932de1484SPankaj Gupta return dax_synchronous(dax_dev);
8032de1484SPankaj Gupta }
81ef510424SDan Williams #else
dax_holder(struct dax_device * dax_dev)828012b866SShiyang Ruan static inline void *dax_holder(struct dax_device *dax_dev)
838012b866SShiyang Ruan {
848012b866SShiyang Ruan return NULL;
858012b866SShiyang Ruan }
alloc_dax(void * private,const struct dax_operations * ops)86fb08a190SChristoph Hellwig static inline struct dax_device *alloc_dax(void *private,
8730c6828aSChristoph Hellwig const struct dax_operations *ops)
88976431b0SDan Williams {
89976431b0SDan Williams /*
90976431b0SDan Williams * Callers should check IS_ENABLED(CONFIG_DAX) to know if this
91976431b0SDan Williams * NULL is an error or expected.
92976431b0SDan Williams */
93976431b0SDan Williams return NULL;
94976431b0SDan Williams }
put_dax(struct dax_device * dax_dev)95ef510424SDan Williams static inline void put_dax(struct dax_device *dax_dev)
96ef510424SDan Williams {
97ef510424SDan Williams }
kill_dax(struct dax_device * dax_dev)98976431b0SDan Williams static inline void kill_dax(struct dax_device *dax_dev)
99976431b0SDan Williams {
100976431b0SDan Williams }
dax_write_cache(struct dax_device * dax_dev,bool wc)101976431b0SDan Williams static inline void dax_write_cache(struct dax_device *dax_dev, bool wc)
102976431b0SDan Williams {
103976431b0SDan Williams }
dax_write_cache_enabled(struct dax_device * dax_dev)104976431b0SDan Williams static inline bool dax_write_cache_enabled(struct dax_device *dax_dev)
105976431b0SDan Williams {
106976431b0SDan Williams return false;
107976431b0SDan Williams }
dax_synchronous(struct dax_device * dax_dev)108fefc1d97SPankaj Gupta static inline bool dax_synchronous(struct dax_device *dax_dev)
109fefc1d97SPankaj Gupta {
110fefc1d97SPankaj Gupta return true;
111fefc1d97SPankaj Gupta }
set_dax_synchronous(struct dax_device * dax_dev)112fefc1d97SPankaj Gupta static inline void set_dax_synchronous(struct dax_device *dax_dev)
113fefc1d97SPankaj Gupta {
114fefc1d97SPankaj Gupta }
daxdev_mapping_supported(struct vm_area_struct * vma,struct dax_device * dax_dev)11532de1484SPankaj Gupta static inline bool daxdev_mapping_supported(struct vm_area_struct *vma,
11632de1484SPankaj Gupta struct dax_device *dax_dev)
11732de1484SPankaj Gupta {
11832de1484SPankaj Gupta return !(vma->vm_flags & VM_SYNC);
11932de1484SPankaj Gupta }
dax_recovery_write(struct dax_device * dax_dev,pgoff_t pgoff,void * addr,size_t bytes,struct iov_iter * i)120047218ecSJane Chu static inline size_t dax_recovery_write(struct dax_device *dax_dev,
121047218ecSJane Chu pgoff_t pgoff, void *addr, size_t bytes, struct iov_iter *i)
122047218ecSJane Chu {
123047218ecSJane Chu return 0;
124047218ecSJane Chu }
125ef510424SDan Williams #endif
126ef510424SDan Williams
1277ac5360cSChristoph Hellwig void set_dax_nocache(struct dax_device *dax_dev);
1287ac5360cSChristoph Hellwig void set_dax_nomc(struct dax_device *dax_dev);
1297ac5360cSChristoph Hellwig
130f44c7763SDan Williams struct writeback_control;
1312ede8923SChristoph Hellwig #if defined(CONFIG_BLOCK) && defined(CONFIG_FS_DAX)
132fb08a190SChristoph Hellwig int dax_add_host(struct dax_device *dax_dev, struct gendisk *disk);
133fb08a190SChristoph Hellwig void dax_remove_host(struct gendisk *disk);
1348012b866SShiyang Ruan struct dax_device *fs_dax_get_by_bdev(struct block_device *bdev, u64 *start_off,
1358012b866SShiyang Ruan void *holder, const struct dax_holder_operations *ops);
1368012b866SShiyang Ruan void fs_put_dax(struct dax_device *dax_dev, void *holder);
137f5705aa8SDan Williams #else
dax_add_host(struct dax_device * dax_dev,struct gendisk * disk)138fb08a190SChristoph Hellwig static inline int dax_add_host(struct dax_device *dax_dev, struct gendisk *disk)
139fb08a190SChristoph Hellwig {
140fb08a190SChristoph Hellwig return 0;
141fb08a190SChristoph Hellwig }
dax_remove_host(struct gendisk * disk)142fb08a190SChristoph Hellwig static inline void dax_remove_host(struct gendisk *disk)
143fb08a190SChristoph Hellwig {
144fb08a190SChristoph Hellwig }
fs_dax_get_by_bdev(struct block_device * bdev,u64 * start_off,void * holder,const struct dax_holder_operations * ops)145cd913c76SChristoph Hellwig static inline struct dax_device *fs_dax_get_by_bdev(struct block_device *bdev,
1468012b866SShiyang Ruan u64 *start_off, void *holder,
1478012b866SShiyang Ruan const struct dax_holder_operations *ops)
14878f35473SDan Williams {
14978f35473SDan Williams return NULL;
15078f35473SDan Williams }
fs_put_dax(struct dax_device * dax_dev,void * holder)1518012b866SShiyang Ruan static inline void fs_put_dax(struct dax_device *dax_dev, void *holder)
1522ede8923SChristoph Hellwig {
1532ede8923SChristoph Hellwig }
1542ede8923SChristoph Hellwig #endif /* CONFIG_BLOCK && CONFIG_FS_DAX */
155f44c7763SDan Williams
1562ede8923SChristoph Hellwig #if IS_ENABLED(CONFIG_FS_DAX)
1572ede8923SChristoph Hellwig int dax_writeback_mapping_range(struct address_space *mapping,
1582ede8923SChristoph Hellwig struct dax_device *dax_dev, struct writeback_control *wbc);
1592ede8923SChristoph Hellwig
1602ede8923SChristoph Hellwig struct page *dax_layout_busy_page(struct address_space *mapping);
1612ede8923SChristoph Hellwig struct page *dax_layout_busy_page_range(struct address_space *mapping, loff_t start, loff_t end);
162*393155f9SMatthew Wilcox (Oracle) dax_entry_t dax_lock_folio(struct folio *folio);
163*393155f9SMatthew Wilcox (Oracle) void dax_unlock_folio(struct folio *folio, dax_entry_t cookie);
1642f437effSShiyang Ruan dax_entry_t dax_lock_mapping_entry(struct address_space *mapping,
1652f437effSShiyang Ruan unsigned long index, struct page **page);
1662f437effSShiyang Ruan void dax_unlock_mapping_entry(struct address_space *mapping,
1672f437effSShiyang Ruan unsigned long index, dax_entry_t cookie);
1682ede8923SChristoph Hellwig #else
dax_layout_busy_page(struct address_space * mapping)1695fac7408SDan Williams static inline struct page *dax_layout_busy_page(struct address_space *mapping)
1705fac7408SDan Williams {
1715fac7408SDan Williams return NULL;
1725fac7408SDan Williams }
1735fac7408SDan Williams
dax_layout_busy_page_range(struct address_space * mapping,pgoff_t start,pgoff_t nr_pages)1746bbdd563SVivek Goyal static inline struct page *dax_layout_busy_page_range(struct address_space *mapping, pgoff_t start, pgoff_t nr_pages)
1756bbdd563SVivek Goyal {
1766bbdd563SVivek Goyal return NULL;
1776bbdd563SVivek Goyal }
1786bbdd563SVivek Goyal
dax_writeback_mapping_range(struct address_space * mapping,struct dax_device * dax_dev,struct writeback_control * wbc)179f44c7763SDan Williams static inline int dax_writeback_mapping_range(struct address_space *mapping,
1803f666c56SVivek Goyal struct dax_device *dax_dev, struct writeback_control *wbc)
181f44c7763SDan Williams {
182f44c7763SDan Williams return -EOPNOTSUPP;
183f44c7763SDan Williams }
184c2a7d2a1SDan Williams
dax_lock_folio(struct folio * folio)185*393155f9SMatthew Wilcox (Oracle) static inline dax_entry_t dax_lock_folio(struct folio *folio)
186c2a7d2a1SDan Williams {
187*393155f9SMatthew Wilcox (Oracle) if (IS_DAX(folio->mapping->host))
18827359fd6SMatthew Wilcox return ~0UL;
18927359fd6SMatthew Wilcox return 0;
190c2a7d2a1SDan Williams }
191c2a7d2a1SDan Williams
dax_unlock_folio(struct folio * folio,dax_entry_t cookie)192*393155f9SMatthew Wilcox (Oracle) static inline void dax_unlock_folio(struct folio *folio, dax_entry_t cookie)
193c2a7d2a1SDan Williams {
194c2a7d2a1SDan Williams }
1952f437effSShiyang Ruan
dax_lock_mapping_entry(struct address_space * mapping,unsigned long index,struct page ** page)1962f437effSShiyang Ruan static inline dax_entry_t dax_lock_mapping_entry(struct address_space *mapping,
1972f437effSShiyang Ruan unsigned long index, struct page **page)
1982f437effSShiyang Ruan {
1992f437effSShiyang Ruan return 0;
2002f437effSShiyang Ruan }
2012f437effSShiyang Ruan
dax_unlock_mapping_entry(struct address_space * mapping,unsigned long index,dax_entry_t cookie)2022f437effSShiyang Ruan static inline void dax_unlock_mapping_entry(struct address_space *mapping,
2032f437effSShiyang Ruan unsigned long index, dax_entry_t cookie)
2042f437effSShiyang Ruan {
2052f437effSShiyang Ruan }
206f5705aa8SDan Williams #endif
207f5705aa8SDan Williams
208d984648eSShiyang Ruan int dax_file_unshare(struct inode *inode, loff_t pos, loff_t len,
209d984648eSShiyang Ruan const struct iomap_ops *ops);
210c6f40468SChristoph Hellwig int dax_zero_range(struct inode *inode, loff_t pos, loff_t len, bool *did_zero,
211c6f40468SChristoph Hellwig const struct iomap_ops *ops);
212c6f40468SChristoph Hellwig int dax_truncate_page(struct inode *inode, loff_t pos, bool *did_zero,
213c6f40468SChristoph Hellwig const struct iomap_ops *ops);
214c6f40468SChristoph Hellwig
215e2ec5128SJan Kara #if IS_ENABLED(CONFIG_DAX)
2167b6be844SDan Williams int dax_read_lock(void);
2177b6be844SDan Williams void dax_read_unlock(int id);
218e2ec5128SJan Kara #else
dax_read_lock(void)219e2ec5128SJan Kara static inline int dax_read_lock(void)
220e2ec5128SJan Kara {
221e2ec5128SJan Kara return 0;
222e2ec5128SJan Kara }
223e2ec5128SJan Kara
dax_read_unlock(int id)224e2ec5128SJan Kara static inline void dax_read_unlock(int id)
225e2ec5128SJan Kara {
226e2ec5128SJan Kara }
227e2ec5128SJan Kara #endif /* CONFIG_DAX */
228c1d6e828SDan Williams bool dax_alive(struct dax_device *dax_dev);
229c1d6e828SDan Williams void *dax_get_private(struct dax_device *dax_dev);
230b0686260SDan Williams long dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, long nr_pages,
231e511c4a3SJane Chu enum dax_access_mode mode, void **kaddr, pfn_t *pfn);
2327e026c8cSDan Williams size_t dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr,
2337e026c8cSDan Williams size_t bytes, struct iov_iter *i);
234b3a9a0c3SDan Williams size_t dax_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr,
235b3a9a0c3SDan Williams size_t bytes, struct iov_iter *i);
236f605a263SVivek Goyal int dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff,
237f605a263SVivek Goyal size_t nr_pages);
2388012b866SShiyang Ruan int dax_holder_notify_failure(struct dax_device *dax_dev, u64 off, u64 len,
2398012b866SShiyang Ruan int mf_flags);
240c3ca015fSMikulas Patocka void dax_flush(struct dax_device *dax_dev, void *addr, size_t size);
2417b6be844SDan Williams
24211c59c92SRoss Zwisler ssize_t dax_iomap_rw(struct kiocb *iocb, struct iov_iter *iter,
2438ff6daa1SChristoph Hellwig const struct iomap_ops *ops);
2441d024e7aSMatthew Wilcox (Oracle) vm_fault_t dax_iomap_fault(struct vm_fault *vmf, unsigned int order,
245c0b24625SJan Kara pfn_t *pfnp, int *errp, const struct iomap_ops *ops);
246ab77dab4SSouptick Joarder vm_fault_t dax_finish_sync_fault(struct vm_fault *vmf,
2471d024e7aSMatthew Wilcox (Oracle) unsigned int order, pfn_t pfn);
248ac401cc7SJan Kara int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index);
249c6dcf52cSJan Kara int dax_invalidate_mapping_entry_sync(struct address_space *mapping,
250c6dcf52cSJan Kara pgoff_t index);
2516f7db389SShiyang Ruan int dax_dedupe_file_range_compare(struct inode *src, loff_t srcoff,
2526f7db389SShiyang Ruan struct inode *dest, loff_t destoff,
2536f7db389SShiyang Ruan loff_t len, bool *is_same,
2546f7db389SShiyang Ruan const struct iomap_ops *ops);
2556f7db389SShiyang Ruan int dax_remap_file_range_prep(struct file *file_in, loff_t pos_in,
2566f7db389SShiyang Ruan struct file *file_out, loff_t pos_out,
2576f7db389SShiyang Ruan loff_t *len, unsigned int remap_flags,
2586f7db389SShiyang Ruan const struct iomap_ops *ops);
dax_mapping(struct address_space * mapping)259f9fe48beSRoss Zwisler static inline bool dax_mapping(struct address_space *mapping)
260f9fe48beSRoss Zwisler {
261f9fe48beSRoss Zwisler return mapping->host && IS_DAX(mapping->host);
262f9fe48beSRoss Zwisler }
2637f6d5b52SRoss Zwisler
2641ea7ca1bSJane Chu /*
2651ea7ca1bSJane Chu * Due to dax's memory and block duo personalities, hwpoison reporting
2661ea7ca1bSJane Chu * takes into consideration which personality is presently visible.
2671ea7ca1bSJane Chu * When dax acts like a block device, such as in block IO, an encounter of
2681ea7ca1bSJane Chu * dax hwpoison is reported as -EIO.
2691ea7ca1bSJane Chu * When dax acts like memory, such as in page fault, a detection of hwpoison
2701ea7ca1bSJane Chu * is reported as -EHWPOISON which leads to VM_FAULT_HWPOISON.
2711ea7ca1bSJane Chu */
dax_mem2blk_err(int err)2721ea7ca1bSJane Chu static inline int dax_mem2blk_err(int err)
2731ea7ca1bSJane Chu {
2741ea7ca1bSJane Chu return (err == -EHWPOISON) ? -EIO : err;
2751ea7ca1bSJane Chu }
2761ea7ca1bSJane Chu
277c01044ccSDan Williams #ifdef CONFIG_DEV_DAX_HMEM_DEVICES
2787dab174eSDan Williams void hmem_register_resource(int target_nid, struct resource *r);
279c01044ccSDan Williams #else
hmem_register_resource(int target_nid,struct resource * r)2807dab174eSDan Williams static inline void hmem_register_resource(int target_nid, struct resource *r)
281c01044ccSDan Williams {
282c01044ccSDan Williams }
283c01044ccSDan Williams #endif
284c01044ccSDan Williams
2857dab174eSDan Williams typedef int (*walk_hmem_fn)(struct device *dev, int target_nid,
2867dab174eSDan Williams const struct resource *res);
2877dab174eSDan Williams int walk_hmem_resources(struct device *dev, walk_hmem_fn fn);
288c94c2acfSMatthew Wilcox #endif
289