16bf9d8f6SLeon Romanovsky /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
2f7c6a7b5SRoland Dreier /*
3f7c6a7b5SRoland Dreier * Copyright (c) 2007 Cisco Systems. All rights reserved.
4368c0159SJianxin Xiong * Copyright (c) 2020 Intel Corporation. All rights reserved.
5f7c6a7b5SRoland Dreier */
6f7c6a7b5SRoland Dreier
7f7c6a7b5SRoland Dreier #ifndef IB_UMEM_H
8f7c6a7b5SRoland Dreier #define IB_UMEM_H
9f7c6a7b5SRoland Dreier
10f7c6a7b5SRoland Dreier #include <linux/list.h>
11f7c6a7b5SRoland Dreier #include <linux/scatterlist.h>
12e8edc6e0SAlexey Dobriyan #include <linux/workqueue.h>
13b0ea0fa5SJason Gunthorpe #include <rdma/ib_verbs.h>
14f7c6a7b5SRoland Dreier
15f7c6a7b5SRoland Dreier struct ib_ucontext;
168ada2c1cSShachar Raindel struct ib_umem_odp;
17368c0159SJianxin Xiong struct dma_buf_attach_ops;
18f7c6a7b5SRoland Dreier
19f7c6a7b5SRoland Dreier struct ib_umem {
2047f725eeSJason Gunthorpe struct ib_device *ibdev;
21d4b4dd1bSJason Gunthorpe struct mm_struct *owning_mm;
22a665aca8SJason Gunthorpe u64 iova;
23f7c6a7b5SRoland Dreier size_t length;
24406f9e5fSHaggai Eran unsigned long address;
25597ecc5aSJason Gunthorpe u32 writable : 1;
26597ecc5aSJason Gunthorpe u32 is_odp : 1;
27368c0159SJianxin Xiong u32 is_dmabuf : 1;
283e302dbcSMaor Gottlieb struct sg_append_table sgt_append;
29f7c6a7b5SRoland Dreier };
30f7c6a7b5SRoland Dreier
31368c0159SJianxin Xiong struct ib_umem_dmabuf {
32368c0159SJianxin Xiong struct ib_umem umem;
33368c0159SJianxin Xiong struct dma_buf_attachment *attach;
34368c0159SJianxin Xiong struct sg_table *sgt;
35368c0159SJianxin Xiong struct scatterlist *first_sg;
36368c0159SJianxin Xiong struct scatterlist *last_sg;
37368c0159SJianxin Xiong unsigned long first_sg_offset;
38368c0159SJianxin Xiong unsigned long last_sg_trim;
39368c0159SJianxin Xiong void *private;
401e4df4a2SGal Pressman u8 pinned : 1;
41368c0159SJianxin Xiong };
42368c0159SJianxin Xiong
to_ib_umem_dmabuf(struct ib_umem * umem)43368c0159SJianxin Xiong static inline struct ib_umem_dmabuf *to_ib_umem_dmabuf(struct ib_umem *umem)
44368c0159SJianxin Xiong {
45368c0159SJianxin Xiong return container_of(umem, struct ib_umem_dmabuf, umem);
46368c0159SJianxin Xiong }
47368c0159SJianxin Xiong
48406f9e5fSHaggai Eran /* Returns the offset of the umem start relative to the first page. */
ib_umem_offset(struct ib_umem * umem)49406f9e5fSHaggai Eran static inline int ib_umem_offset(struct ib_umem *umem)
50406f9e5fSHaggai Eran {
51d2183c6fSJason Gunthorpe return umem->address & ~PAGE_MASK;
52406f9e5fSHaggai Eran }
53406f9e5fSHaggai Eran
ib_umem_dma_offset(struct ib_umem * umem,unsigned long pgsz)54b045db62SJason Gunthorpe static inline unsigned long ib_umem_dma_offset(struct ib_umem *umem,
55b045db62SJason Gunthorpe unsigned long pgsz)
56b045db62SJason Gunthorpe {
5779fbd3e1SMaor Gottlieb return (sg_dma_address(umem->sgt_append.sgt.sgl) + ib_umem_offset(umem)) &
58b045db62SJason Gunthorpe (pgsz - 1);
59b045db62SJason Gunthorpe }
60b045db62SJason Gunthorpe
ib_umem_num_dma_blocks(struct ib_umem * umem,unsigned long pgsz)61a665aca8SJason Gunthorpe static inline size_t ib_umem_num_dma_blocks(struct ib_umem *umem,
62a665aca8SJason Gunthorpe unsigned long pgsz)
63a665aca8SJason Gunthorpe {
64a665aca8SJason Gunthorpe return (size_t)((ALIGN(umem->iova + umem->length, pgsz) -
65a665aca8SJason Gunthorpe ALIGN_DOWN(umem->iova, pgsz))) /
66a665aca8SJason Gunthorpe pgsz;
67a665aca8SJason Gunthorpe }
68a665aca8SJason Gunthorpe
ib_umem_num_pages(struct ib_umem * umem)69406f9e5fSHaggai Eran static inline size_t ib_umem_num_pages(struct ib_umem *umem)
70406f9e5fSHaggai Eran {
71a665aca8SJason Gunthorpe return ib_umem_num_dma_blocks(umem, PAGE_SIZE);
72406f9e5fSHaggai Eran }
73406f9e5fSHaggai Eran
__rdma_umem_block_iter_start(struct ib_block_iter * biter,struct ib_umem * umem,unsigned long pgsz)74ebc24096SJason Gunthorpe static inline void __rdma_umem_block_iter_start(struct ib_block_iter *biter,
75ebc24096SJason Gunthorpe struct ib_umem *umem,
76ebc24096SJason Gunthorpe unsigned long pgsz)
77ebc24096SJason Gunthorpe {
7879fbd3e1SMaor Gottlieb __rdma_block_iter_start(biter, umem->sgt_append.sgt.sgl,
7979fbd3e1SMaor Gottlieb umem->sgt_append.sgt.nents, pgsz);
80*d103c131SMike Marciniszyn biter->__sg_advance = ib_umem_offset(umem) & ~(pgsz - 1);
81*d103c131SMike Marciniszyn biter->__sg_numblocks = ib_umem_num_dma_blocks(umem, pgsz);
82*d103c131SMike Marciniszyn }
83*d103c131SMike Marciniszyn
__rdma_umem_block_iter_next(struct ib_block_iter * biter)84*d103c131SMike Marciniszyn static inline bool __rdma_umem_block_iter_next(struct ib_block_iter *biter)
85*d103c131SMike Marciniszyn {
86*d103c131SMike Marciniszyn return __rdma_block_iter_next(biter) && biter->__sg_numblocks--;
87ebc24096SJason Gunthorpe }
88ebc24096SJason Gunthorpe
89ebc24096SJason Gunthorpe /**
90ebc24096SJason Gunthorpe * rdma_umem_for_each_dma_block - iterate over contiguous DMA blocks of the umem
91ebc24096SJason Gunthorpe * @umem: umem to iterate over
92ebc24096SJason Gunthorpe * @pgsz: Page size to split the list into
93ebc24096SJason Gunthorpe *
94ebc24096SJason Gunthorpe * pgsz must be <= PAGE_SIZE or computed by ib_umem_find_best_pgsz(). The
95ebc24096SJason Gunthorpe * returned DMA blocks will be aligned to pgsz and span the range:
96ebc24096SJason Gunthorpe * ALIGN_DOWN(umem->address, pgsz) to ALIGN(umem->address + umem->length, pgsz)
97a665aca8SJason Gunthorpe *
98a665aca8SJason Gunthorpe * Performs exactly ib_umem_num_dma_blocks() iterations.
99ebc24096SJason Gunthorpe */
100ebc24096SJason Gunthorpe #define rdma_umem_for_each_dma_block(umem, biter, pgsz) \
101ebc24096SJason Gunthorpe for (__rdma_umem_block_iter_start(biter, umem, pgsz); \
102*d103c131SMike Marciniszyn __rdma_umem_block_iter_next(biter);)
103ebc24096SJason Gunthorpe
104f7c6a7b5SRoland Dreier #ifdef CONFIG_INFINIBAND_USER_MEM
105f7c6a7b5SRoland Dreier
106c320e527SMoni Shoua struct ib_umem *ib_umem_get(struct ib_device *device, unsigned long addr,
10772b894b0SChristoph Hellwig size_t size, int access);
108f7c6a7b5SRoland Dreier void ib_umem_release(struct ib_umem *umem);
109c5d76f13SHaggai Eran int ib_umem_copy_from(void *dst, struct ib_umem *umem, size_t offset,
110c5d76f13SHaggai Eran size_t length);
1114a353399SShiraz Saleem unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem,
1124a353399SShiraz Saleem unsigned long pgsz_bitmap,
1134a353399SShiraz Saleem unsigned long virt);
114368c0159SJianxin Xiong
115b045db62SJason Gunthorpe /**
116b045db62SJason Gunthorpe * ib_umem_find_best_pgoff - Find best HW page size
117b045db62SJason Gunthorpe *
118b045db62SJason Gunthorpe * @umem: umem struct
119b045db62SJason Gunthorpe * @pgsz_bitmap bitmap of HW supported page sizes
120b045db62SJason Gunthorpe * @pgoff_bitmask: Mask of bits that can be represented with an offset
121b045db62SJason Gunthorpe *
122b045db62SJason Gunthorpe * This is very similar to ib_umem_find_best_pgsz() except instead of accepting
123b045db62SJason Gunthorpe * an IOVA it accepts a bitmask specifying what address bits can be represented
124b045db62SJason Gunthorpe * with a page offset.
125b045db62SJason Gunthorpe *
126b045db62SJason Gunthorpe * For instance if the HW has multiple page sizes, requires 64 byte alignemnt,
127b045db62SJason Gunthorpe * and can support aligned offsets up to 4032 then pgoff_bitmask would be
128b045db62SJason Gunthorpe * "111111000000".
129b045db62SJason Gunthorpe *
130b045db62SJason Gunthorpe * If the pgoff_bitmask requires either alignment in the low bit or an
131b045db62SJason Gunthorpe * unavailable page size for the high bits, this function returns 0.
132b045db62SJason Gunthorpe */
ib_umem_find_best_pgoff(struct ib_umem * umem,unsigned long pgsz_bitmap,u64 pgoff_bitmask)133b045db62SJason Gunthorpe static inline unsigned long ib_umem_find_best_pgoff(struct ib_umem *umem,
134b045db62SJason Gunthorpe unsigned long pgsz_bitmap,
135b045db62SJason Gunthorpe u64 pgoff_bitmask)
136b045db62SJason Gunthorpe {
13779fbd3e1SMaor Gottlieb struct scatterlist *sg = umem->sgt_append.sgt.sgl;
138b045db62SJason Gunthorpe dma_addr_t dma_addr;
139b045db62SJason Gunthorpe
140b045db62SJason Gunthorpe dma_addr = sg_dma_address(sg) + (umem->address & ~PAGE_MASK);
141b045db62SJason Gunthorpe return ib_umem_find_best_pgsz(umem, pgsz_bitmap,
142b045db62SJason Gunthorpe dma_addr & pgoff_bitmask);
143b045db62SJason Gunthorpe }
144f7c6a7b5SRoland Dreier
145368c0159SJianxin Xiong struct ib_umem_dmabuf *ib_umem_dmabuf_get(struct ib_device *device,
146368c0159SJianxin Xiong unsigned long offset, size_t size,
147368c0159SJianxin Xiong int fd, int access,
148368c0159SJianxin Xiong const struct dma_buf_attach_ops *ops);
1491e4df4a2SGal Pressman struct ib_umem_dmabuf *ib_umem_dmabuf_get_pinned(struct ib_device *device,
1501e4df4a2SGal Pressman unsigned long offset,
1511e4df4a2SGal Pressman size_t size, int fd,
1521e4df4a2SGal Pressman int access);
153368c0159SJianxin Xiong int ib_umem_dmabuf_map_pages(struct ib_umem_dmabuf *umem_dmabuf);
154368c0159SJianxin Xiong void ib_umem_dmabuf_unmap_pages(struct ib_umem_dmabuf *umem_dmabuf);
155368c0159SJianxin Xiong void ib_umem_dmabuf_release(struct ib_umem_dmabuf *umem_dmabuf);
156368c0159SJianxin Xiong
157f7c6a7b5SRoland Dreier #else /* CONFIG_INFINIBAND_USER_MEM */
158f7c6a7b5SRoland Dreier
159f7c6a7b5SRoland Dreier #include <linux/err.h>
160f7c6a7b5SRoland Dreier
ib_umem_get(struct ib_device * device,unsigned long addr,size_t size,int access)161c320e527SMoni Shoua static inline struct ib_umem *ib_umem_get(struct ib_device *device,
162f7c6a7b5SRoland Dreier unsigned long addr, size_t size,
16372b894b0SChristoph Hellwig int access)
164b0ea0fa5SJason Gunthorpe {
165368c0159SJianxin Xiong return ERR_PTR(-EOPNOTSUPP);
166f7c6a7b5SRoland Dreier }
ib_umem_release(struct ib_umem * umem)167f7c6a7b5SRoland Dreier static inline void ib_umem_release(struct ib_umem *umem) { }
ib_umem_copy_from(void * dst,struct ib_umem * umem,size_t offset,size_t length)168c1395a2aSHaggai Eran static inline int ib_umem_copy_from(void *dst, struct ib_umem *umem, size_t offset,
169c1395a2aSHaggai Eran size_t length) {
170368c0159SJianxin Xiong return -EOPNOTSUPP;
171c1395a2aSHaggai Eran }
ib_umem_find_best_pgsz(struct ib_umem * umem,unsigned long pgsz_bitmap,unsigned long virt)17261690d01SJason Gunthorpe static inline unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem,
1734a353399SShiraz Saleem unsigned long pgsz_bitmap,
17461690d01SJason Gunthorpe unsigned long virt)
17561690d01SJason Gunthorpe {
17661690d01SJason Gunthorpe return 0;
1774a353399SShiraz Saleem }
ib_umem_find_best_pgoff(struct ib_umem * umem,unsigned long pgsz_bitmap,u64 pgoff_bitmask)178b045db62SJason Gunthorpe static inline unsigned long ib_umem_find_best_pgoff(struct ib_umem *umem,
179b045db62SJason Gunthorpe unsigned long pgsz_bitmap,
180b045db62SJason Gunthorpe u64 pgoff_bitmask)
181b045db62SJason Gunthorpe {
182b045db62SJason Gunthorpe return 0;
183b045db62SJason Gunthorpe }
184368c0159SJianxin Xiong static inline
ib_umem_dmabuf_get(struct ib_device * device,unsigned long offset,size_t size,int fd,int access,struct dma_buf_attach_ops * ops)185368c0159SJianxin Xiong struct ib_umem_dmabuf *ib_umem_dmabuf_get(struct ib_device *device,
186368c0159SJianxin Xiong unsigned long offset,
187368c0159SJianxin Xiong size_t size, int fd,
188368c0159SJianxin Xiong int access,
189368c0159SJianxin Xiong struct dma_buf_attach_ops *ops)
190368c0159SJianxin Xiong {
191368c0159SJianxin Xiong return ERR_PTR(-EOPNOTSUPP);
192368c0159SJianxin Xiong }
1931e4df4a2SGal Pressman static inline struct ib_umem_dmabuf *
ib_umem_dmabuf_get_pinned(struct ib_device * device,unsigned long offset,size_t size,int fd,int access)1941e4df4a2SGal Pressman ib_umem_dmabuf_get_pinned(struct ib_device *device, unsigned long offset,
1951e4df4a2SGal Pressman size_t size, int fd, int access)
1961e4df4a2SGal Pressman {
1971e4df4a2SGal Pressman return ERR_PTR(-EOPNOTSUPP);
1981e4df4a2SGal Pressman }
ib_umem_dmabuf_map_pages(struct ib_umem_dmabuf * umem_dmabuf)199368c0159SJianxin Xiong static inline int ib_umem_dmabuf_map_pages(struct ib_umem_dmabuf *umem_dmabuf)
200368c0159SJianxin Xiong {
201368c0159SJianxin Xiong return -EOPNOTSUPP;
202368c0159SJianxin Xiong }
ib_umem_dmabuf_unmap_pages(struct ib_umem_dmabuf * umem_dmabuf)203368c0159SJianxin Xiong static inline void ib_umem_dmabuf_unmap_pages(struct ib_umem_dmabuf *umem_dmabuf) { }
ib_umem_dmabuf_release(struct ib_umem_dmabuf * umem_dmabuf)204368c0159SJianxin Xiong static inline void ib_umem_dmabuf_release(struct ib_umem_dmabuf *umem_dmabuf) { }
2054a353399SShiraz Saleem
206f7c6a7b5SRoland Dreier #endif /* CONFIG_INFINIBAND_USER_MEM */
207f7c6a7b5SRoland Dreier #endif /* IB_UMEM_H */
208