1 #ifndef _HFI1_USER_SDMA_H
2 #define _HFI1_USER_SDMA_H
3 /*
4  * Copyright(c) 2015 - 2018 Intel Corporation.
5  *
6  * This file is provided under a dual BSD/GPLv2 license.  When using or
7  * redistributing this file, you may do so under either license.
8  *
9  * GPL LICENSE SUMMARY
10  *
11  * This program is free software; you can redistribute it and/or modify
12  * it under the terms of version 2 of the GNU General Public License as
13  * published by the Free Software Foundation.
14  *
15  * This program is distributed in the hope that it will be useful, but
16  * WITHOUT ANY WARRANTY; without even the implied warranty of
17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
18  * General Public License for more details.
19  *
20  * BSD LICENSE
21  *
22  * Redistribution and use in source and binary forms, with or without
23  * modification, are permitted provided that the following conditions
24  * are met:
25  *
26  *  - Redistributions of source code must retain the above copyright
27  *    notice, this list of conditions and the following disclaimer.
28  *  - Redistributions in binary form must reproduce the above copyright
29  *    notice, this list of conditions and the following disclaimer in
30  *    the documentation and/or other materials provided with the
31  *    distribution.
32  *  - Neither the name of Intel Corporation nor the names of its
33  *    contributors may be used to endorse or promote products derived
34  *    from this software without specific prior written permission.
35  *
36  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
37  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
38  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
39  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
40  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
41  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
42  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
43  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
44  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
45  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
46  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
47  *
48  */
49 #include <linux/device.h>
50 #include <linux/wait.h>
51 
52 #include "common.h"
53 #include "iowait.h"
54 #include "user_exp_rcv.h"
55 
56 /* The maximum number of Data io vectors per message/request */
57 #define MAX_VECTORS_PER_REQ 8
58 /*
59  * Maximum number of packet to send from each message/request
60  * before moving to the next one.
61  */
62 #define MAX_PKTS_PER_QUEUE 16
63 
64 #define num_pages(x) (1 + ((((x) - 1) & PAGE_MASK) >> PAGE_SHIFT))
65 
66 #define req_opcode(x) \
67 	(((x) >> HFI1_SDMA_REQ_OPCODE_SHIFT) & HFI1_SDMA_REQ_OPCODE_MASK)
68 #define req_version(x) \
69 	(((x) >> HFI1_SDMA_REQ_VERSION_SHIFT) & HFI1_SDMA_REQ_OPCODE_MASK)
70 #define req_iovcnt(x) \
71 	(((x) >> HFI1_SDMA_REQ_IOVCNT_SHIFT) & HFI1_SDMA_REQ_IOVCNT_MASK)
72 
73 /* Number of BTH.PSN bits used for sequence number in expected rcvs */
74 #define BTH_SEQ_MASK 0x7ffull
75 
76 #define AHG_KDETH_INTR_SHIFT 12
77 #define AHG_KDETH_SH_SHIFT   13
78 #define AHG_KDETH_ARRAY_SIZE  9
79 
80 #define PBC2LRH(x) ((((x) & 0xfff) << 2) - 4)
81 #define LRH2PBC(x) ((((x) >> 2) + 1) & 0xfff)
82 
83 /**
84  * Build an SDMA AHG header update descriptor and save it to an array.
85  * @arr        - Array to save the descriptor to.
86  * @idx        - Index of the array at which the descriptor will be saved.
87  * @array_size - Size of the array arr.
88  * @dw         - Update index into the header in DWs.
89  * @bit        - Start bit.
90  * @width      - Field width.
91  * @value      - 16 bits of immediate data to write into the field.
92  * Returns -ERANGE if idx is invalid. If successful, returns the next index
93  * (idx + 1) of the array to be used for the next descriptor.
94  */
95 static inline int ahg_header_set(u32 *arr, int idx, size_t array_size,
96 				 u8 dw, u8 bit, u8 width, u16 value)
97 {
98 	if ((size_t)idx >= array_size)
99 		return -ERANGE;
100 	arr[idx++] = sdma_build_ahg_descriptor(value, dw, bit, width);
101 	return idx;
102 }
103 
104 /* Tx request flag bits */
105 #define TXREQ_FLAGS_REQ_ACK   BIT(0)      /* Set the ACK bit in the header */
106 #define TXREQ_FLAGS_REQ_DISABLE_SH BIT(1) /* Disable header suppression */
107 
108 enum pkt_q_sdma_state {
109 	SDMA_PKT_Q_ACTIVE,
110 	SDMA_PKT_Q_DEFERRED,
111 };
112 
113 #define SDMA_IOWAIT_TIMEOUT 1000 /* in milliseconds */
114 
115 #define SDMA_DBG(req, fmt, ...)				     \
116 	hfi1_cdbg(SDMA, "[%u:%u:%u:%u] " fmt, (req)->pq->dd->unit, \
117 		 (req)->pq->ctxt, (req)->pq->subctxt, (req)->info.comp_idx, \
118 		 ##__VA_ARGS__)
119 
120 struct hfi1_user_sdma_pkt_q {
121 	u16 ctxt;
122 	u16 subctxt;
123 	u16 n_max_reqs;
124 	atomic_t n_reqs;
125 	u16 reqidx;
126 	struct hfi1_devdata *dd;
127 	struct kmem_cache *txreq_cache;
128 	struct user_sdma_request *reqs;
129 	unsigned long *req_in_use;
130 	struct iowait busy;
131 	enum pkt_q_sdma_state state;
132 	wait_queue_head_t wait;
133 	unsigned long unpinned;
134 	struct mmu_rb_handler *handler;
135 	atomic_t n_locked;
136 	struct mm_struct *mm;
137 };
138 
139 struct hfi1_user_sdma_comp_q {
140 	u16 nentries;
141 	struct hfi1_sdma_comp_entry *comps;
142 };
143 
144 struct sdma_mmu_node {
145 	struct mmu_rb_node rb;
146 	struct hfi1_user_sdma_pkt_q *pq;
147 	atomic_t refcount;
148 	struct page **pages;
149 	unsigned int npages;
150 };
151 
152 struct user_sdma_iovec {
153 	struct list_head list;
154 	struct iovec iov;
155 	/* number of pages in this vector */
156 	unsigned int npages;
157 	/* array of pinned pages for this vector */
158 	struct page **pages;
159 	/*
160 	 * offset into the virtual address space of the vector at
161 	 * which we last left off.
162 	 */
163 	u64 offset;
164 	struct sdma_mmu_node *node;
165 };
166 
167 /* evict operation argument */
168 struct evict_data {
169 	u32 cleared;	/* count evicted so far */
170 	u32 target;	/* target count to evict */
171 };
172 
173 struct user_sdma_request {
174 	/* This is the original header from user space */
175 	struct hfi1_pkt_header hdr;
176 
177 	/* Read mostly fields */
178 	struct hfi1_user_sdma_pkt_q *pq ____cacheline_aligned_in_smp;
179 	struct hfi1_user_sdma_comp_q *cq;
180 	/*
181 	 * Pointer to the SDMA engine for this request.
182 	 * Since different request could be on different VLs,
183 	 * each request will need it's own engine pointer.
184 	 */
185 	struct sdma_engine *sde;
186 	struct sdma_req_info info;
187 	/* TID array values copied from the tid_iov vector */
188 	u32 *tids;
189 	/* total length of the data in the request */
190 	u32 data_len;
191 	/* number of elements copied to the tids array */
192 	u16 n_tids;
193 	/*
194 	 * We copy the iovs for this request (based on
195 	 * info.iovcnt). These are only the data vectors
196 	 */
197 	u8 data_iovs;
198 	s8 ahg_idx;
199 
200 	/* Writeable fields shared with interrupt */
201 	u16 seqcomp ____cacheline_aligned_in_smp;
202 	u16 seqsubmitted;
203 
204 	/* Send side fields */
205 	struct list_head txps ____cacheline_aligned_in_smp;
206 	u16 seqnum;
207 	/*
208 	 * KDETH.OFFSET (TID) field
209 	 * The offset can cover multiple packets, depending on the
210 	 * size of the TID entry.
211 	 */
212 	u32 tidoffset;
213 	/*
214 	 * KDETH.Offset (Eager) field
215 	 * We need to remember the initial value so the headers
216 	 * can be updated properly.
217 	 */
218 	u32 koffset;
219 	u32 sent;
220 	/* TID index copied from the tid_iov vector */
221 	u16 tididx;
222 	/* progress index moving along the iovs array */
223 	u8 iov_idx;
224 	u8 has_error;
225 
226 	struct user_sdma_iovec iovs[MAX_VECTORS_PER_REQ];
227 } ____cacheline_aligned_in_smp;
228 
229 /*
230  * A single txreq could span up to 3 physical pages when the MTU
231  * is sufficiently large (> 4K). Each of the IOV pointers also
232  * needs it's own set of flags so the vector has been handled
233  * independently of each other.
234  */
235 struct user_sdma_txreq {
236 	/* Packet header for the txreq */
237 	struct hfi1_pkt_header hdr;
238 	struct sdma_txreq txreq;
239 	struct list_head list;
240 	struct user_sdma_request *req;
241 	u16 flags;
242 	u16 seqnum;
243 };
244 
245 int hfi1_user_sdma_alloc_queues(struct hfi1_ctxtdata *uctxt,
246 				struct hfi1_filedata *fd);
247 int hfi1_user_sdma_free_queues(struct hfi1_filedata *fd,
248 			       struct hfi1_ctxtdata *uctxt);
249 int hfi1_user_sdma_process_request(struct hfi1_filedata *fd,
250 				   struct iovec *iovec, unsigned long dim,
251 				   unsigned long *count);
252 
253 #endif /* _HFI1_USER_SDMA_H */
254