1 #ifndef _HFI1_IOWAIT_H
2 #define _HFI1_IOWAIT_H
3 /*
4  * Copyright(c) 2015 - 2018 Intel Corporation.
5  *
6  * This file is provided under a dual BSD/GPLv2 license.  When using or
7  * redistributing this file, you may do so under either license.
8  *
9  * GPL LICENSE SUMMARY
10  *
11  * This program is free software; you can redistribute it and/or modify
12  * it under the terms of version 2 of the GNU General Public License as
13  * published by the Free Software Foundation.
14  *
15  * This program is distributed in the hope that it will be useful, but
16  * WITHOUT ANY WARRANTY; without even the implied warranty of
17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
18  * General Public License for more details.
19  *
20  * BSD LICENSE
21  *
22  * Redistribution and use in source and binary forms, with or without
23  * modification, are permitted provided that the following conditions
24  * are met:
25  *
26  *  - Redistributions of source code must retain the above copyright
27  *    notice, this list of conditions and the following disclaimer.
28  *  - Redistributions in binary form must reproduce the above copyright
29  *    notice, this list of conditions and the following disclaimer in
30  *    the documentation and/or other materials provided with the
31  *    distribution.
32  *  - Neither the name of Intel Corporation nor the names of its
33  *    contributors may be used to endorse or promote products derived
34  *    from this software without specific prior written permission.
35  *
36  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
37  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
38  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
39  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
40  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
41  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
42  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
43  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
44  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
45  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
46  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
47  *
48  */
49 
50 #include <linux/list.h>
51 #include <linux/workqueue.h>
52 #include <linux/wait.h>
53 #include <linux/sched.h>
54 
55 #include "sdma_txreq.h"
56 
57 /*
58  * typedef (*restart_t)() - restart callback
59  * @work: pointer to work structure
60  */
61 typedef void (*restart_t)(struct work_struct *work);
62 
63 #define IOWAIT_PENDING_IB  0x0
64 #define IOWAIT_PENDING_TID 0x1
65 
66 /*
67  * A QP can have multiple Send Engines (SEs).
68  *
69  * The current use case is for supporting a TID RDMA
70  * packet build/xmit mechanism independent from verbs.
71  */
72 #define IOWAIT_SES 2
73 #define IOWAIT_IB_SE 0
74 #define IOWAIT_TID_SE 1
75 
76 struct sdma_txreq;
77 struct sdma_engine;
78 /**
79  * @iowork: the work struct
80  * @tx_head: list of prebuilt packets
81  * @iow: the parent iowait structure
82  *
83  * This structure is the work item (process) specific
84  * details associated with the each of the two SEs of the
85  * QP.
86  *
87  * The workstruct and the queued TXs are unique to each
88  * SE.
89  */
90 struct iowait;
91 struct iowait_work {
92 	struct work_struct iowork;
93 	struct list_head tx_head;
94 	struct iowait *iow;
95 };
96 
97 /**
98  * @list: used to add/insert into QP/PQ wait lists
99  * @tx_head: overflow list of sdma_txreq's
100  * @sleep: no space callback
101  * @wakeup: space callback wakeup
102  * @sdma_drained: sdma count drained
103  * @init_priority: callback to manipulate priority
104  * @lock: lock protected head of wait queue
105  * @iowork: workqueue overhead
106  * @wait_dma: wait for sdma_busy == 0
107  * @wait_pio: wait for pio_busy == 0
108  * @sdma_busy: # of packets in flight
109  * @count: total number of descriptors in tx_head'ed list
110  * @tx_limit: limit for overflow queuing
111  * @tx_count: number of tx entry's in tx_head'ed list
112  * @flags: wait flags (one per QP)
113  * @wait: SE array for multiple legs
114  *
115  * This is to be embedded in user's state structure
116  * (QP or PQ).
117  *
118  * The sleep and wakeup members are a
119  * bit misnamed.   They do not strictly
120  * speaking sleep or wake up, but they
121  * are callbacks for the ULP to implement
122  * what ever queuing/dequeuing of
123  * the embedded iowait and its containing struct
124  * when a resource shortage like SDMA ring space
125  * or PIO credit space is seen.
126  *
127  * Both potentially have locks help
128  * so sleeping is not allowed and it is not
129  * supported to submit txreqs from the wakeup
130  * call directly because of lock conflicts.
131  *
132  * The wait_dma member along with the iow
133  *
134  * The lock field is used by waiters to record
135  * the seqlock_t that guards the list head.
136  * Waiters explicity know that, but the destroy
137  * code that unwaits QPs does not.
138  */
139 struct iowait {
140 	struct list_head list;
141 	int (*sleep)(
142 		struct sdma_engine *sde,
143 		struct iowait_work *wait,
144 		struct sdma_txreq *tx,
145 		uint seq,
146 		bool pkts_sent
147 		);
148 	void (*wakeup)(struct iowait *wait, int reason);
149 	void (*sdma_drained)(struct iowait *wait);
150 	void (*init_priority)(struct iowait *wait);
151 	seqlock_t *lock;
152 	wait_queue_head_t wait_dma;
153 	wait_queue_head_t wait_pio;
154 	atomic_t sdma_busy;
155 	atomic_t pio_busy;
156 	u32 count;
157 	u32 tx_limit;
158 	u32 tx_count;
159 	u8 starved_cnt;
160 	u8 priority;
161 	unsigned long flags;
162 	struct iowait_work wait[IOWAIT_SES];
163 };
164 
165 #define SDMA_AVAIL_REASON 0
166 
167 void iowait_set_flag(struct iowait *wait, u32 flag);
168 bool iowait_flag_set(struct iowait *wait, u32 flag);
169 void iowait_clear_flag(struct iowait *wait, u32 flag);
170 
171 void iowait_init(struct iowait *wait, u32 tx_limit,
172 		 void (*func)(struct work_struct *work),
173 		 void (*tidfunc)(struct work_struct *work),
174 		 int (*sleep)(struct sdma_engine *sde,
175 			      struct iowait_work *wait,
176 			      struct sdma_txreq *tx,
177 			      uint seq,
178 			      bool pkts_sent),
179 		 void (*wakeup)(struct iowait *wait, int reason),
180 		 void (*sdma_drained)(struct iowait *wait),
181 		 void (*init_priority)(struct iowait *wait));
182 
183 /**
184  * iowait_schedule() - schedule the default send engine work
185  * @wait: wait struct to schedule
186  * @wq: workqueue for schedule
187  * @cpu: cpu
188  */
189 static inline bool iowait_schedule(struct iowait *wait,
190 				   struct workqueue_struct *wq, int cpu)
191 {
192 	return !!queue_work_on(cpu, wq, &wait->wait[IOWAIT_IB_SE].iowork);
193 }
194 
195 /**
196  * iowait_tid_schedule - schedule the tid SE
197  * @wait: the iowait structure
198  * @wq: the work queue
199  * @cpu: the cpu
200  */
201 static inline bool iowait_tid_schedule(struct iowait *wait,
202 				       struct workqueue_struct *wq, int cpu)
203 {
204 	return !!queue_work_on(cpu, wq, &wait->wait[IOWAIT_TID_SE].iowork);
205 }
206 
207 /**
208  * iowait_sdma_drain() - wait for DMAs to drain
209  *
210  * @wait: iowait structure
211  *
212  * This will delay until the iowait sdmas have
213  * completed.
214  */
215 static inline void iowait_sdma_drain(struct iowait *wait)
216 {
217 	wait_event(wait->wait_dma, !atomic_read(&wait->sdma_busy));
218 }
219 
220 /**
221  * iowait_sdma_pending() - return sdma pending count
222  *
223  * @wait: iowait structure
224  *
225  */
226 static inline int iowait_sdma_pending(struct iowait *wait)
227 {
228 	return atomic_read(&wait->sdma_busy);
229 }
230 
231 /**
232  * iowait_sdma_inc - note sdma io pending
233  * @wait: iowait structure
234  */
235 static inline void iowait_sdma_inc(struct iowait *wait)
236 {
237 	atomic_inc(&wait->sdma_busy);
238 }
239 
240 /**
241  * iowait_sdma_add - add count to pending
242  * @wait: iowait structure
243  */
244 static inline void iowait_sdma_add(struct iowait *wait, int count)
245 {
246 	atomic_add(count, &wait->sdma_busy);
247 }
248 
249 /**
250  * iowait_sdma_dec - note sdma complete
251  * @wait: iowait structure
252  */
253 static inline int iowait_sdma_dec(struct iowait *wait)
254 {
255 	if (!wait)
256 		return 0;
257 	return atomic_dec_and_test(&wait->sdma_busy);
258 }
259 
260 /**
261  * iowait_pio_drain() - wait for pios to drain
262  *
263  * @wait: iowait structure
264  *
265  * This will delay until the iowait pios have
266  * completed.
267  */
268 static inline void iowait_pio_drain(struct iowait *wait)
269 {
270 	wait_event_timeout(wait->wait_pio,
271 			   !atomic_read(&wait->pio_busy),
272 			   HZ);
273 }
274 
275 /**
276  * iowait_pio_pending() - return pio pending count
277  *
278  * @wait: iowait structure
279  *
280  */
281 static inline int iowait_pio_pending(struct iowait *wait)
282 {
283 	return atomic_read(&wait->pio_busy);
284 }
285 
286 /**
287  * iowait_pio_inc - note pio pending
288  * @wait: iowait structure
289  */
290 static inline void iowait_pio_inc(struct iowait *wait)
291 {
292 	atomic_inc(&wait->pio_busy);
293 }
294 
295 /**
296  * iowait_pio_dec - note pio complete
297  * @wait: iowait structure
298  */
299 static inline int iowait_pio_dec(struct iowait *wait)
300 {
301 	if (!wait)
302 		return 0;
303 	return atomic_dec_and_test(&wait->pio_busy);
304 }
305 
306 /**
307  * iowait_drain_wakeup() - trigger iowait_drain() waiter
308  *
309  * @wait: iowait structure
310  *
311  * This will trigger any waiters.
312  */
313 static inline void iowait_drain_wakeup(struct iowait *wait)
314 {
315 	wake_up(&wait->wait_dma);
316 	wake_up(&wait->wait_pio);
317 	if (wait->sdma_drained)
318 		wait->sdma_drained(wait);
319 }
320 
321 /**
322  * iowait_get_txhead() - get packet off of iowait list
323  *
324  * @wait iowait_work struture
325  */
326 static inline struct sdma_txreq *iowait_get_txhead(struct iowait_work *wait)
327 {
328 	struct sdma_txreq *tx = NULL;
329 
330 	if (!list_empty(&wait->tx_head)) {
331 		tx = list_first_entry(
332 			&wait->tx_head,
333 			struct sdma_txreq,
334 			list);
335 		list_del_init(&tx->list);
336 	}
337 	return tx;
338 }
339 
340 static inline u16 iowait_get_desc(struct iowait_work *w)
341 {
342 	u16 num_desc = 0;
343 	struct sdma_txreq *tx = NULL;
344 
345 	if (!list_empty(&w->tx_head)) {
346 		tx = list_first_entry(&w->tx_head, struct sdma_txreq,
347 				      list);
348 		num_desc = tx->num_desc;
349 		if (tx->flags & SDMA_TXREQ_F_VIP)
350 			w->iow->priority++;
351 	}
352 	return num_desc;
353 }
354 
355 static inline u32 iowait_get_all_desc(struct iowait *w)
356 {
357 	u32 num_desc = 0;
358 
359 	num_desc = iowait_get_desc(&w->wait[IOWAIT_IB_SE]);
360 	num_desc += iowait_get_desc(&w->wait[IOWAIT_TID_SE]);
361 	return num_desc;
362 }
363 
364 static inline void iowait_update_priority(struct iowait_work *w)
365 {
366 	struct sdma_txreq *tx = NULL;
367 
368 	if (!list_empty(&w->tx_head)) {
369 		tx = list_first_entry(&w->tx_head, struct sdma_txreq,
370 				      list);
371 		if (tx->flags & SDMA_TXREQ_F_VIP)
372 			w->iow->priority++;
373 	}
374 }
375 
376 static inline void iowait_update_all_priority(struct iowait *w)
377 {
378 	iowait_update_priority(&w->wait[IOWAIT_IB_SE]);
379 	iowait_update_priority(&w->wait[IOWAIT_TID_SE]);
380 }
381 
382 static inline void iowait_init_priority(struct iowait *w)
383 {
384 	w->priority = 0;
385 	if (w->init_priority)
386 		w->init_priority(w);
387 }
388 
389 static inline void iowait_get_priority(struct iowait *w)
390 {
391 	iowait_init_priority(w);
392 	iowait_update_all_priority(w);
393 }
394 
395 /**
396  * iowait_queue - Put the iowait on a wait queue
397  * @pkts_sent: have some packets been sent before queuing?
398  * @w: the iowait struct
399  * @wait_head: the wait queue
400  *
401  * This function is called to insert an iowait struct into a
402  * wait queue after a resource (eg, sdma descriptor or pio
403  * buffer) is run out.
404  */
405 static inline void iowait_queue(bool pkts_sent, struct iowait *w,
406 				struct list_head *wait_head)
407 {
408 	/*
409 	 * To play fair, insert the iowait at the tail of the wait queue if it
410 	 * has already sent some packets; Otherwise, put it at the head.
411 	 * However, if it has priority packets to send, also put it at the
412 	 * head.
413 	 */
414 	if (pkts_sent)
415 		w->starved_cnt = 0;
416 	else
417 		w->starved_cnt++;
418 
419 	if (w->priority > 0 || !pkts_sent)
420 		list_add(&w->list, wait_head);
421 	else
422 		list_add_tail(&w->list, wait_head);
423 }
424 
425 /**
426  * iowait_starve_clear - clear the wait queue's starve count
427  * @pkts_sent: have some packets been sent?
428  * @w: the iowait struct
429  *
430  * This function is called to clear the starve count. If no
431  * packets have been sent, the starve count will not be cleared.
432  */
433 static inline void iowait_starve_clear(bool pkts_sent, struct iowait *w)
434 {
435 	if (pkts_sent)
436 		w->starved_cnt = 0;
437 }
438 
439 /* Update the top priority index */
440 uint iowait_priority_update_top(struct iowait *w,
441 				struct iowait *top,
442 				uint idx, uint top_idx);
443 
444 /**
445  * iowait_packet_queued() - determine if a packet is queued
446  * @wait: the iowait_work structure
447  */
448 static inline bool iowait_packet_queued(struct iowait_work *wait)
449 {
450 	return !list_empty(&wait->tx_head);
451 }
452 
453 /**
454  * inc_wait_count - increment wait counts
455  * @w: the log work struct
456  * @n: the count
457  */
458 static inline void iowait_inc_wait_count(struct iowait_work *w, u16 n)
459 {
460 	if (!w)
461 		return;
462 	w->iow->tx_count++;
463 	w->iow->count += n;
464 }
465 
466 /**
467  * iowait_get_tid_work - return iowait_work for tid SE
468  * @w: the iowait struct
469  */
470 static inline struct iowait_work *iowait_get_tid_work(struct iowait *w)
471 {
472 	return &w->wait[IOWAIT_TID_SE];
473 }
474 
475 /**
476  * iowait_get_ib_work - return iowait_work for ib SE
477  * @w: the iowait struct
478  */
479 static inline struct iowait_work *iowait_get_ib_work(struct iowait *w)
480 {
481 	return &w->wait[IOWAIT_IB_SE];
482 }
483 
484 /**
485  * iowait_ioww_to_iow - return iowait given iowait_work
486  * @w: the iowait_work struct
487  */
488 static inline struct iowait *iowait_ioww_to_iow(struct iowait_work *w)
489 {
490 	if (likely(w))
491 		return w->iow;
492 	return NULL;
493 }
494 
495 void iowait_cancel_work(struct iowait *w);
496 int iowait_set_work_flag(struct iowait_work *w);
497 
498 #endif
499