1 #ifndef _HFI1_IOWAIT_H
2 #define _HFI1_IOWAIT_H
3 /*
4  * Copyright(c) 2015, 2016 Intel Corporation.
5  *
6  * This file is provided under a dual BSD/GPLv2 license.  When using or
7  * redistributing this file, you may do so under either license.
8  *
9  * GPL LICENSE SUMMARY
10  *
11  * This program is free software; you can redistribute it and/or modify
12  * it under the terms of version 2 of the GNU General Public License as
13  * published by the Free Software Foundation.
14  *
15  * This program is distributed in the hope that it will be useful, but
16  * WITHOUT ANY WARRANTY; without even the implied warranty of
17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
18  * General Public License for more details.
19  *
20  * BSD LICENSE
21  *
22  * Redistribution and use in source and binary forms, with or without
23  * modification, are permitted provided that the following conditions
24  * are met:
25  *
26  *  - Redistributions of source code must retain the above copyright
27  *    notice, this list of conditions and the following disclaimer.
28  *  - Redistributions in binary form must reproduce the above copyright
29  *    notice, this list of conditions and the following disclaimer in
30  *    the documentation and/or other materials provided with the
31  *    distribution.
32  *  - Neither the name of Intel Corporation nor the names of its
33  *    contributors may be used to endorse or promote products derived
34  *    from this software without specific prior written permission.
35  *
36  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
37  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
38  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
39  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
40  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
41  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
42  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
43  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
44  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
45  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
46  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
47  *
48  */
49 
50 #include <linux/list.h>
51 #include <linux/workqueue.h>
52 #include <linux/sched.h>
53 
54 #include "sdma_txreq.h"
55 
56 /*
57  * typedef (*restart_t)() - restart callback
58  * @work: pointer to work structure
59  */
60 typedef void (*restart_t)(struct work_struct *work);
61 
62 struct sdma_txreq;
63 struct sdma_engine;
64 /**
65  * struct iowait - linkage for delayed progress/waiting
66  * @list: used to add/insert into QP/PQ wait lists
67  * @lock: uses to record the list head lock
68  * @tx_head: overflow list of sdma_txreq's
69  * @sleep: no space callback
70  * @wakeup: space callback wakeup
71  * @sdma_drained: sdma count drained
72  * @iowork: workqueue overhead
73  * @wait_dma: wait for sdma_busy == 0
74  * @wait_pio: wait for pio_busy == 0
75  * @sdma_busy: # of packets in flight
76  * @count: total number of descriptors in tx_head'ed list
77  * @tx_limit: limit for overflow queuing
78  * @tx_count: number of tx entry's in tx_head'ed list
79  *
80  * This is to be embedded in user's state structure
81  * (QP or PQ).
82  *
83  * The sleep and wakeup members are a
84  * bit misnamed.   They do not strictly
85  * speaking sleep or wake up, but they
86  * are callbacks for the ULP to implement
87  * what ever queuing/dequeuing of
88  * the embedded iowait and its containing struct
89  * when a resource shortage like SDMA ring space is seen.
90  *
91  * Both potentially have locks help
92  * so sleeping is not allowed.
93  *
94  * The wait_dma member along with the iow
95  *
96  * The lock field is used by waiters to record
97  * the seqlock_t that guards the list head.
98  * Waiters explicity know that, but the destroy
99  * code that unwaits QPs does not.
100  */
101 
102 struct iowait {
103 	struct list_head list;
104 	struct list_head tx_head;
105 	int (*sleep)(
106 		struct sdma_engine *sde,
107 		struct iowait *wait,
108 		struct sdma_txreq *tx,
109 		uint seq,
110 		bool pkts_sent
111 		);
112 	void (*wakeup)(struct iowait *wait, int reason);
113 	void (*sdma_drained)(struct iowait *wait);
114 	seqlock_t *lock;
115 	struct work_struct iowork;
116 	wait_queue_head_t wait_dma;
117 	wait_queue_head_t wait_pio;
118 	atomic_t sdma_busy;
119 	atomic_t pio_busy;
120 	u32 count;
121 	u32 tx_limit;
122 	u32 tx_count;
123 	u8 starved_cnt;
124 };
125 
126 #define SDMA_AVAIL_REASON 0
127 
128 /**
129  * iowait_init() - initialize wait structure
130  * @wait: wait struct to initialize
131  * @tx_limit: limit for overflow queuing
132  * @func: restart function for workqueue
133  * @sleep: sleep function for no space
134  * @resume: wakeup function for no space
135  *
136  * This function initializes the iowait
137  * structure embedded in the QP or PQ.
138  *
139  */
140 
141 static inline void iowait_init(
142 	struct iowait *wait,
143 	u32 tx_limit,
144 	void (*func)(struct work_struct *work),
145 	int (*sleep)(
146 		struct sdma_engine *sde,
147 		struct iowait *wait,
148 		struct sdma_txreq *tx,
149 		uint seq,
150 		bool pkts_sent),
151 	void (*wakeup)(struct iowait *wait, int reason),
152 	void (*sdma_drained)(struct iowait *wait))
153 {
154 	wait->count = 0;
155 	wait->lock = NULL;
156 	INIT_LIST_HEAD(&wait->list);
157 	INIT_LIST_HEAD(&wait->tx_head);
158 	INIT_WORK(&wait->iowork, func);
159 	init_waitqueue_head(&wait->wait_dma);
160 	init_waitqueue_head(&wait->wait_pio);
161 	atomic_set(&wait->sdma_busy, 0);
162 	atomic_set(&wait->pio_busy, 0);
163 	wait->tx_limit = tx_limit;
164 	wait->sleep = sleep;
165 	wait->wakeup = wakeup;
166 	wait->sdma_drained = sdma_drained;
167 }
168 
169 /**
170  * iowait_schedule() - initialize wait structure
171  * @wait: wait struct to schedule
172  * @wq: workqueue for schedule
173  * @cpu: cpu
174  */
175 static inline void iowait_schedule(
176 	struct iowait *wait,
177 	struct workqueue_struct *wq,
178 	int cpu)
179 {
180 	queue_work_on(cpu, wq, &wait->iowork);
181 }
182 
183 /**
184  * iowait_sdma_drain() - wait for DMAs to drain
185  *
186  * @wait: iowait structure
187  *
188  * This will delay until the iowait sdmas have
189  * completed.
190  */
191 static inline void iowait_sdma_drain(struct iowait *wait)
192 {
193 	wait_event(wait->wait_dma, !atomic_read(&wait->sdma_busy));
194 }
195 
196 /**
197  * iowait_sdma_pending() - return sdma pending count
198  *
199  * @wait: iowait structure
200  *
201  */
202 static inline int iowait_sdma_pending(struct iowait *wait)
203 {
204 	return atomic_read(&wait->sdma_busy);
205 }
206 
207 /**
208  * iowait_sdma_inc - note sdma io pending
209  * @wait: iowait structure
210  */
211 static inline void iowait_sdma_inc(struct iowait *wait)
212 {
213 	atomic_inc(&wait->sdma_busy);
214 }
215 
216 /**
217  * iowait_sdma_add - add count to pending
218  * @wait: iowait structure
219  */
220 static inline void iowait_sdma_add(struct iowait *wait, int count)
221 {
222 	atomic_add(count, &wait->sdma_busy);
223 }
224 
225 /**
226  * iowait_sdma_dec - note sdma complete
227  * @wait: iowait structure
228  */
229 static inline int iowait_sdma_dec(struct iowait *wait)
230 {
231 	return atomic_dec_and_test(&wait->sdma_busy);
232 }
233 
234 /**
235  * iowait_pio_drain() - wait for pios to drain
236  *
237  * @wait: iowait structure
238  *
239  * This will delay until the iowait pios have
240  * completed.
241  */
242 static inline void iowait_pio_drain(struct iowait *wait)
243 {
244 	wait_event_timeout(wait->wait_pio,
245 			   !atomic_read(&wait->pio_busy),
246 			   HZ);
247 }
248 
249 /**
250  * iowait_pio_pending() - return pio pending count
251  *
252  * @wait: iowait structure
253  *
254  */
255 static inline int iowait_pio_pending(struct iowait *wait)
256 {
257 	return atomic_read(&wait->pio_busy);
258 }
259 
260 /**
261  * iowait_pio_inc - note pio pending
262  * @wait: iowait structure
263  */
264 static inline void iowait_pio_inc(struct iowait *wait)
265 {
266 	atomic_inc(&wait->pio_busy);
267 }
268 
269 /**
270  * iowait_sdma_dec - note pio complete
271  * @wait: iowait structure
272  */
273 static inline int iowait_pio_dec(struct iowait *wait)
274 {
275 	return atomic_dec_and_test(&wait->pio_busy);
276 }
277 
278 /**
279  * iowait_drain_wakeup() - trigger iowait_drain() waiter
280  *
281  * @wait: iowait structure
282  *
283  * This will trigger any waiters.
284  */
285 static inline void iowait_drain_wakeup(struct iowait *wait)
286 {
287 	wake_up(&wait->wait_dma);
288 	wake_up(&wait->wait_pio);
289 	if (wait->sdma_drained)
290 		wait->sdma_drained(wait);
291 }
292 
293 /**
294  * iowait_get_txhead() - get packet off of iowait list
295  *
296  * @wait wait struture
297  */
298 static inline struct sdma_txreq *iowait_get_txhead(struct iowait *wait)
299 {
300 	struct sdma_txreq *tx = NULL;
301 
302 	if (!list_empty(&wait->tx_head)) {
303 		tx = list_first_entry(
304 			&wait->tx_head,
305 			struct sdma_txreq,
306 			list);
307 		list_del_init(&tx->list);
308 	}
309 	return tx;
310 }
311 
312 /**
313  * iowait_queue - Put the iowait on a wait queue
314  * @pkts_sent: have some packets been sent before queuing?
315  * @w: the iowait struct
316  * @wait_head: the wait queue
317  *
318  * This function is called to insert an iowait struct into a
319  * wait queue after a resource (eg, sdma decriptor or pio
320  * buffer) is run out.
321  */
322 static inline void iowait_queue(bool pkts_sent, struct iowait *w,
323 				struct list_head *wait_head)
324 {
325 	/*
326 	 * To play fair, insert the iowait at the tail of the wait queue if it
327 	 * has already sent some packets; Otherwise, put it at the head.
328 	 */
329 	if (pkts_sent) {
330 		list_add_tail(&w->list, wait_head);
331 		w->starved_cnt = 0;
332 	} else {
333 		list_add(&w->list, wait_head);
334 		w->starved_cnt++;
335 	}
336 }
337 
338 /**
339  * iowait_starve_clear - clear the wait queue's starve count
340  * @pkts_sent: have some packets been sent?
341  * @w: the iowait struct
342  *
343  * This function is called to clear the starve count. If no
344  * packets have been sent, the starve count will not be cleared.
345  */
346 static inline void iowait_starve_clear(bool pkts_sent, struct iowait *w)
347 {
348 	if (pkts_sent)
349 		w->starved_cnt = 0;
350 }
351 
352 /**
353  * iowait_starve_find_max - Find the maximum of the starve count
354  * @w: the iowait struct
355  * @max: a variable containing the max starve count
356  * @idx: the index of the current iowait in an array
357  * @max_idx: a variable containing the array index for the
358  *         iowait entry that has the max starve count
359  *
360  * This function is called to compare the starve count of a
361  * given iowait with the given max starve count. The max starve
362  * count and the index will be updated if the iowait's start
363  * count is larger.
364  */
365 static inline void iowait_starve_find_max(struct iowait *w, u8 *max,
366 					  uint idx, uint *max_idx)
367 {
368 	if (w->starved_cnt > *max) {
369 		*max = w->starved_cnt;
370 		*max_idx = idx;
371 	}
372 }
373 
374 /**
375  * iowait_packet_queued() - determine if a packet is already built
376  * @wait: the wait structure
377  */
378 static inline bool iowait_packet_queued(struct iowait *wait)
379 {
380 	return !list_empty(&wait->tx_head);
381 }
382 
383 #endif
384