1 // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
2 /*
3  * Copyright(c) 2018 Intel Corporation.
4  *
5  */
6 #include "iowait.h"
7 #include "trace_iowait.h"
8 
9 /* 1 priority == 16 starve_cnt */
10 #define IOWAIT_PRIORITY_STARVE_SHIFT 4
11 
iowait_set_flag(struct iowait * wait,u32 flag)12 void iowait_set_flag(struct iowait *wait, u32 flag)
13 {
14 	trace_hfi1_iowait_set(wait, flag);
15 	set_bit(flag, &wait->flags);
16 }
17 
iowait_flag_set(struct iowait * wait,u32 flag)18 bool iowait_flag_set(struct iowait *wait, u32 flag)
19 {
20 	return test_bit(flag, &wait->flags);
21 }
22 
iowait_clear_flag(struct iowait * wait,u32 flag)23 inline void iowait_clear_flag(struct iowait *wait, u32 flag)
24 {
25 	trace_hfi1_iowait_clear(wait, flag);
26 	clear_bit(flag, &wait->flags);
27 }
28 
29 /*
30  * iowait_init() - initialize wait structure
31  * @wait: wait struct to initialize
32  * @tx_limit: limit for overflow queuing
33  * @func: restart function for workqueue
34  * @sleep: sleep function for no space
35  * @resume: wakeup function for no space
36  *
37  * This function initializes the iowait
38  * structure embedded in the QP or PQ.
39  *
40  */
iowait_init(struct iowait * wait,u32 tx_limit,void (* func)(struct work_struct * work),void (* tidfunc)(struct work_struct * work),int (* sleep)(struct sdma_engine * sde,struct iowait_work * wait,struct sdma_txreq * tx,uint seq,bool pkts_sent),void (* wakeup)(struct iowait * wait,int reason),void (* sdma_drained)(struct iowait * wait),void (* init_priority)(struct iowait * wait))41 void iowait_init(struct iowait *wait, u32 tx_limit,
42 		 void (*func)(struct work_struct *work),
43 		 void (*tidfunc)(struct work_struct *work),
44 		 int (*sleep)(struct sdma_engine *sde,
45 			      struct iowait_work *wait,
46 			      struct sdma_txreq *tx,
47 			      uint seq,
48 			      bool pkts_sent),
49 		 void (*wakeup)(struct iowait *wait, int reason),
50 		 void (*sdma_drained)(struct iowait *wait),
51 		 void (*init_priority)(struct iowait *wait))
52 {
53 	int i;
54 
55 	wait->count = 0;
56 	INIT_LIST_HEAD(&wait->list);
57 	init_waitqueue_head(&wait->wait_dma);
58 	init_waitqueue_head(&wait->wait_pio);
59 	atomic_set(&wait->sdma_busy, 0);
60 	atomic_set(&wait->pio_busy, 0);
61 	wait->tx_limit = tx_limit;
62 	wait->sleep = sleep;
63 	wait->wakeup = wakeup;
64 	wait->sdma_drained = sdma_drained;
65 	wait->init_priority = init_priority;
66 	wait->flags = 0;
67 	for (i = 0; i < IOWAIT_SES; i++) {
68 		wait->wait[i].iow = wait;
69 		INIT_LIST_HEAD(&wait->wait[i].tx_head);
70 		if (i == IOWAIT_IB_SE)
71 			INIT_WORK(&wait->wait[i].iowork, func);
72 		else
73 			INIT_WORK(&wait->wait[i].iowork, tidfunc);
74 	}
75 }
76 
77 /**
78  * iowait_cancel_work - cancel all work in iowait
79  * @w: the iowait struct
80  */
iowait_cancel_work(struct iowait * w)81 void iowait_cancel_work(struct iowait *w)
82 {
83 	cancel_work_sync(&iowait_get_ib_work(w)->iowork);
84 	/* Make sure that the iowork for TID RDMA is used */
85 	if (iowait_get_tid_work(w)->iowork.func)
86 		cancel_work_sync(&iowait_get_tid_work(w)->iowork);
87 }
88 
89 /**
90  * iowait_set_work_flag - set work flag based on leg
91  * @w: the iowait work struct
92  */
iowait_set_work_flag(struct iowait_work * w)93 int iowait_set_work_flag(struct iowait_work *w)
94 {
95 	if (w == &w->iow->wait[IOWAIT_IB_SE]) {
96 		iowait_set_flag(w->iow, IOWAIT_PENDING_IB);
97 		return IOWAIT_IB_SE;
98 	}
99 	iowait_set_flag(w->iow, IOWAIT_PENDING_TID);
100 	return IOWAIT_TID_SE;
101 }
102 
103 /**
104  * iowait_priority_update_top - update the top priority entry
105  * @w: the iowait struct
106  * @top: a pointer to the top priority entry
107  * @idx: the index of the current iowait in an array
108  * @top_idx: the array index for the iowait entry that has the top priority
109  *
110  * This function is called to compare the priority of a given
111  * iowait with the given top priority entry. The top index will
112  * be returned.
113  */
iowait_priority_update_top(struct iowait * w,struct iowait * top,uint idx,uint top_idx)114 uint iowait_priority_update_top(struct iowait *w,
115 				struct iowait *top,
116 				uint idx, uint top_idx)
117 {
118 	u8 cnt, tcnt;
119 
120 	/* Convert priority into starve_cnt and compare the total.*/
121 	cnt = (w->priority << IOWAIT_PRIORITY_STARVE_SHIFT) + w->starved_cnt;
122 	tcnt = (top->priority << IOWAIT_PRIORITY_STARVE_SHIFT) +
123 		top->starved_cnt;
124 	if (cnt > tcnt)
125 		return idx;
126 	else
127 		return top_idx;
128 }
129