1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. 4 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. 5 */ 6 7 #include "rxe.h" 8 9 static struct workqueue_struct *rxe_wq; 10 11 int rxe_alloc_wq(void) 12 { 13 rxe_wq = alloc_workqueue("rxe_wq", WQ_UNBOUND, WQ_MAX_ACTIVE); 14 if (!rxe_wq) 15 return -ENOMEM; 16 17 return 0; 18 } 19 20 void rxe_destroy_wq(void) 21 { 22 destroy_workqueue(rxe_wq); 23 } 24 25 /* Check if task is idle i.e. not running, not scheduled in 26 * work queue and not draining. If so move to busy to 27 * reserve a slot in do_task() by setting to busy and taking 28 * a qp reference to cover the gap from now until the task finishes. 29 * state will move out of busy if task returns a non zero value 30 * in do_task(). If state is already busy it is raised to armed 31 * to indicate to do_task that additional pass should be made 32 * over the task. 33 * Context: caller should hold task->lock. 34 * Returns: true if state transitioned from idle to busy else false. 35 */ 36 static bool __reserve_if_idle(struct rxe_task *task) 37 { 38 WARN_ON(rxe_read(task->qp) <= 0); 39 40 if (task->state == TASK_STATE_IDLE) { 41 rxe_get(task->qp); 42 task->state = TASK_STATE_BUSY; 43 task->num_sched++; 44 return true; 45 } 46 47 if (task->state == TASK_STATE_BUSY) 48 task->state = TASK_STATE_ARMED; 49 50 return false; 51 } 52 53 /* check if task is idle or drained and not currently 54 * scheduled in the work queue. This routine is 55 * called by rxe_cleanup_task or rxe_disable_task to 56 * see if the queue is empty. 57 * Context: caller should hold task->lock. 58 * Returns true if done else false. 59 */ 60 static bool __is_done(struct rxe_task *task) 61 { 62 if (work_pending(&task->work)) 63 return false; 64 65 if (task->state == TASK_STATE_IDLE || 66 task->state == TASK_STATE_DRAINED) { 67 return true; 68 } 69 70 return false; 71 } 72 73 /* a locked version of __is_done */ 74 static bool is_done(struct rxe_task *task) 75 { 76 unsigned long flags; 77 int done; 78 79 spin_lock_irqsave(&task->lock, flags); 80 done = __is_done(task); 81 spin_unlock_irqrestore(&task->lock, flags); 82 83 return done; 84 } 85 86 /* do_task is a wrapper for the three tasks (requester, 87 * completer, responder) and calls them in a loop until 88 * they return a non-zero value. It is called either 89 * directly by rxe_run_task or indirectly if rxe_sched_task 90 * schedules the task. They must call __reserve_if_idle to 91 * move the task to busy before calling or scheduling. 92 * The task can also be moved to drained or invalid 93 * by calls to rxe_cleanup_task or rxe_disable_task. 94 * In that case tasks which get here are not executed but 95 * just flushed. The tasks are designed to look to see if 96 * there is work to do and then do part of it before returning 97 * here with a return value of zero until all the work 98 * has been consumed then it returns a non-zero value. 99 * The number of times the task can be run is limited by 100 * max iterations so one task cannot hold the cpu forever. 101 * If the limit is hit and work remains the task is rescheduled. 102 */ 103 static void do_task(struct rxe_task *task) 104 { 105 unsigned int iterations; 106 unsigned long flags; 107 int resched = 0; 108 int cont; 109 int ret; 110 111 WARN_ON(rxe_read(task->qp) <= 0); 112 113 spin_lock_irqsave(&task->lock, flags); 114 if (task->state >= TASK_STATE_DRAINED) { 115 rxe_put(task->qp); 116 task->num_done++; 117 spin_unlock_irqrestore(&task->lock, flags); 118 return; 119 } 120 spin_unlock_irqrestore(&task->lock, flags); 121 122 do { 123 iterations = RXE_MAX_ITERATIONS; 124 cont = 0; 125 126 do { 127 ret = task->func(task->qp); 128 } while (ret == 0 && iterations-- > 0); 129 130 spin_lock_irqsave(&task->lock, flags); 131 /* we're not done yet but we ran out of iterations. 132 * yield the cpu and reschedule the task 133 */ 134 if (!ret) { 135 task->state = TASK_STATE_IDLE; 136 resched = 1; 137 goto exit; 138 } 139 140 switch (task->state) { 141 case TASK_STATE_BUSY: 142 task->state = TASK_STATE_IDLE; 143 break; 144 145 /* someone tried to schedule the task while we 146 * were running, keep going 147 */ 148 case TASK_STATE_ARMED: 149 task->state = TASK_STATE_BUSY; 150 cont = 1; 151 break; 152 153 case TASK_STATE_DRAINING: 154 task->state = TASK_STATE_DRAINED; 155 break; 156 157 default: 158 WARN_ON(1); 159 rxe_dbg_qp(task->qp, "unexpected task state = %d", 160 task->state); 161 task->state = TASK_STATE_IDLE; 162 } 163 164 exit: 165 if (!cont) { 166 task->num_done++; 167 if (WARN_ON(task->num_done != task->num_sched)) 168 rxe_dbg_qp( 169 task->qp, 170 "%ld tasks scheduled, %ld tasks done", 171 task->num_sched, task->num_done); 172 } 173 spin_unlock_irqrestore(&task->lock, flags); 174 } while (cont); 175 176 task->ret = ret; 177 178 if (resched) 179 rxe_sched_task(task); 180 181 rxe_put(task->qp); 182 } 183 184 /* wrapper around do_task to fix argument for work queue */ 185 static void do_work(struct work_struct *work) 186 { 187 do_task(container_of(work, struct rxe_task, work)); 188 } 189 190 int rxe_init_task(struct rxe_task *task, struct rxe_qp *qp, 191 int (*func)(struct rxe_qp *)) 192 { 193 WARN_ON(rxe_read(qp) <= 0); 194 195 task->qp = qp; 196 task->func = func; 197 task->state = TASK_STATE_IDLE; 198 spin_lock_init(&task->lock); 199 INIT_WORK(&task->work, do_work); 200 201 return 0; 202 } 203 204 /* rxe_cleanup_task is only called from rxe_do_qp_cleanup in 205 * process context. The qp is already completed with no 206 * remaining references. Once the queue is drained the 207 * task is moved to invalid and returns. The qp cleanup 208 * code then calls the task functions directly without 209 * using the task struct to drain any late arriving packets 210 * or work requests. 211 */ 212 void rxe_cleanup_task(struct rxe_task *task) 213 { 214 unsigned long flags; 215 216 spin_lock_irqsave(&task->lock, flags); 217 if (!__is_done(task) && task->state < TASK_STATE_DRAINED) { 218 task->state = TASK_STATE_DRAINING; 219 } else { 220 task->state = TASK_STATE_INVALID; 221 spin_unlock_irqrestore(&task->lock, flags); 222 return; 223 } 224 spin_unlock_irqrestore(&task->lock, flags); 225 226 /* now the task cannot be scheduled or run just wait 227 * for the previously scheduled tasks to finish. 228 */ 229 while (!is_done(task)) 230 cond_resched(); 231 232 spin_lock_irqsave(&task->lock, flags); 233 task->state = TASK_STATE_INVALID; 234 spin_unlock_irqrestore(&task->lock, flags); 235 } 236 237 /* run the task inline if it is currently idle 238 * cannot call do_task holding the lock 239 */ 240 void rxe_run_task(struct rxe_task *task) 241 { 242 unsigned long flags; 243 bool run; 244 245 WARN_ON(rxe_read(task->qp) <= 0); 246 247 spin_lock_irqsave(&task->lock, flags); 248 run = __reserve_if_idle(task); 249 spin_unlock_irqrestore(&task->lock, flags); 250 251 if (run) 252 do_task(task); 253 } 254 255 /* schedule the task to run later as a work queue entry. 256 * the queue_work call can be called holding 257 * the lock. 258 */ 259 void rxe_sched_task(struct rxe_task *task) 260 { 261 unsigned long flags; 262 263 WARN_ON(rxe_read(task->qp) <= 0); 264 265 spin_lock_irqsave(&task->lock, flags); 266 if (__reserve_if_idle(task)) 267 queue_work(rxe_wq, &task->work); 268 spin_unlock_irqrestore(&task->lock, flags); 269 } 270 271 /* rxe_disable/enable_task are only called from 272 * rxe_modify_qp in process context. Task is moved 273 * to the drained state by do_task. 274 */ 275 void rxe_disable_task(struct rxe_task *task) 276 { 277 unsigned long flags; 278 279 WARN_ON(rxe_read(task->qp) <= 0); 280 281 spin_lock_irqsave(&task->lock, flags); 282 if (!__is_done(task) && task->state < TASK_STATE_DRAINED) { 283 task->state = TASK_STATE_DRAINING; 284 } else { 285 task->state = TASK_STATE_DRAINED; 286 spin_unlock_irqrestore(&task->lock, flags); 287 return; 288 } 289 spin_unlock_irqrestore(&task->lock, flags); 290 291 while (!is_done(task)) 292 cond_resched(); 293 294 spin_lock_irqsave(&task->lock, flags); 295 task->state = TASK_STATE_DRAINED; 296 spin_unlock_irqrestore(&task->lock, flags); 297 } 298 299 void rxe_enable_task(struct rxe_task *task) 300 { 301 unsigned long flags; 302 303 WARN_ON(rxe_read(task->qp) <= 0); 304 305 spin_lock_irqsave(&task->lock, flags); 306 if (task->state == TASK_STATE_INVALID) { 307 spin_unlock_irqrestore(&task->lock, flags); 308 return; 309 } 310 311 task->state = TASK_STATE_IDLE; 312 spin_unlock_irqrestore(&task->lock, flags); 313 } 314