1 /* 2 * Handle async block request by crypto hardware engine. 3 * 4 * Copyright (C) 2016 Linaro, Inc. 5 * 6 * Author: Baolin Wang <baolin.wang@linaro.org> 7 * 8 * This program is free software; you can redistribute it and/or modify it 9 * under the terms of the GNU General Public License as published by the Free 10 * Software Foundation; either version 2 of the License, or (at your option) 11 * any later version. 12 * 13 */ 14 15 #include <linux/err.h> 16 #include <linux/delay.h> 17 #include <crypto/engine.h> 18 #include <crypto/internal/hash.h> 19 #include "internal.h" 20 21 #define CRYPTO_ENGINE_MAX_QLEN 10 22 23 /** 24 * crypto_pump_requests - dequeue one request from engine queue to process 25 * @engine: the hardware engine 26 * @in_kthread: true if we are in the context of the request pump thread 27 * 28 * This function checks if there is any request in the engine queue that 29 * needs processing and if so call out to the driver to initialize hardware 30 * and handle each request. 31 */ 32 static void crypto_pump_requests(struct crypto_engine *engine, 33 bool in_kthread) 34 { 35 struct crypto_async_request *async_req, *backlog; 36 struct ahash_request *hreq; 37 struct ablkcipher_request *breq; 38 unsigned long flags; 39 bool was_busy = false; 40 int ret, rtype; 41 42 spin_lock_irqsave(&engine->queue_lock, flags); 43 44 /* Make sure we are not already running a request */ 45 if (engine->cur_req) 46 goto out; 47 48 /* If another context is idling then defer */ 49 if (engine->idling) { 50 kthread_queue_work(engine->kworker, &engine->pump_requests); 51 goto out; 52 } 53 54 /* Check if the engine queue is idle */ 55 if (!crypto_queue_len(&engine->queue) || !engine->running) { 56 if (!engine->busy) 57 goto out; 58 59 /* Only do teardown in the thread */ 60 if (!in_kthread) { 61 kthread_queue_work(engine->kworker, 62 &engine->pump_requests); 63 goto out; 64 } 65 66 engine->busy = false; 67 engine->idling = true; 68 spin_unlock_irqrestore(&engine->queue_lock, flags); 69 70 if (engine->unprepare_crypt_hardware && 71 engine->unprepare_crypt_hardware(engine)) 72 pr_err("failed to unprepare crypt hardware\n"); 73 74 spin_lock_irqsave(&engine->queue_lock, flags); 75 engine->idling = false; 76 goto out; 77 } 78 79 /* Get the fist request from the engine queue to handle */ 80 backlog = crypto_get_backlog(&engine->queue); 81 async_req = crypto_dequeue_request(&engine->queue); 82 if (!async_req) 83 goto out; 84 85 engine->cur_req = async_req; 86 if (backlog) 87 backlog->complete(backlog, -EINPROGRESS); 88 89 if (engine->busy) 90 was_busy = true; 91 else 92 engine->busy = true; 93 94 spin_unlock_irqrestore(&engine->queue_lock, flags); 95 96 rtype = crypto_tfm_alg_type(engine->cur_req->tfm); 97 /* Until here we get the request need to be encrypted successfully */ 98 if (!was_busy && engine->prepare_crypt_hardware) { 99 ret = engine->prepare_crypt_hardware(engine); 100 if (ret) { 101 pr_err("failed to prepare crypt hardware\n"); 102 goto req_err; 103 } 104 } 105 106 switch (rtype) { 107 case CRYPTO_ALG_TYPE_AHASH: 108 hreq = ahash_request_cast(engine->cur_req); 109 if (engine->prepare_hash_request) { 110 ret = engine->prepare_hash_request(engine, hreq); 111 if (ret) { 112 pr_err("failed to prepare request: %d\n", ret); 113 goto req_err; 114 } 115 engine->cur_req_prepared = true; 116 } 117 ret = engine->hash_one_request(engine, hreq); 118 if (ret) { 119 pr_err("failed to hash one request from queue\n"); 120 goto req_err; 121 } 122 return; 123 case CRYPTO_ALG_TYPE_ABLKCIPHER: 124 breq = ablkcipher_request_cast(engine->cur_req); 125 if (engine->prepare_cipher_request) { 126 ret = engine->prepare_cipher_request(engine, breq); 127 if (ret) { 128 pr_err("failed to prepare request: %d\n", ret); 129 goto req_err; 130 } 131 engine->cur_req_prepared = true; 132 } 133 ret = engine->cipher_one_request(engine, breq); 134 if (ret) { 135 pr_err("failed to cipher one request from queue\n"); 136 goto req_err; 137 } 138 return; 139 default: 140 pr_err("failed to prepare request of unknown type\n"); 141 return; 142 } 143 144 req_err: 145 switch (rtype) { 146 case CRYPTO_ALG_TYPE_AHASH: 147 hreq = ahash_request_cast(engine->cur_req); 148 crypto_finalize_hash_request(engine, hreq, ret); 149 break; 150 case CRYPTO_ALG_TYPE_ABLKCIPHER: 151 breq = ablkcipher_request_cast(engine->cur_req); 152 crypto_finalize_cipher_request(engine, breq, ret); 153 break; 154 } 155 return; 156 157 out: 158 spin_unlock_irqrestore(&engine->queue_lock, flags); 159 } 160 161 static void crypto_pump_work(struct kthread_work *work) 162 { 163 struct crypto_engine *engine = 164 container_of(work, struct crypto_engine, pump_requests); 165 166 crypto_pump_requests(engine, true); 167 } 168 169 /** 170 * crypto_transfer_cipher_request - transfer the new request into the 171 * enginequeue 172 * @engine: the hardware engine 173 * @req: the request need to be listed into the engine queue 174 */ 175 int crypto_transfer_cipher_request(struct crypto_engine *engine, 176 struct ablkcipher_request *req, 177 bool need_pump) 178 { 179 unsigned long flags; 180 int ret; 181 182 spin_lock_irqsave(&engine->queue_lock, flags); 183 184 if (!engine->running) { 185 spin_unlock_irqrestore(&engine->queue_lock, flags); 186 return -ESHUTDOWN; 187 } 188 189 ret = ablkcipher_enqueue_request(&engine->queue, req); 190 191 if (!engine->busy && need_pump) 192 kthread_queue_work(engine->kworker, &engine->pump_requests); 193 194 spin_unlock_irqrestore(&engine->queue_lock, flags); 195 return ret; 196 } 197 EXPORT_SYMBOL_GPL(crypto_transfer_cipher_request); 198 199 /** 200 * crypto_transfer_cipher_request_to_engine - transfer one request to list 201 * into the engine queue 202 * @engine: the hardware engine 203 * @req: the request need to be listed into the engine queue 204 */ 205 int crypto_transfer_cipher_request_to_engine(struct crypto_engine *engine, 206 struct ablkcipher_request *req) 207 { 208 return crypto_transfer_cipher_request(engine, req, true); 209 } 210 EXPORT_SYMBOL_GPL(crypto_transfer_cipher_request_to_engine); 211 212 /** 213 * crypto_transfer_hash_request - transfer the new request into the 214 * enginequeue 215 * @engine: the hardware engine 216 * @req: the request need to be listed into the engine queue 217 */ 218 int crypto_transfer_hash_request(struct crypto_engine *engine, 219 struct ahash_request *req, bool need_pump) 220 { 221 unsigned long flags; 222 int ret; 223 224 spin_lock_irqsave(&engine->queue_lock, flags); 225 226 if (!engine->running) { 227 spin_unlock_irqrestore(&engine->queue_lock, flags); 228 return -ESHUTDOWN; 229 } 230 231 ret = ahash_enqueue_request(&engine->queue, req); 232 233 if (!engine->busy && need_pump) 234 kthread_queue_work(engine->kworker, &engine->pump_requests); 235 236 spin_unlock_irqrestore(&engine->queue_lock, flags); 237 return ret; 238 } 239 EXPORT_SYMBOL_GPL(crypto_transfer_hash_request); 240 241 /** 242 * crypto_transfer_hash_request_to_engine - transfer one request to list 243 * into the engine queue 244 * @engine: the hardware engine 245 * @req: the request need to be listed into the engine queue 246 */ 247 int crypto_transfer_hash_request_to_engine(struct crypto_engine *engine, 248 struct ahash_request *req) 249 { 250 return crypto_transfer_hash_request(engine, req, true); 251 } 252 EXPORT_SYMBOL_GPL(crypto_transfer_hash_request_to_engine); 253 254 /** 255 * crypto_finalize_cipher_request - finalize one request if the request is done 256 * @engine: the hardware engine 257 * @req: the request need to be finalized 258 * @err: error number 259 */ 260 void crypto_finalize_cipher_request(struct crypto_engine *engine, 261 struct ablkcipher_request *req, int err) 262 { 263 unsigned long flags; 264 bool finalize_cur_req = false; 265 int ret; 266 267 spin_lock_irqsave(&engine->queue_lock, flags); 268 if (engine->cur_req == &req->base) 269 finalize_cur_req = true; 270 spin_unlock_irqrestore(&engine->queue_lock, flags); 271 272 if (finalize_cur_req) { 273 if (engine->cur_req_prepared && 274 engine->unprepare_cipher_request) { 275 ret = engine->unprepare_cipher_request(engine, req); 276 if (ret) 277 pr_err("failed to unprepare request\n"); 278 } 279 spin_lock_irqsave(&engine->queue_lock, flags); 280 engine->cur_req = NULL; 281 engine->cur_req_prepared = false; 282 spin_unlock_irqrestore(&engine->queue_lock, flags); 283 } 284 285 req->base.complete(&req->base, err); 286 287 kthread_queue_work(engine->kworker, &engine->pump_requests); 288 } 289 EXPORT_SYMBOL_GPL(crypto_finalize_cipher_request); 290 291 /** 292 * crypto_finalize_hash_request - finalize one request if the request is done 293 * @engine: the hardware engine 294 * @req: the request need to be finalized 295 * @err: error number 296 */ 297 void crypto_finalize_hash_request(struct crypto_engine *engine, 298 struct ahash_request *req, int err) 299 { 300 unsigned long flags; 301 bool finalize_cur_req = false; 302 int ret; 303 304 spin_lock_irqsave(&engine->queue_lock, flags); 305 if (engine->cur_req == &req->base) 306 finalize_cur_req = true; 307 spin_unlock_irqrestore(&engine->queue_lock, flags); 308 309 if (finalize_cur_req) { 310 if (engine->cur_req_prepared && 311 engine->unprepare_hash_request) { 312 ret = engine->unprepare_hash_request(engine, req); 313 if (ret) 314 pr_err("failed to unprepare request\n"); 315 } 316 spin_lock_irqsave(&engine->queue_lock, flags); 317 engine->cur_req = NULL; 318 engine->cur_req_prepared = false; 319 spin_unlock_irqrestore(&engine->queue_lock, flags); 320 } 321 322 req->base.complete(&req->base, err); 323 324 kthread_queue_work(engine->kworker, &engine->pump_requests); 325 } 326 EXPORT_SYMBOL_GPL(crypto_finalize_hash_request); 327 328 /** 329 * crypto_engine_start - start the hardware engine 330 * @engine: the hardware engine need to be started 331 * 332 * Return 0 on success, else on fail. 333 */ 334 int crypto_engine_start(struct crypto_engine *engine) 335 { 336 unsigned long flags; 337 338 spin_lock_irqsave(&engine->queue_lock, flags); 339 340 if (engine->running || engine->busy) { 341 spin_unlock_irqrestore(&engine->queue_lock, flags); 342 return -EBUSY; 343 } 344 345 engine->running = true; 346 spin_unlock_irqrestore(&engine->queue_lock, flags); 347 348 kthread_queue_work(engine->kworker, &engine->pump_requests); 349 350 return 0; 351 } 352 EXPORT_SYMBOL_GPL(crypto_engine_start); 353 354 /** 355 * crypto_engine_stop - stop the hardware engine 356 * @engine: the hardware engine need to be stopped 357 * 358 * Return 0 on success, else on fail. 359 */ 360 int crypto_engine_stop(struct crypto_engine *engine) 361 { 362 unsigned long flags; 363 unsigned int limit = 500; 364 int ret = 0; 365 366 spin_lock_irqsave(&engine->queue_lock, flags); 367 368 /* 369 * If the engine queue is not empty or the engine is on busy state, 370 * we need to wait for a while to pump the requests of engine queue. 371 */ 372 while ((crypto_queue_len(&engine->queue) || engine->busy) && limit--) { 373 spin_unlock_irqrestore(&engine->queue_lock, flags); 374 msleep(20); 375 spin_lock_irqsave(&engine->queue_lock, flags); 376 } 377 378 if (crypto_queue_len(&engine->queue) || engine->busy) 379 ret = -EBUSY; 380 else 381 engine->running = false; 382 383 spin_unlock_irqrestore(&engine->queue_lock, flags); 384 385 if (ret) 386 pr_warn("could not stop engine\n"); 387 388 return ret; 389 } 390 EXPORT_SYMBOL_GPL(crypto_engine_stop); 391 392 /** 393 * crypto_engine_alloc_init - allocate crypto hardware engine structure and 394 * initialize it. 395 * @dev: the device attached with one hardware engine 396 * @rt: whether this queue is set to run as a realtime task 397 * 398 * This must be called from context that can sleep. 399 * Return: the crypto engine structure on success, else NULL. 400 */ 401 struct crypto_engine *crypto_engine_alloc_init(struct device *dev, bool rt) 402 { 403 struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 }; 404 struct crypto_engine *engine; 405 406 if (!dev) 407 return NULL; 408 409 engine = devm_kzalloc(dev, sizeof(*engine), GFP_KERNEL); 410 if (!engine) 411 return NULL; 412 413 engine->rt = rt; 414 engine->running = false; 415 engine->busy = false; 416 engine->idling = false; 417 engine->cur_req_prepared = false; 418 engine->priv_data = dev; 419 snprintf(engine->name, sizeof(engine->name), 420 "%s-engine", dev_name(dev)); 421 422 crypto_init_queue(&engine->queue, CRYPTO_ENGINE_MAX_QLEN); 423 spin_lock_init(&engine->queue_lock); 424 425 engine->kworker = kthread_create_worker(0, "%s", engine->name); 426 if (IS_ERR(engine->kworker)) { 427 dev_err(dev, "failed to create crypto request pump task\n"); 428 return NULL; 429 } 430 kthread_init_work(&engine->pump_requests, crypto_pump_work); 431 432 if (engine->rt) { 433 dev_info(dev, "will run requests pump with realtime priority\n"); 434 sched_setscheduler(engine->kworker->task, SCHED_FIFO, ¶m); 435 } 436 437 return engine; 438 } 439 EXPORT_SYMBOL_GPL(crypto_engine_alloc_init); 440 441 /** 442 * crypto_engine_exit - free the resources of hardware engine when exit 443 * @engine: the hardware engine need to be freed 444 * 445 * Return 0 for success. 446 */ 447 int crypto_engine_exit(struct crypto_engine *engine) 448 { 449 int ret; 450 451 ret = crypto_engine_stop(engine); 452 if (ret) 453 return ret; 454 455 kthread_destroy_worker(engine->kworker); 456 457 return 0; 458 } 459 EXPORT_SYMBOL_GPL(crypto_engine_exit); 460 461 MODULE_LICENSE("GPL"); 462 MODULE_DESCRIPTION("Crypto hardware engine framework"); 463