1 // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
2 /* Copyright(c) 2014 - 2020 Intel Corporation */
3 #include <linux/mutex.h>
4 #include <linux/list.h>
5 #include <linux/bitops.h>
6 #include <linux/delay.h>
7 #include "adf_accel_devices.h"
8 #include "adf_cfg.h"
9 #include "adf_common_drv.h"
10 #include "adf_dbgfs.h"
11 
12 static LIST_HEAD(service_table);
13 static DEFINE_MUTEX(service_lock);
14 
15 static void adf_service_add(struct service_hndl *service)
16 {
17 	mutex_lock(&service_lock);
18 	list_add(&service->list, &service_table);
19 	mutex_unlock(&service_lock);
20 }
21 
22 int adf_service_register(struct service_hndl *service)
23 {
24 	memset(service->init_status, 0, sizeof(service->init_status));
25 	memset(service->start_status, 0, sizeof(service->start_status));
26 	adf_service_add(service);
27 	return 0;
28 }
29 
30 static void adf_service_remove(struct service_hndl *service)
31 {
32 	mutex_lock(&service_lock);
33 	list_del(&service->list);
34 	mutex_unlock(&service_lock);
35 }
36 
37 int adf_service_unregister(struct service_hndl *service)
38 {
39 	int i;
40 
41 	for (i = 0; i < ARRAY_SIZE(service->init_status); i++) {
42 		if (service->init_status[i] || service->start_status[i]) {
43 			pr_err("QAT: Could not remove active service\n");
44 			return -EFAULT;
45 		}
46 	}
47 	adf_service_remove(service);
48 	return 0;
49 }
50 
51 /**
52  * adf_dev_init() - Init data structures and services for the given accel device
53  * @accel_dev: Pointer to acceleration device.
54  *
55  * Initialize the ring data structures and the admin comms and arbitration
56  * services.
57  *
58  * Return: 0 on success, error code otherwise.
59  */
60 static int adf_dev_init(struct adf_accel_dev *accel_dev)
61 {
62 	struct service_hndl *service;
63 	struct list_head *list_itr;
64 	struct adf_hw_device_data *hw_data = accel_dev->hw_device;
65 	int ret;
66 
67 	if (!hw_data) {
68 		dev_err(&GET_DEV(accel_dev),
69 			"Failed to init device - hw_data not set\n");
70 		return -EFAULT;
71 	}
72 
73 	if (!test_bit(ADF_STATUS_CONFIGURED, &accel_dev->status) &&
74 	    !accel_dev->is_vf) {
75 		dev_err(&GET_DEV(accel_dev), "Device not configured\n");
76 		return -EFAULT;
77 	}
78 
79 	if (adf_init_etr_data(accel_dev)) {
80 		dev_err(&GET_DEV(accel_dev), "Failed initialize etr\n");
81 		return -EFAULT;
82 	}
83 
84 	if (hw_data->init_device && hw_data->init_device(accel_dev)) {
85 		dev_err(&GET_DEV(accel_dev), "Failed to initialize device\n");
86 		return -EFAULT;
87 	}
88 
89 	if (hw_data->init_admin_comms && hw_data->init_admin_comms(accel_dev)) {
90 		dev_err(&GET_DEV(accel_dev), "Failed initialize admin comms\n");
91 		return -EFAULT;
92 	}
93 
94 	if (hw_data->init_arb && hw_data->init_arb(accel_dev)) {
95 		dev_err(&GET_DEV(accel_dev), "Failed initialize hw arbiter\n");
96 		return -EFAULT;
97 	}
98 
99 	if (adf_ae_init(accel_dev)) {
100 		dev_err(&GET_DEV(accel_dev),
101 			"Failed to initialise Acceleration Engine\n");
102 		return -EFAULT;
103 	}
104 	set_bit(ADF_STATUS_AE_INITIALISED, &accel_dev->status);
105 
106 	if (adf_ae_fw_load(accel_dev)) {
107 		dev_err(&GET_DEV(accel_dev),
108 			"Failed to load acceleration FW\n");
109 		return -EFAULT;
110 	}
111 	set_bit(ADF_STATUS_AE_UCODE_LOADED, &accel_dev->status);
112 
113 	if (hw_data->alloc_irq(accel_dev)) {
114 		dev_err(&GET_DEV(accel_dev), "Failed to allocate interrupts\n");
115 		return -EFAULT;
116 	}
117 	set_bit(ADF_STATUS_IRQ_ALLOCATED, &accel_dev->status);
118 
119 	hw_data->enable_ints(accel_dev);
120 	hw_data->enable_error_correction(accel_dev);
121 
122 	ret = hw_data->pfvf_ops.enable_comms(accel_dev);
123 	if (ret)
124 		return ret;
125 
126 	if (!test_bit(ADF_STATUS_CONFIGURED, &accel_dev->status) &&
127 	    accel_dev->is_vf) {
128 		if (qat_crypto_vf_dev_config(accel_dev))
129 			return -EFAULT;
130 	}
131 
132 	/*
133 	 * Subservice initialisation is divided into two stages: init and start.
134 	 * This is to facilitate any ordering dependencies between services
135 	 * prior to starting any of the accelerators.
136 	 */
137 	list_for_each(list_itr, &service_table) {
138 		service = list_entry(list_itr, struct service_hndl, list);
139 		if (service->event_hld(accel_dev, ADF_EVENT_INIT)) {
140 			dev_err(&GET_DEV(accel_dev),
141 				"Failed to initialise service %s\n",
142 				service->name);
143 			return -EFAULT;
144 		}
145 		set_bit(accel_dev->accel_id, service->init_status);
146 	}
147 
148 	return 0;
149 }
150 
151 /**
152  * adf_dev_start() - Start acceleration service for the given accel device
153  * @accel_dev:    Pointer to acceleration device.
154  *
155  * Function notifies all the registered services that the acceleration device
156  * is ready to be used.
157  * To be used by QAT device specific drivers.
158  *
159  * Return: 0 on success, error code otherwise.
160  */
161 static int adf_dev_start(struct adf_accel_dev *accel_dev)
162 {
163 	struct adf_hw_device_data *hw_data = accel_dev->hw_device;
164 	struct service_hndl *service;
165 	struct list_head *list_itr;
166 
167 	set_bit(ADF_STATUS_STARTING, &accel_dev->status);
168 
169 	if (adf_ae_start(accel_dev)) {
170 		dev_err(&GET_DEV(accel_dev), "AE Start Failed\n");
171 		return -EFAULT;
172 	}
173 	set_bit(ADF_STATUS_AE_STARTED, &accel_dev->status);
174 
175 	if (hw_data->send_admin_init(accel_dev)) {
176 		dev_err(&GET_DEV(accel_dev), "Failed to send init message\n");
177 		return -EFAULT;
178 	}
179 
180 	/* Set ssm watch dog timer */
181 	if (hw_data->set_ssm_wdtimer)
182 		hw_data->set_ssm_wdtimer(accel_dev);
183 
184 	/* Enable Power Management */
185 	if (hw_data->enable_pm && hw_data->enable_pm(accel_dev)) {
186 		dev_err(&GET_DEV(accel_dev), "Failed to configure Power Management\n");
187 		return -EFAULT;
188 	}
189 
190 	list_for_each(list_itr, &service_table) {
191 		service = list_entry(list_itr, struct service_hndl, list);
192 		if (service->event_hld(accel_dev, ADF_EVENT_START)) {
193 			dev_err(&GET_DEV(accel_dev),
194 				"Failed to start service %s\n",
195 				service->name);
196 			return -EFAULT;
197 		}
198 		set_bit(accel_dev->accel_id, service->start_status);
199 	}
200 
201 	clear_bit(ADF_STATUS_STARTING, &accel_dev->status);
202 	set_bit(ADF_STATUS_STARTED, &accel_dev->status);
203 
204 	if (!list_empty(&accel_dev->crypto_list) &&
205 	    (qat_algs_register() || qat_asym_algs_register())) {
206 		dev_err(&GET_DEV(accel_dev),
207 			"Failed to register crypto algs\n");
208 		set_bit(ADF_STATUS_STARTING, &accel_dev->status);
209 		clear_bit(ADF_STATUS_STARTED, &accel_dev->status);
210 		return -EFAULT;
211 	}
212 
213 	if (!list_empty(&accel_dev->compression_list) && qat_comp_algs_register()) {
214 		dev_err(&GET_DEV(accel_dev),
215 			"Failed to register compression algs\n");
216 		set_bit(ADF_STATUS_STARTING, &accel_dev->status);
217 		clear_bit(ADF_STATUS_STARTED, &accel_dev->status);
218 		return -EFAULT;
219 	}
220 
221 	adf_dbgfs_add(accel_dev);
222 
223 	return 0;
224 }
225 
226 /**
227  * adf_dev_stop() - Stop acceleration service for the given accel device
228  * @accel_dev:    Pointer to acceleration device.
229  *
230  * Function notifies all the registered services that the acceleration device
231  * is shuting down.
232  * To be used by QAT device specific drivers.
233  *
234  * Return: void
235  */
236 static void adf_dev_stop(struct adf_accel_dev *accel_dev)
237 {
238 	struct service_hndl *service;
239 	struct list_head *list_itr;
240 	bool wait = false;
241 	int ret;
242 
243 	if (!adf_dev_started(accel_dev) &&
244 	    !test_bit(ADF_STATUS_STARTING, &accel_dev->status))
245 		return;
246 
247 	adf_dbgfs_rm(accel_dev);
248 
249 	clear_bit(ADF_STATUS_STARTING, &accel_dev->status);
250 	clear_bit(ADF_STATUS_STARTED, &accel_dev->status);
251 
252 	if (!list_empty(&accel_dev->crypto_list)) {
253 		qat_algs_unregister();
254 		qat_asym_algs_unregister();
255 	}
256 
257 	if (!list_empty(&accel_dev->compression_list))
258 		qat_comp_algs_unregister();
259 
260 	list_for_each(list_itr, &service_table) {
261 		service = list_entry(list_itr, struct service_hndl, list);
262 		if (!test_bit(accel_dev->accel_id, service->start_status))
263 			continue;
264 		ret = service->event_hld(accel_dev, ADF_EVENT_STOP);
265 		if (!ret) {
266 			clear_bit(accel_dev->accel_id, service->start_status);
267 		} else if (ret == -EAGAIN) {
268 			wait = true;
269 			clear_bit(accel_dev->accel_id, service->start_status);
270 		}
271 	}
272 
273 	if (wait)
274 		msleep(100);
275 
276 	if (test_bit(ADF_STATUS_AE_STARTED, &accel_dev->status)) {
277 		if (adf_ae_stop(accel_dev))
278 			dev_err(&GET_DEV(accel_dev), "failed to stop AE\n");
279 		else
280 			clear_bit(ADF_STATUS_AE_STARTED, &accel_dev->status);
281 	}
282 }
283 
284 /**
285  * adf_dev_shutdown() - shutdown acceleration services and data strucutures
286  * @accel_dev: Pointer to acceleration device
287  *
288  * Cleanup the ring data structures and the admin comms and arbitration
289  * services.
290  */
291 static void adf_dev_shutdown(struct adf_accel_dev *accel_dev)
292 {
293 	struct adf_hw_device_data *hw_data = accel_dev->hw_device;
294 	struct service_hndl *service;
295 	struct list_head *list_itr;
296 
297 	if (!hw_data) {
298 		dev_err(&GET_DEV(accel_dev),
299 			"QAT: Failed to shutdown device - hw_data not set\n");
300 		return;
301 	}
302 
303 	if (test_bit(ADF_STATUS_AE_UCODE_LOADED, &accel_dev->status)) {
304 		adf_ae_fw_release(accel_dev);
305 		clear_bit(ADF_STATUS_AE_UCODE_LOADED, &accel_dev->status);
306 	}
307 
308 	if (test_bit(ADF_STATUS_AE_INITIALISED, &accel_dev->status)) {
309 		if (adf_ae_shutdown(accel_dev))
310 			dev_err(&GET_DEV(accel_dev),
311 				"Failed to shutdown Accel Engine\n");
312 		else
313 			clear_bit(ADF_STATUS_AE_INITIALISED,
314 				  &accel_dev->status);
315 	}
316 
317 	list_for_each(list_itr, &service_table) {
318 		service = list_entry(list_itr, struct service_hndl, list);
319 		if (!test_bit(accel_dev->accel_id, service->init_status))
320 			continue;
321 		if (service->event_hld(accel_dev, ADF_EVENT_SHUTDOWN))
322 			dev_err(&GET_DEV(accel_dev),
323 				"Failed to shutdown service %s\n",
324 				service->name);
325 		else
326 			clear_bit(accel_dev->accel_id, service->init_status);
327 	}
328 
329 	hw_data->disable_iov(accel_dev);
330 
331 	if (test_bit(ADF_STATUS_IRQ_ALLOCATED, &accel_dev->status)) {
332 		hw_data->free_irq(accel_dev);
333 		clear_bit(ADF_STATUS_IRQ_ALLOCATED, &accel_dev->status);
334 	}
335 
336 	/* Delete configuration only if not restarting */
337 	if (!test_bit(ADF_STATUS_RESTARTING, &accel_dev->status))
338 		adf_cfg_del_all(accel_dev);
339 
340 	if (hw_data->exit_arb)
341 		hw_data->exit_arb(accel_dev);
342 
343 	if (hw_data->exit_admin_comms)
344 		hw_data->exit_admin_comms(accel_dev);
345 
346 	adf_cleanup_etr_data(accel_dev);
347 	adf_dev_restore(accel_dev);
348 }
349 
350 int adf_dev_restarting_notify(struct adf_accel_dev *accel_dev)
351 {
352 	struct service_hndl *service;
353 	struct list_head *list_itr;
354 
355 	list_for_each(list_itr, &service_table) {
356 		service = list_entry(list_itr, struct service_hndl, list);
357 		if (service->event_hld(accel_dev, ADF_EVENT_RESTARTING))
358 			dev_err(&GET_DEV(accel_dev),
359 				"Failed to restart service %s.\n",
360 				service->name);
361 	}
362 	return 0;
363 }
364 
365 int adf_dev_restarted_notify(struct adf_accel_dev *accel_dev)
366 {
367 	struct service_hndl *service;
368 	struct list_head *list_itr;
369 
370 	list_for_each(list_itr, &service_table) {
371 		service = list_entry(list_itr, struct service_hndl, list);
372 		if (service->event_hld(accel_dev, ADF_EVENT_RESTARTED))
373 			dev_err(&GET_DEV(accel_dev),
374 				"Failed to restart service %s.\n",
375 				service->name);
376 	}
377 	return 0;
378 }
379 
380 static int adf_dev_shutdown_cache_cfg(struct adf_accel_dev *accel_dev)
381 {
382 	char services[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = {0};
383 	int ret;
384 
385 	ret = adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC,
386 				      ADF_SERVICES_ENABLED, services);
387 
388 	adf_dev_stop(accel_dev);
389 	adf_dev_shutdown(accel_dev);
390 
391 	if (!ret) {
392 		ret = adf_cfg_section_add(accel_dev, ADF_GENERAL_SEC);
393 		if (ret)
394 			return ret;
395 
396 		ret = adf_cfg_add_key_value_param(accel_dev, ADF_GENERAL_SEC,
397 						  ADF_SERVICES_ENABLED,
398 						  services, ADF_STR);
399 		if (ret)
400 			return ret;
401 	}
402 
403 	return 0;
404 }
405 
406 int adf_dev_down(struct adf_accel_dev *accel_dev, bool reconfig)
407 {
408 	int ret = 0;
409 
410 	if (!accel_dev)
411 		return -EINVAL;
412 
413 	mutex_lock(&accel_dev->state_lock);
414 
415 	if (!adf_dev_started(accel_dev)) {
416 		dev_info(&GET_DEV(accel_dev), "Device qat_dev%d already down\n",
417 			 accel_dev->accel_id);
418 		ret = -EINVAL;
419 		goto out;
420 	}
421 
422 	if (reconfig) {
423 		ret = adf_dev_shutdown_cache_cfg(accel_dev);
424 		goto out;
425 	}
426 
427 	adf_dev_stop(accel_dev);
428 	adf_dev_shutdown(accel_dev);
429 
430 out:
431 	mutex_unlock(&accel_dev->state_lock);
432 	return ret;
433 }
434 EXPORT_SYMBOL_GPL(adf_dev_down);
435 
436 int adf_dev_up(struct adf_accel_dev *accel_dev, bool config)
437 {
438 	int ret = 0;
439 
440 	if (!accel_dev)
441 		return -EINVAL;
442 
443 	mutex_lock(&accel_dev->state_lock);
444 
445 	if (adf_dev_started(accel_dev)) {
446 		dev_info(&GET_DEV(accel_dev), "Device qat_dev%d already up\n",
447 			 accel_dev->accel_id);
448 		ret = -EALREADY;
449 		goto out;
450 	}
451 
452 	if (config && GET_HW_DATA(accel_dev)->dev_config) {
453 		ret = GET_HW_DATA(accel_dev)->dev_config(accel_dev);
454 		if (unlikely(ret))
455 			goto out;
456 	}
457 
458 	ret = adf_dev_init(accel_dev);
459 	if (unlikely(ret))
460 		goto out;
461 
462 	ret = adf_dev_start(accel_dev);
463 
464 out:
465 	mutex_unlock(&accel_dev->state_lock);
466 	return ret;
467 }
468 EXPORT_SYMBOL_GPL(adf_dev_up);
469 
470 int adf_dev_restart(struct adf_accel_dev *accel_dev)
471 {
472 	int ret = 0;
473 
474 	if (!accel_dev)
475 		return -EFAULT;
476 
477 	adf_dev_down(accel_dev, false);
478 
479 	ret = adf_dev_up(accel_dev, false);
480 	/* if device is already up return success*/
481 	if (ret == -EALREADY)
482 		return 0;
483 
484 	return ret;
485 }
486 EXPORT_SYMBOL_GPL(adf_dev_restart);
487