1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright(c) 2023 Intel Corporation */
3 
4 #include <linux/dev_printk.h>
5 #include <linux/dma-mapping.h>
6 #include <linux/export.h>
7 #include <linux/kernel.h>
8 #include <linux/kstrtox.h>
9 #include <linux/overflow.h>
10 #include <linux/string.h>
11 #include <linux/slab.h>
12 #include <linux/types.h>
13 #include <asm/errno.h>
14 #include "adf_accel_devices.h"
15 #include "adf_cfg.h"
16 #include "adf_cfg_strings.h"
17 #include "adf_clock.h"
18 #include "adf_common_drv.h"
19 #include "adf_heartbeat.h"
20 #include "adf_transport_internal.h"
21 #include "icp_qat_fw_init_admin.h"
22 
23 #define ADF_HB_EMPTY_SIG 0xA5A5A5A5
24 
25 /* Heartbeat counter pair */
26 struct hb_cnt_pair {
27 	__u16 resp_heartbeat_cnt;
28 	__u16 req_heartbeat_cnt;
29 };
30 
adf_hb_check_polling_freq(struct adf_accel_dev * accel_dev)31 static int adf_hb_check_polling_freq(struct adf_accel_dev *accel_dev)
32 {
33 	u64 curr_time = adf_clock_get_current_time();
34 	u64 polling_time = curr_time - accel_dev->heartbeat->last_hb_check_time;
35 
36 	if (polling_time < accel_dev->heartbeat->hb_timer) {
37 		dev_warn(&GET_DEV(accel_dev),
38 			 "HB polling too frequent. Configured HB timer %d ms\n",
39 			 accel_dev->heartbeat->hb_timer);
40 		return -EINVAL;
41 	}
42 
43 	accel_dev->heartbeat->last_hb_check_time = curr_time;
44 	return 0;
45 }
46 
47 /**
48  * validate_hb_ctrs_cnt() - checks if the number of heartbeat counters should
49  * be updated by one to support the currently loaded firmware.
50  * @accel_dev: Pointer to acceleration device.
51  *
52  * Return:
53  * * true - hb_ctrs must increased by ADF_NUM_PKE_STRAND
54  * * false - no changes needed
55  */
validate_hb_ctrs_cnt(struct adf_accel_dev * accel_dev)56 static bool validate_hb_ctrs_cnt(struct adf_accel_dev *accel_dev)
57 {
58 	const size_t hb_ctrs = accel_dev->hw_device->num_hb_ctrs;
59 	const size_t max_aes = accel_dev->hw_device->num_engines;
60 	const size_t hb_struct_size = sizeof(struct hb_cnt_pair);
61 	const size_t exp_diff_size = array3_size(ADF_NUM_PKE_STRAND, max_aes,
62 						 hb_struct_size);
63 	const size_t dev_ctrs = size_mul(max_aes, hb_ctrs);
64 	const size_t stats_size = size_mul(dev_ctrs, hb_struct_size);
65 	const u32 exp_diff_cnt = exp_diff_size / sizeof(u32);
66 	const u32 stats_el_cnt = stats_size / sizeof(u32);
67 	struct hb_cnt_pair *hb_stats = accel_dev->heartbeat->dma.virt_addr;
68 	const u32 *mem_to_chk = (u32 *)(hb_stats + dev_ctrs);
69 	u32 el_diff_cnt = 0;
70 	int i;
71 
72 	/* count how many bytes are different from pattern */
73 	for (i = 0; i < stats_el_cnt; i++) {
74 		if (mem_to_chk[i] == ADF_HB_EMPTY_SIG)
75 			break;
76 
77 		el_diff_cnt++;
78 	}
79 
80 	return el_diff_cnt && el_diff_cnt == exp_diff_cnt;
81 }
82 
adf_heartbeat_check_ctrs(struct adf_accel_dev * accel_dev)83 void adf_heartbeat_check_ctrs(struct adf_accel_dev *accel_dev)
84 {
85 	struct hb_cnt_pair *hb_stats = accel_dev->heartbeat->dma.virt_addr;
86 	const size_t hb_ctrs = accel_dev->hw_device->num_hb_ctrs;
87 	const size_t max_aes = accel_dev->hw_device->num_engines;
88 	const size_t dev_ctrs = size_mul(max_aes, hb_ctrs);
89 	const size_t stats_size = size_mul(dev_ctrs, sizeof(struct hb_cnt_pair));
90 	const size_t mem_items_to_fill = size_mul(stats_size, 2) / sizeof(u32);
91 
92 	/* fill hb stats memory with pattern */
93 	memset32((uint32_t *)hb_stats, ADF_HB_EMPTY_SIG, mem_items_to_fill);
94 	accel_dev->heartbeat->ctrs_cnt_checked = false;
95 }
96 EXPORT_SYMBOL_GPL(adf_heartbeat_check_ctrs);
97 
get_timer_ticks(struct adf_accel_dev * accel_dev,unsigned int * value)98 static int get_timer_ticks(struct adf_accel_dev *accel_dev, unsigned int *value)
99 {
100 	char timer_str[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = { };
101 	u32 timer_ms = ADF_CFG_HB_TIMER_DEFAULT_MS;
102 	int cfg_read_status;
103 	u32 ticks;
104 	int ret;
105 
106 	cfg_read_status = adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC,
107 						  ADF_HEARTBEAT_TIMER, timer_str);
108 	if (cfg_read_status == 0) {
109 		if (kstrtouint(timer_str, 10, &timer_ms))
110 			dev_dbg(&GET_DEV(accel_dev),
111 				"kstrtouint failed to parse the %s, param value",
112 				ADF_HEARTBEAT_TIMER);
113 	}
114 
115 	if (timer_ms < ADF_CFG_HB_TIMER_MIN_MS) {
116 		dev_err(&GET_DEV(accel_dev), "Timer cannot be less than %u\n",
117 			ADF_CFG_HB_TIMER_MIN_MS);
118 		return -EINVAL;
119 	}
120 
121 	/*
122 	 * On 4xxx devices adf_timer is responsible for HB updates and
123 	 * its period is fixed to 200ms
124 	 */
125 	if (accel_dev->timer)
126 		timer_ms = ADF_CFG_HB_TIMER_MIN_MS;
127 
128 	ret = adf_heartbeat_ms_to_ticks(accel_dev, timer_ms, &ticks);
129 	if (ret)
130 		return ret;
131 
132 	adf_heartbeat_save_cfg_param(accel_dev, timer_ms);
133 
134 	accel_dev->heartbeat->hb_timer = timer_ms;
135 	*value = ticks;
136 
137 	return 0;
138 }
139 
check_ae(struct hb_cnt_pair * curr,struct hb_cnt_pair * prev,u16 * count,const size_t hb_ctrs)140 static int check_ae(struct hb_cnt_pair *curr, struct hb_cnt_pair *prev,
141 		    u16 *count, const size_t hb_ctrs)
142 {
143 	size_t thr;
144 
145 	/* loop through all threads in AE */
146 	for (thr = 0; thr < hb_ctrs; thr++) {
147 		u16 req = curr[thr].req_heartbeat_cnt;
148 		u16 resp = curr[thr].resp_heartbeat_cnt;
149 		u16 last = prev[thr].resp_heartbeat_cnt;
150 
151 		if ((thr == ADF_AE_ADMIN_THREAD || req != resp) && resp == last) {
152 			u16 retry = ++count[thr];
153 
154 			if (retry >= ADF_CFG_HB_COUNT_THRESHOLD)
155 				return -EIO;
156 
157 		} else {
158 			count[thr] = 0;
159 		}
160 	}
161 	return 0;
162 }
163 
adf_hb_get_status(struct adf_accel_dev * accel_dev)164 static int adf_hb_get_status(struct adf_accel_dev *accel_dev)
165 {
166 	struct adf_hw_device_data *hw_device = accel_dev->hw_device;
167 	struct hb_cnt_pair *live_stats, *last_stats, *curr_stats;
168 	const size_t hb_ctrs = hw_device->num_hb_ctrs;
169 	const unsigned long ae_mask = hw_device->ae_mask;
170 	const size_t max_aes = hw_device->num_engines;
171 	const size_t dev_ctrs = size_mul(max_aes, hb_ctrs);
172 	const size_t stats_size = size_mul(dev_ctrs, sizeof(*curr_stats));
173 	struct hb_cnt_pair *ae_curr_p, *ae_prev_p;
174 	u16 *count_fails, *ae_count_p;
175 	size_t ae_offset;
176 	size_t ae = 0;
177 	int ret = 0;
178 
179 	if (!accel_dev->heartbeat->ctrs_cnt_checked) {
180 		if (validate_hb_ctrs_cnt(accel_dev))
181 			hw_device->num_hb_ctrs += ADF_NUM_PKE_STRAND;
182 
183 		accel_dev->heartbeat->ctrs_cnt_checked = true;
184 	}
185 
186 	live_stats = accel_dev->heartbeat->dma.virt_addr;
187 	last_stats = live_stats + dev_ctrs;
188 	count_fails = (u16 *)(last_stats + dev_ctrs);
189 
190 	curr_stats = kmemdup(live_stats, stats_size, GFP_KERNEL);
191 	if (!curr_stats)
192 		return -ENOMEM;
193 
194 	/* loop through active AEs */
195 	for_each_set_bit(ae, &ae_mask, max_aes) {
196 		ae_offset = size_mul(ae, hb_ctrs);
197 		ae_curr_p = curr_stats + ae_offset;
198 		ae_prev_p = last_stats + ae_offset;
199 		ae_count_p = count_fails + ae_offset;
200 
201 		ret = check_ae(ae_curr_p, ae_prev_p, ae_count_p, hb_ctrs);
202 		if (ret)
203 			break;
204 	}
205 
206 	/* Copy current stats for the next iteration */
207 	memcpy(last_stats, curr_stats, stats_size);
208 	kfree(curr_stats);
209 
210 	return ret;
211 }
212 
adf_heartbeat_status(struct adf_accel_dev * accel_dev,enum adf_device_heartbeat_status * hb_status)213 void adf_heartbeat_status(struct adf_accel_dev *accel_dev,
214 			  enum adf_device_heartbeat_status *hb_status)
215 {
216 	struct adf_heartbeat *hb;
217 
218 	if (!adf_dev_started(accel_dev) ||
219 	    test_bit(ADF_STATUS_RESTARTING, &accel_dev->status)) {
220 		*hb_status = HB_DEV_UNRESPONSIVE;
221 		return;
222 	}
223 
224 	if (adf_hb_check_polling_freq(accel_dev) == -EINVAL) {
225 		*hb_status = HB_DEV_UNSUPPORTED;
226 		return;
227 	}
228 
229 	hb = accel_dev->heartbeat;
230 	hb->hb_sent_counter++;
231 
232 	if (adf_hb_get_status(accel_dev)) {
233 		dev_err(&GET_DEV(accel_dev),
234 			"Heartbeat ERROR: QAT is not responding.\n");
235 		*hb_status = HB_DEV_UNRESPONSIVE;
236 		hb->hb_failed_counter++;
237 		return;
238 	}
239 
240 	*hb_status = HB_DEV_ALIVE;
241 }
242 
adf_heartbeat_ms_to_ticks(struct adf_accel_dev * accel_dev,unsigned int time_ms,u32 * value)243 int adf_heartbeat_ms_to_ticks(struct adf_accel_dev *accel_dev, unsigned int time_ms,
244 			      u32 *value)
245 {
246 	struct adf_hw_device_data *hw_data = accel_dev->hw_device;
247 	u32 clk_per_sec;
248 
249 	/* HB clock may be different than AE clock */
250 	if (!hw_data->get_hb_clock)
251 		return -EINVAL;
252 
253 	clk_per_sec = hw_data->get_hb_clock(hw_data);
254 	*value = time_ms * (clk_per_sec / MSEC_PER_SEC);
255 
256 	return 0;
257 }
258 
adf_heartbeat_save_cfg_param(struct adf_accel_dev * accel_dev,unsigned int timer_ms)259 int adf_heartbeat_save_cfg_param(struct adf_accel_dev *accel_dev,
260 				 unsigned int timer_ms)
261 {
262 	char timer_str[ADF_CFG_MAX_VAL_LEN_IN_BYTES];
263 
264 	snprintf(timer_str, sizeof(timer_str), "%u", timer_ms);
265 	return adf_cfg_add_key_value_param(accel_dev, ADF_GENERAL_SEC,
266 					  ADF_HEARTBEAT_TIMER, timer_str,
267 					  ADF_STR);
268 }
269 EXPORT_SYMBOL_GPL(adf_heartbeat_save_cfg_param);
270 
adf_heartbeat_init(struct adf_accel_dev * accel_dev)271 int adf_heartbeat_init(struct adf_accel_dev *accel_dev)
272 {
273 	struct adf_heartbeat *hb;
274 
275 	hb = kzalloc(sizeof(*hb), GFP_KERNEL);
276 	if (!hb)
277 		goto err_ret;
278 
279 	hb->dma.virt_addr = dma_alloc_coherent(&GET_DEV(accel_dev), PAGE_SIZE,
280 					       &hb->dma.phy_addr, GFP_KERNEL);
281 	if (!hb->dma.virt_addr)
282 		goto err_free;
283 
284 	/*
285 	 * Default set this flag as true to avoid unnecessary checks,
286 	 * it will be reset on platforms that need such a check
287 	 */
288 	hb->ctrs_cnt_checked = true;
289 	accel_dev->heartbeat = hb;
290 
291 	return 0;
292 
293 err_free:
294 	kfree(hb);
295 err_ret:
296 	return -ENOMEM;
297 }
298 
adf_heartbeat_start(struct adf_accel_dev * accel_dev)299 int adf_heartbeat_start(struct adf_accel_dev *accel_dev)
300 {
301 	unsigned int timer_ticks;
302 	int ret;
303 
304 	if (!accel_dev->heartbeat) {
305 		dev_warn(&GET_DEV(accel_dev), "Heartbeat instance not found!");
306 		return -EFAULT;
307 	}
308 
309 	if (accel_dev->hw_device->check_hb_ctrs)
310 		accel_dev->hw_device->check_hb_ctrs(accel_dev);
311 
312 	ret = get_timer_ticks(accel_dev, &timer_ticks);
313 	if (ret)
314 		return ret;
315 
316 	ret = adf_send_admin_hb_timer(accel_dev, timer_ticks);
317 	if (ret)
318 		dev_warn(&GET_DEV(accel_dev), "Heartbeat not supported!");
319 
320 	return ret;
321 }
322 
adf_heartbeat_shutdown(struct adf_accel_dev * accel_dev)323 void adf_heartbeat_shutdown(struct adf_accel_dev *accel_dev)
324 {
325 	struct adf_heartbeat *hb = accel_dev->heartbeat;
326 
327 	if (!hb)
328 		return;
329 
330 	if (hb->dma.virt_addr)
331 		dma_free_coherent(&GET_DEV(accel_dev), PAGE_SIZE,
332 				  hb->dma.virt_addr, hb->dma.phy_addr);
333 
334 	kfree(hb);
335 	accel_dev->heartbeat = NULL;
336 }
337