1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright(c) 2023 Intel Corporation */
3
4 #include <linux/dev_printk.h>
5 #include <linux/dma-mapping.h>
6 #include <linux/export.h>
7 #include <linux/kernel.h>
8 #include <linux/kstrtox.h>
9 #include <linux/overflow.h>
10 #include <linux/string.h>
11 #include <linux/slab.h>
12 #include <linux/types.h>
13 #include <asm/errno.h>
14 #include "adf_accel_devices.h"
15 #include "adf_cfg.h"
16 #include "adf_cfg_strings.h"
17 #include "adf_clock.h"
18 #include "adf_common_drv.h"
19 #include "adf_heartbeat.h"
20 #include "adf_transport_internal.h"
21 #include "icp_qat_fw_init_admin.h"
22
23 #define ADF_HB_EMPTY_SIG 0xA5A5A5A5
24
25 /* Heartbeat counter pair */
26 struct hb_cnt_pair {
27 __u16 resp_heartbeat_cnt;
28 __u16 req_heartbeat_cnt;
29 };
30
adf_hb_check_polling_freq(struct adf_accel_dev * accel_dev)31 static int adf_hb_check_polling_freq(struct adf_accel_dev *accel_dev)
32 {
33 u64 curr_time = adf_clock_get_current_time();
34 u64 polling_time = curr_time - accel_dev->heartbeat->last_hb_check_time;
35
36 if (polling_time < accel_dev->heartbeat->hb_timer) {
37 dev_warn(&GET_DEV(accel_dev),
38 "HB polling too frequent. Configured HB timer %d ms\n",
39 accel_dev->heartbeat->hb_timer);
40 return -EINVAL;
41 }
42
43 accel_dev->heartbeat->last_hb_check_time = curr_time;
44 return 0;
45 }
46
47 /**
48 * validate_hb_ctrs_cnt() - checks if the number of heartbeat counters should
49 * be updated by one to support the currently loaded firmware.
50 * @accel_dev: Pointer to acceleration device.
51 *
52 * Return:
53 * * true - hb_ctrs must increased by ADF_NUM_PKE_STRAND
54 * * false - no changes needed
55 */
validate_hb_ctrs_cnt(struct adf_accel_dev * accel_dev)56 static bool validate_hb_ctrs_cnt(struct adf_accel_dev *accel_dev)
57 {
58 const size_t hb_ctrs = accel_dev->hw_device->num_hb_ctrs;
59 const size_t max_aes = accel_dev->hw_device->num_engines;
60 const size_t hb_struct_size = sizeof(struct hb_cnt_pair);
61 const size_t exp_diff_size = array3_size(ADF_NUM_PKE_STRAND, max_aes,
62 hb_struct_size);
63 const size_t dev_ctrs = size_mul(max_aes, hb_ctrs);
64 const size_t stats_size = size_mul(dev_ctrs, hb_struct_size);
65 const u32 exp_diff_cnt = exp_diff_size / sizeof(u32);
66 const u32 stats_el_cnt = stats_size / sizeof(u32);
67 struct hb_cnt_pair *hb_stats = accel_dev->heartbeat->dma.virt_addr;
68 const u32 *mem_to_chk = (u32 *)(hb_stats + dev_ctrs);
69 u32 el_diff_cnt = 0;
70 int i;
71
72 /* count how many bytes are different from pattern */
73 for (i = 0; i < stats_el_cnt; i++) {
74 if (mem_to_chk[i] == ADF_HB_EMPTY_SIG)
75 break;
76
77 el_diff_cnt++;
78 }
79
80 return el_diff_cnt && el_diff_cnt == exp_diff_cnt;
81 }
82
adf_heartbeat_check_ctrs(struct adf_accel_dev * accel_dev)83 void adf_heartbeat_check_ctrs(struct adf_accel_dev *accel_dev)
84 {
85 struct hb_cnt_pair *hb_stats = accel_dev->heartbeat->dma.virt_addr;
86 const size_t hb_ctrs = accel_dev->hw_device->num_hb_ctrs;
87 const size_t max_aes = accel_dev->hw_device->num_engines;
88 const size_t dev_ctrs = size_mul(max_aes, hb_ctrs);
89 const size_t stats_size = size_mul(dev_ctrs, sizeof(struct hb_cnt_pair));
90 const size_t mem_items_to_fill = size_mul(stats_size, 2) / sizeof(u32);
91
92 /* fill hb stats memory with pattern */
93 memset32((uint32_t *)hb_stats, ADF_HB_EMPTY_SIG, mem_items_to_fill);
94 accel_dev->heartbeat->ctrs_cnt_checked = false;
95 }
96 EXPORT_SYMBOL_GPL(adf_heartbeat_check_ctrs);
97
get_timer_ticks(struct adf_accel_dev * accel_dev,unsigned int * value)98 static int get_timer_ticks(struct adf_accel_dev *accel_dev, unsigned int *value)
99 {
100 char timer_str[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = { };
101 u32 timer_ms = ADF_CFG_HB_TIMER_DEFAULT_MS;
102 int cfg_read_status;
103 u32 ticks;
104 int ret;
105
106 cfg_read_status = adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC,
107 ADF_HEARTBEAT_TIMER, timer_str);
108 if (cfg_read_status == 0) {
109 if (kstrtouint(timer_str, 10, &timer_ms))
110 dev_dbg(&GET_DEV(accel_dev),
111 "kstrtouint failed to parse the %s, param value",
112 ADF_HEARTBEAT_TIMER);
113 }
114
115 if (timer_ms < ADF_CFG_HB_TIMER_MIN_MS) {
116 dev_err(&GET_DEV(accel_dev), "Timer cannot be less than %u\n",
117 ADF_CFG_HB_TIMER_MIN_MS);
118 return -EINVAL;
119 }
120
121 /*
122 * On 4xxx devices adf_timer is responsible for HB updates and
123 * its period is fixed to 200ms
124 */
125 if (accel_dev->timer)
126 timer_ms = ADF_CFG_HB_TIMER_MIN_MS;
127
128 ret = adf_heartbeat_ms_to_ticks(accel_dev, timer_ms, &ticks);
129 if (ret)
130 return ret;
131
132 adf_heartbeat_save_cfg_param(accel_dev, timer_ms);
133
134 accel_dev->heartbeat->hb_timer = timer_ms;
135 *value = ticks;
136
137 return 0;
138 }
139
check_ae(struct hb_cnt_pair * curr,struct hb_cnt_pair * prev,u16 * count,const size_t hb_ctrs)140 static int check_ae(struct hb_cnt_pair *curr, struct hb_cnt_pair *prev,
141 u16 *count, const size_t hb_ctrs)
142 {
143 size_t thr;
144
145 /* loop through all threads in AE */
146 for (thr = 0; thr < hb_ctrs; thr++) {
147 u16 req = curr[thr].req_heartbeat_cnt;
148 u16 resp = curr[thr].resp_heartbeat_cnt;
149 u16 last = prev[thr].resp_heartbeat_cnt;
150
151 if ((thr == ADF_AE_ADMIN_THREAD || req != resp) && resp == last) {
152 u16 retry = ++count[thr];
153
154 if (retry >= ADF_CFG_HB_COUNT_THRESHOLD)
155 return -EIO;
156
157 } else {
158 count[thr] = 0;
159 }
160 }
161 return 0;
162 }
163
adf_hb_get_status(struct adf_accel_dev * accel_dev)164 static int adf_hb_get_status(struct adf_accel_dev *accel_dev)
165 {
166 struct adf_hw_device_data *hw_device = accel_dev->hw_device;
167 struct hb_cnt_pair *live_stats, *last_stats, *curr_stats;
168 const size_t hb_ctrs = hw_device->num_hb_ctrs;
169 const unsigned long ae_mask = hw_device->ae_mask;
170 const size_t max_aes = hw_device->num_engines;
171 const size_t dev_ctrs = size_mul(max_aes, hb_ctrs);
172 const size_t stats_size = size_mul(dev_ctrs, sizeof(*curr_stats));
173 struct hb_cnt_pair *ae_curr_p, *ae_prev_p;
174 u16 *count_fails, *ae_count_p;
175 size_t ae_offset;
176 size_t ae = 0;
177 int ret = 0;
178
179 if (!accel_dev->heartbeat->ctrs_cnt_checked) {
180 if (validate_hb_ctrs_cnt(accel_dev))
181 hw_device->num_hb_ctrs += ADF_NUM_PKE_STRAND;
182
183 accel_dev->heartbeat->ctrs_cnt_checked = true;
184 }
185
186 live_stats = accel_dev->heartbeat->dma.virt_addr;
187 last_stats = live_stats + dev_ctrs;
188 count_fails = (u16 *)(last_stats + dev_ctrs);
189
190 curr_stats = kmemdup(live_stats, stats_size, GFP_KERNEL);
191 if (!curr_stats)
192 return -ENOMEM;
193
194 /* loop through active AEs */
195 for_each_set_bit(ae, &ae_mask, max_aes) {
196 ae_offset = size_mul(ae, hb_ctrs);
197 ae_curr_p = curr_stats + ae_offset;
198 ae_prev_p = last_stats + ae_offset;
199 ae_count_p = count_fails + ae_offset;
200
201 ret = check_ae(ae_curr_p, ae_prev_p, ae_count_p, hb_ctrs);
202 if (ret)
203 break;
204 }
205
206 /* Copy current stats for the next iteration */
207 memcpy(last_stats, curr_stats, stats_size);
208 kfree(curr_stats);
209
210 return ret;
211 }
212
adf_heartbeat_status(struct adf_accel_dev * accel_dev,enum adf_device_heartbeat_status * hb_status)213 void adf_heartbeat_status(struct adf_accel_dev *accel_dev,
214 enum adf_device_heartbeat_status *hb_status)
215 {
216 struct adf_heartbeat *hb;
217
218 if (!adf_dev_started(accel_dev) ||
219 test_bit(ADF_STATUS_RESTARTING, &accel_dev->status)) {
220 *hb_status = HB_DEV_UNRESPONSIVE;
221 return;
222 }
223
224 if (adf_hb_check_polling_freq(accel_dev) == -EINVAL) {
225 *hb_status = HB_DEV_UNSUPPORTED;
226 return;
227 }
228
229 hb = accel_dev->heartbeat;
230 hb->hb_sent_counter++;
231
232 if (adf_hb_get_status(accel_dev)) {
233 dev_err(&GET_DEV(accel_dev),
234 "Heartbeat ERROR: QAT is not responding.\n");
235 *hb_status = HB_DEV_UNRESPONSIVE;
236 hb->hb_failed_counter++;
237 return;
238 }
239
240 *hb_status = HB_DEV_ALIVE;
241 }
242
adf_heartbeat_ms_to_ticks(struct adf_accel_dev * accel_dev,unsigned int time_ms,u32 * value)243 int adf_heartbeat_ms_to_ticks(struct adf_accel_dev *accel_dev, unsigned int time_ms,
244 u32 *value)
245 {
246 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
247 u32 clk_per_sec;
248
249 /* HB clock may be different than AE clock */
250 if (!hw_data->get_hb_clock)
251 return -EINVAL;
252
253 clk_per_sec = hw_data->get_hb_clock(hw_data);
254 *value = time_ms * (clk_per_sec / MSEC_PER_SEC);
255
256 return 0;
257 }
258
adf_heartbeat_save_cfg_param(struct adf_accel_dev * accel_dev,unsigned int timer_ms)259 int adf_heartbeat_save_cfg_param(struct adf_accel_dev *accel_dev,
260 unsigned int timer_ms)
261 {
262 char timer_str[ADF_CFG_MAX_VAL_LEN_IN_BYTES];
263
264 snprintf(timer_str, sizeof(timer_str), "%u", timer_ms);
265 return adf_cfg_add_key_value_param(accel_dev, ADF_GENERAL_SEC,
266 ADF_HEARTBEAT_TIMER, timer_str,
267 ADF_STR);
268 }
269 EXPORT_SYMBOL_GPL(adf_heartbeat_save_cfg_param);
270
adf_heartbeat_init(struct adf_accel_dev * accel_dev)271 int adf_heartbeat_init(struct adf_accel_dev *accel_dev)
272 {
273 struct adf_heartbeat *hb;
274
275 hb = kzalloc(sizeof(*hb), GFP_KERNEL);
276 if (!hb)
277 goto err_ret;
278
279 hb->dma.virt_addr = dma_alloc_coherent(&GET_DEV(accel_dev), PAGE_SIZE,
280 &hb->dma.phy_addr, GFP_KERNEL);
281 if (!hb->dma.virt_addr)
282 goto err_free;
283
284 /*
285 * Default set this flag as true to avoid unnecessary checks,
286 * it will be reset on platforms that need such a check
287 */
288 hb->ctrs_cnt_checked = true;
289 accel_dev->heartbeat = hb;
290
291 return 0;
292
293 err_free:
294 kfree(hb);
295 err_ret:
296 return -ENOMEM;
297 }
298
adf_heartbeat_start(struct adf_accel_dev * accel_dev)299 int adf_heartbeat_start(struct adf_accel_dev *accel_dev)
300 {
301 unsigned int timer_ticks;
302 int ret;
303
304 if (!accel_dev->heartbeat) {
305 dev_warn(&GET_DEV(accel_dev), "Heartbeat instance not found!");
306 return -EFAULT;
307 }
308
309 if (accel_dev->hw_device->check_hb_ctrs)
310 accel_dev->hw_device->check_hb_ctrs(accel_dev);
311
312 ret = get_timer_ticks(accel_dev, &timer_ticks);
313 if (ret)
314 return ret;
315
316 ret = adf_send_admin_hb_timer(accel_dev, timer_ticks);
317 if (ret)
318 dev_warn(&GET_DEV(accel_dev), "Heartbeat not supported!");
319
320 return ret;
321 }
322
adf_heartbeat_shutdown(struct adf_accel_dev * accel_dev)323 void adf_heartbeat_shutdown(struct adf_accel_dev *accel_dev)
324 {
325 struct adf_heartbeat *hb = accel_dev->heartbeat;
326
327 if (!hb)
328 return;
329
330 if (hb->dma.virt_addr)
331 dma_free_coherent(&GET_DEV(accel_dev), PAGE_SIZE,
332 hb->dma.virt_addr, hb->dma.phy_addr);
333
334 kfree(hb);
335 accel_dev->heartbeat = NULL;
336 }
337