1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright(c) 2023 Intel Corporation */ 3 4 #include <linux/dev_printk.h> 5 #include <linux/dma-mapping.h> 6 #include <linux/export.h> 7 #include <linux/kernel.h> 8 #include <linux/kstrtox.h> 9 #include <linux/overflow.h> 10 #include <linux/string.h> 11 #include <linux/slab.h> 12 #include <linux/types.h> 13 #include <asm/errno.h> 14 #include "adf_accel_devices.h" 15 #include "adf_cfg.h" 16 #include "adf_cfg_strings.h" 17 #include "adf_clock.h" 18 #include "adf_common_drv.h" 19 #include "adf_heartbeat.h" 20 #include "adf_transport_internal.h" 21 #include "icp_qat_fw_init_admin.h" 22 23 #define ADF_HB_EMPTY_SIG 0xA5A5A5A5 24 25 /* Heartbeat counter pair */ 26 struct hb_cnt_pair { 27 __u16 resp_heartbeat_cnt; 28 __u16 req_heartbeat_cnt; 29 }; 30 adf_hb_check_polling_freq(struct adf_accel_dev * accel_dev)31 static int adf_hb_check_polling_freq(struct adf_accel_dev *accel_dev) 32 { 33 u64 curr_time = adf_clock_get_current_time(); 34 u64 polling_time = curr_time - accel_dev->heartbeat->last_hb_check_time; 35 36 if (polling_time < accel_dev->heartbeat->hb_timer) { 37 dev_warn(&GET_DEV(accel_dev), 38 "HB polling too frequent. Configured HB timer %d ms\n", 39 accel_dev->heartbeat->hb_timer); 40 return -EINVAL; 41 } 42 43 accel_dev->heartbeat->last_hb_check_time = curr_time; 44 return 0; 45 } 46 47 /** 48 * validate_hb_ctrs_cnt() - checks if the number of heartbeat counters should 49 * be updated by one to support the currently loaded firmware. 50 * @accel_dev: Pointer to acceleration device. 51 * 52 * Return: 53 * * true - hb_ctrs must increased by ADF_NUM_PKE_STRAND 54 * * false - no changes needed 55 */ validate_hb_ctrs_cnt(struct adf_accel_dev * accel_dev)56 static bool validate_hb_ctrs_cnt(struct adf_accel_dev *accel_dev) 57 { 58 const size_t hb_ctrs = accel_dev->hw_device->num_hb_ctrs; 59 const size_t max_aes = accel_dev->hw_device->num_engines; 60 const size_t hb_struct_size = sizeof(struct hb_cnt_pair); 61 const size_t exp_diff_size = array3_size(ADF_NUM_PKE_STRAND, max_aes, 62 hb_struct_size); 63 const size_t dev_ctrs = size_mul(max_aes, hb_ctrs); 64 const size_t stats_size = size_mul(dev_ctrs, hb_struct_size); 65 const u32 exp_diff_cnt = exp_diff_size / sizeof(u32); 66 const u32 stats_el_cnt = stats_size / sizeof(u32); 67 struct hb_cnt_pair *hb_stats = accel_dev->heartbeat->dma.virt_addr; 68 const u32 *mem_to_chk = (u32 *)(hb_stats + dev_ctrs); 69 u32 el_diff_cnt = 0; 70 int i; 71 72 /* count how many bytes are different from pattern */ 73 for (i = 0; i < stats_el_cnt; i++) { 74 if (mem_to_chk[i] == ADF_HB_EMPTY_SIG) 75 break; 76 77 el_diff_cnt++; 78 } 79 80 return el_diff_cnt && el_diff_cnt == exp_diff_cnt; 81 } 82 adf_heartbeat_check_ctrs(struct adf_accel_dev * accel_dev)83 void adf_heartbeat_check_ctrs(struct adf_accel_dev *accel_dev) 84 { 85 struct hb_cnt_pair *hb_stats = accel_dev->heartbeat->dma.virt_addr; 86 const size_t hb_ctrs = accel_dev->hw_device->num_hb_ctrs; 87 const size_t max_aes = accel_dev->hw_device->num_engines; 88 const size_t dev_ctrs = size_mul(max_aes, hb_ctrs); 89 const size_t stats_size = size_mul(dev_ctrs, sizeof(struct hb_cnt_pair)); 90 const size_t mem_items_to_fill = size_mul(stats_size, 2) / sizeof(u32); 91 92 /* fill hb stats memory with pattern */ 93 memset32((uint32_t *)hb_stats, ADF_HB_EMPTY_SIG, mem_items_to_fill); 94 accel_dev->heartbeat->ctrs_cnt_checked = false; 95 } 96 EXPORT_SYMBOL_GPL(adf_heartbeat_check_ctrs); 97 get_timer_ticks(struct adf_accel_dev * accel_dev,unsigned int * value)98 static int get_timer_ticks(struct adf_accel_dev *accel_dev, unsigned int *value) 99 { 100 char timer_str[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = { }; 101 u32 timer_ms = ADF_CFG_HB_TIMER_DEFAULT_MS; 102 int cfg_read_status; 103 u32 ticks; 104 int ret; 105 106 cfg_read_status = adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC, 107 ADF_HEARTBEAT_TIMER, timer_str); 108 if (cfg_read_status == 0) { 109 if (kstrtouint(timer_str, 10, &timer_ms)) 110 dev_dbg(&GET_DEV(accel_dev), 111 "kstrtouint failed to parse the %s, param value", 112 ADF_HEARTBEAT_TIMER); 113 } 114 115 if (timer_ms < ADF_CFG_HB_TIMER_MIN_MS) { 116 dev_err(&GET_DEV(accel_dev), "Timer cannot be less than %u\n", 117 ADF_CFG_HB_TIMER_MIN_MS); 118 return -EINVAL; 119 } 120 121 /* 122 * On 4xxx devices adf_timer is responsible for HB updates and 123 * its period is fixed to 200ms 124 */ 125 if (accel_dev->timer) 126 timer_ms = ADF_CFG_HB_TIMER_MIN_MS; 127 128 ret = adf_heartbeat_ms_to_ticks(accel_dev, timer_ms, &ticks); 129 if (ret) 130 return ret; 131 132 adf_heartbeat_save_cfg_param(accel_dev, timer_ms); 133 134 accel_dev->heartbeat->hb_timer = timer_ms; 135 *value = ticks; 136 137 return 0; 138 } 139 check_ae(struct hb_cnt_pair * curr,struct hb_cnt_pair * prev,u16 * count,const size_t hb_ctrs)140 static int check_ae(struct hb_cnt_pair *curr, struct hb_cnt_pair *prev, 141 u16 *count, const size_t hb_ctrs) 142 { 143 size_t thr; 144 145 /* loop through all threads in AE */ 146 for (thr = 0; thr < hb_ctrs; thr++) { 147 u16 req = curr[thr].req_heartbeat_cnt; 148 u16 resp = curr[thr].resp_heartbeat_cnt; 149 u16 last = prev[thr].resp_heartbeat_cnt; 150 151 if ((thr == ADF_AE_ADMIN_THREAD || req != resp) && resp == last) { 152 u16 retry = ++count[thr]; 153 154 if (retry >= ADF_CFG_HB_COUNT_THRESHOLD) 155 return -EIO; 156 157 } else { 158 count[thr] = 0; 159 } 160 } 161 return 0; 162 } 163 adf_hb_get_status(struct adf_accel_dev * accel_dev)164 static int adf_hb_get_status(struct adf_accel_dev *accel_dev) 165 { 166 struct adf_hw_device_data *hw_device = accel_dev->hw_device; 167 struct hb_cnt_pair *live_stats, *last_stats, *curr_stats; 168 const size_t hb_ctrs = hw_device->num_hb_ctrs; 169 const unsigned long ae_mask = hw_device->ae_mask; 170 const size_t max_aes = hw_device->num_engines; 171 const size_t dev_ctrs = size_mul(max_aes, hb_ctrs); 172 const size_t stats_size = size_mul(dev_ctrs, sizeof(*curr_stats)); 173 struct hb_cnt_pair *ae_curr_p, *ae_prev_p; 174 u16 *count_fails, *ae_count_p; 175 size_t ae_offset; 176 size_t ae = 0; 177 int ret = 0; 178 179 if (!accel_dev->heartbeat->ctrs_cnt_checked) { 180 if (validate_hb_ctrs_cnt(accel_dev)) 181 hw_device->num_hb_ctrs += ADF_NUM_PKE_STRAND; 182 183 accel_dev->heartbeat->ctrs_cnt_checked = true; 184 } 185 186 live_stats = accel_dev->heartbeat->dma.virt_addr; 187 last_stats = live_stats + dev_ctrs; 188 count_fails = (u16 *)(last_stats + dev_ctrs); 189 190 curr_stats = kmemdup(live_stats, stats_size, GFP_KERNEL); 191 if (!curr_stats) 192 return -ENOMEM; 193 194 /* loop through active AEs */ 195 for_each_set_bit(ae, &ae_mask, max_aes) { 196 ae_offset = size_mul(ae, hb_ctrs); 197 ae_curr_p = curr_stats + ae_offset; 198 ae_prev_p = last_stats + ae_offset; 199 ae_count_p = count_fails + ae_offset; 200 201 ret = check_ae(ae_curr_p, ae_prev_p, ae_count_p, hb_ctrs); 202 if (ret) 203 break; 204 } 205 206 /* Copy current stats for the next iteration */ 207 memcpy(last_stats, curr_stats, stats_size); 208 kfree(curr_stats); 209 210 return ret; 211 } 212 adf_heartbeat_status(struct adf_accel_dev * accel_dev,enum adf_device_heartbeat_status * hb_status)213 void adf_heartbeat_status(struct adf_accel_dev *accel_dev, 214 enum adf_device_heartbeat_status *hb_status) 215 { 216 struct adf_heartbeat *hb; 217 218 if (!adf_dev_started(accel_dev) || 219 test_bit(ADF_STATUS_RESTARTING, &accel_dev->status)) { 220 *hb_status = HB_DEV_UNRESPONSIVE; 221 return; 222 } 223 224 if (adf_hb_check_polling_freq(accel_dev) == -EINVAL) { 225 *hb_status = HB_DEV_UNSUPPORTED; 226 return; 227 } 228 229 hb = accel_dev->heartbeat; 230 hb->hb_sent_counter++; 231 232 if (adf_hb_get_status(accel_dev)) { 233 dev_err(&GET_DEV(accel_dev), 234 "Heartbeat ERROR: QAT is not responding.\n"); 235 *hb_status = HB_DEV_UNRESPONSIVE; 236 hb->hb_failed_counter++; 237 return; 238 } 239 240 *hb_status = HB_DEV_ALIVE; 241 } 242 adf_heartbeat_ms_to_ticks(struct adf_accel_dev * accel_dev,unsigned int time_ms,u32 * value)243 int adf_heartbeat_ms_to_ticks(struct adf_accel_dev *accel_dev, unsigned int time_ms, 244 u32 *value) 245 { 246 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 247 u32 clk_per_sec; 248 249 /* HB clock may be different than AE clock */ 250 if (!hw_data->get_hb_clock) 251 return -EINVAL; 252 253 clk_per_sec = hw_data->get_hb_clock(hw_data); 254 *value = time_ms * (clk_per_sec / MSEC_PER_SEC); 255 256 return 0; 257 } 258 adf_heartbeat_save_cfg_param(struct adf_accel_dev * accel_dev,unsigned int timer_ms)259 int adf_heartbeat_save_cfg_param(struct adf_accel_dev *accel_dev, 260 unsigned int timer_ms) 261 { 262 char timer_str[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; 263 264 snprintf(timer_str, sizeof(timer_str), "%u", timer_ms); 265 return adf_cfg_add_key_value_param(accel_dev, ADF_GENERAL_SEC, 266 ADF_HEARTBEAT_TIMER, timer_str, 267 ADF_STR); 268 } 269 EXPORT_SYMBOL_GPL(adf_heartbeat_save_cfg_param); 270 adf_heartbeat_init(struct adf_accel_dev * accel_dev)271 int adf_heartbeat_init(struct adf_accel_dev *accel_dev) 272 { 273 struct adf_heartbeat *hb; 274 275 hb = kzalloc(sizeof(*hb), GFP_KERNEL); 276 if (!hb) 277 goto err_ret; 278 279 hb->dma.virt_addr = dma_alloc_coherent(&GET_DEV(accel_dev), PAGE_SIZE, 280 &hb->dma.phy_addr, GFP_KERNEL); 281 if (!hb->dma.virt_addr) 282 goto err_free; 283 284 /* 285 * Default set this flag as true to avoid unnecessary checks, 286 * it will be reset on platforms that need such a check 287 */ 288 hb->ctrs_cnt_checked = true; 289 accel_dev->heartbeat = hb; 290 291 return 0; 292 293 err_free: 294 kfree(hb); 295 err_ret: 296 return -ENOMEM; 297 } 298 adf_heartbeat_start(struct adf_accel_dev * accel_dev)299 int adf_heartbeat_start(struct adf_accel_dev *accel_dev) 300 { 301 unsigned int timer_ticks; 302 int ret; 303 304 if (!accel_dev->heartbeat) { 305 dev_warn(&GET_DEV(accel_dev), "Heartbeat instance not found!"); 306 return -EFAULT; 307 } 308 309 if (accel_dev->hw_device->check_hb_ctrs) 310 accel_dev->hw_device->check_hb_ctrs(accel_dev); 311 312 ret = get_timer_ticks(accel_dev, &timer_ticks); 313 if (ret) 314 return ret; 315 316 ret = adf_send_admin_hb_timer(accel_dev, timer_ticks); 317 if (ret) 318 dev_warn(&GET_DEV(accel_dev), "Heartbeat not supported!"); 319 320 return ret; 321 } 322 adf_heartbeat_shutdown(struct adf_accel_dev * accel_dev)323 void adf_heartbeat_shutdown(struct adf_accel_dev *accel_dev) 324 { 325 struct adf_heartbeat *hb = accel_dev->heartbeat; 326 327 if (!hb) 328 return; 329 330 if (hb->dma.virt_addr) 331 dma_free_coherent(&GET_DEV(accel_dev), PAGE_SIZE, 332 hb->dma.virt_addr, hb->dma.phy_addr); 333 334 kfree(hb); 335 accel_dev->heartbeat = NULL; 336 } 337