1 // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) 2 /* Copyright(c) 2021 Intel Corporation */ 3 #include <linux/delay.h> 4 #include <linux/iopoll.h> 5 #include <linux/mutex.h> 6 #include <linux/types.h> 7 #include "adf_accel_devices.h" 8 #include "adf_common_drv.h" 9 #include "adf_gen2_pfvf.h" 10 #include "adf_pfvf_msg.h" 11 #include "adf_pfvf_pf_proto.h" 12 #include "adf_pfvf_vf_proto.h" 13 #include "adf_pfvf_utils.h" 14 15 /* VF2PF interrupts */ 16 #define ADF_GEN2_VF_MSK 0xFFFF 17 #define ADF_GEN2_ERR_REG_VF2PF(vf_src) (((vf_src) & 0x01FFFE00) >> 9) 18 #define ADF_GEN2_ERR_MSK_VF2PF(vf_mask) (((vf_mask) & ADF_GEN2_VF_MSK) << 9) 19 20 #define ADF_GEN2_PF_PF2VF_OFFSET(i) (0x3A000 + 0x280 + ((i) * 0x04)) 21 #define ADF_GEN2_VF_PF2VF_OFFSET 0x200 22 23 #define ADF_GEN2_CSR_IN_USE 0x6AC2 24 #define ADF_GEN2_CSR_IN_USE_MASK 0xFFFE 25 26 enum gen2_csr_pos { 27 ADF_GEN2_CSR_PF2VF_OFFSET = 0, 28 ADF_GEN2_CSR_VF2PF_OFFSET = 16, 29 }; 30 31 #define ADF_PFVF_GEN2_MSGTYPE_SHIFT 2 32 #define ADF_PFVF_GEN2_MSGTYPE_MASK 0x0F 33 #define ADF_PFVF_GEN2_MSGDATA_SHIFT 6 34 #define ADF_PFVF_GEN2_MSGDATA_MASK 0x3FF 35 36 static const struct pfvf_csr_format csr_gen2_fmt = { 37 { ADF_PFVF_GEN2_MSGTYPE_SHIFT, ADF_PFVF_GEN2_MSGTYPE_MASK }, 38 { ADF_PFVF_GEN2_MSGDATA_SHIFT, ADF_PFVF_GEN2_MSGDATA_MASK }, 39 }; 40 41 #define ADF_PFVF_MSG_RETRY_DELAY 5 42 #define ADF_PFVF_MSG_MAX_RETRIES 3 43 44 static u32 adf_gen2_pf_get_pfvf_offset(u32 i) 45 { 46 return ADF_GEN2_PF_PF2VF_OFFSET(i); 47 } 48 49 static u32 adf_gen2_vf_get_pfvf_offset(u32 i) 50 { 51 return ADF_GEN2_VF_PF2VF_OFFSET; 52 } 53 54 static void adf_gen2_enable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask) 55 { 56 /* Enable VF2PF Messaging Ints - VFs 0 through 15 per vf_mask[15:0] */ 57 if (vf_mask & ADF_GEN2_VF_MSK) { 58 u32 val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3) 59 & ~ADF_GEN2_ERR_MSK_VF2PF(vf_mask); 60 ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, val); 61 } 62 } 63 64 static void adf_gen2_disable_all_vf2pf_interrupts(void __iomem *pmisc_addr) 65 { 66 /* Disable VF2PF interrupts for VFs 0 through 15 per vf_mask[15:0] */ 67 u32 val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3) 68 | ADF_GEN2_ERR_MSK_VF2PF(ADF_GEN2_VF_MSK); 69 ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, val); 70 } 71 72 static u32 adf_gen2_disable_pending_vf2pf_interrupts(void __iomem *pmisc_addr) 73 { 74 u32 sources, disabled, pending; 75 u32 errsou3, errmsk3; 76 77 /* Get the interrupt sources triggered by VFs */ 78 errsou3 = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRSOU3); 79 sources = ADF_GEN2_ERR_REG_VF2PF(errsou3); 80 81 if (!sources) 82 return 0; 83 84 /* Get the already disabled interrupts */ 85 errmsk3 = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3); 86 disabled = ADF_GEN2_ERR_REG_VF2PF(errmsk3); 87 88 pending = sources & ~disabled; 89 if (!pending) 90 return 0; 91 92 /* Due to HW limitations, when disabling the interrupts, we can't 93 * just disable the requested sources, as this would lead to missed 94 * interrupts if ERRSOU3 changes just before writing to ERRMSK3. 95 * To work around it, disable all and re-enable only the sources that 96 * are not in vf_mask and were not already disabled. Re-enabling will 97 * trigger a new interrupt for the sources that have changed in the 98 * meantime, if any. 99 */ 100 errmsk3 |= ADF_GEN2_ERR_MSK_VF2PF(ADF_GEN2_VF_MSK); 101 ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, errmsk3); 102 103 errmsk3 &= ADF_GEN2_ERR_MSK_VF2PF(sources | disabled); 104 ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, errmsk3); 105 106 /* Return the sources of the (new) interrupt(s) */ 107 return pending; 108 } 109 110 static u32 gen2_csr_get_int_bit(enum gen2_csr_pos offset) 111 { 112 return ADF_PFVF_INT << offset; 113 } 114 115 static u32 gen2_csr_msg_to_position(u32 csr_msg, enum gen2_csr_pos offset) 116 { 117 return (csr_msg & 0xFFFF) << offset; 118 } 119 120 static u32 gen2_csr_msg_from_position(u32 csr_val, enum gen2_csr_pos offset) 121 { 122 return (csr_val >> offset) & 0xFFFF; 123 } 124 125 static bool gen2_csr_is_in_use(u32 msg, enum gen2_csr_pos offset) 126 { 127 return ((msg >> offset) & ADF_GEN2_CSR_IN_USE_MASK) == ADF_GEN2_CSR_IN_USE; 128 } 129 130 static void gen2_csr_clear_in_use(u32 *msg, enum gen2_csr_pos offset) 131 { 132 *msg &= ~(ADF_GEN2_CSR_IN_USE_MASK << offset); 133 } 134 135 static void gen2_csr_set_in_use(u32 *msg, enum gen2_csr_pos offset) 136 { 137 *msg |= (ADF_GEN2_CSR_IN_USE << offset); 138 } 139 140 static bool is_legacy_user_pfvf_message(u32 msg) 141 { 142 return !(msg & ADF_PFVF_MSGORIGIN_SYSTEM); 143 } 144 145 static bool is_pf2vf_notification(u8 msg_type) 146 { 147 switch (msg_type) { 148 case ADF_PF2VF_MSGTYPE_RESTARTING: 149 return true; 150 default: 151 return false; 152 } 153 } 154 155 static bool is_vf2pf_notification(u8 msg_type) 156 { 157 switch (msg_type) { 158 case ADF_VF2PF_MSGTYPE_INIT: 159 case ADF_VF2PF_MSGTYPE_SHUTDOWN: 160 return true; 161 default: 162 return false; 163 } 164 } 165 166 struct pfvf_gen2_params { 167 u32 pfvf_offset; 168 struct mutex *csr_lock; /* lock preventing concurrent access of CSR */ 169 enum gen2_csr_pos local_offset; 170 enum gen2_csr_pos remote_offset; 171 bool (*is_notification_message)(u8 msg_type); 172 u8 compat_ver; 173 }; 174 175 static int adf_gen2_pfvf_send(struct adf_accel_dev *accel_dev, 176 struct pfvf_message msg, 177 struct pfvf_gen2_params *params) 178 { 179 void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); 180 enum gen2_csr_pos remote_offset = params->remote_offset; 181 enum gen2_csr_pos local_offset = params->local_offset; 182 unsigned int retries = ADF_PFVF_MSG_MAX_RETRIES; 183 struct mutex *lock = params->csr_lock; 184 u32 pfvf_offset = params->pfvf_offset; 185 u32 int_bit; 186 u32 csr_val; 187 u32 csr_msg; 188 int ret; 189 190 /* Gen2 messages, both PF->VF and VF->PF, are all 16 bits long. This 191 * allows us to build and read messages as if they where all 0 based. 192 * However, send and receive are in a single shared 32 bits register, 193 * so we need to shift and/or mask the message half before decoding 194 * it and after encoding it. Which one to shift depends on the 195 * direction. 196 */ 197 198 int_bit = gen2_csr_get_int_bit(local_offset); 199 200 csr_msg = adf_pfvf_csr_msg_of(accel_dev, msg, &csr_gen2_fmt); 201 if (unlikely(!csr_msg)) 202 return -EINVAL; 203 204 /* Prepare for CSR format, shifting the wire message in place and 205 * setting the in use pattern 206 */ 207 csr_msg = gen2_csr_msg_to_position(csr_msg, local_offset); 208 gen2_csr_set_in_use(&csr_msg, remote_offset); 209 210 mutex_lock(lock); 211 212 start: 213 /* Check if the PFVF CSR is in use by remote function */ 214 csr_val = ADF_CSR_RD(pmisc_addr, pfvf_offset); 215 if (gen2_csr_is_in_use(csr_val, local_offset)) { 216 dev_dbg(&GET_DEV(accel_dev), 217 "PFVF CSR in use by remote function\n"); 218 goto retry; 219 } 220 221 /* Attempt to get ownership of the PFVF CSR */ 222 ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_msg | int_bit); 223 224 /* Wait for confirmation from remote func it received the message */ 225 ret = read_poll_timeout(ADF_CSR_RD, csr_val, !(csr_val & int_bit), 226 ADF_PFVF_MSG_ACK_DELAY_US, 227 ADF_PFVF_MSG_ACK_MAX_DELAY_US, 228 true, pmisc_addr, pfvf_offset); 229 if (unlikely(ret < 0)) { 230 dev_dbg(&GET_DEV(accel_dev), "ACK not received from remote\n"); 231 csr_val &= ~int_bit; 232 } 233 234 /* For fire-and-forget notifications, the receiver does not clear 235 * the in-use pattern. This is used to detect collisions. 236 */ 237 if (params->is_notification_message(msg.type) && csr_val != csr_msg) { 238 /* Collision must have overwritten the message */ 239 dev_err(&GET_DEV(accel_dev), 240 "Collision on notification - PFVF CSR overwritten by remote function\n"); 241 goto retry; 242 } 243 244 /* If the far side did not clear the in-use pattern it is either 245 * 1) Notification - message left intact to detect collision 246 * 2) Older protocol (compatibility version < 3) on the far side 247 * where the sender is responsible for clearing the in-use 248 * pattern after the received has acknowledged receipt. 249 * In either case, clear the in-use pattern now. 250 */ 251 if (gen2_csr_is_in_use(csr_val, remote_offset)) { 252 gen2_csr_clear_in_use(&csr_val, remote_offset); 253 ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_val); 254 } 255 256 out: 257 mutex_unlock(lock); 258 return ret; 259 260 retry: 261 if (--retries) { 262 msleep(ADF_PFVF_MSG_RETRY_DELAY); 263 goto start; 264 } else { 265 ret = -EBUSY; 266 goto out; 267 } 268 } 269 270 static struct pfvf_message adf_gen2_pfvf_recv(struct adf_accel_dev *accel_dev, 271 struct pfvf_gen2_params *params) 272 { 273 void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); 274 enum gen2_csr_pos remote_offset = params->remote_offset; 275 enum gen2_csr_pos local_offset = params->local_offset; 276 u32 pfvf_offset = params->pfvf_offset; 277 struct pfvf_message msg = { 0 }; 278 u32 int_bit; 279 u32 csr_val; 280 u16 csr_msg; 281 282 int_bit = gen2_csr_get_int_bit(local_offset); 283 284 /* Read message */ 285 csr_val = ADF_CSR_RD(pmisc_addr, pfvf_offset); 286 if (!(csr_val & int_bit)) { 287 dev_info(&GET_DEV(accel_dev), 288 "Spurious PFVF interrupt, msg 0x%.8x. Ignored\n", csr_val); 289 return msg; 290 } 291 292 /* Extract the message from the CSR */ 293 csr_msg = gen2_csr_msg_from_position(csr_val, local_offset); 294 295 /* Ignore legacy non-system (non-kernel) messages */ 296 if (unlikely(is_legacy_user_pfvf_message(csr_msg))) { 297 dev_dbg(&GET_DEV(accel_dev), 298 "Ignored non-system message (0x%.8x);\n", csr_val); 299 /* Because this must be a legacy message, the far side 300 * must clear the in-use pattern, so don't do it. 301 */ 302 return msg; 303 } 304 305 /* Return the pfvf_message format */ 306 msg = adf_pfvf_message_of(accel_dev, csr_msg, &csr_gen2_fmt); 307 308 /* The in-use pattern is not cleared for notifications (so that 309 * it can be used for collision detection) or older implementations 310 */ 311 if (params->compat_ver >= ADF_PFVF_COMPAT_FAST_ACK && 312 !params->is_notification_message(msg.type)) 313 gen2_csr_clear_in_use(&csr_val, remote_offset); 314 315 /* To ACK, clear the INT bit */ 316 csr_val &= ~int_bit; 317 ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_val); 318 319 return msg; 320 } 321 322 static int adf_gen2_pf2vf_send(struct adf_accel_dev *accel_dev, struct pfvf_message msg, 323 u32 pfvf_offset, struct mutex *csr_lock) 324 { 325 struct pfvf_gen2_params params = { 326 .csr_lock = csr_lock, 327 .pfvf_offset = pfvf_offset, 328 .local_offset = ADF_GEN2_CSR_PF2VF_OFFSET, 329 .remote_offset = ADF_GEN2_CSR_VF2PF_OFFSET, 330 .is_notification_message = is_pf2vf_notification, 331 }; 332 333 return adf_gen2_pfvf_send(accel_dev, msg, ¶ms); 334 } 335 336 static int adf_gen2_vf2pf_send(struct adf_accel_dev *accel_dev, struct pfvf_message msg, 337 u32 pfvf_offset, struct mutex *csr_lock) 338 { 339 struct pfvf_gen2_params params = { 340 .csr_lock = csr_lock, 341 .pfvf_offset = pfvf_offset, 342 .local_offset = ADF_GEN2_CSR_VF2PF_OFFSET, 343 .remote_offset = ADF_GEN2_CSR_PF2VF_OFFSET, 344 .is_notification_message = is_vf2pf_notification, 345 }; 346 347 return adf_gen2_pfvf_send(accel_dev, msg, ¶ms); 348 } 349 350 static struct pfvf_message adf_gen2_pf2vf_recv(struct adf_accel_dev *accel_dev, 351 u32 pfvf_offset, u8 compat_ver) 352 { 353 struct pfvf_gen2_params params = { 354 .pfvf_offset = pfvf_offset, 355 .local_offset = ADF_GEN2_CSR_PF2VF_OFFSET, 356 .remote_offset = ADF_GEN2_CSR_VF2PF_OFFSET, 357 .is_notification_message = is_pf2vf_notification, 358 .compat_ver = compat_ver, 359 }; 360 361 return adf_gen2_pfvf_recv(accel_dev, ¶ms); 362 } 363 364 static struct pfvf_message adf_gen2_vf2pf_recv(struct adf_accel_dev *accel_dev, 365 u32 pfvf_offset, u8 compat_ver) 366 { 367 struct pfvf_gen2_params params = { 368 .pfvf_offset = pfvf_offset, 369 .local_offset = ADF_GEN2_CSR_VF2PF_OFFSET, 370 .remote_offset = ADF_GEN2_CSR_PF2VF_OFFSET, 371 .is_notification_message = is_vf2pf_notification, 372 .compat_ver = compat_ver, 373 }; 374 375 return adf_gen2_pfvf_recv(accel_dev, ¶ms); 376 } 377 378 void adf_gen2_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops) 379 { 380 pfvf_ops->enable_comms = adf_enable_pf2vf_comms; 381 pfvf_ops->get_pf2vf_offset = adf_gen2_pf_get_pfvf_offset; 382 pfvf_ops->get_vf2pf_offset = adf_gen2_pf_get_pfvf_offset; 383 pfvf_ops->enable_vf2pf_interrupts = adf_gen2_enable_vf2pf_interrupts; 384 pfvf_ops->disable_all_vf2pf_interrupts = adf_gen2_disable_all_vf2pf_interrupts; 385 pfvf_ops->disable_pending_vf2pf_interrupts = adf_gen2_disable_pending_vf2pf_interrupts; 386 pfvf_ops->send_msg = adf_gen2_pf2vf_send; 387 pfvf_ops->recv_msg = adf_gen2_vf2pf_recv; 388 } 389 EXPORT_SYMBOL_GPL(adf_gen2_init_pf_pfvf_ops); 390 391 void adf_gen2_init_vf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops) 392 { 393 pfvf_ops->enable_comms = adf_enable_vf2pf_comms; 394 pfvf_ops->get_pf2vf_offset = adf_gen2_vf_get_pfvf_offset; 395 pfvf_ops->get_vf2pf_offset = adf_gen2_vf_get_pfvf_offset; 396 pfvf_ops->send_msg = adf_gen2_vf2pf_send; 397 pfvf_ops->recv_msg = adf_gen2_pf2vf_recv; 398 } 399 EXPORT_SYMBOL_GPL(adf_gen2_init_vf_pfvf_ops); 400