1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Data Object Exchange 4 * PCIe r6.0, sec 6.30 DOE 5 * 6 * Copyright (C) 2021 Huawei 7 * Jonathan Cameron <Jonathan.Cameron@huawei.com> 8 * 9 * Copyright (C) 2022 Intel Corporation 10 * Ira Weiny <ira.weiny@intel.com> 11 */ 12 13 #define dev_fmt(fmt) "DOE: " fmt 14 15 #include <linux/bitfield.h> 16 #include <linux/delay.h> 17 #include <linux/jiffies.h> 18 #include <linux/mutex.h> 19 #include <linux/pci.h> 20 #include <linux/pci-doe.h> 21 #include <linux/workqueue.h> 22 23 #define PCI_DOE_PROTOCOL_DISCOVERY 0 24 25 /* Timeout of 1 second from 6.30.2 Operation, PCI Spec r6.0 */ 26 #define PCI_DOE_TIMEOUT HZ 27 #define PCI_DOE_POLL_INTERVAL (PCI_DOE_TIMEOUT / 128) 28 29 #define PCI_DOE_FLAG_CANCEL 0 30 #define PCI_DOE_FLAG_DEAD 1 31 32 /* Max data object length is 2^18 dwords */ 33 #define PCI_DOE_MAX_LENGTH (1 << 18) 34 35 /** 36 * struct pci_doe_mb - State for a single DOE mailbox 37 * 38 * This state is used to manage a single DOE mailbox capability. All fields 39 * should be considered opaque to the consumers and the structure passed into 40 * the helpers below after being created by devm_pci_doe_create() 41 * 42 * @pdev: PCI device this mailbox belongs to 43 * @cap_offset: Capability offset 44 * @prots: Array of protocols supported (encoded as long values) 45 * @wq: Wait queue for work item 46 * @work_queue: Queue of pci_doe_work items 47 * @flags: Bit array of PCI_DOE_FLAG_* flags 48 */ 49 struct pci_doe_mb { 50 struct pci_dev *pdev; 51 u16 cap_offset; 52 struct xarray prots; 53 54 wait_queue_head_t wq; 55 struct workqueue_struct *work_queue; 56 unsigned long flags; 57 }; 58 59 static int pci_doe_wait(struct pci_doe_mb *doe_mb, unsigned long timeout) 60 { 61 if (wait_event_timeout(doe_mb->wq, 62 test_bit(PCI_DOE_FLAG_CANCEL, &doe_mb->flags), 63 timeout)) 64 return -EIO; 65 return 0; 66 } 67 68 static void pci_doe_write_ctrl(struct pci_doe_mb *doe_mb, u32 val) 69 { 70 struct pci_dev *pdev = doe_mb->pdev; 71 int offset = doe_mb->cap_offset; 72 73 pci_write_config_dword(pdev, offset + PCI_DOE_CTRL, val); 74 } 75 76 static int pci_doe_abort(struct pci_doe_mb *doe_mb) 77 { 78 struct pci_dev *pdev = doe_mb->pdev; 79 int offset = doe_mb->cap_offset; 80 unsigned long timeout_jiffies; 81 82 pci_dbg(pdev, "[%x] Issuing Abort\n", offset); 83 84 timeout_jiffies = jiffies + PCI_DOE_TIMEOUT; 85 pci_doe_write_ctrl(doe_mb, PCI_DOE_CTRL_ABORT); 86 87 do { 88 int rc; 89 u32 val; 90 91 rc = pci_doe_wait(doe_mb, PCI_DOE_POLL_INTERVAL); 92 if (rc) 93 return rc; 94 pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val); 95 96 /* Abort success! */ 97 if (!FIELD_GET(PCI_DOE_STATUS_ERROR, val) && 98 !FIELD_GET(PCI_DOE_STATUS_BUSY, val)) 99 return 0; 100 101 } while (!time_after(jiffies, timeout_jiffies)); 102 103 /* Abort has timed out and the MB is dead */ 104 pci_err(pdev, "[%x] ABORT timed out\n", offset); 105 return -EIO; 106 } 107 108 static int pci_doe_send_req(struct pci_doe_mb *doe_mb, 109 struct pci_doe_task *task) 110 { 111 struct pci_dev *pdev = doe_mb->pdev; 112 int offset = doe_mb->cap_offset; 113 size_t length; 114 u32 val; 115 int i; 116 117 /* 118 * Check the DOE busy bit is not set. If it is set, this could indicate 119 * someone other than Linux (e.g. firmware) is using the mailbox. Note 120 * it is expected that firmware and OS will negotiate access rights via 121 * an, as yet to be defined, method. 122 */ 123 pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val); 124 if (FIELD_GET(PCI_DOE_STATUS_BUSY, val)) 125 return -EBUSY; 126 127 if (FIELD_GET(PCI_DOE_STATUS_ERROR, val)) 128 return -EIO; 129 130 /* Length is 2 DW of header + length of payload in DW */ 131 length = 2 + task->request_pl_sz / sizeof(__le32); 132 if (length > PCI_DOE_MAX_LENGTH) 133 return -EIO; 134 if (length == PCI_DOE_MAX_LENGTH) 135 length = 0; 136 137 /* Write DOE Header */ 138 val = FIELD_PREP(PCI_DOE_DATA_OBJECT_HEADER_1_VID, task->prot.vid) | 139 FIELD_PREP(PCI_DOE_DATA_OBJECT_HEADER_1_TYPE, task->prot.type); 140 pci_write_config_dword(pdev, offset + PCI_DOE_WRITE, val); 141 pci_write_config_dword(pdev, offset + PCI_DOE_WRITE, 142 FIELD_PREP(PCI_DOE_DATA_OBJECT_HEADER_2_LENGTH, 143 length)); 144 for (i = 0; i < task->request_pl_sz / sizeof(__le32); i++) 145 pci_write_config_dword(pdev, offset + PCI_DOE_WRITE, 146 le32_to_cpu(task->request_pl[i])); 147 148 pci_doe_write_ctrl(doe_mb, PCI_DOE_CTRL_GO); 149 150 return 0; 151 } 152 153 static bool pci_doe_data_obj_ready(struct pci_doe_mb *doe_mb) 154 { 155 struct pci_dev *pdev = doe_mb->pdev; 156 int offset = doe_mb->cap_offset; 157 u32 val; 158 159 pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val); 160 if (FIELD_GET(PCI_DOE_STATUS_DATA_OBJECT_READY, val)) 161 return true; 162 return false; 163 } 164 165 static int pci_doe_recv_resp(struct pci_doe_mb *doe_mb, struct pci_doe_task *task) 166 { 167 struct pci_dev *pdev = doe_mb->pdev; 168 int offset = doe_mb->cap_offset; 169 size_t length, payload_length; 170 u32 val; 171 int i; 172 173 /* Read the first dword to get the protocol */ 174 pci_read_config_dword(pdev, offset + PCI_DOE_READ, &val); 175 if ((FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_1_VID, val) != task->prot.vid) || 176 (FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_1_TYPE, val) != task->prot.type)) { 177 dev_err_ratelimited(&pdev->dev, "[%x] expected [VID, Protocol] = [%04x, %02x], got [%04x, %02x]\n", 178 doe_mb->cap_offset, task->prot.vid, task->prot.type, 179 FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_1_VID, val), 180 FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_1_TYPE, val)); 181 return -EIO; 182 } 183 184 pci_write_config_dword(pdev, offset + PCI_DOE_READ, 0); 185 /* Read the second dword to get the length */ 186 pci_read_config_dword(pdev, offset + PCI_DOE_READ, &val); 187 pci_write_config_dword(pdev, offset + PCI_DOE_READ, 0); 188 189 length = FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_2_LENGTH, val); 190 /* A value of 0x0 indicates max data object length */ 191 if (!length) 192 length = PCI_DOE_MAX_LENGTH; 193 if (length < 2) 194 return -EIO; 195 196 /* First 2 dwords have already been read */ 197 length -= 2; 198 payload_length = min(length, task->response_pl_sz / sizeof(__le32)); 199 /* Read the rest of the response payload */ 200 for (i = 0; i < payload_length; i++) { 201 pci_read_config_dword(pdev, offset + PCI_DOE_READ, &val); 202 task->response_pl[i] = cpu_to_le32(val); 203 /* Prior to the last ack, ensure Data Object Ready */ 204 if (i == (payload_length - 1) && !pci_doe_data_obj_ready(doe_mb)) 205 return -EIO; 206 pci_write_config_dword(pdev, offset + PCI_DOE_READ, 0); 207 } 208 209 /* Flush excess length */ 210 for (; i < length; i++) { 211 pci_read_config_dword(pdev, offset + PCI_DOE_READ, &val); 212 pci_write_config_dword(pdev, offset + PCI_DOE_READ, 0); 213 } 214 215 /* Final error check to pick up on any since Data Object Ready */ 216 pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val); 217 if (FIELD_GET(PCI_DOE_STATUS_ERROR, val)) 218 return -EIO; 219 220 return min(length, task->response_pl_sz / sizeof(__le32)) * sizeof(__le32); 221 } 222 223 static void signal_task_complete(struct pci_doe_task *task, int rv) 224 { 225 task->rv = rv; 226 task->complete(task); 227 destroy_work_on_stack(&task->work); 228 } 229 230 static void signal_task_abort(struct pci_doe_task *task, int rv) 231 { 232 struct pci_doe_mb *doe_mb = task->doe_mb; 233 struct pci_dev *pdev = doe_mb->pdev; 234 235 if (pci_doe_abort(doe_mb)) { 236 /* 237 * If the device can't process an abort; set the mailbox dead 238 * - no more submissions 239 */ 240 pci_err(pdev, "[%x] Abort failed marking mailbox dead\n", 241 doe_mb->cap_offset); 242 set_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags); 243 } 244 signal_task_complete(task, rv); 245 } 246 247 static void doe_statemachine_work(struct work_struct *work) 248 { 249 struct pci_doe_task *task = container_of(work, struct pci_doe_task, 250 work); 251 struct pci_doe_mb *doe_mb = task->doe_mb; 252 struct pci_dev *pdev = doe_mb->pdev; 253 int offset = doe_mb->cap_offset; 254 unsigned long timeout_jiffies; 255 u32 val; 256 int rc; 257 258 if (test_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags)) { 259 signal_task_complete(task, -EIO); 260 return; 261 } 262 263 /* Send request */ 264 rc = pci_doe_send_req(doe_mb, task); 265 if (rc) { 266 /* 267 * The specification does not provide any guidance on how to 268 * resolve conflicting requests from other entities. 269 * Furthermore, it is likely that busy will not be detected 270 * most of the time. Flag any detection of status busy with an 271 * error. 272 */ 273 if (rc == -EBUSY) 274 dev_err_ratelimited(&pdev->dev, "[%x] busy detected; another entity is sending conflicting requests\n", 275 offset); 276 signal_task_abort(task, rc); 277 return; 278 } 279 280 timeout_jiffies = jiffies + PCI_DOE_TIMEOUT; 281 /* Poll for response */ 282 retry_resp: 283 pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val); 284 if (FIELD_GET(PCI_DOE_STATUS_ERROR, val)) { 285 signal_task_abort(task, -EIO); 286 return; 287 } 288 289 if (!FIELD_GET(PCI_DOE_STATUS_DATA_OBJECT_READY, val)) { 290 if (time_after(jiffies, timeout_jiffies)) { 291 signal_task_abort(task, -EIO); 292 return; 293 } 294 rc = pci_doe_wait(doe_mb, PCI_DOE_POLL_INTERVAL); 295 if (rc) { 296 signal_task_abort(task, rc); 297 return; 298 } 299 goto retry_resp; 300 } 301 302 rc = pci_doe_recv_resp(doe_mb, task); 303 if (rc < 0) { 304 signal_task_abort(task, rc); 305 return; 306 } 307 308 signal_task_complete(task, rc); 309 } 310 311 static void pci_doe_task_complete(struct pci_doe_task *task) 312 { 313 complete(task->private); 314 } 315 316 static int pci_doe_discovery(struct pci_doe_mb *doe_mb, u8 *index, u16 *vid, 317 u8 *protocol) 318 { 319 u32 request_pl = FIELD_PREP(PCI_DOE_DATA_OBJECT_DISC_REQ_3_INDEX, 320 *index); 321 __le32 request_pl_le = cpu_to_le32(request_pl); 322 __le32 response_pl_le; 323 u32 response_pl; 324 DECLARE_COMPLETION_ONSTACK(c); 325 struct pci_doe_task task = { 326 .prot.vid = PCI_VENDOR_ID_PCI_SIG, 327 .prot.type = PCI_DOE_PROTOCOL_DISCOVERY, 328 .request_pl = &request_pl_le, 329 .request_pl_sz = sizeof(request_pl), 330 .response_pl = &response_pl_le, 331 .response_pl_sz = sizeof(response_pl), 332 .complete = pci_doe_task_complete, 333 .private = &c, 334 }; 335 int rc; 336 337 rc = pci_doe_submit_task(doe_mb, &task); 338 if (rc < 0) 339 return rc; 340 341 wait_for_completion(&c); 342 343 if (task.rv != sizeof(response_pl)) 344 return -EIO; 345 346 response_pl = le32_to_cpu(response_pl_le); 347 *vid = FIELD_GET(PCI_DOE_DATA_OBJECT_DISC_RSP_3_VID, response_pl); 348 *protocol = FIELD_GET(PCI_DOE_DATA_OBJECT_DISC_RSP_3_PROTOCOL, 349 response_pl); 350 *index = FIELD_GET(PCI_DOE_DATA_OBJECT_DISC_RSP_3_NEXT_INDEX, 351 response_pl); 352 353 return 0; 354 } 355 356 static void *pci_doe_xa_prot_entry(u16 vid, u8 prot) 357 { 358 return xa_mk_value((vid << 8) | prot); 359 } 360 361 static int pci_doe_cache_protocols(struct pci_doe_mb *doe_mb) 362 { 363 u8 index = 0; 364 u8 xa_idx = 0; 365 366 do { 367 int rc; 368 u16 vid; 369 u8 prot; 370 371 rc = pci_doe_discovery(doe_mb, &index, &vid, &prot); 372 if (rc) 373 return rc; 374 375 pci_dbg(doe_mb->pdev, 376 "[%x] Found protocol %d vid: %x prot: %x\n", 377 doe_mb->cap_offset, xa_idx, vid, prot); 378 379 rc = xa_insert(&doe_mb->prots, xa_idx++, 380 pci_doe_xa_prot_entry(vid, prot), GFP_KERNEL); 381 if (rc) 382 return rc; 383 } while (index); 384 385 return 0; 386 } 387 388 static void pci_doe_xa_destroy(void *mb) 389 { 390 struct pci_doe_mb *doe_mb = mb; 391 392 xa_destroy(&doe_mb->prots); 393 } 394 395 static void pci_doe_destroy_workqueue(void *mb) 396 { 397 struct pci_doe_mb *doe_mb = mb; 398 399 destroy_workqueue(doe_mb->work_queue); 400 } 401 402 static void pci_doe_flush_mb(void *mb) 403 { 404 struct pci_doe_mb *doe_mb = mb; 405 406 /* Stop all pending work items from starting */ 407 set_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags); 408 409 /* Cancel an in progress work item, if necessary */ 410 set_bit(PCI_DOE_FLAG_CANCEL, &doe_mb->flags); 411 wake_up(&doe_mb->wq); 412 413 /* Flush all work items */ 414 flush_workqueue(doe_mb->work_queue); 415 } 416 417 /** 418 * pcim_doe_create_mb() - Create a DOE mailbox object 419 * 420 * @pdev: PCI device to create the DOE mailbox for 421 * @cap_offset: Offset of the DOE mailbox 422 * 423 * Create a single mailbox object to manage the mailbox protocol at the 424 * cap_offset specified. 425 * 426 * RETURNS: created mailbox object on success 427 * ERR_PTR(-errno) on failure 428 */ 429 struct pci_doe_mb *pcim_doe_create_mb(struct pci_dev *pdev, u16 cap_offset) 430 { 431 struct pci_doe_mb *doe_mb; 432 struct device *dev = &pdev->dev; 433 int rc; 434 435 doe_mb = devm_kzalloc(dev, sizeof(*doe_mb), GFP_KERNEL); 436 if (!doe_mb) 437 return ERR_PTR(-ENOMEM); 438 439 doe_mb->pdev = pdev; 440 doe_mb->cap_offset = cap_offset; 441 init_waitqueue_head(&doe_mb->wq); 442 443 xa_init(&doe_mb->prots); 444 rc = devm_add_action(dev, pci_doe_xa_destroy, doe_mb); 445 if (rc) 446 return ERR_PTR(rc); 447 448 doe_mb->work_queue = alloc_ordered_workqueue("%s %s DOE [%x]", 0, 449 dev_driver_string(&pdev->dev), 450 pci_name(pdev), 451 doe_mb->cap_offset); 452 if (!doe_mb->work_queue) { 453 pci_err(pdev, "[%x] failed to allocate work queue\n", 454 doe_mb->cap_offset); 455 return ERR_PTR(-ENOMEM); 456 } 457 rc = devm_add_action_or_reset(dev, pci_doe_destroy_workqueue, doe_mb); 458 if (rc) 459 return ERR_PTR(rc); 460 461 /* Reset the mailbox by issuing an abort */ 462 rc = pci_doe_abort(doe_mb); 463 if (rc) { 464 pci_err(pdev, "[%x] failed to reset mailbox with abort command : %d\n", 465 doe_mb->cap_offset, rc); 466 return ERR_PTR(rc); 467 } 468 469 /* 470 * The state machine and the mailbox should be in sync now; 471 * Set up mailbox flush prior to using the mailbox to query protocols. 472 */ 473 rc = devm_add_action_or_reset(dev, pci_doe_flush_mb, doe_mb); 474 if (rc) 475 return ERR_PTR(rc); 476 477 rc = pci_doe_cache_protocols(doe_mb); 478 if (rc) { 479 pci_err(pdev, "[%x] failed to cache protocols : %d\n", 480 doe_mb->cap_offset, rc); 481 return ERR_PTR(rc); 482 } 483 484 return doe_mb; 485 } 486 EXPORT_SYMBOL_GPL(pcim_doe_create_mb); 487 488 /** 489 * pci_doe_supports_prot() - Return if the DOE instance supports the given 490 * protocol 491 * @doe_mb: DOE mailbox capability to query 492 * @vid: Protocol Vendor ID 493 * @type: Protocol type 494 * 495 * RETURNS: True if the DOE mailbox supports the protocol specified 496 */ 497 bool pci_doe_supports_prot(struct pci_doe_mb *doe_mb, u16 vid, u8 type) 498 { 499 unsigned long index; 500 void *entry; 501 502 /* The discovery protocol must always be supported */ 503 if (vid == PCI_VENDOR_ID_PCI_SIG && type == PCI_DOE_PROTOCOL_DISCOVERY) 504 return true; 505 506 xa_for_each(&doe_mb->prots, index, entry) 507 if (entry == pci_doe_xa_prot_entry(vid, type)) 508 return true; 509 510 return false; 511 } 512 EXPORT_SYMBOL_GPL(pci_doe_supports_prot); 513 514 /** 515 * pci_doe_submit_task() - Submit a task to be processed by the state machine 516 * 517 * @doe_mb: DOE mailbox capability to submit to 518 * @task: task to be queued 519 * 520 * Submit a DOE task (request/response) to the DOE mailbox to be processed. 521 * Returns upon queueing the task object. If the queue is full this function 522 * will sleep until there is room in the queue. 523 * 524 * task->complete will be called when the state machine is done processing this 525 * task. 526 * 527 * @task must be allocated on the stack. 528 * 529 * Excess data will be discarded. 530 * 531 * RETURNS: 0 when task has been successfully queued, -ERRNO on error 532 */ 533 int pci_doe_submit_task(struct pci_doe_mb *doe_mb, struct pci_doe_task *task) 534 { 535 if (!pci_doe_supports_prot(doe_mb, task->prot.vid, task->prot.type)) 536 return -EINVAL; 537 538 /* 539 * DOE requests must be a whole number of DW and the response needs to 540 * be big enough for at least 1 DW 541 */ 542 if (task->request_pl_sz % sizeof(__le32) || 543 task->response_pl_sz < sizeof(__le32)) 544 return -EINVAL; 545 546 if (test_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags)) 547 return -EIO; 548 549 task->doe_mb = doe_mb; 550 INIT_WORK_ONSTACK(&task->work, doe_statemachine_work); 551 queue_work(doe_mb->work_queue, &task->work); 552 return 0; 553 } 554 EXPORT_SYMBOL_GPL(pci_doe_submit_task); 555