Lines Matching +full:i3c +full:- +full:master +full:- +full:v1
1 // SPDX-License-Identifier: BSD-3-Clause
7 * Note: The I3C HCI v2.0 spec is still in flux. The IBI support is based on
8 * v1.x of the spec and v2.0 will likely be split out.
13 #include <linux/dma-mapping.h>
15 #include <linux/i3c/master.h>
40 #define rhs_reg_read(r) readl(hci->RHS_regs + (RHS_##r))
41 #define rhs_reg_write(r, v) writel(v, hci->RHS_regs + (RHS_##r))
52 * Ring Header (Per-Ring Bundle)
55 #define rh_reg_read(r) readl(rh->regs + (RH_##r))
56 #define rh_reg_write(r, v) writel(v, rh->regs + (RH_##r))
157 /* trickery to avoid compiler warnings on 32-bit build targets */ in hi32()
167 struct hci_rings_data *rings = hci->io_data; in hci_dma_cleanup()
174 for (i = 0; i < rings->total; i++) { in hci_dma_cleanup()
175 rh = &rings->headers[i]; in hci_dma_cleanup()
182 if (rh->xfer) in hci_dma_cleanup()
183 dma_free_coherent(&hci->master.dev, in hci_dma_cleanup()
184 rh->xfer_struct_sz * rh->xfer_entries, in hci_dma_cleanup()
185 rh->xfer, rh->xfer_dma); in hci_dma_cleanup()
186 if (rh->resp) in hci_dma_cleanup()
187 dma_free_coherent(&hci->master.dev, in hci_dma_cleanup()
188 rh->resp_struct_sz * rh->xfer_entries, in hci_dma_cleanup()
189 rh->resp, rh->resp_dma); in hci_dma_cleanup()
190 kfree(rh->src_xfers); in hci_dma_cleanup()
191 if (rh->ibi_status) in hci_dma_cleanup()
192 dma_free_coherent(&hci->master.dev, in hci_dma_cleanup()
193 rh->ibi_status_sz * rh->ibi_status_entries, in hci_dma_cleanup()
194 rh->ibi_status, rh->ibi_status_dma); in hci_dma_cleanup()
195 if (rh->ibi_data_dma) in hci_dma_cleanup()
196 dma_unmap_single(&hci->master.dev, rh->ibi_data_dma, in hci_dma_cleanup()
197 rh->ibi_chunk_sz * rh->ibi_chunks_total, in hci_dma_cleanup()
199 kfree(rh->ibi_data); in hci_dma_cleanup()
205 hci->io_data = NULL; in hci_dma_cleanup()
219 dev_info(&hci->master.dev, "%d DMA rings available\n", nr_rings); in hci_dma_init()
221 dev_err(&hci->master.dev, "number of rings should be <= 8\n"); in hci_dma_init()
228 return -ENOMEM; in hci_dma_init()
229 hci->io_data = rings; in hci_dma_init()
230 rings->total = nr_rings; in hci_dma_init()
232 for (i = 0; i < rings->total; i++) { in hci_dma_init()
235 dev_info(&hci->master.dev, "Ring %d at offset %#x\n", i, offset); in hci_dma_init()
236 ret = -EINVAL; in hci_dma_init()
239 rh = &rings->headers[i]; in hci_dma_init()
240 rh->regs = hci->base_regs + offset; in hci_dma_init()
241 spin_lock_init(&rh->lock); in hci_dma_init()
242 init_completion(&rh->op_done); in hci_dma_init()
244 rh->xfer_entries = XFER_RING_ENTRIES; in hci_dma_init()
247 rh->xfer_struct_sz = FIELD_GET(CR_XFER_STRUCT_SIZE, regval); in hci_dma_init()
248 rh->resp_struct_sz = FIELD_GET(CR_RESP_STRUCT_SIZE, regval); in hci_dma_init()
250 rh->xfer_struct_sz, rh->resp_struct_sz); in hci_dma_init()
251 xfers_sz = rh->xfer_struct_sz * rh->xfer_entries; in hci_dma_init()
252 resps_sz = rh->resp_struct_sz * rh->xfer_entries; in hci_dma_init()
254 rh->xfer = dma_alloc_coherent(&hci->master.dev, xfers_sz, in hci_dma_init()
255 &rh->xfer_dma, GFP_KERNEL); in hci_dma_init()
256 rh->resp = dma_alloc_coherent(&hci->master.dev, resps_sz, in hci_dma_init()
257 &rh->resp_dma, GFP_KERNEL); in hci_dma_init()
258 rh->src_xfers = in hci_dma_init()
259 kmalloc_array(rh->xfer_entries, sizeof(*rh->src_xfers), in hci_dma_init()
261 ret = -ENOMEM; in hci_dma_init()
262 if (!rh->xfer || !rh->resp || !rh->src_xfers) in hci_dma_init()
265 rh_reg_write(CMD_RING_BASE_LO, lo32(rh->xfer_dma)); in hci_dma_init()
266 rh_reg_write(CMD_RING_BASE_HI, hi32(rh->xfer_dma)); in hci_dma_init()
267 rh_reg_write(RESP_RING_BASE_LO, lo32(rh->resp_dma)); in hci_dma_init()
268 rh_reg_write(RESP_RING_BASE_HI, hi32(rh->resp_dma)); in hci_dma_init()
270 regval = FIELD_PREP(CR_RING_SIZE, rh->xfer_entries); in hci_dma_init()
288 rh->ibi_status_sz = FIELD_GET(IBI_STATUS_STRUCT_SIZE, regval); in hci_dma_init()
289 rh->ibi_status_entries = IBI_STATUS_RING_ENTRIES; in hci_dma_init()
290 rh->ibi_chunks_total = IBI_CHUNK_POOL_SIZE; in hci_dma_init()
292 rh->ibi_chunk_sz = dma_get_cache_alignment(); in hci_dma_init()
293 rh->ibi_chunk_sz *= IBI_CHUNK_CACHELINES; in hci_dma_init()
294 if (rh->ibi_chunk_sz > 256) { in hci_dma_init()
295 ret = -EINVAL; in hci_dma_init()
299 ibi_status_ring_sz = rh->ibi_status_sz * rh->ibi_status_entries; in hci_dma_init()
300 ibi_data_ring_sz = rh->ibi_chunk_sz * rh->ibi_chunks_total; in hci_dma_init()
302 rh->ibi_status = in hci_dma_init()
303 dma_alloc_coherent(&hci->master.dev, ibi_status_ring_sz, in hci_dma_init()
304 &rh->ibi_status_dma, GFP_KERNEL); in hci_dma_init()
305 rh->ibi_data = kmalloc(ibi_data_ring_sz, GFP_KERNEL); in hci_dma_init()
306 ret = -ENOMEM; in hci_dma_init()
307 if (!rh->ibi_status || !rh->ibi_data) in hci_dma_init()
309 rh->ibi_data_dma = in hci_dma_init()
310 dma_map_single(&hci->master.dev, rh->ibi_data, in hci_dma_init()
312 if (dma_mapping_error(&hci->master.dev, rh->ibi_data_dma)) { in hci_dma_init()
313 rh->ibi_data_dma = 0; in hci_dma_init()
314 ret = -ENOMEM; in hci_dma_init()
319 rh->ibi_status_entries) | in hci_dma_init()
321 ilog2(rh->ibi_chunk_sz) - 2) | in hci_dma_init()
323 rh->ibi_chunks_total); in hci_dma_init()
334 regval = FIELD_PREP(MAX_HEADER_COUNT, rings->total); in hci_dma_init()
351 if (!xfer->data) in hci_dma_unmap_xfer()
353 dma_unmap_single(&hci->master.dev, in hci_dma_unmap_xfer()
354 xfer->data_dma, xfer->data_len, in hci_dma_unmap_xfer()
355 xfer->rnw ? DMA_FROM_DEVICE : DMA_TO_DEVICE); in hci_dma_unmap_xfer()
362 struct hci_rings_data *rings = hci->io_data; in hci_dma_queue_xfer()
369 rh = &rings->headers[ring]; in hci_dma_queue_xfer()
375 u32 *ring_data = rh->xfer + rh->xfer_struct_sz * enqueue_ptr; in hci_dma_queue_xfer()
378 *ring_data++ = xfer->cmd_desc[0]; in hci_dma_queue_xfer()
379 *ring_data++ = xfer->cmd_desc[1]; in hci_dma_queue_xfer()
380 if (hci->cmd == &mipi_i3c_hci_cmd_v2) { in hci_dma_queue_xfer()
381 *ring_data++ = xfer->cmd_desc[2]; in hci_dma_queue_xfer()
382 *ring_data++ = xfer->cmd_desc[3]; in hci_dma_queue_xfer()
386 if (!xfer->data) in hci_dma_queue_xfer()
387 xfer->data_len = 0; in hci_dma_queue_xfer()
389 FIELD_PREP(DATA_BUF_BLOCK_SIZE, xfer->data_len) | in hci_dma_queue_xfer()
390 ((i == n - 1) ? DATA_BUF_IOC : 0); in hci_dma_queue_xfer()
393 if (xfer->data) { in hci_dma_queue_xfer()
394 xfer->data_dma = in hci_dma_queue_xfer()
395 dma_map_single(&hci->master.dev, in hci_dma_queue_xfer()
396 xfer->data, in hci_dma_queue_xfer()
397 xfer->data_len, in hci_dma_queue_xfer()
398 xfer->rnw ? in hci_dma_queue_xfer()
401 if (dma_mapping_error(&hci->master.dev, in hci_dma_queue_xfer()
402 xfer->data_dma)) { in hci_dma_queue_xfer()
404 return -ENOMEM; in hci_dma_queue_xfer()
406 *ring_data++ = lo32(xfer->data_dma); in hci_dma_queue_xfer()
407 *ring_data++ = hi32(xfer->data_dma); in hci_dma_queue_xfer()
414 rh->src_xfers[enqueue_ptr] = xfer; in hci_dma_queue_xfer()
416 xfer->ring_number = ring; in hci_dma_queue_xfer()
417 xfer->ring_entry = enqueue_ptr; in hci_dma_queue_xfer()
419 enqueue_ptr = (enqueue_ptr + 1) % rh->xfer_entries; in hci_dma_queue_xfer()
429 return -EBUSY; in hci_dma_queue_xfer()
434 spin_lock_irq(&rh->lock); in hci_dma_queue_xfer()
439 spin_unlock_irq(&rh->lock); in hci_dma_queue_xfer()
447 struct hci_rings_data *rings = hci->io_data; in hci_dma_dequeue_xfer()
448 struct hci_rh_data *rh = &rings->headers[xfer_list[0].ring_number]; in hci_dma_dequeue_xfer()
454 if (wait_for_completion_timeout(&rh->op_done, HZ) == 0) { in hci_dma_dequeue_xfer()
459 dev_crit(&hci->master.dev, "unable to abort the ring\n"); in hci_dma_dequeue_xfer()
465 int idx = xfer->ring_entry; in hci_dma_dequeue_xfer()
470 * descriptor entries with a no-op. in hci_dma_dequeue_xfer()
473 u32 *ring_data = rh->xfer + rh->xfer_struct_sz * idx; in hci_dma_dequeue_xfer()
475 /* store no-op cmd descriptor */ in hci_dma_dequeue_xfer()
478 if (hci->cmd == &mipi_i3c_hci_cmd_v2) { in hci_dma_dequeue_xfer()
484 rh->src_xfers[idx] = NULL; in hci_dma_dequeue_xfer()
502 unsigned int tid, done_ptr = rh->done_ptr; in hci_dma_xfer_done()
510 ring_resp = rh->resp + rh->resp_struct_sz * done_ptr; in hci_dma_xfer_done()
515 xfer = rh->src_xfers[done_ptr]; in hci_dma_xfer_done()
520 xfer->ring_entry = -1; in hci_dma_xfer_done()
521 xfer->response = resp; in hci_dma_xfer_done()
522 if (tid != xfer->cmd_tid) { in hci_dma_xfer_done()
523 dev_err(&hci->master.dev, in hci_dma_xfer_done()
525 tid, xfer->cmd_tid); in hci_dma_xfer_done()
528 if (xfer->completion) in hci_dma_xfer_done()
529 complete(xfer->completion); in hci_dma_xfer_done()
532 done_ptr = (done_ptr + 1) % rh->xfer_entries; in hci_dma_xfer_done()
533 rh->done_ptr = done_ptr; in hci_dma_xfer_done()
537 spin_lock(&rh->lock); in hci_dma_xfer_done()
542 spin_unlock(&rh->lock); in hci_dma_xfer_done()
554 return -ENOMEM; in hci_dma_request_ibi()
560 dev_ibi->pool = pool; in hci_dma_request_ibi()
561 dev_ibi->max_len = req->max_payload_len; in hci_dma_request_ibi()
562 dev_data->ibi_data = dev_ibi; in hci_dma_request_ibi()
569 struct hci_dma_dev_ibi_data *dev_ibi = dev_data->ibi_data; in hci_dma_free_ibi()
571 dev_data->ibi_data = NULL; in hci_dma_free_ibi()
572 i3c_generic_ibi_free_pool(dev_ibi->pool); in hci_dma_free_ibi()
581 struct hci_dma_dev_ibi_data *dev_ibi = dev_data->ibi_data; in hci_dma_recycle_ibi_slot()
583 i3c_generic_ibi_recycle_slot(dev_ibi->pool, slot); in hci_dma_recycle_ibi_slot()
606 ibi_addr = -1; in hci_dma_process_ibi()
609 last_ptr = -1; in hci_dma_process_ibi()
613 ptr = (ptr + 1) % rh->ibi_status_entries) { in hci_dma_process_ibi()
617 ring_ibi_status = rh->ibi_status + rh->ibi_status_sz * ptr; in hci_dma_process_ibi()
625 } else if (ibi_addr == -1) { in hci_dma_process_ibi()
635 ibi_size += chunks * rh->ibi_chunk_sz; in hci_dma_process_ibi()
645 if (last_ptr == -1) { in hci_dma_process_ibi()
651 deq_ptr %= rh->ibi_status_entries; in hci_dma_process_ibi()
654 dev_err(&hci->master.dev, "IBI error from %#x\n", ibi_addr); in hci_dma_process_ibi()
661 dev_err(&hci->master.dev, in hci_dma_process_ibi()
667 dev_ibi = dev_data->ibi_data; in hci_dma_process_ibi()
668 if (ibi_size > dev_ibi->max_len) { in hci_dma_process_ibi()
669 dev_err(&hci->master.dev, "IBI payload too big (%d > %d)\n", in hci_dma_process_ibi()
670 ibi_size, dev_ibi->max_len); in hci_dma_process_ibi()
675 * This ring model is not suitable for zero-copy processing of IBIs. in hci_dma_process_ibi()
676 * We have the data chunk ring wrap-around to deal with, meaning in hci_dma_process_ibi()
684 slot = i3c_generic_ibi_get_free_slot(dev_ibi->pool); in hci_dma_process_ibi()
686 dev_err(&hci->master.dev, "no free slot for IBI\n"); in hci_dma_process_ibi()
691 ibi_data_offset = rh->ibi_chunk_sz * rh->ibi_chunk_ptr; in hci_dma_process_ibi()
692 ring_ibi_data = rh->ibi_data + ibi_data_offset; in hci_dma_process_ibi()
693 ring_ibi_data_dma = rh->ibi_data_dma + ibi_data_offset; in hci_dma_process_ibi()
694 first_part = (rh->ibi_chunks_total - rh->ibi_chunk_ptr) in hci_dma_process_ibi()
695 * rh->ibi_chunk_sz; in hci_dma_process_ibi()
698 dma_sync_single_for_cpu(&hci->master.dev, ring_ibi_data_dma, in hci_dma_process_ibi()
700 memcpy(slot->data, ring_ibi_data, first_part); in hci_dma_process_ibi()
705 ring_ibi_data = rh->ibi_data; in hci_dma_process_ibi()
706 ring_ibi_data_dma = rh->ibi_data_dma; in hci_dma_process_ibi()
707 dma_sync_single_for_cpu(&hci->master.dev, ring_ibi_data_dma, in hci_dma_process_ibi()
708 ibi_size - first_part, DMA_FROM_DEVICE); in hci_dma_process_ibi()
709 memcpy(slot->data + first_part, ring_ibi_data, in hci_dma_process_ibi()
710 ibi_size - first_part); in hci_dma_process_ibi()
714 slot->dev = dev; in hci_dma_process_ibi()
715 slot->len = ibi_size; in hci_dma_process_ibi()
720 spin_lock(&rh->lock); in hci_dma_process_ibi()
725 spin_unlock(&rh->lock); in hci_dma_process_ibi()
728 rh->ibi_chunk_ptr += ibi_chunks; in hci_dma_process_ibi()
729 rh->ibi_chunk_ptr %= rh->ibi_chunks_total; in hci_dma_process_ibi()
737 struct hci_rings_data *rings = hci->io_data; in hci_dma_irq_handler()
741 for (i = 0; mask && i < rings->total; i++) { in hci_dma_irq_handler()
749 rh = &rings->headers[i]; in hci_dma_irq_handler()
761 complete(&rh->op_done); in hci_dma_irq_handler()
764 dev_notice_ratelimited(&hci->master.dev, in hci_dma_irq_handler()
767 dev_warn_ratelimited(&hci->master.dev, in hci_dma_irq_handler()
770 dev_err_ratelimited(&hci->master.dev, in hci_dma_irq_handler()