1 // SPDX-License-Identifier: BSD-3-Clause 2 /* 3 * Copyright (c) 2020, MIPI Alliance, Inc. 4 * 5 * Author: Nicolas Pitre <npitre@baylibre.com> 6 */ 7 8 #include <linux/bitfield.h> 9 #include <linux/device.h> 10 #include <linux/errno.h> 11 #include <linux/i3c/master.h> 12 #include <linux/io.h> 13 14 #include "hci.h" 15 #include "cmd.h" 16 #include "ibi.h" 17 18 19 /* 20 * PIO Access Area 21 */ 22 23 #define pio_reg_read(r) readl(hci->PIO_regs + (PIO_##r)) 24 #define pio_reg_write(r, v) writel(v, hci->PIO_regs + (PIO_##r)) 25 26 #define PIO_COMMAND_QUEUE_PORT 0x00 27 #define PIO_RESPONSE_QUEUE_PORT 0x04 28 #define PIO_XFER_DATA_PORT 0x08 29 #define PIO_IBI_PORT 0x0c 30 31 #define PIO_QUEUE_THLD_CTRL 0x10 32 #define QUEUE_IBI_STATUS_THLD GENMASK(31, 24) 33 #define QUEUE_IBI_DATA_THLD GENMASK(23, 16) 34 #define QUEUE_RESP_BUF_THLD GENMASK(15, 8) 35 #define QUEUE_CMD_EMPTY_BUF_THLD GENMASK(7, 0) 36 37 #define PIO_DATA_BUFFER_THLD_CTRL 0x14 38 #define DATA_RX_START_THLD GENMASK(26, 24) 39 #define DATA_TX_START_THLD GENMASK(18, 16) 40 #define DATA_RX_BUF_THLD GENMASK(10, 8) 41 #define DATA_TX_BUF_THLD GENMASK(2, 0) 42 43 #define PIO_QUEUE_SIZE 0x18 44 #define TX_DATA_BUFFER_SIZE GENMASK(31, 24) 45 #define RX_DATA_BUFFER_SIZE GENMASK(23, 16) 46 #define IBI_STATUS_SIZE GENMASK(15, 8) 47 #define CR_QUEUE_SIZE GENMASK(7, 0) 48 49 #define PIO_INTR_STATUS 0x20 50 #define PIO_INTR_STATUS_ENABLE 0x24 51 #define PIO_INTR_SIGNAL_ENABLE 0x28 52 #define PIO_INTR_FORCE 0x2c 53 #define STAT_TRANSFER_BLOCKED BIT(25) 54 #define STAT_PERR_RESP_UFLOW BIT(24) 55 #define STAT_PERR_CMD_OFLOW BIT(23) 56 #define STAT_PERR_IBI_UFLOW BIT(22) 57 #define STAT_PERR_RX_UFLOW BIT(21) 58 #define STAT_PERR_TX_OFLOW BIT(20) 59 #define STAT_ERR_RESP_QUEUE_FULL BIT(19) 60 #define STAT_WARN_RESP_QUEUE_FULL BIT(18) 61 #define STAT_ERR_IBI_QUEUE_FULL BIT(17) 62 #define STAT_WARN_IBI_QUEUE_FULL BIT(16) 63 #define STAT_ERR_RX_DATA_FULL BIT(15) 64 #define STAT_WARN_RX_DATA_FULL BIT(14) 65 #define STAT_ERR_TX_DATA_EMPTY BIT(13) 66 #define STAT_WARN_TX_DATA_EMPTY BIT(12) 67 #define STAT_TRANSFER_ERR BIT(9) 68 #define STAT_WARN_INS_STOP_MODE BIT(7) 69 #define STAT_TRANSFER_ABORT BIT(5) 70 #define STAT_RESP_READY BIT(4) 71 #define STAT_CMD_QUEUE_READY BIT(3) 72 #define STAT_IBI_STATUS_THLD BIT(2) 73 #define STAT_RX_THLD BIT(1) 74 #define STAT_TX_THLD BIT(0) 75 76 #define PIO_QUEUE_CUR_STATUS 0x38 77 #define CUR_IBI_Q_LEVEL GENMASK(28, 20) 78 #define CUR_RESP_Q_LEVEL GENMASK(18, 10) 79 #define CUR_CMD_Q_EMPTY_LEVEL GENMASK(8, 0) 80 81 #define PIO_DATA_BUFFER_CUR_STATUS 0x3c 82 #define CUR_RX_BUF_LVL GENMASK(26, 16) 83 #define CUR_TX_BUF_LVL GENMASK(10, 0) 84 85 /* 86 * Handy status bit combinations 87 */ 88 89 #define STAT_LATENCY_WARNINGS (STAT_WARN_RESP_QUEUE_FULL | \ 90 STAT_WARN_IBI_QUEUE_FULL | \ 91 STAT_WARN_RX_DATA_FULL | \ 92 STAT_WARN_TX_DATA_EMPTY | \ 93 STAT_WARN_INS_STOP_MODE) 94 95 #define STAT_LATENCY_ERRORS (STAT_ERR_RESP_QUEUE_FULL | \ 96 STAT_ERR_IBI_QUEUE_FULL | \ 97 STAT_ERR_RX_DATA_FULL | \ 98 STAT_ERR_TX_DATA_EMPTY) 99 100 #define STAT_PROG_ERRORS (STAT_TRANSFER_BLOCKED | \ 101 STAT_PERR_RESP_UFLOW | \ 102 STAT_PERR_CMD_OFLOW | \ 103 STAT_PERR_IBI_UFLOW | \ 104 STAT_PERR_RX_UFLOW | \ 105 STAT_PERR_TX_OFLOW) 106 107 #define STAT_ALL_ERRORS (STAT_TRANSFER_ABORT | \ 108 STAT_TRANSFER_ERR | \ 109 STAT_LATENCY_ERRORS | \ 110 STAT_PROG_ERRORS) 111 112 struct hci_pio_dev_ibi_data { 113 struct i3c_generic_ibi_pool *pool; 114 unsigned int max_len; 115 }; 116 117 struct hci_pio_ibi_data { 118 struct i3c_ibi_slot *slot; 119 void *data_ptr; 120 unsigned int addr; 121 unsigned int seg_len, seg_cnt; 122 unsigned int max_len; 123 bool last_seg; 124 }; 125 126 struct hci_pio_data { 127 spinlock_t lock; 128 struct hci_xfer *curr_xfer, *xfer_queue; 129 struct hci_xfer *curr_rx, *rx_queue; 130 struct hci_xfer *curr_tx, *tx_queue; 131 struct hci_xfer *curr_resp, *resp_queue; 132 struct hci_pio_ibi_data ibi; 133 unsigned int rx_thresh_size, tx_thresh_size; 134 unsigned int max_ibi_thresh; 135 u32 reg_queue_thresh; 136 u32 enabled_irqs; 137 }; 138 139 static int hci_pio_init(struct i3c_hci *hci) 140 { 141 struct hci_pio_data *pio; 142 u32 val, size_val, rx_thresh, tx_thresh, ibi_val; 143 144 pio = kzalloc(sizeof(*pio), GFP_KERNEL); 145 if (!pio) 146 return -ENOMEM; 147 148 hci->io_data = pio; 149 spin_lock_init(&pio->lock); 150 151 size_val = pio_reg_read(QUEUE_SIZE); 152 dev_info(&hci->master.dev, "CMD/RESP FIFO = %ld entries\n", 153 FIELD_GET(CR_QUEUE_SIZE, size_val)); 154 dev_info(&hci->master.dev, "IBI FIFO = %ld bytes\n", 155 4 * FIELD_GET(IBI_STATUS_SIZE, size_val)); 156 dev_info(&hci->master.dev, "RX data FIFO = %d bytes\n", 157 4 * (2 << FIELD_GET(RX_DATA_BUFFER_SIZE, size_val))); 158 dev_info(&hci->master.dev, "TX data FIFO = %d bytes\n", 159 4 * (2 << FIELD_GET(TX_DATA_BUFFER_SIZE, size_val))); 160 161 /* 162 * Let's initialize data thresholds to half of the actual FIFO size. 163 * The start thresholds aren't used (set to 0) as the FIFO is always 164 * serviced before the corresponding command is queued. 165 */ 166 rx_thresh = FIELD_GET(RX_DATA_BUFFER_SIZE, size_val); 167 tx_thresh = FIELD_GET(TX_DATA_BUFFER_SIZE, size_val); 168 if (hci->version_major == 1) { 169 /* those are expressed as 2^[n+1), so just sub 1 if not 0 */ 170 if (rx_thresh) 171 rx_thresh -= 1; 172 if (tx_thresh) 173 tx_thresh -= 1; 174 pio->rx_thresh_size = 2 << rx_thresh; 175 pio->tx_thresh_size = 2 << tx_thresh; 176 } else { 177 /* size is 2^(n+1) and threshold is 2^n i.e. already halved */ 178 pio->rx_thresh_size = 1 << rx_thresh; 179 pio->tx_thresh_size = 1 << tx_thresh; 180 } 181 val = FIELD_PREP(DATA_RX_BUF_THLD, rx_thresh) | 182 FIELD_PREP(DATA_TX_BUF_THLD, tx_thresh); 183 pio_reg_write(DATA_BUFFER_THLD_CTRL, val); 184 185 /* 186 * Let's raise an interrupt as soon as there is one free cmd slot 187 * or one available response or IBI. For IBI data let's use half the 188 * IBI queue size within allowed bounds. 189 */ 190 ibi_val = FIELD_GET(IBI_STATUS_SIZE, size_val); 191 pio->max_ibi_thresh = clamp_val(ibi_val/2, 1, 63); 192 val = FIELD_PREP(QUEUE_IBI_STATUS_THLD, 1) | 193 FIELD_PREP(QUEUE_IBI_DATA_THLD, pio->max_ibi_thresh) | 194 FIELD_PREP(QUEUE_RESP_BUF_THLD, 1) | 195 FIELD_PREP(QUEUE_CMD_EMPTY_BUF_THLD, 1); 196 pio_reg_write(QUEUE_THLD_CTRL, val); 197 pio->reg_queue_thresh = val; 198 199 /* Disable all IRQs but allow all status bits */ 200 pio_reg_write(INTR_SIGNAL_ENABLE, 0x0); 201 pio_reg_write(INTR_STATUS_ENABLE, 0xffffffff); 202 203 /* Always accept error interrupts (will be activated on first xfer) */ 204 pio->enabled_irqs = STAT_ALL_ERRORS; 205 206 return 0; 207 } 208 209 static void hci_pio_cleanup(struct i3c_hci *hci) 210 { 211 struct hci_pio_data *pio = hci->io_data; 212 213 pio_reg_write(INTR_SIGNAL_ENABLE, 0x0); 214 215 if (pio) { 216 DBG("status = %#x/%#x", 217 pio_reg_read(INTR_STATUS), pio_reg_read(INTR_SIGNAL_ENABLE)); 218 BUG_ON(pio->curr_xfer); 219 BUG_ON(pio->curr_rx); 220 BUG_ON(pio->curr_tx); 221 BUG_ON(pio->curr_resp); 222 kfree(pio); 223 hci->io_data = NULL; 224 } 225 } 226 227 static void hci_pio_write_cmd(struct i3c_hci *hci, struct hci_xfer *xfer) 228 { 229 DBG("cmd_desc[%d] = 0x%08x", 0, xfer->cmd_desc[0]); 230 DBG("cmd_desc[%d] = 0x%08x", 1, xfer->cmd_desc[1]); 231 pio_reg_write(COMMAND_QUEUE_PORT, xfer->cmd_desc[0]); 232 pio_reg_write(COMMAND_QUEUE_PORT, xfer->cmd_desc[1]); 233 if (hci->cmd == &mipi_i3c_hci_cmd_v2) { 234 DBG("cmd_desc[%d] = 0x%08x", 2, xfer->cmd_desc[2]); 235 DBG("cmd_desc[%d] = 0x%08x", 3, xfer->cmd_desc[3]); 236 pio_reg_write(COMMAND_QUEUE_PORT, xfer->cmd_desc[2]); 237 pio_reg_write(COMMAND_QUEUE_PORT, xfer->cmd_desc[3]); 238 } 239 } 240 241 static bool hci_pio_do_rx(struct i3c_hci *hci, struct hci_pio_data *pio) 242 { 243 struct hci_xfer *xfer = pio->curr_rx; 244 unsigned int nr_words; 245 u32 *p; 246 247 p = xfer->data; 248 p += (xfer->data_len - xfer->data_left) / 4; 249 250 while (xfer->data_left >= 4) { 251 /* bail out if FIFO hasn't reached the threshold value yet */ 252 if (!(pio_reg_read(INTR_STATUS) & STAT_RX_THLD)) 253 return false; 254 nr_words = min(xfer->data_left / 4, pio->rx_thresh_size); 255 /* extract data from FIFO */ 256 xfer->data_left -= nr_words * 4; 257 DBG("now %d left %d", nr_words * 4, xfer->data_left); 258 while (nr_words--) 259 *p++ = pio_reg_read(XFER_DATA_PORT); 260 } 261 262 /* trailing data is retrieved upon response reception */ 263 return !xfer->data_left; 264 } 265 266 static void hci_pio_do_trailing_rx(struct i3c_hci *hci, 267 struct hci_pio_data *pio, unsigned int count) 268 { 269 struct hci_xfer *xfer = pio->curr_rx; 270 u32 *p; 271 272 DBG("%d remaining", count); 273 274 p = xfer->data; 275 p += (xfer->data_len - xfer->data_left) / 4; 276 277 if (count >= 4) { 278 unsigned int nr_words = count / 4; 279 /* extract data from FIFO */ 280 xfer->data_left -= nr_words * 4; 281 DBG("now %d left %d", nr_words * 4, xfer->data_left); 282 while (nr_words--) 283 *p++ = pio_reg_read(XFER_DATA_PORT); 284 } 285 286 count &= 3; 287 if (count) { 288 /* 289 * There are trailing bytes in the last word. 290 * Fetch it and extract bytes in an endian independent way. 291 * Unlike the TX case, we must not write memory past the 292 * end of the destination buffer. 293 */ 294 u8 *p_byte = (u8 *)p; 295 u32 data = pio_reg_read(XFER_DATA_PORT); 296 297 xfer->data_word_before_partial = data; 298 xfer->data_left -= count; 299 data = (__force u32) cpu_to_le32(data); 300 while (count--) { 301 *p_byte++ = data; 302 data >>= 8; 303 } 304 } 305 } 306 307 static bool hci_pio_do_tx(struct i3c_hci *hci, struct hci_pio_data *pio) 308 { 309 struct hci_xfer *xfer = pio->curr_tx; 310 unsigned int nr_words; 311 u32 *p; 312 313 p = xfer->data; 314 p += (xfer->data_len - xfer->data_left) / 4; 315 316 while (xfer->data_left >= 4) { 317 /* bail out if FIFO free space is below set threshold */ 318 if (!(pio_reg_read(INTR_STATUS) & STAT_TX_THLD)) 319 return false; 320 /* we can fill up to that TX threshold */ 321 nr_words = min(xfer->data_left / 4, pio->tx_thresh_size); 322 /* push data into the FIFO */ 323 xfer->data_left -= nr_words * 4; 324 DBG("now %d left %d", nr_words * 4, xfer->data_left); 325 while (nr_words--) 326 pio_reg_write(XFER_DATA_PORT, *p++); 327 } 328 329 if (xfer->data_left) { 330 /* 331 * There are trailing bytes to send. We can simply load 332 * them from memory as a word which will keep those bytes 333 * in their proper place even on a BE system. This will 334 * also get some bytes past the actual buffer but no one 335 * should care as they won't be sent out. 336 */ 337 if (!(pio_reg_read(INTR_STATUS) & STAT_TX_THLD)) 338 return false; 339 DBG("trailing %d", xfer->data_left); 340 pio_reg_write(XFER_DATA_PORT, *p); 341 xfer->data_left = 0; 342 } 343 344 return true; 345 } 346 347 static bool hci_pio_process_rx(struct i3c_hci *hci, struct hci_pio_data *pio) 348 { 349 while (pio->curr_rx && hci_pio_do_rx(hci, pio)) 350 pio->curr_rx = pio->curr_rx->next_data; 351 return !pio->curr_rx; 352 } 353 354 static bool hci_pio_process_tx(struct i3c_hci *hci, struct hci_pio_data *pio) 355 { 356 while (pio->curr_tx && hci_pio_do_tx(hci, pio)) 357 pio->curr_tx = pio->curr_tx->next_data; 358 return !pio->curr_tx; 359 } 360 361 static void hci_pio_queue_data(struct i3c_hci *hci, struct hci_pio_data *pio) 362 { 363 struct hci_xfer *xfer = pio->curr_xfer; 364 struct hci_xfer *prev_queue_tail; 365 366 if (!xfer->data) { 367 xfer->data_len = xfer->data_left = 0; 368 return; 369 } 370 371 if (xfer->rnw) { 372 prev_queue_tail = pio->rx_queue; 373 pio->rx_queue = xfer; 374 if (pio->curr_rx) { 375 prev_queue_tail->next_data = xfer; 376 } else { 377 pio->curr_rx = xfer; 378 if (!hci_pio_process_rx(hci, pio)) 379 pio->enabled_irqs |= STAT_RX_THLD; 380 } 381 } else { 382 prev_queue_tail = pio->tx_queue; 383 pio->tx_queue = xfer; 384 if (pio->curr_tx) { 385 prev_queue_tail->next_data = xfer; 386 } else { 387 pio->curr_tx = xfer; 388 if (!hci_pio_process_tx(hci, pio)) 389 pio->enabled_irqs |= STAT_TX_THLD; 390 } 391 } 392 } 393 394 static void hci_pio_push_to_next_rx(struct i3c_hci *hci, struct hci_xfer *xfer, 395 unsigned int words_to_keep) 396 { 397 u32 *from = xfer->data; 398 u32 from_last; 399 unsigned int received, count; 400 401 received = (xfer->data_len - xfer->data_left) / 4; 402 if ((xfer->data_len - xfer->data_left) & 3) { 403 from_last = xfer->data_word_before_partial; 404 received += 1; 405 } else { 406 from_last = from[received]; 407 } 408 from += words_to_keep; 409 count = received - words_to_keep; 410 411 while (count) { 412 unsigned int room, left, chunk, bytes_to_move; 413 u32 last_word; 414 415 xfer = xfer->next_data; 416 if (!xfer) { 417 dev_err(&hci->master.dev, "pushing RX data to unexistent xfer\n"); 418 return; 419 } 420 421 room = DIV_ROUND_UP(xfer->data_len, 4); 422 left = DIV_ROUND_UP(xfer->data_left, 4); 423 chunk = min(count, room); 424 if (chunk > left) { 425 hci_pio_push_to_next_rx(hci, xfer, chunk - left); 426 left = chunk; 427 xfer->data_left = left * 4; 428 } 429 430 bytes_to_move = xfer->data_len - xfer->data_left; 431 if (bytes_to_move & 3) { 432 /* preserve word to become partial */ 433 u32 *p = xfer->data; 434 435 xfer->data_word_before_partial = p[bytes_to_move / 4]; 436 } 437 memmove(xfer->data + chunk, xfer->data, bytes_to_move); 438 439 /* treat last word specially because of partial word issues */ 440 chunk -= 1; 441 442 memcpy(xfer->data, from, chunk * 4); 443 xfer->data_left -= chunk * 4; 444 from += chunk; 445 count -= chunk; 446 447 last_word = (count == 1) ? from_last : *from++; 448 if (xfer->data_left < 4) { 449 /* 450 * Like in hci_pio_do_trailing_rx(), preserve original 451 * word to be stored partially then store bytes it 452 * in an endian independent way. 453 */ 454 u8 *p_byte = xfer->data; 455 456 p_byte += chunk * 4; 457 xfer->data_word_before_partial = last_word; 458 last_word = (__force u32) cpu_to_le32(last_word); 459 while (xfer->data_left--) { 460 *p_byte++ = last_word; 461 last_word >>= 8; 462 } 463 } else { 464 u32 *p = xfer->data; 465 466 p[chunk] = last_word; 467 xfer->data_left -= 4; 468 } 469 count--; 470 } 471 } 472 473 static void hci_pio_err(struct i3c_hci *hci, struct hci_pio_data *pio, 474 u32 status); 475 476 static bool hci_pio_process_resp(struct i3c_hci *hci, struct hci_pio_data *pio) 477 { 478 while (pio->curr_resp && 479 (pio_reg_read(INTR_STATUS) & STAT_RESP_READY)) { 480 struct hci_xfer *xfer = pio->curr_resp; 481 u32 resp = pio_reg_read(RESPONSE_QUEUE_PORT); 482 unsigned int tid = RESP_TID(resp); 483 484 DBG("resp = 0x%08x", resp); 485 if (tid != xfer->cmd_tid) { 486 dev_err(&hci->master.dev, 487 "response tid=%d when expecting %d\n", 488 tid, xfer->cmd_tid); 489 /* let's pretend it is a prog error... any of them */ 490 hci_pio_err(hci, pio, STAT_PROG_ERRORS); 491 return false; 492 } 493 xfer->response = resp; 494 495 if (pio->curr_rx == xfer) { 496 /* 497 * Response availability implies RX completion. 498 * Retrieve trailing RX data if any. 499 * Note that short reads are possible. 500 */ 501 unsigned int received, expected, to_keep; 502 503 received = xfer->data_len - xfer->data_left; 504 expected = RESP_DATA_LENGTH(xfer->response); 505 if (expected > received) { 506 hci_pio_do_trailing_rx(hci, pio, 507 expected - received); 508 } else if (received > expected) { 509 /* we consumed data meant for next xfer */ 510 to_keep = DIV_ROUND_UP(expected, 4); 511 hci_pio_push_to_next_rx(hci, xfer, to_keep); 512 } 513 514 /* then process the RX list pointer */ 515 if (hci_pio_process_rx(hci, pio)) 516 pio->enabled_irqs &= ~STAT_RX_THLD; 517 } 518 519 /* 520 * We're about to give back ownership of the xfer structure 521 * to the waiting instance. Make sure no reference to it 522 * still exists. 523 */ 524 if (pio->curr_rx == xfer) { 525 DBG("short RX ?"); 526 pio->curr_rx = pio->curr_rx->next_data; 527 } else if (pio->curr_tx == xfer) { 528 DBG("short TX ?"); 529 pio->curr_tx = pio->curr_tx->next_data; 530 } else if (xfer->data_left) { 531 DBG("PIO xfer count = %d after response", 532 xfer->data_left); 533 } 534 535 pio->curr_resp = xfer->next_resp; 536 if (xfer->completion) 537 complete(xfer->completion); 538 } 539 return !pio->curr_resp; 540 } 541 542 static void hci_pio_queue_resp(struct i3c_hci *hci, struct hci_pio_data *pio) 543 { 544 struct hci_xfer *xfer = pio->curr_xfer; 545 struct hci_xfer *prev_queue_tail; 546 547 if (!(xfer->cmd_desc[0] & CMD_0_ROC)) 548 return; 549 550 prev_queue_tail = pio->resp_queue; 551 pio->resp_queue = xfer; 552 if (pio->curr_resp) { 553 prev_queue_tail->next_resp = xfer; 554 } else { 555 pio->curr_resp = xfer; 556 if (!hci_pio_process_resp(hci, pio)) 557 pio->enabled_irqs |= STAT_RESP_READY; 558 } 559 } 560 561 static bool hci_pio_process_cmd(struct i3c_hci *hci, struct hci_pio_data *pio) 562 { 563 while (pio->curr_xfer && 564 (pio_reg_read(INTR_STATUS) & STAT_CMD_QUEUE_READY)) { 565 /* 566 * Always process the data FIFO before sending the command 567 * so needed TX data or RX space is available upfront. 568 */ 569 hci_pio_queue_data(hci, pio); 570 /* 571 * Then queue our response request. This will also process 572 * the response FIFO in case it got suddenly filled up 573 * with results from previous commands. 574 */ 575 hci_pio_queue_resp(hci, pio); 576 /* 577 * Finally send the command. 578 */ 579 hci_pio_write_cmd(hci, pio->curr_xfer); 580 /* 581 * And move on. 582 */ 583 pio->curr_xfer = pio->curr_xfer->next_xfer; 584 } 585 return !pio->curr_xfer; 586 } 587 588 static int hci_pio_queue_xfer(struct i3c_hci *hci, struct hci_xfer *xfer, int n) 589 { 590 struct hci_pio_data *pio = hci->io_data; 591 struct hci_xfer *prev_queue_tail; 592 int i; 593 594 DBG("n = %d", n); 595 596 /* link xfer instances together and initialize data count */ 597 for (i = 0; i < n; i++) { 598 xfer[i].next_xfer = (i + 1 < n) ? &xfer[i + 1] : NULL; 599 xfer[i].next_data = NULL; 600 xfer[i].next_resp = NULL; 601 xfer[i].data_left = xfer[i].data_len; 602 } 603 604 spin_lock_irq(&pio->lock); 605 prev_queue_tail = pio->xfer_queue; 606 pio->xfer_queue = &xfer[n - 1]; 607 if (pio->curr_xfer) { 608 prev_queue_tail->next_xfer = xfer; 609 } else { 610 pio->curr_xfer = xfer; 611 if (!hci_pio_process_cmd(hci, pio)) 612 pio->enabled_irqs |= STAT_CMD_QUEUE_READY; 613 pio_reg_write(INTR_SIGNAL_ENABLE, pio->enabled_irqs); 614 DBG("status = %#x/%#x", 615 pio_reg_read(INTR_STATUS), pio_reg_read(INTR_SIGNAL_ENABLE)); 616 } 617 spin_unlock_irq(&pio->lock); 618 return 0; 619 } 620 621 static bool hci_pio_dequeue_xfer_common(struct i3c_hci *hci, 622 struct hci_pio_data *pio, 623 struct hci_xfer *xfer, int n) 624 { 625 struct hci_xfer *p, **p_prev_next; 626 int i; 627 628 /* 629 * To safely dequeue a transfer request, it must be either entirely 630 * processed, or not yet processed at all. If our request tail is 631 * reachable from either the data or resp list that means the command 632 * was submitted and not yet completed. 633 */ 634 for (p = pio->curr_resp; p; p = p->next_resp) 635 for (i = 0; i < n; i++) 636 if (p == &xfer[i]) 637 goto pio_screwed; 638 for (p = pio->curr_rx; p; p = p->next_data) 639 for (i = 0; i < n; i++) 640 if (p == &xfer[i]) 641 goto pio_screwed; 642 for (p = pio->curr_tx; p; p = p->next_data) 643 for (i = 0; i < n; i++) 644 if (p == &xfer[i]) 645 goto pio_screwed; 646 647 /* 648 * The command was completed, or wasn't yet submitted. 649 * Unlink it from the que if the later. 650 */ 651 p_prev_next = &pio->curr_xfer; 652 for (p = pio->curr_xfer; p; p = p->next_xfer) { 653 if (p == &xfer[0]) { 654 *p_prev_next = xfer[n - 1].next_xfer; 655 break; 656 } 657 p_prev_next = &p->next_xfer; 658 } 659 660 /* return true if we actually unqueued something */ 661 return !!p; 662 663 pio_screwed: 664 /* 665 * Life is tough. We must invalidate the hardware state and 666 * discard everything that is still queued. 667 */ 668 for (p = pio->curr_resp; p; p = p->next_resp) { 669 p->response = FIELD_PREP(RESP_ERR_FIELD, RESP_ERR_HC_TERMINATED); 670 if (p->completion) 671 complete(p->completion); 672 } 673 for (p = pio->curr_xfer; p; p = p->next_xfer) { 674 p->response = FIELD_PREP(RESP_ERR_FIELD, RESP_ERR_HC_TERMINATED); 675 if (p->completion) 676 complete(p->completion); 677 } 678 pio->curr_xfer = pio->curr_rx = pio->curr_tx = pio->curr_resp = NULL; 679 680 return true; 681 } 682 683 static bool hci_pio_dequeue_xfer(struct i3c_hci *hci, struct hci_xfer *xfer, int n) 684 { 685 struct hci_pio_data *pio = hci->io_data; 686 int ret; 687 688 spin_lock_irq(&pio->lock); 689 DBG("n=%d status=%#x/%#x", n, 690 pio_reg_read(INTR_STATUS), pio_reg_read(INTR_SIGNAL_ENABLE)); 691 DBG("main_status = %#x/%#x", 692 readl(hci->base_regs + 0x20), readl(hci->base_regs + 0x28)); 693 694 ret = hci_pio_dequeue_xfer_common(hci, pio, xfer, n); 695 spin_unlock_irq(&pio->lock); 696 return ret; 697 } 698 699 static void hci_pio_err(struct i3c_hci *hci, struct hci_pio_data *pio, 700 u32 status) 701 { 702 /* TODO: this ought to be more sophisticated eventually */ 703 704 if (pio_reg_read(INTR_STATUS) & STAT_RESP_READY) { 705 /* this may happen when an error is signaled with ROC unset */ 706 u32 resp = pio_reg_read(RESPONSE_QUEUE_PORT); 707 708 dev_err(&hci->master.dev, 709 "orphan response (%#x) on error\n", resp); 710 } 711 712 /* dump states on programming errors */ 713 if (status & STAT_PROG_ERRORS) { 714 u32 queue = pio_reg_read(QUEUE_CUR_STATUS); 715 u32 data = pio_reg_read(DATA_BUFFER_CUR_STATUS); 716 717 dev_err(&hci->master.dev, 718 "prog error %#lx (C/R/I = %ld/%ld/%ld, TX/RX = %ld/%ld)\n", 719 status & STAT_PROG_ERRORS, 720 FIELD_GET(CUR_CMD_Q_EMPTY_LEVEL, queue), 721 FIELD_GET(CUR_RESP_Q_LEVEL, queue), 722 FIELD_GET(CUR_IBI_Q_LEVEL, queue), 723 FIELD_GET(CUR_TX_BUF_LVL, data), 724 FIELD_GET(CUR_RX_BUF_LVL, data)); 725 } 726 727 /* just bust out everything with pending responses for now */ 728 hci_pio_dequeue_xfer_common(hci, pio, pio->curr_resp, 1); 729 /* ... and half-way TX transfers if any */ 730 if (pio->curr_tx && pio->curr_tx->data_left != pio->curr_tx->data_len) 731 hci_pio_dequeue_xfer_common(hci, pio, pio->curr_tx, 1); 732 /* then reset the hardware */ 733 mipi_i3c_hci_pio_reset(hci); 734 mipi_i3c_hci_resume(hci); 735 736 DBG("status=%#x/%#x", 737 pio_reg_read(INTR_STATUS), pio_reg_read(INTR_SIGNAL_ENABLE)); 738 } 739 740 static void hci_pio_set_ibi_thresh(struct i3c_hci *hci, 741 struct hci_pio_data *pio, 742 unsigned int thresh_val) 743 { 744 u32 regval = pio->reg_queue_thresh; 745 746 regval &= ~QUEUE_IBI_STATUS_THLD; 747 regval |= FIELD_PREP(QUEUE_IBI_STATUS_THLD, thresh_val); 748 /* write the threshold reg only if it changes */ 749 if (regval != pio->reg_queue_thresh) { 750 pio_reg_write(QUEUE_THLD_CTRL, regval); 751 pio->reg_queue_thresh = regval; 752 DBG("%d", thresh_val); 753 } 754 } 755 756 static bool hci_pio_get_ibi_segment(struct i3c_hci *hci, 757 struct hci_pio_data *pio) 758 { 759 struct hci_pio_ibi_data *ibi = &pio->ibi; 760 unsigned int nr_words, thresh_val; 761 u32 *p; 762 763 p = ibi->data_ptr; 764 p += (ibi->seg_len - ibi->seg_cnt) / 4; 765 766 while ((nr_words = ibi->seg_cnt/4)) { 767 /* determine our IBI queue threshold value */ 768 thresh_val = min(nr_words, pio->max_ibi_thresh); 769 hci_pio_set_ibi_thresh(hci, pio, thresh_val); 770 /* bail out if we don't have that amount of data ready */ 771 if (!(pio_reg_read(INTR_STATUS) & STAT_IBI_STATUS_THLD)) 772 return false; 773 /* extract the data from the IBI port */ 774 nr_words = thresh_val; 775 ibi->seg_cnt -= nr_words * 4; 776 DBG("now %d left %d", nr_words * 4, ibi->seg_cnt); 777 while (nr_words--) 778 *p++ = pio_reg_read(IBI_PORT); 779 } 780 781 if (ibi->seg_cnt) { 782 /* 783 * There are trailing bytes in the last word. 784 * Fetch it and extract bytes in an endian independent way. 785 * Unlike the TX case, we must not write past the end of 786 * the destination buffer. 787 */ 788 u32 data; 789 u8 *p_byte = (u8 *)p; 790 791 hci_pio_set_ibi_thresh(hci, pio, 1); 792 if (!(pio_reg_read(INTR_STATUS) & STAT_IBI_STATUS_THLD)) 793 return false; 794 DBG("trailing %d", ibi->seg_cnt); 795 data = pio_reg_read(IBI_PORT); 796 data = (__force u32) cpu_to_le32(data); 797 while (ibi->seg_cnt--) { 798 *p_byte++ = data; 799 data >>= 8; 800 } 801 } 802 803 return true; 804 } 805 806 static bool hci_pio_prep_new_ibi(struct i3c_hci *hci, struct hci_pio_data *pio) 807 { 808 struct hci_pio_ibi_data *ibi = &pio->ibi; 809 struct i3c_dev_desc *dev; 810 struct i3c_hci_dev_data *dev_data; 811 struct hci_pio_dev_ibi_data *dev_ibi; 812 u32 ibi_status; 813 814 /* 815 * We have a new IBI. Try to set up its payload retrieval. 816 * When returning true, the IBI data has to be consumed whether 817 * or not we are set up to capture it. If we return true with 818 * ibi->slot == NULL that means the data payload has to be 819 * drained out of the IBI port and dropped. 820 */ 821 822 ibi_status = pio_reg_read(IBI_PORT); 823 DBG("status = %#x", ibi_status); 824 ibi->addr = FIELD_GET(IBI_TARGET_ADDR, ibi_status); 825 if (ibi_status & IBI_ERROR) { 826 dev_err(&hci->master.dev, "IBI error from %#x\n", ibi->addr); 827 return false; 828 } 829 830 ibi->last_seg = ibi_status & IBI_LAST_STATUS; 831 ibi->seg_len = FIELD_GET(IBI_DATA_LENGTH, ibi_status); 832 ibi->seg_cnt = ibi->seg_len; 833 834 dev = i3c_hci_addr_to_dev(hci, ibi->addr); 835 if (!dev) { 836 dev_err(&hci->master.dev, 837 "IBI for unknown device %#x\n", ibi->addr); 838 return true; 839 } 840 841 dev_data = i3c_dev_get_master_data(dev); 842 dev_ibi = dev_data->ibi_data; 843 ibi->max_len = dev_ibi->max_len; 844 845 if (ibi->seg_len > ibi->max_len) { 846 dev_err(&hci->master.dev, "IBI payload too big (%d > %d)\n", 847 ibi->seg_len, ibi->max_len); 848 return true; 849 } 850 851 ibi->slot = i3c_generic_ibi_get_free_slot(dev_ibi->pool); 852 if (!ibi->slot) { 853 dev_err(&hci->master.dev, "no free slot for IBI\n"); 854 } else { 855 ibi->slot->len = 0; 856 ibi->data_ptr = ibi->slot->data; 857 } 858 return true; 859 } 860 861 static void hci_pio_free_ibi_slot(struct i3c_hci *hci, struct hci_pio_data *pio) 862 { 863 struct hci_pio_ibi_data *ibi = &pio->ibi; 864 struct hci_pio_dev_ibi_data *dev_ibi; 865 866 if (ibi->slot) { 867 dev_ibi = ibi->slot->dev->common.master_priv; 868 i3c_generic_ibi_recycle_slot(dev_ibi->pool, ibi->slot); 869 ibi->slot = NULL; 870 } 871 } 872 873 static bool hci_pio_process_ibi(struct i3c_hci *hci, struct hci_pio_data *pio) 874 { 875 struct hci_pio_ibi_data *ibi = &pio->ibi; 876 877 if (!ibi->slot && !ibi->seg_cnt && ibi->last_seg) 878 if (!hci_pio_prep_new_ibi(hci, pio)) 879 return false; 880 881 for (;;) { 882 u32 ibi_status; 883 unsigned int ibi_addr; 884 885 if (ibi->slot) { 886 if (!hci_pio_get_ibi_segment(hci, pio)) 887 return false; 888 ibi->slot->len += ibi->seg_len; 889 ibi->data_ptr += ibi->seg_len; 890 if (ibi->last_seg) { 891 /* was the last segment: submit it and leave */ 892 i3c_master_queue_ibi(ibi->slot->dev, ibi->slot); 893 ibi->slot = NULL; 894 hci_pio_set_ibi_thresh(hci, pio, 1); 895 return true; 896 } 897 } else if (ibi->seg_cnt) { 898 /* 899 * No slot but a non-zero count. This is the result 900 * of some error and the payload must be drained. 901 * This normally does not happen therefore no need 902 * to be extra optimized here. 903 */ 904 hci_pio_set_ibi_thresh(hci, pio, 1); 905 do { 906 if (!(pio_reg_read(INTR_STATUS) & STAT_IBI_STATUS_THLD)) 907 return false; 908 pio_reg_read(IBI_PORT); 909 } while (--ibi->seg_cnt); 910 if (ibi->last_seg) 911 return true; 912 } 913 914 /* try to move to the next segment right away */ 915 hci_pio_set_ibi_thresh(hci, pio, 1); 916 if (!(pio_reg_read(INTR_STATUS) & STAT_IBI_STATUS_THLD)) 917 return false; 918 ibi_status = pio_reg_read(IBI_PORT); 919 ibi_addr = FIELD_GET(IBI_TARGET_ADDR, ibi_status); 920 if (ibi->addr != ibi_addr) { 921 /* target address changed before last segment */ 922 dev_err(&hci->master.dev, 923 "unexp IBI address changed from %d to %d\n", 924 ibi->addr, ibi_addr); 925 hci_pio_free_ibi_slot(hci, pio); 926 } 927 ibi->last_seg = ibi_status & IBI_LAST_STATUS; 928 ibi->seg_len = FIELD_GET(IBI_DATA_LENGTH, ibi_status); 929 ibi->seg_cnt = ibi->seg_len; 930 if (ibi->slot && ibi->slot->len + ibi->seg_len > ibi->max_len) { 931 dev_err(&hci->master.dev, 932 "IBI payload too big (%d > %d)\n", 933 ibi->slot->len + ibi->seg_len, ibi->max_len); 934 hci_pio_free_ibi_slot(hci, pio); 935 } 936 } 937 938 return false; 939 } 940 941 static int hci_pio_request_ibi(struct i3c_hci *hci, struct i3c_dev_desc *dev, 942 const struct i3c_ibi_setup *req) 943 { 944 struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev); 945 struct i3c_generic_ibi_pool *pool; 946 struct hci_pio_dev_ibi_data *dev_ibi; 947 948 dev_ibi = kmalloc(sizeof(*dev_ibi), GFP_KERNEL); 949 if (!dev_ibi) 950 return -ENOMEM; 951 pool = i3c_generic_ibi_alloc_pool(dev, req); 952 if (IS_ERR(pool)) { 953 kfree(dev_ibi); 954 return PTR_ERR(pool); 955 } 956 dev_ibi->pool = pool; 957 dev_ibi->max_len = req->max_payload_len; 958 dev_data->ibi_data = dev_ibi; 959 return 0; 960 } 961 962 static void hci_pio_free_ibi(struct i3c_hci *hci, struct i3c_dev_desc *dev) 963 { 964 struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev); 965 struct hci_pio_dev_ibi_data *dev_ibi = dev_data->ibi_data; 966 967 dev_data->ibi_data = NULL; 968 i3c_generic_ibi_free_pool(dev_ibi->pool); 969 kfree(dev_ibi); 970 } 971 972 static void hci_pio_recycle_ibi_slot(struct i3c_hci *hci, 973 struct i3c_dev_desc *dev, 974 struct i3c_ibi_slot *slot) 975 { 976 struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev); 977 struct hci_pio_dev_ibi_data *dev_ibi = dev_data->ibi_data; 978 979 i3c_generic_ibi_recycle_slot(dev_ibi->pool, slot); 980 } 981 982 static bool hci_pio_irq_handler(struct i3c_hci *hci, unsigned int unused) 983 { 984 struct hci_pio_data *pio = hci->io_data; 985 u32 status; 986 987 spin_lock(&pio->lock); 988 status = pio_reg_read(INTR_STATUS); 989 DBG("(in) status: %#x/%#x", status, pio->enabled_irqs); 990 status &= pio->enabled_irqs | STAT_LATENCY_WARNINGS; 991 if (!status) { 992 spin_unlock(&pio->lock); 993 return false; 994 } 995 996 if (status & STAT_IBI_STATUS_THLD) 997 hci_pio_process_ibi(hci, pio); 998 999 if (status & STAT_RX_THLD) 1000 if (hci_pio_process_rx(hci, pio)) 1001 pio->enabled_irqs &= ~STAT_RX_THLD; 1002 if (status & STAT_TX_THLD) 1003 if (hci_pio_process_tx(hci, pio)) 1004 pio->enabled_irqs &= ~STAT_TX_THLD; 1005 if (status & STAT_RESP_READY) 1006 if (hci_pio_process_resp(hci, pio)) 1007 pio->enabled_irqs &= ~STAT_RESP_READY; 1008 1009 if (unlikely(status & STAT_LATENCY_WARNINGS)) { 1010 pio_reg_write(INTR_STATUS, status & STAT_LATENCY_WARNINGS); 1011 dev_warn_ratelimited(&hci->master.dev, 1012 "encountered warning condition %#lx\n", 1013 status & STAT_LATENCY_WARNINGS); 1014 } 1015 1016 if (unlikely(status & STAT_ALL_ERRORS)) { 1017 pio_reg_write(INTR_STATUS, status & STAT_ALL_ERRORS); 1018 hci_pio_err(hci, pio, status & STAT_ALL_ERRORS); 1019 } 1020 1021 if (status & STAT_CMD_QUEUE_READY) 1022 if (hci_pio_process_cmd(hci, pio)) 1023 pio->enabled_irqs &= ~STAT_CMD_QUEUE_READY; 1024 1025 pio_reg_write(INTR_SIGNAL_ENABLE, pio->enabled_irqs); 1026 DBG("(out) status: %#x/%#x", 1027 pio_reg_read(INTR_STATUS), pio_reg_read(INTR_SIGNAL_ENABLE)); 1028 spin_unlock(&pio->lock); 1029 return true; 1030 } 1031 1032 const struct hci_io_ops mipi_i3c_hci_pio = { 1033 .init = hci_pio_init, 1034 .cleanup = hci_pio_cleanup, 1035 .queue_xfer = hci_pio_queue_xfer, 1036 .dequeue_xfer = hci_pio_dequeue_xfer, 1037 .irq_handler = hci_pio_irq_handler, 1038 .request_ibi = hci_pio_request_ibi, 1039 .free_ibi = hci_pio_free_ibi, 1040 .recycle_ibi_slot = hci_pio_recycle_ibi_slot, 1041 }; 1042