1 /* Altera TSE SGDMA and MSGDMA Linux driver 2 * Copyright (C) 2014 Altera Corporation. All rights reserved 3 * 4 * This program is free software; you can redistribute it and/or modify it 5 * under the terms and conditions of the GNU General Public License, 6 * version 2, as published by the Free Software Foundation. 7 * 8 * This program is distributed in the hope it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 11 * more details. 12 * 13 * You should have received a copy of the GNU General Public License along with 14 * this program. If not, see <http://www.gnu.org/licenses/>. 15 */ 16 17 #include <linux/list.h> 18 #include "altera_utils.h" 19 #include "altera_tse.h" 20 #include "altera_sgdmahw.h" 21 #include "altera_sgdma.h" 22 23 static void sgdma_setup_descrip(struct sgdma_descrip __iomem *desc, 24 struct sgdma_descrip __iomem *ndesc, 25 dma_addr_t ndesc_phys, 26 dma_addr_t raddr, 27 dma_addr_t waddr, 28 u16 length, 29 int generate_eop, 30 int rfixed, 31 int wfixed); 32 33 static int sgdma_async_write(struct altera_tse_private *priv, 34 struct sgdma_descrip __iomem *desc); 35 36 static int sgdma_async_read(struct altera_tse_private *priv); 37 38 static dma_addr_t 39 sgdma_txphysaddr(struct altera_tse_private *priv, 40 struct sgdma_descrip __iomem *desc); 41 42 static dma_addr_t 43 sgdma_rxphysaddr(struct altera_tse_private *priv, 44 struct sgdma_descrip __iomem *desc); 45 46 static int sgdma_txbusy(struct altera_tse_private *priv); 47 48 static int sgdma_rxbusy(struct altera_tse_private *priv); 49 50 static void 51 queue_tx(struct altera_tse_private *priv, struct tse_buffer *buffer); 52 53 static void 54 queue_rx(struct altera_tse_private *priv, struct tse_buffer *buffer); 55 56 static struct tse_buffer * 57 dequeue_tx(struct altera_tse_private *priv); 58 59 static struct tse_buffer * 60 dequeue_rx(struct altera_tse_private *priv); 61 62 static struct tse_buffer * 63 queue_rx_peekhead(struct altera_tse_private *priv); 64 65 int sgdma_initialize(struct altera_tse_private *priv) 66 { 67 priv->txctrlreg = SGDMA_CTRLREG_ILASTD | 68 SGDMA_CTRLREG_INTEN; 69 70 priv->rxctrlreg = SGDMA_CTRLREG_IDESCRIP | 71 SGDMA_CTRLREG_INTEN | 72 SGDMA_CTRLREG_ILASTD; 73 74 priv->sgdmadesclen = sizeof(struct sgdma_descrip); 75 76 INIT_LIST_HEAD(&priv->txlisthd); 77 INIT_LIST_HEAD(&priv->rxlisthd); 78 79 priv->rxdescphys = (dma_addr_t) 0; 80 priv->txdescphys = (dma_addr_t) 0; 81 82 priv->rxdescphys = dma_map_single(priv->device, 83 (void __force *)priv->rx_dma_desc, 84 priv->rxdescmem, DMA_BIDIRECTIONAL); 85 86 if (dma_mapping_error(priv->device, priv->rxdescphys)) { 87 sgdma_uninitialize(priv); 88 netdev_err(priv->dev, "error mapping rx descriptor memory\n"); 89 return -EINVAL; 90 } 91 92 priv->txdescphys = dma_map_single(priv->device, 93 (void __force *)priv->tx_dma_desc, 94 priv->txdescmem, DMA_TO_DEVICE); 95 96 if (dma_mapping_error(priv->device, priv->txdescphys)) { 97 sgdma_uninitialize(priv); 98 netdev_err(priv->dev, "error mapping tx descriptor memory\n"); 99 return -EINVAL; 100 } 101 102 /* Initialize descriptor memory to all 0's, sync memory to cache */ 103 memset_io(priv->tx_dma_desc, 0, priv->txdescmem); 104 memset_io(priv->rx_dma_desc, 0, priv->rxdescmem); 105 106 dma_sync_single_for_device(priv->device, priv->txdescphys, 107 priv->txdescmem, DMA_TO_DEVICE); 108 109 dma_sync_single_for_device(priv->device, priv->rxdescphys, 110 priv->rxdescmem, DMA_TO_DEVICE); 111 112 return 0; 113 } 114 115 void sgdma_uninitialize(struct altera_tse_private *priv) 116 { 117 if (priv->rxdescphys) 118 dma_unmap_single(priv->device, priv->rxdescphys, 119 priv->rxdescmem, DMA_BIDIRECTIONAL); 120 121 if (priv->txdescphys) 122 dma_unmap_single(priv->device, priv->txdescphys, 123 priv->txdescmem, DMA_TO_DEVICE); 124 } 125 126 /* This function resets the SGDMA controller and clears the 127 * descriptor memory used for transmits and receives. 128 */ 129 void sgdma_reset(struct altera_tse_private *priv) 130 { 131 /* Initialize descriptor memory to 0 */ 132 memset_io(priv->tx_dma_desc, 0, priv->txdescmem); 133 memset_io(priv->rx_dma_desc, 0, priv->rxdescmem); 134 135 csrwr32(SGDMA_CTRLREG_RESET, priv->tx_dma_csr, sgdma_csroffs(control)); 136 csrwr32(0, priv->tx_dma_csr, sgdma_csroffs(control)); 137 138 csrwr32(SGDMA_CTRLREG_RESET, priv->rx_dma_csr, sgdma_csroffs(control)); 139 csrwr32(0, priv->rx_dma_csr, sgdma_csroffs(control)); 140 } 141 142 /* For SGDMA, interrupts remain enabled after initially enabling, 143 * so no need to provide implementations for abstract enable 144 * and disable 145 */ 146 147 void sgdma_enable_rxirq(struct altera_tse_private *priv) 148 { 149 } 150 151 void sgdma_enable_txirq(struct altera_tse_private *priv) 152 { 153 } 154 155 void sgdma_disable_rxirq(struct altera_tse_private *priv) 156 { 157 } 158 159 void sgdma_disable_txirq(struct altera_tse_private *priv) 160 { 161 } 162 163 void sgdma_clear_rxirq(struct altera_tse_private *priv) 164 { 165 tse_set_bit(priv->rx_dma_csr, sgdma_csroffs(control), 166 SGDMA_CTRLREG_CLRINT); 167 } 168 169 void sgdma_clear_txirq(struct altera_tse_private *priv) 170 { 171 tse_set_bit(priv->tx_dma_csr, sgdma_csroffs(control), 172 SGDMA_CTRLREG_CLRINT); 173 } 174 175 /* transmits buffer through SGDMA. Returns number of buffers 176 * transmitted, 0 if not possible. 177 * 178 * tx_lock is held by the caller 179 */ 180 int sgdma_tx_buffer(struct altera_tse_private *priv, struct tse_buffer *buffer) 181 { 182 struct sgdma_descrip __iomem *descbase = 183 (struct sgdma_descrip __iomem *)priv->tx_dma_desc; 184 185 struct sgdma_descrip __iomem *cdesc = &descbase[0]; 186 struct sgdma_descrip __iomem *ndesc = &descbase[1]; 187 188 /* wait 'til the tx sgdma is ready for the next transmit request */ 189 if (sgdma_txbusy(priv)) 190 return 0; 191 192 sgdma_setup_descrip(cdesc, /* current descriptor */ 193 ndesc, /* next descriptor */ 194 sgdma_txphysaddr(priv, ndesc), 195 buffer->dma_addr, /* address of packet to xmit */ 196 0, /* write addr 0 for tx dma */ 197 buffer->len, /* length of packet */ 198 SGDMA_CONTROL_EOP, /* Generate EOP */ 199 0, /* read fixed */ 200 SGDMA_CONTROL_WR_FIXED); /* Generate SOP */ 201 202 sgdma_async_write(priv, cdesc); 203 204 /* enqueue the request to the pending transmit queue */ 205 queue_tx(priv, buffer); 206 207 return 1; 208 } 209 210 211 /* tx_lock held to protect access to queued tx list 212 */ 213 u32 sgdma_tx_completions(struct altera_tse_private *priv) 214 { 215 u32 ready = 0; 216 217 if (!sgdma_txbusy(priv) && 218 ((csrrd8(priv->tx_dma_desc, sgdma_descroffs(control)) 219 & SGDMA_CONTROL_HW_OWNED) == 0) && 220 (dequeue_tx(priv))) { 221 ready = 1; 222 } 223 224 return ready; 225 } 226 227 void sgdma_start_rxdma(struct altera_tse_private *priv) 228 { 229 sgdma_async_read(priv); 230 } 231 232 void sgdma_add_rx_desc(struct altera_tse_private *priv, 233 struct tse_buffer *rxbuffer) 234 { 235 queue_rx(priv, rxbuffer); 236 } 237 238 /* status is returned on upper 16 bits, 239 * length is returned in lower 16 bits 240 */ 241 u32 sgdma_rx_status(struct altera_tse_private *priv) 242 { 243 struct sgdma_descrip __iomem *base = 244 (struct sgdma_descrip __iomem *)priv->rx_dma_desc; 245 struct sgdma_descrip __iomem *desc = NULL; 246 struct tse_buffer *rxbuffer = NULL; 247 unsigned int rxstatus = 0; 248 249 u32 sts = csrrd32(priv->rx_dma_csr, sgdma_csroffs(status)); 250 251 desc = &base[0]; 252 if (sts & SGDMA_STSREG_EOP) { 253 unsigned int pktlength = 0; 254 unsigned int pktstatus = 0; 255 dma_sync_single_for_cpu(priv->device, 256 priv->rxdescphys, 257 priv->sgdmadesclen, 258 DMA_FROM_DEVICE); 259 260 pktlength = csrrd16(desc, sgdma_descroffs(bytes_xferred)); 261 pktstatus = csrrd8(desc, sgdma_descroffs(status)); 262 rxstatus = pktstatus & ~SGDMA_STATUS_EOP; 263 rxstatus = rxstatus << 16; 264 rxstatus |= (pktlength & 0xffff); 265 266 if (rxstatus) { 267 csrwr8(0, desc, sgdma_descroffs(status)); 268 269 rxbuffer = dequeue_rx(priv); 270 if (rxbuffer == NULL) 271 netdev_info(priv->dev, 272 "sgdma rx and rx queue empty!\n"); 273 274 /* Clear control */ 275 csrwr32(0, priv->rx_dma_csr, sgdma_csroffs(control)); 276 /* clear status */ 277 csrwr32(0xf, priv->rx_dma_csr, sgdma_csroffs(status)); 278 279 /* kick the rx sgdma after reaping this descriptor */ 280 sgdma_async_read(priv); 281 282 } else { 283 /* If the SGDMA indicated an end of packet on recv, 284 * then it's expected that the rxstatus from the 285 * descriptor is non-zero - meaning a valid packet 286 * with a nonzero length, or an error has been 287 * indicated. if not, then all we can do is signal 288 * an error and return no packet received. Most likely 289 * there is a system design error, or an error in the 290 * underlying kernel (cache or cache management problem) 291 */ 292 netdev_err(priv->dev, 293 "SGDMA RX Error Info: %x, %x, %x\n", 294 sts, csrrd8(desc, sgdma_descroffs(status)), 295 rxstatus); 296 } 297 } else if (sts == 0) { 298 sgdma_async_read(priv); 299 } 300 301 return rxstatus; 302 } 303 304 305 /* Private functions */ 306 static void sgdma_setup_descrip(struct sgdma_descrip __iomem *desc, 307 struct sgdma_descrip __iomem *ndesc, 308 dma_addr_t ndesc_phys, 309 dma_addr_t raddr, 310 dma_addr_t waddr, 311 u16 length, 312 int generate_eop, 313 int rfixed, 314 int wfixed) 315 { 316 /* Clear the next descriptor as not owned by hardware */ 317 318 u32 ctrl = csrrd8(ndesc, sgdma_descroffs(control)); 319 ctrl &= ~SGDMA_CONTROL_HW_OWNED; 320 csrwr8(ctrl, ndesc, sgdma_descroffs(control)); 321 322 ctrl = SGDMA_CONTROL_HW_OWNED; 323 ctrl |= generate_eop; 324 ctrl |= rfixed; 325 ctrl |= wfixed; 326 327 /* Channel is implicitly zero, initialized to 0 by default */ 328 csrwr32(lower_32_bits(raddr), desc, sgdma_descroffs(raddr)); 329 csrwr32(lower_32_bits(waddr), desc, sgdma_descroffs(waddr)); 330 331 csrwr32(0, desc, sgdma_descroffs(pad1)); 332 csrwr32(0, desc, sgdma_descroffs(pad2)); 333 csrwr32(lower_32_bits(ndesc_phys), desc, sgdma_descroffs(next)); 334 335 csrwr8(ctrl, desc, sgdma_descroffs(control)); 336 csrwr8(0, desc, sgdma_descroffs(status)); 337 csrwr8(0, desc, sgdma_descroffs(wburst)); 338 csrwr8(0, desc, sgdma_descroffs(rburst)); 339 csrwr16(length, desc, sgdma_descroffs(bytes)); 340 csrwr16(0, desc, sgdma_descroffs(bytes_xferred)); 341 } 342 343 /* If hardware is busy, don't restart async read. 344 * if status register is 0 - meaning initial state, restart async read, 345 * probably for the first time when populating a receive buffer. 346 * If read status indicate not busy and a status, restart the async 347 * DMA read. 348 */ 349 static int sgdma_async_read(struct altera_tse_private *priv) 350 { 351 struct sgdma_descrip __iomem *descbase = 352 (struct sgdma_descrip __iomem *)priv->rx_dma_desc; 353 354 struct sgdma_descrip __iomem *cdesc = &descbase[0]; 355 struct sgdma_descrip __iomem *ndesc = &descbase[1]; 356 357 struct tse_buffer *rxbuffer = NULL; 358 359 if (!sgdma_rxbusy(priv)) { 360 rxbuffer = queue_rx_peekhead(priv); 361 if (rxbuffer == NULL) { 362 netdev_err(priv->dev, "no rx buffers available\n"); 363 return 0; 364 } 365 366 sgdma_setup_descrip(cdesc, /* current descriptor */ 367 ndesc, /* next descriptor */ 368 sgdma_rxphysaddr(priv, ndesc), 369 0, /* read addr 0 for rx dma */ 370 rxbuffer->dma_addr, /* write addr for rx dma */ 371 0, /* read 'til EOP */ 372 0, /* EOP: NA for rx dma */ 373 0, /* read fixed: NA for rx dma */ 374 0); /* SOP: NA for rx DMA */ 375 376 dma_sync_single_for_device(priv->device, 377 priv->rxdescphys, 378 priv->sgdmadesclen, 379 DMA_TO_DEVICE); 380 381 csrwr32(lower_32_bits(sgdma_rxphysaddr(priv, cdesc)), 382 priv->rx_dma_csr, 383 sgdma_csroffs(next_descrip)); 384 385 csrwr32((priv->rxctrlreg | SGDMA_CTRLREG_START), 386 priv->rx_dma_csr, 387 sgdma_csroffs(control)); 388 389 return 1; 390 } 391 392 return 0; 393 } 394 395 static int sgdma_async_write(struct altera_tse_private *priv, 396 struct sgdma_descrip __iomem *desc) 397 { 398 if (sgdma_txbusy(priv)) 399 return 0; 400 401 /* clear control and status */ 402 csrwr32(0, priv->tx_dma_csr, sgdma_csroffs(control)); 403 csrwr32(0x1f, priv->tx_dma_csr, sgdma_csroffs(status)); 404 405 dma_sync_single_for_device(priv->device, priv->txdescphys, 406 priv->sgdmadesclen, DMA_TO_DEVICE); 407 408 csrwr32(lower_32_bits(sgdma_txphysaddr(priv, desc)), 409 priv->tx_dma_csr, 410 sgdma_csroffs(next_descrip)); 411 412 csrwr32((priv->txctrlreg | SGDMA_CTRLREG_START), 413 priv->tx_dma_csr, 414 sgdma_csroffs(control)); 415 416 return 1; 417 } 418 419 static dma_addr_t 420 sgdma_txphysaddr(struct altera_tse_private *priv, 421 struct sgdma_descrip __iomem *desc) 422 { 423 dma_addr_t paddr = priv->txdescmem_busaddr; 424 uintptr_t offs = (uintptr_t)desc - (uintptr_t)priv->tx_dma_desc; 425 return (dma_addr_t)((uintptr_t)paddr + offs); 426 } 427 428 static dma_addr_t 429 sgdma_rxphysaddr(struct altera_tse_private *priv, 430 struct sgdma_descrip __iomem *desc) 431 { 432 dma_addr_t paddr = priv->rxdescmem_busaddr; 433 uintptr_t offs = (uintptr_t)desc - (uintptr_t)priv->rx_dma_desc; 434 return (dma_addr_t)((uintptr_t)paddr + offs); 435 } 436 437 #define list_remove_head(list, entry, type, member) \ 438 do { \ 439 entry = NULL; \ 440 if (!list_empty(list)) { \ 441 entry = list_entry((list)->next, type, member); \ 442 list_del_init(&entry->member); \ 443 } \ 444 } while (0) 445 446 #define list_peek_head(list, entry, type, member) \ 447 do { \ 448 entry = NULL; \ 449 if (!list_empty(list)) { \ 450 entry = list_entry((list)->next, type, member); \ 451 } \ 452 } while (0) 453 454 /* adds a tse_buffer to the tail of a tx buffer list. 455 * assumes the caller is managing and holding a mutual exclusion 456 * primitive to avoid simultaneous pushes/pops to the list. 457 */ 458 static void 459 queue_tx(struct altera_tse_private *priv, struct tse_buffer *buffer) 460 { 461 list_add_tail(&buffer->lh, &priv->txlisthd); 462 } 463 464 465 /* adds a tse_buffer to the tail of a rx buffer list 466 * assumes the caller is managing and holding a mutual exclusion 467 * primitive to avoid simultaneous pushes/pops to the list. 468 */ 469 static void 470 queue_rx(struct altera_tse_private *priv, struct tse_buffer *buffer) 471 { 472 list_add_tail(&buffer->lh, &priv->rxlisthd); 473 } 474 475 /* dequeues a tse_buffer from the transmit buffer list, otherwise 476 * returns NULL if empty. 477 * assumes the caller is managing and holding a mutual exclusion 478 * primitive to avoid simultaneous pushes/pops to the list. 479 */ 480 static struct tse_buffer * 481 dequeue_tx(struct altera_tse_private *priv) 482 { 483 struct tse_buffer *buffer = NULL; 484 list_remove_head(&priv->txlisthd, buffer, struct tse_buffer, lh); 485 return buffer; 486 } 487 488 /* dequeues a tse_buffer from the receive buffer list, otherwise 489 * returns NULL if empty 490 * assumes the caller is managing and holding a mutual exclusion 491 * primitive to avoid simultaneous pushes/pops to the list. 492 */ 493 static struct tse_buffer * 494 dequeue_rx(struct altera_tse_private *priv) 495 { 496 struct tse_buffer *buffer = NULL; 497 list_remove_head(&priv->rxlisthd, buffer, struct tse_buffer, lh); 498 return buffer; 499 } 500 501 /* dequeues a tse_buffer from the receive buffer list, otherwise 502 * returns NULL if empty 503 * assumes the caller is managing and holding a mutual exclusion 504 * primitive to avoid simultaneous pushes/pops to the list while the 505 * head is being examined. 506 */ 507 static struct tse_buffer * 508 queue_rx_peekhead(struct altera_tse_private *priv) 509 { 510 struct tse_buffer *buffer = NULL; 511 list_peek_head(&priv->rxlisthd, buffer, struct tse_buffer, lh); 512 return buffer; 513 } 514 515 /* check and return rx sgdma status without polling 516 */ 517 static int sgdma_rxbusy(struct altera_tse_private *priv) 518 { 519 return csrrd32(priv->rx_dma_csr, sgdma_csroffs(status)) 520 & SGDMA_STSREG_BUSY; 521 } 522 523 /* waits for the tx sgdma to finish it's current operation, returns 0 524 * when it transitions to nonbusy, returns 1 if the operation times out 525 */ 526 static int sgdma_txbusy(struct altera_tse_private *priv) 527 { 528 int delay = 0; 529 530 /* if DMA is busy, wait for current transactino to finish */ 531 while ((csrrd32(priv->tx_dma_csr, sgdma_csroffs(status)) 532 & SGDMA_STSREG_BUSY) && (delay++ < 100)) 533 udelay(1); 534 535 if (csrrd32(priv->tx_dma_csr, sgdma_csroffs(status)) 536 & SGDMA_STSREG_BUSY) { 537 netdev_err(priv->dev, "timeout waiting for tx dma\n"); 538 return 1; 539 } 540 return 0; 541 } 542