1 /* 2 * Copyright (C) 2015-2016 Marvell International Ltd. 3 4 * This program is free software: you can redistribute it and/or 5 * modify it under the terms of the GNU General Public License as 6 * published by the Free Software Foundation, either version 2 of the 7 * License, or any later version. 8 * 9 * This program is distributed in the hope that it will be useful, but 10 * WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 * General Public License for more details. 13 */ 14 15 #include <linux/clk.h> 16 #include <linux/dma-mapping.h> 17 #include <linux/interrupt.h> 18 #include <linux/io.h> 19 #include <linux/module.h> 20 #include <linux/msi.h> 21 #include <linux/of.h> 22 #include <linux/of_irq.h> 23 #include <linux/platform_device.h> 24 #include <linux/spinlock.h> 25 26 #include "dmaengine.h" 27 28 /* DMA Engine Registers */ 29 #define MV_XOR_V2_DMA_DESQ_BALR_OFF 0x000 30 #define MV_XOR_V2_DMA_DESQ_BAHR_OFF 0x004 31 #define MV_XOR_V2_DMA_DESQ_SIZE_OFF 0x008 32 #define MV_XOR_V2_DMA_DESQ_DONE_OFF 0x00C 33 #define MV_XOR_V2_DMA_DESQ_DONE_PENDING_MASK 0x7FFF 34 #define MV_XOR_V2_DMA_DESQ_DONE_PENDING_SHIFT 0 35 #define MV_XOR_V2_DMA_DESQ_DONE_READ_PTR_MASK 0x1FFF 36 #define MV_XOR_V2_DMA_DESQ_DONE_READ_PTR_SHIFT 16 37 #define MV_XOR_V2_DMA_DESQ_ARATTR_OFF 0x010 38 #define MV_XOR_V2_DMA_DESQ_ATTR_CACHE_MASK 0x3F3F 39 #define MV_XOR_V2_DMA_DESQ_ATTR_OUTER_SHAREABLE 0x202 40 #define MV_XOR_V2_DMA_DESQ_ATTR_CACHEABLE 0x3C3C 41 #define MV_XOR_V2_DMA_IMSG_CDAT_OFF 0x014 42 #define MV_XOR_V2_DMA_IMSG_THRD_OFF 0x018 43 #define MV_XOR_V2_DMA_IMSG_THRD_MASK 0x7FFF 44 #define MV_XOR_V2_DMA_IMSG_THRD_SHIFT 0x0 45 #define MV_XOR_V2_DMA_DESQ_AWATTR_OFF 0x01C 46 /* Same flags as MV_XOR_V2_DMA_DESQ_ARATTR_OFF */ 47 #define MV_XOR_V2_DMA_DESQ_ALLOC_OFF 0x04C 48 #define MV_XOR_V2_DMA_DESQ_ALLOC_WRPTR_MASK 0xFFFF 49 #define MV_XOR_V2_DMA_DESQ_ALLOC_WRPTR_SHIFT 16 50 #define MV_XOR_V2_DMA_IMSG_BALR_OFF 0x050 51 #define MV_XOR_V2_DMA_IMSG_BAHR_OFF 0x054 52 #define MV_XOR_V2_DMA_DESQ_CTRL_OFF 0x100 53 #define MV_XOR_V2_DMA_DESQ_CTRL_32B 1 54 #define MV_XOR_V2_DMA_DESQ_CTRL_128B 7 55 #define MV_XOR_V2_DMA_DESQ_STOP_OFF 0x800 56 #define MV_XOR_V2_DMA_DESQ_DEALLOC_OFF 0x804 57 #define MV_XOR_V2_DMA_DESQ_ADD_OFF 0x808 58 59 /* XOR Global registers */ 60 #define MV_XOR_V2_GLOB_BW_CTRL 0x4 61 #define MV_XOR_V2_GLOB_BW_CTRL_NUM_OSTD_RD_SHIFT 0 62 #define MV_XOR_V2_GLOB_BW_CTRL_NUM_OSTD_RD_VAL 64 63 #define MV_XOR_V2_GLOB_BW_CTRL_NUM_OSTD_WR_SHIFT 8 64 #define MV_XOR_V2_GLOB_BW_CTRL_NUM_OSTD_WR_VAL 8 65 #define MV_XOR_V2_GLOB_BW_CTRL_RD_BURST_LEN_SHIFT 12 66 #define MV_XOR_V2_GLOB_BW_CTRL_RD_BURST_LEN_VAL 4 67 #define MV_XOR_V2_GLOB_BW_CTRL_WR_BURST_LEN_SHIFT 16 68 #define MV_XOR_V2_GLOB_BW_CTRL_WR_BURST_LEN_VAL 4 69 #define MV_XOR_V2_GLOB_PAUSE 0x014 70 #define MV_XOR_V2_GLOB_PAUSE_AXI_TIME_DIS_VAL 0x8 71 #define MV_XOR_V2_GLOB_SYS_INT_CAUSE 0x200 72 #define MV_XOR_V2_GLOB_SYS_INT_MASK 0x204 73 #define MV_XOR_V2_GLOB_MEM_INT_CAUSE 0x220 74 #define MV_XOR_V2_GLOB_MEM_INT_MASK 0x224 75 76 #define MV_XOR_V2_MIN_DESC_SIZE 32 77 #define MV_XOR_V2_EXT_DESC_SIZE 128 78 79 #define MV_XOR_V2_DESC_RESERVED_SIZE 12 80 #define MV_XOR_V2_DESC_BUFF_D_ADDR_SIZE 12 81 82 #define MV_XOR_V2_CMD_LINE_NUM_MAX_D_BUF 8 83 84 /* 85 * Descriptors queue size. With 32 bytes descriptors, up to 2^14 86 * descriptors are allowed, with 128 bytes descriptors, up to 2^12 87 * descriptors are allowed. This driver uses 128 bytes descriptors, 88 * but experimentation has shown that a set of 1024 descriptors is 89 * sufficient to reach a good level of performance. 90 */ 91 #define MV_XOR_V2_DESC_NUM 1024 92 93 /** 94 * struct mv_xor_v2_descriptor - DMA HW descriptor 95 * @desc_id: used by S/W and is not affected by H/W. 96 * @flags: error and status flags 97 * @crc32_result: CRC32 calculation result 98 * @desc_ctrl: operation mode and control flags 99 * @buff_size: amount of bytes to be processed 100 * @fill_pattern_src_addr: Fill-Pattern or Source-Address and 101 * AW-Attributes 102 * @data_buff_addr: Source (and might be RAID6 destination) 103 * addresses of data buffers in RAID5 and RAID6 104 * @reserved: reserved 105 */ 106 struct mv_xor_v2_descriptor { 107 u16 desc_id; 108 u16 flags; 109 u32 crc32_result; 110 u32 desc_ctrl; 111 112 /* Definitions for desc_ctrl */ 113 #define DESC_NUM_ACTIVE_D_BUF_SHIFT 22 114 #define DESC_OP_MODE_SHIFT 28 115 #define DESC_OP_MODE_NOP 0 /* Idle operation */ 116 #define DESC_OP_MODE_MEMCPY 1 /* Pure-DMA operation */ 117 #define DESC_OP_MODE_MEMSET 2 /* Mem-Fill operation */ 118 #define DESC_OP_MODE_MEMINIT 3 /* Mem-Init operation */ 119 #define DESC_OP_MODE_MEM_COMPARE 4 /* Mem-Compare operation */ 120 #define DESC_OP_MODE_CRC32 5 /* CRC32 calculation */ 121 #define DESC_OP_MODE_XOR 6 /* RAID5 (XOR) operation */ 122 #define DESC_OP_MODE_RAID6 7 /* RAID6 P&Q-generation */ 123 #define DESC_OP_MODE_RAID6_REC 8 /* RAID6 Recovery */ 124 #define DESC_Q_BUFFER_ENABLE BIT(16) 125 #define DESC_P_BUFFER_ENABLE BIT(17) 126 #define DESC_IOD BIT(27) 127 128 u32 buff_size; 129 u32 fill_pattern_src_addr[4]; 130 u32 data_buff_addr[MV_XOR_V2_DESC_BUFF_D_ADDR_SIZE]; 131 u32 reserved[MV_XOR_V2_DESC_RESERVED_SIZE]; 132 }; 133 134 /** 135 * struct mv_xor_v2_device - implements a xor device 136 * @lock: lock for the engine 137 * @dma_base: memory mapped DMA register base 138 * @glob_base: memory mapped global register base 139 * @irq_tasklet: 140 * @free_sw_desc: linked list of free SW descriptors 141 * @dmadev: dma device 142 * @dmachan: dma channel 143 * @hw_desq: HW descriptors queue 144 * @hw_desq_virt: virtual address of DESCQ 145 * @sw_desq: SW descriptors queue 146 * @desc_size: HW descriptor size 147 * @npendings: number of pending descriptors (for which tx_submit has 148 * been called, but not yet issue_pending) 149 */ 150 struct mv_xor_v2_device { 151 spinlock_t lock; 152 void __iomem *dma_base; 153 void __iomem *glob_base; 154 struct clk *clk; 155 struct tasklet_struct irq_tasklet; 156 struct list_head free_sw_desc; 157 struct dma_device dmadev; 158 struct dma_chan dmachan; 159 dma_addr_t hw_desq; 160 struct mv_xor_v2_descriptor *hw_desq_virt; 161 struct mv_xor_v2_sw_desc *sw_desq; 162 int desc_size; 163 unsigned int npendings; 164 }; 165 166 /** 167 * struct mv_xor_v2_sw_desc - implements a xor SW descriptor 168 * @idx: descriptor index 169 * @async_tx: support for the async_tx api 170 * @hw_desc: assosiated HW descriptor 171 * @free_list: node of the free SW descriprots list 172 */ 173 struct mv_xor_v2_sw_desc { 174 int idx; 175 struct dma_async_tx_descriptor async_tx; 176 struct mv_xor_v2_descriptor hw_desc; 177 struct list_head free_list; 178 }; 179 180 /* 181 * Fill the data buffers to a HW descriptor 182 */ 183 static void mv_xor_v2_set_data_buffers(struct mv_xor_v2_device *xor_dev, 184 struct mv_xor_v2_descriptor *desc, 185 dma_addr_t src, int index) 186 { 187 int arr_index = ((index >> 1) * 3); 188 189 /* 190 * Fill the buffer's addresses to the descriptor. 191 * 192 * The format of the buffers address for 2 sequential buffers 193 * X and X + 1: 194 * 195 * First word: Buffer-DX-Address-Low[31:0] 196 * Second word: Buffer-DX+1-Address-Low[31:0] 197 * Third word: DX+1-Buffer-Address-High[47:32] [31:16] 198 * DX-Buffer-Address-High[47:32] [15:0] 199 */ 200 if ((index & 0x1) == 0) { 201 desc->data_buff_addr[arr_index] = lower_32_bits(src); 202 203 desc->data_buff_addr[arr_index + 2] &= ~0xFFFF; 204 desc->data_buff_addr[arr_index + 2] |= 205 upper_32_bits(src) & 0xFFFF; 206 } else { 207 desc->data_buff_addr[arr_index + 1] = 208 lower_32_bits(src); 209 210 desc->data_buff_addr[arr_index + 2] &= ~0xFFFF0000; 211 desc->data_buff_addr[arr_index + 2] |= 212 (upper_32_bits(src) & 0xFFFF) << 16; 213 } 214 } 215 216 /* 217 * Return the next available index in the DESQ. 218 */ 219 static int mv_xor_v2_get_desq_write_ptr(struct mv_xor_v2_device *xor_dev) 220 { 221 /* read the index for the next available descriptor in the DESQ */ 222 u32 reg = readl(xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_ALLOC_OFF); 223 224 return ((reg >> MV_XOR_V2_DMA_DESQ_ALLOC_WRPTR_SHIFT) 225 & MV_XOR_V2_DMA_DESQ_ALLOC_WRPTR_MASK); 226 } 227 228 /* 229 * notify the engine of new descriptors, and update the available index. 230 */ 231 static void mv_xor_v2_add_desc_to_desq(struct mv_xor_v2_device *xor_dev, 232 int num_of_desc) 233 { 234 /* write the number of new descriptors in the DESQ. */ 235 writel(num_of_desc, xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_ADD_OFF); 236 } 237 238 /* 239 * free HW descriptors 240 */ 241 static void mv_xor_v2_free_desc_from_desq(struct mv_xor_v2_device *xor_dev, 242 int num_of_desc) 243 { 244 /* write the number of new descriptors in the DESQ. */ 245 writel(num_of_desc, xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_DEALLOC_OFF); 246 } 247 248 /* 249 * Set descriptor size 250 * Return the HW descriptor size in bytes 251 */ 252 static int mv_xor_v2_set_desc_size(struct mv_xor_v2_device *xor_dev) 253 { 254 writel(MV_XOR_V2_DMA_DESQ_CTRL_128B, 255 xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_CTRL_OFF); 256 257 return MV_XOR_V2_EXT_DESC_SIZE; 258 } 259 260 /* 261 * Set the IMSG threshold 262 */ 263 static inline 264 void mv_xor_v2_set_imsg_thrd(struct mv_xor_v2_device *xor_dev, int thrd_val) 265 { 266 u32 reg; 267 268 reg = readl(xor_dev->dma_base + MV_XOR_V2_DMA_IMSG_THRD_OFF); 269 270 reg &= (~MV_XOR_V2_DMA_IMSG_THRD_MASK << MV_XOR_V2_DMA_IMSG_THRD_SHIFT); 271 reg |= (thrd_val << MV_XOR_V2_DMA_IMSG_THRD_SHIFT); 272 273 writel(reg, xor_dev->dma_base + MV_XOR_V2_DMA_IMSG_THRD_OFF); 274 } 275 276 static irqreturn_t mv_xor_v2_interrupt_handler(int irq, void *data) 277 { 278 struct mv_xor_v2_device *xor_dev = data; 279 unsigned int ndescs; 280 u32 reg; 281 282 reg = readl(xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_DONE_OFF); 283 284 ndescs = ((reg >> MV_XOR_V2_DMA_DESQ_DONE_PENDING_SHIFT) & 285 MV_XOR_V2_DMA_DESQ_DONE_PENDING_MASK); 286 287 /* No descriptors to process */ 288 if (!ndescs) 289 return IRQ_NONE; 290 291 /* 292 * Update IMSG threshold, to disable new IMSG interrupts until 293 * end of the tasklet 294 */ 295 mv_xor_v2_set_imsg_thrd(xor_dev, MV_XOR_V2_DESC_NUM); 296 297 /* schedule a tasklet to handle descriptors callbacks */ 298 tasklet_schedule(&xor_dev->irq_tasklet); 299 300 return IRQ_HANDLED; 301 } 302 303 /* 304 * submit a descriptor to the DMA engine 305 */ 306 static dma_cookie_t 307 mv_xor_v2_tx_submit(struct dma_async_tx_descriptor *tx) 308 { 309 int desq_ptr; 310 void *dest_hw_desc; 311 dma_cookie_t cookie; 312 struct mv_xor_v2_sw_desc *sw_desc = 313 container_of(tx, struct mv_xor_v2_sw_desc, async_tx); 314 struct mv_xor_v2_device *xor_dev = 315 container_of(tx->chan, struct mv_xor_v2_device, dmachan); 316 317 dev_dbg(xor_dev->dmadev.dev, 318 "%s sw_desc %p: async_tx %p\n", 319 __func__, sw_desc, &sw_desc->async_tx); 320 321 /* assign coookie */ 322 spin_lock_bh(&xor_dev->lock); 323 cookie = dma_cookie_assign(tx); 324 325 /* get the next available slot in the DESQ */ 326 desq_ptr = mv_xor_v2_get_desq_write_ptr(xor_dev); 327 328 /* copy the HW descriptor from the SW descriptor to the DESQ */ 329 dest_hw_desc = xor_dev->hw_desq_virt + desq_ptr; 330 331 memcpy(dest_hw_desc, &sw_desc->hw_desc, xor_dev->desc_size); 332 333 xor_dev->npendings++; 334 335 spin_unlock_bh(&xor_dev->lock); 336 337 return cookie; 338 } 339 340 /* 341 * Prepare a SW descriptor 342 */ 343 static struct mv_xor_v2_sw_desc * 344 mv_xor_v2_prep_sw_desc(struct mv_xor_v2_device *xor_dev) 345 { 346 struct mv_xor_v2_sw_desc *sw_desc; 347 348 /* Lock the channel */ 349 spin_lock_bh(&xor_dev->lock); 350 351 if (list_empty(&xor_dev->free_sw_desc)) { 352 spin_unlock_bh(&xor_dev->lock); 353 /* schedule tasklet to free some descriptors */ 354 tasklet_schedule(&xor_dev->irq_tasklet); 355 return NULL; 356 } 357 358 /* get a free SW descriptor from the SW DESQ */ 359 sw_desc = list_first_entry(&xor_dev->free_sw_desc, 360 struct mv_xor_v2_sw_desc, free_list); 361 list_del(&sw_desc->free_list); 362 363 /* Release the channel */ 364 spin_unlock_bh(&xor_dev->lock); 365 366 /* set the async tx descriptor */ 367 dma_async_tx_descriptor_init(&sw_desc->async_tx, &xor_dev->dmachan); 368 sw_desc->async_tx.tx_submit = mv_xor_v2_tx_submit; 369 async_tx_ack(&sw_desc->async_tx); 370 371 return sw_desc; 372 } 373 374 /* 375 * Prepare a HW descriptor for a memcpy operation 376 */ 377 static struct dma_async_tx_descriptor * 378 mv_xor_v2_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, 379 dma_addr_t src, size_t len, unsigned long flags) 380 { 381 struct mv_xor_v2_sw_desc *sw_desc; 382 struct mv_xor_v2_descriptor *hw_descriptor; 383 struct mv_xor_v2_device *xor_dev; 384 385 xor_dev = container_of(chan, struct mv_xor_v2_device, dmachan); 386 387 dev_dbg(xor_dev->dmadev.dev, 388 "%s len: %zu src %pad dest %pad flags: %ld\n", 389 __func__, len, &src, &dest, flags); 390 391 sw_desc = mv_xor_v2_prep_sw_desc(xor_dev); 392 393 sw_desc->async_tx.flags = flags; 394 395 /* set the HW descriptor */ 396 hw_descriptor = &sw_desc->hw_desc; 397 398 /* save the SW descriptor ID to restore when operation is done */ 399 hw_descriptor->desc_id = sw_desc->idx; 400 401 /* Set the MEMCPY control word */ 402 hw_descriptor->desc_ctrl = 403 DESC_OP_MODE_MEMCPY << DESC_OP_MODE_SHIFT; 404 405 if (flags & DMA_PREP_INTERRUPT) 406 hw_descriptor->desc_ctrl |= DESC_IOD; 407 408 /* Set source address */ 409 hw_descriptor->fill_pattern_src_addr[0] = lower_32_bits(src); 410 hw_descriptor->fill_pattern_src_addr[1] = 411 upper_32_bits(src) & 0xFFFF; 412 413 /* Set Destination address */ 414 hw_descriptor->fill_pattern_src_addr[2] = lower_32_bits(dest); 415 hw_descriptor->fill_pattern_src_addr[3] = 416 upper_32_bits(dest) & 0xFFFF; 417 418 /* Set buffers size */ 419 hw_descriptor->buff_size = len; 420 421 /* return the async tx descriptor */ 422 return &sw_desc->async_tx; 423 } 424 425 /* 426 * Prepare a HW descriptor for a XOR operation 427 */ 428 static struct dma_async_tx_descriptor * 429 mv_xor_v2_prep_dma_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src, 430 unsigned int src_cnt, size_t len, unsigned long flags) 431 { 432 struct mv_xor_v2_sw_desc *sw_desc; 433 struct mv_xor_v2_descriptor *hw_descriptor; 434 struct mv_xor_v2_device *xor_dev = 435 container_of(chan, struct mv_xor_v2_device, dmachan); 436 int i; 437 438 if (src_cnt > MV_XOR_V2_CMD_LINE_NUM_MAX_D_BUF || src_cnt < 1) 439 return NULL; 440 441 dev_dbg(xor_dev->dmadev.dev, 442 "%s src_cnt: %d len: %zu dest %pad flags: %ld\n", 443 __func__, src_cnt, len, &dest, flags); 444 445 sw_desc = mv_xor_v2_prep_sw_desc(xor_dev); 446 447 sw_desc->async_tx.flags = flags; 448 449 /* set the HW descriptor */ 450 hw_descriptor = &sw_desc->hw_desc; 451 452 /* save the SW descriptor ID to restore when operation is done */ 453 hw_descriptor->desc_id = sw_desc->idx; 454 455 /* Set the XOR control word */ 456 hw_descriptor->desc_ctrl = 457 DESC_OP_MODE_XOR << DESC_OP_MODE_SHIFT; 458 hw_descriptor->desc_ctrl |= DESC_P_BUFFER_ENABLE; 459 460 if (flags & DMA_PREP_INTERRUPT) 461 hw_descriptor->desc_ctrl |= DESC_IOD; 462 463 /* Set the data buffers */ 464 for (i = 0; i < src_cnt; i++) 465 mv_xor_v2_set_data_buffers(xor_dev, hw_descriptor, src[i], i); 466 467 hw_descriptor->desc_ctrl |= 468 src_cnt << DESC_NUM_ACTIVE_D_BUF_SHIFT; 469 470 /* Set Destination address */ 471 hw_descriptor->fill_pattern_src_addr[2] = lower_32_bits(dest); 472 hw_descriptor->fill_pattern_src_addr[3] = 473 upper_32_bits(dest) & 0xFFFF; 474 475 /* Set buffers size */ 476 hw_descriptor->buff_size = len; 477 478 /* return the async tx descriptor */ 479 return &sw_desc->async_tx; 480 } 481 482 /* 483 * Prepare a HW descriptor for interrupt operation. 484 */ 485 static struct dma_async_tx_descriptor * 486 mv_xor_v2_prep_dma_interrupt(struct dma_chan *chan, unsigned long flags) 487 { 488 struct mv_xor_v2_sw_desc *sw_desc; 489 struct mv_xor_v2_descriptor *hw_descriptor; 490 struct mv_xor_v2_device *xor_dev = 491 container_of(chan, struct mv_xor_v2_device, dmachan); 492 493 sw_desc = mv_xor_v2_prep_sw_desc(xor_dev); 494 495 /* set the HW descriptor */ 496 hw_descriptor = &sw_desc->hw_desc; 497 498 /* save the SW descriptor ID to restore when operation is done */ 499 hw_descriptor->desc_id = sw_desc->idx; 500 501 /* Set the INTERRUPT control word */ 502 hw_descriptor->desc_ctrl = 503 DESC_OP_MODE_NOP << DESC_OP_MODE_SHIFT; 504 hw_descriptor->desc_ctrl |= DESC_IOD; 505 506 /* return the async tx descriptor */ 507 return &sw_desc->async_tx; 508 } 509 510 /* 511 * push pending transactions to hardware 512 */ 513 static void mv_xor_v2_issue_pending(struct dma_chan *chan) 514 { 515 struct mv_xor_v2_device *xor_dev = 516 container_of(chan, struct mv_xor_v2_device, dmachan); 517 518 spin_lock_bh(&xor_dev->lock); 519 520 /* 521 * update the engine with the number of descriptors to 522 * process 523 */ 524 mv_xor_v2_add_desc_to_desq(xor_dev, xor_dev->npendings); 525 xor_dev->npendings = 0; 526 527 /* Activate the channel */ 528 writel(0, xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_STOP_OFF); 529 530 spin_unlock_bh(&xor_dev->lock); 531 } 532 533 static inline 534 int mv_xor_v2_get_pending_params(struct mv_xor_v2_device *xor_dev, 535 int *pending_ptr) 536 { 537 u32 reg; 538 539 reg = readl(xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_DONE_OFF); 540 541 /* get the next pending descriptor index */ 542 *pending_ptr = ((reg >> MV_XOR_V2_DMA_DESQ_DONE_READ_PTR_SHIFT) & 543 MV_XOR_V2_DMA_DESQ_DONE_READ_PTR_MASK); 544 545 /* get the number of descriptors pending handle */ 546 return ((reg >> MV_XOR_V2_DMA_DESQ_DONE_PENDING_SHIFT) & 547 MV_XOR_V2_DMA_DESQ_DONE_PENDING_MASK); 548 } 549 550 /* 551 * handle the descriptors after HW process 552 */ 553 static void mv_xor_v2_tasklet(unsigned long data) 554 { 555 struct mv_xor_v2_device *xor_dev = (struct mv_xor_v2_device *) data; 556 int pending_ptr, num_of_pending, i; 557 struct mv_xor_v2_descriptor *next_pending_hw_desc = NULL; 558 struct mv_xor_v2_sw_desc *next_pending_sw_desc = NULL; 559 560 dev_dbg(xor_dev->dmadev.dev, "%s %d\n", __func__, __LINE__); 561 562 /* get the pending descriptors parameters */ 563 num_of_pending = mv_xor_v2_get_pending_params(xor_dev, &pending_ptr); 564 565 /* next HW descriptor */ 566 next_pending_hw_desc = xor_dev->hw_desq_virt + pending_ptr; 567 568 /* loop over free descriptors */ 569 for (i = 0; i < num_of_pending; i++) { 570 571 if (pending_ptr > MV_XOR_V2_DESC_NUM) 572 pending_ptr = 0; 573 574 if (next_pending_sw_desc != NULL) 575 next_pending_hw_desc++; 576 577 /* get the SW descriptor related to the HW descriptor */ 578 next_pending_sw_desc = 579 &xor_dev->sw_desq[next_pending_hw_desc->desc_id]; 580 581 /* call the callback */ 582 if (next_pending_sw_desc->async_tx.cookie > 0) { 583 /* 584 * update the channel's completed cookie - no 585 * lock is required the IMSG threshold provide 586 * the locking 587 */ 588 dma_cookie_complete(&next_pending_sw_desc->async_tx); 589 590 if (next_pending_sw_desc->async_tx.callback) 591 next_pending_sw_desc->async_tx.callback( 592 next_pending_sw_desc->async_tx.callback_param); 593 594 dma_descriptor_unmap(&next_pending_sw_desc->async_tx); 595 } 596 597 dma_run_dependencies(&next_pending_sw_desc->async_tx); 598 599 /* Lock the channel */ 600 spin_lock_bh(&xor_dev->lock); 601 602 /* add the SW descriptor to the free descriptors list */ 603 list_add(&next_pending_sw_desc->free_list, 604 &xor_dev->free_sw_desc); 605 606 /* Release the channel */ 607 spin_unlock_bh(&xor_dev->lock); 608 609 /* increment the next descriptor */ 610 pending_ptr++; 611 } 612 613 if (num_of_pending != 0) { 614 /* free the descriptores */ 615 mv_xor_v2_free_desc_from_desq(xor_dev, num_of_pending); 616 } 617 618 /* Update IMSG threshold, to enable new IMSG interrupts */ 619 mv_xor_v2_set_imsg_thrd(xor_dev, 0); 620 } 621 622 /* 623 * Set DMA Interrupt-message (IMSG) parameters 624 */ 625 static void mv_xor_v2_set_msi_msg(struct msi_desc *desc, struct msi_msg *msg) 626 { 627 struct mv_xor_v2_device *xor_dev = dev_get_drvdata(desc->dev); 628 629 writel(msg->address_lo, 630 xor_dev->dma_base + MV_XOR_V2_DMA_IMSG_BALR_OFF); 631 writel(msg->address_hi & 0xFFFF, 632 xor_dev->dma_base + MV_XOR_V2_DMA_IMSG_BAHR_OFF); 633 writel(msg->data, 634 xor_dev->dma_base + MV_XOR_V2_DMA_IMSG_CDAT_OFF); 635 } 636 637 static int mv_xor_v2_descq_init(struct mv_xor_v2_device *xor_dev) 638 { 639 u32 reg; 640 641 /* write the DESQ size to the DMA engine */ 642 writel(MV_XOR_V2_DESC_NUM, 643 xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_SIZE_OFF); 644 645 /* write the DESQ address to the DMA enngine*/ 646 writel(xor_dev->hw_desq & 0xFFFFFFFF, 647 xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_BALR_OFF); 648 writel((xor_dev->hw_desq & 0xFFFF00000000) >> 32, 649 xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_BAHR_OFF); 650 651 /* enable the DMA engine */ 652 writel(0, xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_STOP_OFF); 653 654 /* 655 * This is a temporary solution, until we activate the 656 * SMMU. Set the attributes for reading & writing data buffers 657 * & descriptors to: 658 * 659 * - OuterShareable - Snoops will be performed on CPU caches 660 * - Enable cacheable - Bufferable, Modifiable, Other Allocate 661 * and Allocate 662 */ 663 reg = readl(xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_ARATTR_OFF); 664 reg &= ~MV_XOR_V2_DMA_DESQ_ATTR_CACHE_MASK; 665 reg |= MV_XOR_V2_DMA_DESQ_ATTR_OUTER_SHAREABLE | 666 MV_XOR_V2_DMA_DESQ_ATTR_CACHEABLE; 667 writel(reg, xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_ARATTR_OFF); 668 669 reg = readl(xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_AWATTR_OFF); 670 reg &= ~MV_XOR_V2_DMA_DESQ_ATTR_CACHE_MASK; 671 reg |= MV_XOR_V2_DMA_DESQ_ATTR_OUTER_SHAREABLE | 672 MV_XOR_V2_DMA_DESQ_ATTR_CACHEABLE; 673 writel(reg, xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_AWATTR_OFF); 674 675 /* BW CTRL - set values to optimize the XOR performance: 676 * 677 * - Set WrBurstLen & RdBurstLen - the unit will issue 678 * maximum of 256B write/read transactions. 679 * - Limit the number of outstanding write & read data 680 * (OBB/IBB) requests to the maximal value. 681 */ 682 reg = ((MV_XOR_V2_GLOB_BW_CTRL_NUM_OSTD_RD_VAL << 683 MV_XOR_V2_GLOB_BW_CTRL_NUM_OSTD_RD_SHIFT) | 684 (MV_XOR_V2_GLOB_BW_CTRL_NUM_OSTD_WR_VAL << 685 MV_XOR_V2_GLOB_BW_CTRL_NUM_OSTD_WR_SHIFT) | 686 (MV_XOR_V2_GLOB_BW_CTRL_RD_BURST_LEN_VAL << 687 MV_XOR_V2_GLOB_BW_CTRL_RD_BURST_LEN_SHIFT) | 688 (MV_XOR_V2_GLOB_BW_CTRL_WR_BURST_LEN_VAL << 689 MV_XOR_V2_GLOB_BW_CTRL_WR_BURST_LEN_SHIFT)); 690 writel(reg, xor_dev->glob_base + MV_XOR_V2_GLOB_BW_CTRL); 691 692 /* Disable the AXI timer feature */ 693 reg = readl(xor_dev->glob_base + MV_XOR_V2_GLOB_PAUSE); 694 reg |= MV_XOR_V2_GLOB_PAUSE_AXI_TIME_DIS_VAL; 695 writel(reg, xor_dev->glob_base + MV_XOR_V2_GLOB_PAUSE); 696 697 return 0; 698 } 699 700 static int mv_xor_v2_probe(struct platform_device *pdev) 701 { 702 struct mv_xor_v2_device *xor_dev; 703 struct resource *res; 704 int i, ret = 0; 705 struct dma_device *dma_dev; 706 struct mv_xor_v2_sw_desc *sw_desc; 707 struct msi_desc *msi_desc; 708 709 BUILD_BUG_ON(sizeof(struct mv_xor_v2_descriptor) != 710 MV_XOR_V2_EXT_DESC_SIZE); 711 712 xor_dev = devm_kzalloc(&pdev->dev, sizeof(*xor_dev), GFP_KERNEL); 713 if (!xor_dev) 714 return -ENOMEM; 715 716 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 717 xor_dev->dma_base = devm_ioremap_resource(&pdev->dev, res); 718 if (IS_ERR(xor_dev->dma_base)) 719 return PTR_ERR(xor_dev->dma_base); 720 721 res = platform_get_resource(pdev, IORESOURCE_MEM, 1); 722 xor_dev->glob_base = devm_ioremap_resource(&pdev->dev, res); 723 if (IS_ERR(xor_dev->glob_base)) 724 return PTR_ERR(xor_dev->glob_base); 725 726 platform_set_drvdata(pdev, xor_dev); 727 728 xor_dev->clk = devm_clk_get(&pdev->dev, NULL); 729 if (IS_ERR(xor_dev->clk) && PTR_ERR(xor_dev->clk) == -EPROBE_DEFER) 730 return -EPROBE_DEFER; 731 if (!IS_ERR(xor_dev->clk)) { 732 ret = clk_prepare_enable(xor_dev->clk); 733 if (ret) 734 return ret; 735 } 736 737 ret = platform_msi_domain_alloc_irqs(&pdev->dev, 1, 738 mv_xor_v2_set_msi_msg); 739 if (ret) 740 goto disable_clk; 741 742 msi_desc = first_msi_entry(&pdev->dev); 743 if (!msi_desc) 744 goto free_msi_irqs; 745 746 ret = devm_request_irq(&pdev->dev, msi_desc->irq, 747 mv_xor_v2_interrupt_handler, 0, 748 dev_name(&pdev->dev), xor_dev); 749 if (ret) 750 goto free_msi_irqs; 751 752 tasklet_init(&xor_dev->irq_tasklet, mv_xor_v2_tasklet, 753 (unsigned long) xor_dev); 754 755 xor_dev->desc_size = mv_xor_v2_set_desc_size(xor_dev); 756 757 dma_cookie_init(&xor_dev->dmachan); 758 759 /* 760 * allocate coherent memory for hardware descriptors 761 * note: writecombine gives slightly better performance, but 762 * requires that we explicitly flush the writes 763 */ 764 xor_dev->hw_desq_virt = 765 dma_alloc_coherent(&pdev->dev, 766 xor_dev->desc_size * MV_XOR_V2_DESC_NUM, 767 &xor_dev->hw_desq, GFP_KERNEL); 768 if (!xor_dev->hw_desq_virt) { 769 ret = -ENOMEM; 770 goto free_msi_irqs; 771 } 772 773 /* alloc memory for the SW descriptors */ 774 xor_dev->sw_desq = devm_kzalloc(&pdev->dev, sizeof(*sw_desc) * 775 MV_XOR_V2_DESC_NUM, GFP_KERNEL); 776 if (!xor_dev->sw_desq) { 777 ret = -ENOMEM; 778 goto free_hw_desq; 779 } 780 781 spin_lock_init(&xor_dev->lock); 782 783 /* init the free SW descriptors list */ 784 INIT_LIST_HEAD(&xor_dev->free_sw_desc); 785 786 /* add all SW descriptors to the free list */ 787 for (i = 0; i < MV_XOR_V2_DESC_NUM; i++) { 788 xor_dev->sw_desq[i].idx = i; 789 list_add(&xor_dev->sw_desq[i].free_list, 790 &xor_dev->free_sw_desc); 791 } 792 793 dma_dev = &xor_dev->dmadev; 794 795 /* set DMA capabilities */ 796 dma_cap_zero(dma_dev->cap_mask); 797 dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); 798 dma_cap_set(DMA_XOR, dma_dev->cap_mask); 799 dma_cap_set(DMA_INTERRUPT, dma_dev->cap_mask); 800 801 /* init dma link list */ 802 INIT_LIST_HEAD(&dma_dev->channels); 803 804 /* set base routines */ 805 dma_dev->device_tx_status = dma_cookie_status; 806 dma_dev->device_issue_pending = mv_xor_v2_issue_pending; 807 dma_dev->dev = &pdev->dev; 808 809 dma_dev->device_prep_dma_memcpy = mv_xor_v2_prep_dma_memcpy; 810 dma_dev->device_prep_dma_interrupt = mv_xor_v2_prep_dma_interrupt; 811 dma_dev->max_xor = 8; 812 dma_dev->device_prep_dma_xor = mv_xor_v2_prep_dma_xor; 813 814 xor_dev->dmachan.device = dma_dev; 815 816 list_add_tail(&xor_dev->dmachan.device_node, 817 &dma_dev->channels); 818 819 mv_xor_v2_descq_init(xor_dev); 820 821 ret = dma_async_device_register(dma_dev); 822 if (ret) 823 goto free_hw_desq; 824 825 dev_notice(&pdev->dev, "Marvell Version 2 XOR driver\n"); 826 827 return 0; 828 829 free_hw_desq: 830 dma_free_coherent(&pdev->dev, 831 xor_dev->desc_size * MV_XOR_V2_DESC_NUM, 832 xor_dev->hw_desq_virt, xor_dev->hw_desq); 833 free_msi_irqs: 834 platform_msi_domain_free_irqs(&pdev->dev); 835 disable_clk: 836 if (!IS_ERR(xor_dev->clk)) 837 clk_disable_unprepare(xor_dev->clk); 838 return ret; 839 } 840 841 static int mv_xor_v2_remove(struct platform_device *pdev) 842 { 843 struct mv_xor_v2_device *xor_dev = platform_get_drvdata(pdev); 844 845 dma_async_device_unregister(&xor_dev->dmadev); 846 847 dma_free_coherent(&pdev->dev, 848 xor_dev->desc_size * MV_XOR_V2_DESC_NUM, 849 xor_dev->hw_desq_virt, xor_dev->hw_desq); 850 851 platform_msi_domain_free_irqs(&pdev->dev); 852 853 clk_disable_unprepare(xor_dev->clk); 854 855 return 0; 856 } 857 858 #ifdef CONFIG_OF 859 static const struct of_device_id mv_xor_v2_dt_ids[] = { 860 { .compatible = "marvell,xor-v2", }, 861 {}, 862 }; 863 MODULE_DEVICE_TABLE(of, mv_xor_v2_dt_ids); 864 #endif 865 866 static struct platform_driver mv_xor_v2_driver = { 867 .probe = mv_xor_v2_probe, 868 .remove = mv_xor_v2_remove, 869 .driver = { 870 .name = "mv_xor_v2", 871 .of_match_table = of_match_ptr(mv_xor_v2_dt_ids), 872 }, 873 }; 874 875 module_platform_driver(mv_xor_v2_driver); 876 877 MODULE_DESCRIPTION("DMA engine driver for Marvell's Version 2 of XOR engine"); 878 MODULE_LICENSE("GPL"); 879