1 /* 2 * Copyright (C) 2015-2016 Marvell International Ltd. 3 4 * This program is free software: you can redistribute it and/or 5 * modify it under the terms of the GNU General Public License as 6 * published by the Free Software Foundation, either version 2 of the 7 * License, or any later version. 8 * 9 * This program is distributed in the hope that it will be useful, but 10 * WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 * General Public License for more details. 13 */ 14 15 #include <linux/clk.h> 16 #include <linux/dma-mapping.h> 17 #include <linux/interrupt.h> 18 #include <linux/io.h> 19 #include <linux/module.h> 20 #include <linux/msi.h> 21 #include <linux/of.h> 22 #include <linux/of_irq.h> 23 #include <linux/platform_device.h> 24 #include <linux/spinlock.h> 25 26 #include "dmaengine.h" 27 28 /* DMA Engine Registers */ 29 #define MV_XOR_V2_DMA_DESQ_BALR_OFF 0x000 30 #define MV_XOR_V2_DMA_DESQ_BAHR_OFF 0x004 31 #define MV_XOR_V2_DMA_DESQ_SIZE_OFF 0x008 32 #define MV_XOR_V2_DMA_DESQ_DONE_OFF 0x00C 33 #define MV_XOR_V2_DMA_DESQ_DONE_PENDING_MASK 0x7FFF 34 #define MV_XOR_V2_DMA_DESQ_DONE_PENDING_SHIFT 0 35 #define MV_XOR_V2_DMA_DESQ_DONE_READ_PTR_MASK 0x1FFF 36 #define MV_XOR_V2_DMA_DESQ_DONE_READ_PTR_SHIFT 16 37 #define MV_XOR_V2_DMA_DESQ_ARATTR_OFF 0x010 38 #define MV_XOR_V2_DMA_DESQ_ATTR_CACHE_MASK 0x3F3F 39 #define MV_XOR_V2_DMA_DESQ_ATTR_OUTER_SHAREABLE 0x202 40 #define MV_XOR_V2_DMA_DESQ_ATTR_CACHEABLE 0x3C3C 41 #define MV_XOR_V2_DMA_IMSG_CDAT_OFF 0x014 42 #define MV_XOR_V2_DMA_IMSG_THRD_OFF 0x018 43 #define MV_XOR_V2_DMA_IMSG_THRD_MASK 0x7FFF 44 #define MV_XOR_V2_DMA_IMSG_THRD_SHIFT 0x0 45 #define MV_XOR_V2_DMA_IMSG_TIMER_EN BIT(18) 46 #define MV_XOR_V2_DMA_DESQ_AWATTR_OFF 0x01C 47 /* Same flags as MV_XOR_V2_DMA_DESQ_ARATTR_OFF */ 48 #define MV_XOR_V2_DMA_DESQ_ALLOC_OFF 0x04C 49 #define MV_XOR_V2_DMA_DESQ_ALLOC_WRPTR_MASK 0xFFFF 50 #define MV_XOR_V2_DMA_DESQ_ALLOC_WRPTR_SHIFT 16 51 #define MV_XOR_V2_DMA_IMSG_BALR_OFF 0x050 52 #define MV_XOR_V2_DMA_IMSG_BAHR_OFF 0x054 53 #define MV_XOR_V2_DMA_DESQ_CTRL_OFF 0x100 54 #define MV_XOR_V2_DMA_DESQ_CTRL_32B 1 55 #define MV_XOR_V2_DMA_DESQ_CTRL_128B 7 56 #define MV_XOR_V2_DMA_DESQ_STOP_OFF 0x800 57 #define MV_XOR_V2_DMA_DESQ_DEALLOC_OFF 0x804 58 #define MV_XOR_V2_DMA_DESQ_ADD_OFF 0x808 59 #define MV_XOR_V2_DMA_IMSG_TMOT 0x810 60 #define MV_XOR_V2_DMA_IMSG_TIMER_THRD_MASK 0x1FFF 61 #define MV_XOR_V2_DMA_IMSG_TIMER_THRD_SHIFT 0 62 63 /* XOR Global registers */ 64 #define MV_XOR_V2_GLOB_BW_CTRL 0x4 65 #define MV_XOR_V2_GLOB_BW_CTRL_NUM_OSTD_RD_SHIFT 0 66 #define MV_XOR_V2_GLOB_BW_CTRL_NUM_OSTD_RD_VAL 64 67 #define MV_XOR_V2_GLOB_BW_CTRL_NUM_OSTD_WR_SHIFT 8 68 #define MV_XOR_V2_GLOB_BW_CTRL_NUM_OSTD_WR_VAL 8 69 #define MV_XOR_V2_GLOB_BW_CTRL_RD_BURST_LEN_SHIFT 12 70 #define MV_XOR_V2_GLOB_BW_CTRL_RD_BURST_LEN_VAL 4 71 #define MV_XOR_V2_GLOB_BW_CTRL_WR_BURST_LEN_SHIFT 16 72 #define MV_XOR_V2_GLOB_BW_CTRL_WR_BURST_LEN_VAL 4 73 #define MV_XOR_V2_GLOB_PAUSE 0x014 74 #define MV_XOR_V2_GLOB_PAUSE_AXI_TIME_DIS_VAL 0x8 75 #define MV_XOR_V2_GLOB_SYS_INT_CAUSE 0x200 76 #define MV_XOR_V2_GLOB_SYS_INT_MASK 0x204 77 #define MV_XOR_V2_GLOB_MEM_INT_CAUSE 0x220 78 #define MV_XOR_V2_GLOB_MEM_INT_MASK 0x224 79 80 #define MV_XOR_V2_MIN_DESC_SIZE 32 81 #define MV_XOR_V2_EXT_DESC_SIZE 128 82 83 #define MV_XOR_V2_DESC_RESERVED_SIZE 12 84 #define MV_XOR_V2_DESC_BUFF_D_ADDR_SIZE 12 85 86 #define MV_XOR_V2_CMD_LINE_NUM_MAX_D_BUF 8 87 88 /* 89 * Descriptors queue size. With 32 bytes descriptors, up to 2^14 90 * descriptors are allowed, with 128 bytes descriptors, up to 2^12 91 * descriptors are allowed. This driver uses 128 bytes descriptors, 92 * but experimentation has shown that a set of 1024 descriptors is 93 * sufficient to reach a good level of performance. 94 */ 95 #define MV_XOR_V2_DESC_NUM 1024 96 97 /* 98 * Threshold values for descriptors and timeout, determined by 99 * experimentation as giving a good level of performance. 100 */ 101 #define MV_XOR_V2_DONE_IMSG_THRD 0x14 102 #define MV_XOR_V2_TIMER_THRD 0xB0 103 104 /** 105 * struct mv_xor_v2_descriptor - DMA HW descriptor 106 * @desc_id: used by S/W and is not affected by H/W. 107 * @flags: error and status flags 108 * @crc32_result: CRC32 calculation result 109 * @desc_ctrl: operation mode and control flags 110 * @buff_size: amount of bytes to be processed 111 * @fill_pattern_src_addr: Fill-Pattern or Source-Address and 112 * AW-Attributes 113 * @data_buff_addr: Source (and might be RAID6 destination) 114 * addresses of data buffers in RAID5 and RAID6 115 * @reserved: reserved 116 */ 117 struct mv_xor_v2_descriptor { 118 u16 desc_id; 119 u16 flags; 120 u32 crc32_result; 121 u32 desc_ctrl; 122 123 /* Definitions for desc_ctrl */ 124 #define DESC_NUM_ACTIVE_D_BUF_SHIFT 22 125 #define DESC_OP_MODE_SHIFT 28 126 #define DESC_OP_MODE_NOP 0 /* Idle operation */ 127 #define DESC_OP_MODE_MEMCPY 1 /* Pure-DMA operation */ 128 #define DESC_OP_MODE_MEMSET 2 /* Mem-Fill operation */ 129 #define DESC_OP_MODE_MEMINIT 3 /* Mem-Init operation */ 130 #define DESC_OP_MODE_MEM_COMPARE 4 /* Mem-Compare operation */ 131 #define DESC_OP_MODE_CRC32 5 /* CRC32 calculation */ 132 #define DESC_OP_MODE_XOR 6 /* RAID5 (XOR) operation */ 133 #define DESC_OP_MODE_RAID6 7 /* RAID6 P&Q-generation */ 134 #define DESC_OP_MODE_RAID6_REC 8 /* RAID6 Recovery */ 135 #define DESC_Q_BUFFER_ENABLE BIT(16) 136 #define DESC_P_BUFFER_ENABLE BIT(17) 137 #define DESC_IOD BIT(27) 138 139 u32 buff_size; 140 u32 fill_pattern_src_addr[4]; 141 u32 data_buff_addr[MV_XOR_V2_DESC_BUFF_D_ADDR_SIZE]; 142 u32 reserved[MV_XOR_V2_DESC_RESERVED_SIZE]; 143 }; 144 145 /** 146 * struct mv_xor_v2_device - implements a xor device 147 * @lock: lock for the engine 148 * @dma_base: memory mapped DMA register base 149 * @glob_base: memory mapped global register base 150 * @irq_tasklet: 151 * @free_sw_desc: linked list of free SW descriptors 152 * @dmadev: dma device 153 * @dmachan: dma channel 154 * @hw_desq: HW descriptors queue 155 * @hw_desq_virt: virtual address of DESCQ 156 * @sw_desq: SW descriptors queue 157 * @desc_size: HW descriptor size 158 * @npendings: number of pending descriptors (for which tx_submit has 159 * been called, but not yet issue_pending) 160 */ 161 struct mv_xor_v2_device { 162 spinlock_t lock; 163 void __iomem *dma_base; 164 void __iomem *glob_base; 165 struct clk *clk; 166 struct clk *reg_clk; 167 struct tasklet_struct irq_tasklet; 168 struct list_head free_sw_desc; 169 struct dma_device dmadev; 170 struct dma_chan dmachan; 171 dma_addr_t hw_desq; 172 struct mv_xor_v2_descriptor *hw_desq_virt; 173 struct mv_xor_v2_sw_desc *sw_desq; 174 int desc_size; 175 unsigned int npendings; 176 unsigned int hw_queue_idx; 177 struct msi_desc *msi_desc; 178 }; 179 180 /** 181 * struct mv_xor_v2_sw_desc - implements a xor SW descriptor 182 * @idx: descriptor index 183 * @async_tx: support for the async_tx api 184 * @hw_desc: assosiated HW descriptor 185 * @free_list: node of the free SW descriprots list 186 */ 187 struct mv_xor_v2_sw_desc { 188 int idx; 189 struct dma_async_tx_descriptor async_tx; 190 struct mv_xor_v2_descriptor hw_desc; 191 struct list_head free_list; 192 }; 193 194 /* 195 * Fill the data buffers to a HW descriptor 196 */ 197 static void mv_xor_v2_set_data_buffers(struct mv_xor_v2_device *xor_dev, 198 struct mv_xor_v2_descriptor *desc, 199 dma_addr_t src, int index) 200 { 201 int arr_index = ((index >> 1) * 3); 202 203 /* 204 * Fill the buffer's addresses to the descriptor. 205 * 206 * The format of the buffers address for 2 sequential buffers 207 * X and X + 1: 208 * 209 * First word: Buffer-DX-Address-Low[31:0] 210 * Second word: Buffer-DX+1-Address-Low[31:0] 211 * Third word: DX+1-Buffer-Address-High[47:32] [31:16] 212 * DX-Buffer-Address-High[47:32] [15:0] 213 */ 214 if ((index & 0x1) == 0) { 215 desc->data_buff_addr[arr_index] = lower_32_bits(src); 216 217 desc->data_buff_addr[arr_index + 2] &= ~0xFFFF; 218 desc->data_buff_addr[arr_index + 2] |= 219 upper_32_bits(src) & 0xFFFF; 220 } else { 221 desc->data_buff_addr[arr_index + 1] = 222 lower_32_bits(src); 223 224 desc->data_buff_addr[arr_index + 2] &= ~0xFFFF0000; 225 desc->data_buff_addr[arr_index + 2] |= 226 (upper_32_bits(src) & 0xFFFF) << 16; 227 } 228 } 229 230 /* 231 * notify the engine of new descriptors, and update the available index. 232 */ 233 static void mv_xor_v2_add_desc_to_desq(struct mv_xor_v2_device *xor_dev, 234 int num_of_desc) 235 { 236 /* write the number of new descriptors in the DESQ. */ 237 writel(num_of_desc, xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_ADD_OFF); 238 } 239 240 /* 241 * free HW descriptors 242 */ 243 static void mv_xor_v2_free_desc_from_desq(struct mv_xor_v2_device *xor_dev, 244 int num_of_desc) 245 { 246 /* write the number of new descriptors in the DESQ. */ 247 writel(num_of_desc, xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_DEALLOC_OFF); 248 } 249 250 /* 251 * Set descriptor size 252 * Return the HW descriptor size in bytes 253 */ 254 static int mv_xor_v2_set_desc_size(struct mv_xor_v2_device *xor_dev) 255 { 256 writel(MV_XOR_V2_DMA_DESQ_CTRL_128B, 257 xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_CTRL_OFF); 258 259 return MV_XOR_V2_EXT_DESC_SIZE; 260 } 261 262 /* 263 * Set the IMSG threshold 264 */ 265 static inline 266 void mv_xor_v2_enable_imsg_thrd(struct mv_xor_v2_device *xor_dev) 267 { 268 u32 reg; 269 270 /* Configure threshold of number of descriptors, and enable timer */ 271 reg = readl(xor_dev->dma_base + MV_XOR_V2_DMA_IMSG_THRD_OFF); 272 reg &= (~MV_XOR_V2_DMA_IMSG_THRD_MASK << MV_XOR_V2_DMA_IMSG_THRD_SHIFT); 273 reg |= (MV_XOR_V2_DONE_IMSG_THRD << MV_XOR_V2_DMA_IMSG_THRD_SHIFT); 274 reg |= MV_XOR_V2_DMA_IMSG_TIMER_EN; 275 writel(reg, xor_dev->dma_base + MV_XOR_V2_DMA_IMSG_THRD_OFF); 276 277 /* Configure Timer Threshold */ 278 reg = readl(xor_dev->dma_base + MV_XOR_V2_DMA_IMSG_TMOT); 279 reg &= (~MV_XOR_V2_DMA_IMSG_TIMER_THRD_MASK << 280 MV_XOR_V2_DMA_IMSG_TIMER_THRD_SHIFT); 281 reg |= (MV_XOR_V2_TIMER_THRD << MV_XOR_V2_DMA_IMSG_TIMER_THRD_SHIFT); 282 writel(reg, xor_dev->dma_base + MV_XOR_V2_DMA_IMSG_TMOT); 283 } 284 285 static irqreturn_t mv_xor_v2_interrupt_handler(int irq, void *data) 286 { 287 struct mv_xor_v2_device *xor_dev = data; 288 unsigned int ndescs; 289 u32 reg; 290 291 reg = readl(xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_DONE_OFF); 292 293 ndescs = ((reg >> MV_XOR_V2_DMA_DESQ_DONE_PENDING_SHIFT) & 294 MV_XOR_V2_DMA_DESQ_DONE_PENDING_MASK); 295 296 /* No descriptors to process */ 297 if (!ndescs) 298 return IRQ_NONE; 299 300 /* schedule a tasklet to handle descriptors callbacks */ 301 tasklet_schedule(&xor_dev->irq_tasklet); 302 303 return IRQ_HANDLED; 304 } 305 306 /* 307 * submit a descriptor to the DMA engine 308 */ 309 static dma_cookie_t 310 mv_xor_v2_tx_submit(struct dma_async_tx_descriptor *tx) 311 { 312 void *dest_hw_desc; 313 dma_cookie_t cookie; 314 struct mv_xor_v2_sw_desc *sw_desc = 315 container_of(tx, struct mv_xor_v2_sw_desc, async_tx); 316 struct mv_xor_v2_device *xor_dev = 317 container_of(tx->chan, struct mv_xor_v2_device, dmachan); 318 319 dev_dbg(xor_dev->dmadev.dev, 320 "%s sw_desc %p: async_tx %p\n", 321 __func__, sw_desc, &sw_desc->async_tx); 322 323 /* assign coookie */ 324 spin_lock_bh(&xor_dev->lock); 325 cookie = dma_cookie_assign(tx); 326 327 /* copy the HW descriptor from the SW descriptor to the DESQ */ 328 dest_hw_desc = xor_dev->hw_desq_virt + xor_dev->hw_queue_idx; 329 330 memcpy(dest_hw_desc, &sw_desc->hw_desc, xor_dev->desc_size); 331 332 xor_dev->npendings++; 333 xor_dev->hw_queue_idx++; 334 if (xor_dev->hw_queue_idx >= MV_XOR_V2_DESC_NUM) 335 xor_dev->hw_queue_idx = 0; 336 337 spin_unlock_bh(&xor_dev->lock); 338 339 return cookie; 340 } 341 342 /* 343 * Prepare a SW descriptor 344 */ 345 static struct mv_xor_v2_sw_desc * 346 mv_xor_v2_prep_sw_desc(struct mv_xor_v2_device *xor_dev) 347 { 348 struct mv_xor_v2_sw_desc *sw_desc; 349 bool found = false; 350 351 /* Lock the channel */ 352 spin_lock_bh(&xor_dev->lock); 353 354 if (list_empty(&xor_dev->free_sw_desc)) { 355 spin_unlock_bh(&xor_dev->lock); 356 /* schedule tasklet to free some descriptors */ 357 tasklet_schedule(&xor_dev->irq_tasklet); 358 return NULL; 359 } 360 361 list_for_each_entry(sw_desc, &xor_dev->free_sw_desc, free_list) { 362 if (async_tx_test_ack(&sw_desc->async_tx)) { 363 found = true; 364 break; 365 } 366 } 367 368 if (!found) { 369 spin_unlock_bh(&xor_dev->lock); 370 return NULL; 371 } 372 373 list_del(&sw_desc->free_list); 374 375 /* Release the channel */ 376 spin_unlock_bh(&xor_dev->lock); 377 378 return sw_desc; 379 } 380 381 /* 382 * Prepare a HW descriptor for a memcpy operation 383 */ 384 static struct dma_async_tx_descriptor * 385 mv_xor_v2_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, 386 dma_addr_t src, size_t len, unsigned long flags) 387 { 388 struct mv_xor_v2_sw_desc *sw_desc; 389 struct mv_xor_v2_descriptor *hw_descriptor; 390 struct mv_xor_v2_device *xor_dev; 391 392 xor_dev = container_of(chan, struct mv_xor_v2_device, dmachan); 393 394 dev_dbg(xor_dev->dmadev.dev, 395 "%s len: %zu src %pad dest %pad flags: %ld\n", 396 __func__, len, &src, &dest, flags); 397 398 sw_desc = mv_xor_v2_prep_sw_desc(xor_dev); 399 if (!sw_desc) 400 return NULL; 401 402 sw_desc->async_tx.flags = flags; 403 404 /* set the HW descriptor */ 405 hw_descriptor = &sw_desc->hw_desc; 406 407 /* save the SW descriptor ID to restore when operation is done */ 408 hw_descriptor->desc_id = sw_desc->idx; 409 410 /* Set the MEMCPY control word */ 411 hw_descriptor->desc_ctrl = 412 DESC_OP_MODE_MEMCPY << DESC_OP_MODE_SHIFT; 413 414 if (flags & DMA_PREP_INTERRUPT) 415 hw_descriptor->desc_ctrl |= DESC_IOD; 416 417 /* Set source address */ 418 hw_descriptor->fill_pattern_src_addr[0] = lower_32_bits(src); 419 hw_descriptor->fill_pattern_src_addr[1] = 420 upper_32_bits(src) & 0xFFFF; 421 422 /* Set Destination address */ 423 hw_descriptor->fill_pattern_src_addr[2] = lower_32_bits(dest); 424 hw_descriptor->fill_pattern_src_addr[3] = 425 upper_32_bits(dest) & 0xFFFF; 426 427 /* Set buffers size */ 428 hw_descriptor->buff_size = len; 429 430 /* return the async tx descriptor */ 431 return &sw_desc->async_tx; 432 } 433 434 /* 435 * Prepare a HW descriptor for a XOR operation 436 */ 437 static struct dma_async_tx_descriptor * 438 mv_xor_v2_prep_dma_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src, 439 unsigned int src_cnt, size_t len, unsigned long flags) 440 { 441 struct mv_xor_v2_sw_desc *sw_desc; 442 struct mv_xor_v2_descriptor *hw_descriptor; 443 struct mv_xor_v2_device *xor_dev = 444 container_of(chan, struct mv_xor_v2_device, dmachan); 445 int i; 446 447 if (src_cnt > MV_XOR_V2_CMD_LINE_NUM_MAX_D_BUF || src_cnt < 1) 448 return NULL; 449 450 dev_dbg(xor_dev->dmadev.dev, 451 "%s src_cnt: %d len: %zu dest %pad flags: %ld\n", 452 __func__, src_cnt, len, &dest, flags); 453 454 sw_desc = mv_xor_v2_prep_sw_desc(xor_dev); 455 if (!sw_desc) 456 return NULL; 457 458 sw_desc->async_tx.flags = flags; 459 460 /* set the HW descriptor */ 461 hw_descriptor = &sw_desc->hw_desc; 462 463 /* save the SW descriptor ID to restore when operation is done */ 464 hw_descriptor->desc_id = sw_desc->idx; 465 466 /* Set the XOR control word */ 467 hw_descriptor->desc_ctrl = 468 DESC_OP_MODE_XOR << DESC_OP_MODE_SHIFT; 469 hw_descriptor->desc_ctrl |= DESC_P_BUFFER_ENABLE; 470 471 if (flags & DMA_PREP_INTERRUPT) 472 hw_descriptor->desc_ctrl |= DESC_IOD; 473 474 /* Set the data buffers */ 475 for (i = 0; i < src_cnt; i++) 476 mv_xor_v2_set_data_buffers(xor_dev, hw_descriptor, src[i], i); 477 478 hw_descriptor->desc_ctrl |= 479 src_cnt << DESC_NUM_ACTIVE_D_BUF_SHIFT; 480 481 /* Set Destination address */ 482 hw_descriptor->fill_pattern_src_addr[2] = lower_32_bits(dest); 483 hw_descriptor->fill_pattern_src_addr[3] = 484 upper_32_bits(dest) & 0xFFFF; 485 486 /* Set buffers size */ 487 hw_descriptor->buff_size = len; 488 489 /* return the async tx descriptor */ 490 return &sw_desc->async_tx; 491 } 492 493 /* 494 * Prepare a HW descriptor for interrupt operation. 495 */ 496 static struct dma_async_tx_descriptor * 497 mv_xor_v2_prep_dma_interrupt(struct dma_chan *chan, unsigned long flags) 498 { 499 struct mv_xor_v2_sw_desc *sw_desc; 500 struct mv_xor_v2_descriptor *hw_descriptor; 501 struct mv_xor_v2_device *xor_dev = 502 container_of(chan, struct mv_xor_v2_device, dmachan); 503 504 sw_desc = mv_xor_v2_prep_sw_desc(xor_dev); 505 if (!sw_desc) 506 return NULL; 507 508 /* set the HW descriptor */ 509 hw_descriptor = &sw_desc->hw_desc; 510 511 /* save the SW descriptor ID to restore when operation is done */ 512 hw_descriptor->desc_id = sw_desc->idx; 513 514 /* Set the INTERRUPT control word */ 515 hw_descriptor->desc_ctrl = 516 DESC_OP_MODE_NOP << DESC_OP_MODE_SHIFT; 517 hw_descriptor->desc_ctrl |= DESC_IOD; 518 519 /* return the async tx descriptor */ 520 return &sw_desc->async_tx; 521 } 522 523 /* 524 * push pending transactions to hardware 525 */ 526 static void mv_xor_v2_issue_pending(struct dma_chan *chan) 527 { 528 struct mv_xor_v2_device *xor_dev = 529 container_of(chan, struct mv_xor_v2_device, dmachan); 530 531 spin_lock_bh(&xor_dev->lock); 532 533 /* 534 * update the engine with the number of descriptors to 535 * process 536 */ 537 mv_xor_v2_add_desc_to_desq(xor_dev, xor_dev->npendings); 538 xor_dev->npendings = 0; 539 540 spin_unlock_bh(&xor_dev->lock); 541 } 542 543 static inline 544 int mv_xor_v2_get_pending_params(struct mv_xor_v2_device *xor_dev, 545 int *pending_ptr) 546 { 547 u32 reg; 548 549 reg = readl(xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_DONE_OFF); 550 551 /* get the next pending descriptor index */ 552 *pending_ptr = ((reg >> MV_XOR_V2_DMA_DESQ_DONE_READ_PTR_SHIFT) & 553 MV_XOR_V2_DMA_DESQ_DONE_READ_PTR_MASK); 554 555 /* get the number of descriptors pending handle */ 556 return ((reg >> MV_XOR_V2_DMA_DESQ_DONE_PENDING_SHIFT) & 557 MV_XOR_V2_DMA_DESQ_DONE_PENDING_MASK); 558 } 559 560 /* 561 * handle the descriptors after HW process 562 */ 563 static void mv_xor_v2_tasklet(unsigned long data) 564 { 565 struct mv_xor_v2_device *xor_dev = (struct mv_xor_v2_device *) data; 566 int pending_ptr, num_of_pending, i; 567 struct mv_xor_v2_sw_desc *next_pending_sw_desc = NULL; 568 569 dev_dbg(xor_dev->dmadev.dev, "%s %d\n", __func__, __LINE__); 570 571 /* get the pending descriptors parameters */ 572 num_of_pending = mv_xor_v2_get_pending_params(xor_dev, &pending_ptr); 573 574 /* loop over free descriptors */ 575 for (i = 0; i < num_of_pending; i++) { 576 struct mv_xor_v2_descriptor *next_pending_hw_desc = 577 xor_dev->hw_desq_virt + pending_ptr; 578 579 /* get the SW descriptor related to the HW descriptor */ 580 next_pending_sw_desc = 581 &xor_dev->sw_desq[next_pending_hw_desc->desc_id]; 582 583 /* call the callback */ 584 if (next_pending_sw_desc->async_tx.cookie > 0) { 585 /* 586 * update the channel's completed cookie - no 587 * lock is required the IMSG threshold provide 588 * the locking 589 */ 590 dma_cookie_complete(&next_pending_sw_desc->async_tx); 591 592 dma_descriptor_unmap(&next_pending_sw_desc->async_tx); 593 dmaengine_desc_get_callback_invoke( 594 &next_pending_sw_desc->async_tx, NULL); 595 } 596 597 dma_run_dependencies(&next_pending_sw_desc->async_tx); 598 599 /* Lock the channel */ 600 spin_lock_bh(&xor_dev->lock); 601 602 /* add the SW descriptor to the free descriptors list */ 603 list_add(&next_pending_sw_desc->free_list, 604 &xor_dev->free_sw_desc); 605 606 /* Release the channel */ 607 spin_unlock_bh(&xor_dev->lock); 608 609 /* increment the next descriptor */ 610 pending_ptr++; 611 if (pending_ptr >= MV_XOR_V2_DESC_NUM) 612 pending_ptr = 0; 613 } 614 615 if (num_of_pending != 0) { 616 /* free the descriptores */ 617 mv_xor_v2_free_desc_from_desq(xor_dev, num_of_pending); 618 } 619 } 620 621 /* 622 * Set DMA Interrupt-message (IMSG) parameters 623 */ 624 static void mv_xor_v2_set_msi_msg(struct msi_desc *desc, struct msi_msg *msg) 625 { 626 struct mv_xor_v2_device *xor_dev = dev_get_drvdata(desc->dev); 627 628 writel(msg->address_lo, 629 xor_dev->dma_base + MV_XOR_V2_DMA_IMSG_BALR_OFF); 630 writel(msg->address_hi & 0xFFFF, 631 xor_dev->dma_base + MV_XOR_V2_DMA_IMSG_BAHR_OFF); 632 writel(msg->data, 633 xor_dev->dma_base + MV_XOR_V2_DMA_IMSG_CDAT_OFF); 634 } 635 636 static int mv_xor_v2_descq_init(struct mv_xor_v2_device *xor_dev) 637 { 638 u32 reg; 639 640 /* write the DESQ size to the DMA engine */ 641 writel(MV_XOR_V2_DESC_NUM, 642 xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_SIZE_OFF); 643 644 /* write the DESQ address to the DMA enngine*/ 645 writel(lower_32_bits(xor_dev->hw_desq), 646 xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_BALR_OFF); 647 writel(upper_32_bits(xor_dev->hw_desq), 648 xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_BAHR_OFF); 649 650 /* 651 * This is a temporary solution, until we activate the 652 * SMMU. Set the attributes for reading & writing data buffers 653 * & descriptors to: 654 * 655 * - OuterShareable - Snoops will be performed on CPU caches 656 * - Enable cacheable - Bufferable, Modifiable, Other Allocate 657 * and Allocate 658 */ 659 reg = readl(xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_ARATTR_OFF); 660 reg &= ~MV_XOR_V2_DMA_DESQ_ATTR_CACHE_MASK; 661 reg |= MV_XOR_V2_DMA_DESQ_ATTR_OUTER_SHAREABLE | 662 MV_XOR_V2_DMA_DESQ_ATTR_CACHEABLE; 663 writel(reg, xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_ARATTR_OFF); 664 665 reg = readl(xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_AWATTR_OFF); 666 reg &= ~MV_XOR_V2_DMA_DESQ_ATTR_CACHE_MASK; 667 reg |= MV_XOR_V2_DMA_DESQ_ATTR_OUTER_SHAREABLE | 668 MV_XOR_V2_DMA_DESQ_ATTR_CACHEABLE; 669 writel(reg, xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_AWATTR_OFF); 670 671 /* BW CTRL - set values to optimize the XOR performance: 672 * 673 * - Set WrBurstLen & RdBurstLen - the unit will issue 674 * maximum of 256B write/read transactions. 675 * - Limit the number of outstanding write & read data 676 * (OBB/IBB) requests to the maximal value. 677 */ 678 reg = ((MV_XOR_V2_GLOB_BW_CTRL_NUM_OSTD_RD_VAL << 679 MV_XOR_V2_GLOB_BW_CTRL_NUM_OSTD_RD_SHIFT) | 680 (MV_XOR_V2_GLOB_BW_CTRL_NUM_OSTD_WR_VAL << 681 MV_XOR_V2_GLOB_BW_CTRL_NUM_OSTD_WR_SHIFT) | 682 (MV_XOR_V2_GLOB_BW_CTRL_RD_BURST_LEN_VAL << 683 MV_XOR_V2_GLOB_BW_CTRL_RD_BURST_LEN_SHIFT) | 684 (MV_XOR_V2_GLOB_BW_CTRL_WR_BURST_LEN_VAL << 685 MV_XOR_V2_GLOB_BW_CTRL_WR_BURST_LEN_SHIFT)); 686 writel(reg, xor_dev->glob_base + MV_XOR_V2_GLOB_BW_CTRL); 687 688 /* Disable the AXI timer feature */ 689 reg = readl(xor_dev->glob_base + MV_XOR_V2_GLOB_PAUSE); 690 reg |= MV_XOR_V2_GLOB_PAUSE_AXI_TIME_DIS_VAL; 691 writel(reg, xor_dev->glob_base + MV_XOR_V2_GLOB_PAUSE); 692 693 /* enable the DMA engine */ 694 writel(0, xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_STOP_OFF); 695 696 return 0; 697 } 698 699 static int mv_xor_v2_suspend(struct platform_device *dev, pm_message_t state) 700 { 701 struct mv_xor_v2_device *xor_dev = platform_get_drvdata(dev); 702 703 /* Set this bit to disable to stop the XOR unit. */ 704 writel(0x1, xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_STOP_OFF); 705 706 return 0; 707 } 708 709 static int mv_xor_v2_resume(struct platform_device *dev) 710 { 711 struct mv_xor_v2_device *xor_dev = platform_get_drvdata(dev); 712 713 mv_xor_v2_set_desc_size(xor_dev); 714 mv_xor_v2_enable_imsg_thrd(xor_dev); 715 mv_xor_v2_descq_init(xor_dev); 716 717 return 0; 718 } 719 720 static int mv_xor_v2_probe(struct platform_device *pdev) 721 { 722 struct mv_xor_v2_device *xor_dev; 723 struct resource *res; 724 int i, ret = 0; 725 struct dma_device *dma_dev; 726 struct mv_xor_v2_sw_desc *sw_desc; 727 struct msi_desc *msi_desc; 728 729 BUILD_BUG_ON(sizeof(struct mv_xor_v2_descriptor) != 730 MV_XOR_V2_EXT_DESC_SIZE); 731 732 xor_dev = devm_kzalloc(&pdev->dev, sizeof(*xor_dev), GFP_KERNEL); 733 if (!xor_dev) 734 return -ENOMEM; 735 736 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 737 xor_dev->dma_base = devm_ioremap_resource(&pdev->dev, res); 738 if (IS_ERR(xor_dev->dma_base)) 739 return PTR_ERR(xor_dev->dma_base); 740 741 res = platform_get_resource(pdev, IORESOURCE_MEM, 1); 742 xor_dev->glob_base = devm_ioremap_resource(&pdev->dev, res); 743 if (IS_ERR(xor_dev->glob_base)) 744 return PTR_ERR(xor_dev->glob_base); 745 746 platform_set_drvdata(pdev, xor_dev); 747 748 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40)); 749 if (ret) 750 return ret; 751 752 xor_dev->reg_clk = devm_clk_get(&pdev->dev, "reg"); 753 if (PTR_ERR(xor_dev->reg_clk) != -ENOENT) { 754 if (!IS_ERR(xor_dev->reg_clk)) { 755 ret = clk_prepare_enable(xor_dev->reg_clk); 756 if (ret) 757 return ret; 758 } else { 759 return PTR_ERR(xor_dev->reg_clk); 760 } 761 } 762 763 xor_dev->clk = devm_clk_get(&pdev->dev, NULL); 764 if (IS_ERR(xor_dev->clk) && PTR_ERR(xor_dev->clk) == -EPROBE_DEFER) { 765 ret = EPROBE_DEFER; 766 goto disable_reg_clk; 767 } 768 if (!IS_ERR(xor_dev->clk)) { 769 ret = clk_prepare_enable(xor_dev->clk); 770 if (ret) 771 goto disable_reg_clk; 772 } 773 774 ret = platform_msi_domain_alloc_irqs(&pdev->dev, 1, 775 mv_xor_v2_set_msi_msg); 776 if (ret) 777 goto disable_clk; 778 779 msi_desc = first_msi_entry(&pdev->dev); 780 if (!msi_desc) 781 goto free_msi_irqs; 782 xor_dev->msi_desc = msi_desc; 783 784 ret = devm_request_irq(&pdev->dev, msi_desc->irq, 785 mv_xor_v2_interrupt_handler, 0, 786 dev_name(&pdev->dev), xor_dev); 787 if (ret) 788 goto free_msi_irqs; 789 790 tasklet_init(&xor_dev->irq_tasklet, mv_xor_v2_tasklet, 791 (unsigned long) xor_dev); 792 793 xor_dev->desc_size = mv_xor_v2_set_desc_size(xor_dev); 794 795 dma_cookie_init(&xor_dev->dmachan); 796 797 /* 798 * allocate coherent memory for hardware descriptors 799 * note: writecombine gives slightly better performance, but 800 * requires that we explicitly flush the writes 801 */ 802 xor_dev->hw_desq_virt = 803 dma_alloc_coherent(&pdev->dev, 804 xor_dev->desc_size * MV_XOR_V2_DESC_NUM, 805 &xor_dev->hw_desq, GFP_KERNEL); 806 if (!xor_dev->hw_desq_virt) { 807 ret = -ENOMEM; 808 goto free_msi_irqs; 809 } 810 811 /* alloc memory for the SW descriptors */ 812 xor_dev->sw_desq = devm_kcalloc(&pdev->dev, 813 MV_XOR_V2_DESC_NUM, sizeof(*sw_desc), 814 GFP_KERNEL); 815 if (!xor_dev->sw_desq) { 816 ret = -ENOMEM; 817 goto free_hw_desq; 818 } 819 820 spin_lock_init(&xor_dev->lock); 821 822 /* init the free SW descriptors list */ 823 INIT_LIST_HEAD(&xor_dev->free_sw_desc); 824 825 /* add all SW descriptors to the free list */ 826 for (i = 0; i < MV_XOR_V2_DESC_NUM; i++) { 827 struct mv_xor_v2_sw_desc *sw_desc = 828 xor_dev->sw_desq + i; 829 sw_desc->idx = i; 830 dma_async_tx_descriptor_init(&sw_desc->async_tx, 831 &xor_dev->dmachan); 832 sw_desc->async_tx.tx_submit = mv_xor_v2_tx_submit; 833 async_tx_ack(&sw_desc->async_tx); 834 835 list_add(&sw_desc->free_list, 836 &xor_dev->free_sw_desc); 837 } 838 839 dma_dev = &xor_dev->dmadev; 840 841 /* set DMA capabilities */ 842 dma_cap_zero(dma_dev->cap_mask); 843 dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); 844 dma_cap_set(DMA_XOR, dma_dev->cap_mask); 845 dma_cap_set(DMA_INTERRUPT, dma_dev->cap_mask); 846 847 /* init dma link list */ 848 INIT_LIST_HEAD(&dma_dev->channels); 849 850 /* set base routines */ 851 dma_dev->device_tx_status = dma_cookie_status; 852 dma_dev->device_issue_pending = mv_xor_v2_issue_pending; 853 dma_dev->dev = &pdev->dev; 854 855 dma_dev->device_prep_dma_memcpy = mv_xor_v2_prep_dma_memcpy; 856 dma_dev->device_prep_dma_interrupt = mv_xor_v2_prep_dma_interrupt; 857 dma_dev->max_xor = 8; 858 dma_dev->device_prep_dma_xor = mv_xor_v2_prep_dma_xor; 859 860 xor_dev->dmachan.device = dma_dev; 861 862 list_add_tail(&xor_dev->dmachan.device_node, 863 &dma_dev->channels); 864 865 mv_xor_v2_enable_imsg_thrd(xor_dev); 866 867 mv_xor_v2_descq_init(xor_dev); 868 869 ret = dma_async_device_register(dma_dev); 870 if (ret) 871 goto free_hw_desq; 872 873 dev_notice(&pdev->dev, "Marvell Version 2 XOR driver\n"); 874 875 return 0; 876 877 free_hw_desq: 878 dma_free_coherent(&pdev->dev, 879 xor_dev->desc_size * MV_XOR_V2_DESC_NUM, 880 xor_dev->hw_desq_virt, xor_dev->hw_desq); 881 free_msi_irqs: 882 platform_msi_domain_free_irqs(&pdev->dev); 883 disable_clk: 884 clk_disable_unprepare(xor_dev->clk); 885 disable_reg_clk: 886 clk_disable_unprepare(xor_dev->reg_clk); 887 return ret; 888 } 889 890 static int mv_xor_v2_remove(struct platform_device *pdev) 891 { 892 struct mv_xor_v2_device *xor_dev = platform_get_drvdata(pdev); 893 894 dma_async_device_unregister(&xor_dev->dmadev); 895 896 dma_free_coherent(&pdev->dev, 897 xor_dev->desc_size * MV_XOR_V2_DESC_NUM, 898 xor_dev->hw_desq_virt, xor_dev->hw_desq); 899 900 devm_free_irq(&pdev->dev, xor_dev->msi_desc->irq, xor_dev); 901 902 platform_msi_domain_free_irqs(&pdev->dev); 903 904 tasklet_kill(&xor_dev->irq_tasklet); 905 906 clk_disable_unprepare(xor_dev->clk); 907 908 return 0; 909 } 910 911 #ifdef CONFIG_OF 912 static const struct of_device_id mv_xor_v2_dt_ids[] = { 913 { .compatible = "marvell,xor-v2", }, 914 {}, 915 }; 916 MODULE_DEVICE_TABLE(of, mv_xor_v2_dt_ids); 917 #endif 918 919 static struct platform_driver mv_xor_v2_driver = { 920 .probe = mv_xor_v2_probe, 921 .suspend = mv_xor_v2_suspend, 922 .resume = mv_xor_v2_resume, 923 .remove = mv_xor_v2_remove, 924 .driver = { 925 .name = "mv_xor_v2", 926 .of_match_table = of_match_ptr(mv_xor_v2_dt_ids), 927 }, 928 }; 929 930 module_platform_driver(mv_xor_v2_driver); 931 932 MODULE_DESCRIPTION("DMA engine driver for Marvell's Version 2 of XOR engine"); 933 MODULE_LICENSE("GPL"); 934