1ff7b0479SSaeed Bishara /* 2ff7b0479SSaeed Bishara * offload engine driver for the Marvell XOR engine 3ff7b0479SSaeed Bishara * Copyright (C) 2007, 2008, Marvell International Ltd. 4ff7b0479SSaeed Bishara * 5ff7b0479SSaeed Bishara * This program is free software; you can redistribute it and/or modify it 6ff7b0479SSaeed Bishara * under the terms and conditions of the GNU General Public License, 7ff7b0479SSaeed Bishara * version 2, as published by the Free Software Foundation. 8ff7b0479SSaeed Bishara * 9ff7b0479SSaeed Bishara * This program is distributed in the hope it will be useful, but WITHOUT 10ff7b0479SSaeed Bishara * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11ff7b0479SSaeed Bishara * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12ff7b0479SSaeed Bishara * more details. 13ff7b0479SSaeed Bishara * 14ff7b0479SSaeed Bishara * You should have received a copy of the GNU General Public License along with 15ff7b0479SSaeed Bishara * this program; if not, write to the Free Software Foundation, Inc., 16ff7b0479SSaeed Bishara * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. 17ff7b0479SSaeed Bishara */ 18ff7b0479SSaeed Bishara 19ff7b0479SSaeed Bishara #include <linux/init.h> 20ff7b0479SSaeed Bishara #include <linux/module.h> 21ff7b0479SSaeed Bishara #include <linux/async_tx.h> 22ff7b0479SSaeed Bishara #include <linux/delay.h> 23ff7b0479SSaeed Bishara #include <linux/dma-mapping.h> 24ff7b0479SSaeed Bishara #include <linux/spinlock.h> 25ff7b0479SSaeed Bishara #include <linux/interrupt.h> 26ff7b0479SSaeed Bishara #include <linux/platform_device.h> 27ff7b0479SSaeed Bishara #include <linux/memory.h> 28ff7b0479SSaeed Bishara #include <asm/plat-orion/mv_xor.h> 29ff7b0479SSaeed Bishara #include "mv_xor.h" 30ff7b0479SSaeed Bishara 31ff7b0479SSaeed Bishara static void mv_xor_issue_pending(struct dma_chan *chan); 32ff7b0479SSaeed Bishara 33ff7b0479SSaeed Bishara #define to_mv_xor_chan(chan) \ 34ff7b0479SSaeed Bishara container_of(chan, struct mv_xor_chan, common) 35ff7b0479SSaeed Bishara 36ff7b0479SSaeed Bishara #define to_mv_xor_device(dev) \ 37ff7b0479SSaeed Bishara container_of(dev, struct mv_xor_device, common) 38ff7b0479SSaeed Bishara 39ff7b0479SSaeed Bishara #define to_mv_xor_slot(tx) \ 40ff7b0479SSaeed Bishara container_of(tx, struct mv_xor_desc_slot, async_tx) 41ff7b0479SSaeed Bishara 42ff7b0479SSaeed Bishara static void mv_desc_init(struct mv_xor_desc_slot *desc, unsigned long flags) 43ff7b0479SSaeed Bishara { 44ff7b0479SSaeed Bishara struct mv_xor_desc *hw_desc = desc->hw_desc; 45ff7b0479SSaeed Bishara 46ff7b0479SSaeed Bishara hw_desc->status = (1 << 31); 47ff7b0479SSaeed Bishara hw_desc->phy_next_desc = 0; 48ff7b0479SSaeed Bishara hw_desc->desc_command = (1 << 31); 49ff7b0479SSaeed Bishara } 50ff7b0479SSaeed Bishara 51ff7b0479SSaeed Bishara static u32 mv_desc_get_dest_addr(struct mv_xor_desc_slot *desc) 52ff7b0479SSaeed Bishara { 53ff7b0479SSaeed Bishara struct mv_xor_desc *hw_desc = desc->hw_desc; 54ff7b0479SSaeed Bishara return hw_desc->phy_dest_addr; 55ff7b0479SSaeed Bishara } 56ff7b0479SSaeed Bishara 57ff7b0479SSaeed Bishara static u32 mv_desc_get_src_addr(struct mv_xor_desc_slot *desc, 58ff7b0479SSaeed Bishara int src_idx) 59ff7b0479SSaeed Bishara { 60ff7b0479SSaeed Bishara struct mv_xor_desc *hw_desc = desc->hw_desc; 61ff7b0479SSaeed Bishara return hw_desc->phy_src_addr[src_idx]; 62ff7b0479SSaeed Bishara } 63ff7b0479SSaeed Bishara 64ff7b0479SSaeed Bishara 65ff7b0479SSaeed Bishara static void mv_desc_set_byte_count(struct mv_xor_desc_slot *desc, 66ff7b0479SSaeed Bishara u32 byte_count) 67ff7b0479SSaeed Bishara { 68ff7b0479SSaeed Bishara struct mv_xor_desc *hw_desc = desc->hw_desc; 69ff7b0479SSaeed Bishara hw_desc->byte_count = byte_count; 70ff7b0479SSaeed Bishara } 71ff7b0479SSaeed Bishara 72ff7b0479SSaeed Bishara static void mv_desc_set_next_desc(struct mv_xor_desc_slot *desc, 73ff7b0479SSaeed Bishara u32 next_desc_addr) 74ff7b0479SSaeed Bishara { 75ff7b0479SSaeed Bishara struct mv_xor_desc *hw_desc = desc->hw_desc; 76ff7b0479SSaeed Bishara BUG_ON(hw_desc->phy_next_desc); 77ff7b0479SSaeed Bishara hw_desc->phy_next_desc = next_desc_addr; 78ff7b0479SSaeed Bishara } 79ff7b0479SSaeed Bishara 80ff7b0479SSaeed Bishara static void mv_desc_clear_next_desc(struct mv_xor_desc_slot *desc) 81ff7b0479SSaeed Bishara { 82ff7b0479SSaeed Bishara struct mv_xor_desc *hw_desc = desc->hw_desc; 83ff7b0479SSaeed Bishara hw_desc->phy_next_desc = 0; 84ff7b0479SSaeed Bishara } 85ff7b0479SSaeed Bishara 86ff7b0479SSaeed Bishara static void mv_desc_set_block_fill_val(struct mv_xor_desc_slot *desc, u32 val) 87ff7b0479SSaeed Bishara { 88ff7b0479SSaeed Bishara desc->value = val; 89ff7b0479SSaeed Bishara } 90ff7b0479SSaeed Bishara 91ff7b0479SSaeed Bishara static void mv_desc_set_dest_addr(struct mv_xor_desc_slot *desc, 92ff7b0479SSaeed Bishara dma_addr_t addr) 93ff7b0479SSaeed Bishara { 94ff7b0479SSaeed Bishara struct mv_xor_desc *hw_desc = desc->hw_desc; 95ff7b0479SSaeed Bishara hw_desc->phy_dest_addr = addr; 96ff7b0479SSaeed Bishara } 97ff7b0479SSaeed Bishara 98ff7b0479SSaeed Bishara static int mv_chan_memset_slot_count(size_t len) 99ff7b0479SSaeed Bishara { 100ff7b0479SSaeed Bishara return 1; 101ff7b0479SSaeed Bishara } 102ff7b0479SSaeed Bishara 103ff7b0479SSaeed Bishara #define mv_chan_memcpy_slot_count(c) mv_chan_memset_slot_count(c) 104ff7b0479SSaeed Bishara 105ff7b0479SSaeed Bishara static void mv_desc_set_src_addr(struct mv_xor_desc_slot *desc, 106ff7b0479SSaeed Bishara int index, dma_addr_t addr) 107ff7b0479SSaeed Bishara { 108ff7b0479SSaeed Bishara struct mv_xor_desc *hw_desc = desc->hw_desc; 109ff7b0479SSaeed Bishara hw_desc->phy_src_addr[index] = addr; 110ff7b0479SSaeed Bishara if (desc->type == DMA_XOR) 111ff7b0479SSaeed Bishara hw_desc->desc_command |= (1 << index); 112ff7b0479SSaeed Bishara } 113ff7b0479SSaeed Bishara 114ff7b0479SSaeed Bishara static u32 mv_chan_get_current_desc(struct mv_xor_chan *chan) 115ff7b0479SSaeed Bishara { 116ff7b0479SSaeed Bishara return __raw_readl(XOR_CURR_DESC(chan)); 117ff7b0479SSaeed Bishara } 118ff7b0479SSaeed Bishara 119ff7b0479SSaeed Bishara static void mv_chan_set_next_descriptor(struct mv_xor_chan *chan, 120ff7b0479SSaeed Bishara u32 next_desc_addr) 121ff7b0479SSaeed Bishara { 122ff7b0479SSaeed Bishara __raw_writel(next_desc_addr, XOR_NEXT_DESC(chan)); 123ff7b0479SSaeed Bishara } 124ff7b0479SSaeed Bishara 125ff7b0479SSaeed Bishara static void mv_chan_set_dest_pointer(struct mv_xor_chan *chan, u32 desc_addr) 126ff7b0479SSaeed Bishara { 127ff7b0479SSaeed Bishara __raw_writel(desc_addr, XOR_DEST_POINTER(chan)); 128ff7b0479SSaeed Bishara } 129ff7b0479SSaeed Bishara 130ff7b0479SSaeed Bishara static void mv_chan_set_block_size(struct mv_xor_chan *chan, u32 block_size) 131ff7b0479SSaeed Bishara { 132ff7b0479SSaeed Bishara __raw_writel(block_size, XOR_BLOCK_SIZE(chan)); 133ff7b0479SSaeed Bishara } 134ff7b0479SSaeed Bishara 135ff7b0479SSaeed Bishara static void mv_chan_set_value(struct mv_xor_chan *chan, u32 value) 136ff7b0479SSaeed Bishara { 137ff7b0479SSaeed Bishara __raw_writel(value, XOR_INIT_VALUE_LOW(chan)); 138ff7b0479SSaeed Bishara __raw_writel(value, XOR_INIT_VALUE_HIGH(chan)); 139ff7b0479SSaeed Bishara } 140ff7b0479SSaeed Bishara 141ff7b0479SSaeed Bishara static void mv_chan_unmask_interrupts(struct mv_xor_chan *chan) 142ff7b0479SSaeed Bishara { 143ff7b0479SSaeed Bishara u32 val = __raw_readl(XOR_INTR_MASK(chan)); 144ff7b0479SSaeed Bishara val |= XOR_INTR_MASK_VALUE << (chan->idx * 16); 145ff7b0479SSaeed Bishara __raw_writel(val, XOR_INTR_MASK(chan)); 146ff7b0479SSaeed Bishara } 147ff7b0479SSaeed Bishara 148ff7b0479SSaeed Bishara static u32 mv_chan_get_intr_cause(struct mv_xor_chan *chan) 149ff7b0479SSaeed Bishara { 150ff7b0479SSaeed Bishara u32 intr_cause = __raw_readl(XOR_INTR_CAUSE(chan)); 151ff7b0479SSaeed Bishara intr_cause = (intr_cause >> (chan->idx * 16)) & 0xFFFF; 152ff7b0479SSaeed Bishara return intr_cause; 153ff7b0479SSaeed Bishara } 154ff7b0479SSaeed Bishara 155ff7b0479SSaeed Bishara static int mv_is_err_intr(u32 intr_cause) 156ff7b0479SSaeed Bishara { 157ff7b0479SSaeed Bishara if (intr_cause & ((1<<4)|(1<<5)|(1<<6)|(1<<7)|(1<<8)|(1<<9))) 158ff7b0479SSaeed Bishara return 1; 159ff7b0479SSaeed Bishara 160ff7b0479SSaeed Bishara return 0; 161ff7b0479SSaeed Bishara } 162ff7b0479SSaeed Bishara 163ff7b0479SSaeed Bishara static void mv_xor_device_clear_eoc_cause(struct mv_xor_chan *chan) 164ff7b0479SSaeed Bishara { 165ff7b0479SSaeed Bishara u32 val = (1 << (1 + (chan->idx * 16))); 166ff7b0479SSaeed Bishara dev_dbg(chan->device->common.dev, "%s, val 0x%08x\n", __func__, val); 167ff7b0479SSaeed Bishara __raw_writel(val, XOR_INTR_CAUSE(chan)); 168ff7b0479SSaeed Bishara } 169ff7b0479SSaeed Bishara 170ff7b0479SSaeed Bishara static void mv_xor_device_clear_err_status(struct mv_xor_chan *chan) 171ff7b0479SSaeed Bishara { 172ff7b0479SSaeed Bishara u32 val = 0xFFFF0000 >> (chan->idx * 16); 173ff7b0479SSaeed Bishara __raw_writel(val, XOR_INTR_CAUSE(chan)); 174ff7b0479SSaeed Bishara } 175ff7b0479SSaeed Bishara 176ff7b0479SSaeed Bishara static int mv_can_chain(struct mv_xor_desc_slot *desc) 177ff7b0479SSaeed Bishara { 178ff7b0479SSaeed Bishara struct mv_xor_desc_slot *chain_old_tail = list_entry( 179ff7b0479SSaeed Bishara desc->chain_node.prev, struct mv_xor_desc_slot, chain_node); 180ff7b0479SSaeed Bishara 181ff7b0479SSaeed Bishara if (chain_old_tail->type != desc->type) 182ff7b0479SSaeed Bishara return 0; 183ff7b0479SSaeed Bishara if (desc->type == DMA_MEMSET) 184ff7b0479SSaeed Bishara return 0; 185ff7b0479SSaeed Bishara 186ff7b0479SSaeed Bishara return 1; 187ff7b0479SSaeed Bishara } 188ff7b0479SSaeed Bishara 189ff7b0479SSaeed Bishara static void mv_set_mode(struct mv_xor_chan *chan, 190ff7b0479SSaeed Bishara enum dma_transaction_type type) 191ff7b0479SSaeed Bishara { 192ff7b0479SSaeed Bishara u32 op_mode; 193ff7b0479SSaeed Bishara u32 config = __raw_readl(XOR_CONFIG(chan)); 194ff7b0479SSaeed Bishara 195ff7b0479SSaeed Bishara switch (type) { 196ff7b0479SSaeed Bishara case DMA_XOR: 197ff7b0479SSaeed Bishara op_mode = XOR_OPERATION_MODE_XOR; 198ff7b0479SSaeed Bishara break; 199ff7b0479SSaeed Bishara case DMA_MEMCPY: 200ff7b0479SSaeed Bishara op_mode = XOR_OPERATION_MODE_MEMCPY; 201ff7b0479SSaeed Bishara break; 202ff7b0479SSaeed Bishara case DMA_MEMSET: 203ff7b0479SSaeed Bishara op_mode = XOR_OPERATION_MODE_MEMSET; 204ff7b0479SSaeed Bishara break; 205ff7b0479SSaeed Bishara default: 206ff7b0479SSaeed Bishara dev_printk(KERN_ERR, chan->device->common.dev, 207ff7b0479SSaeed Bishara "error: unsupported operation %d.\n", 208ff7b0479SSaeed Bishara type); 209ff7b0479SSaeed Bishara BUG(); 210ff7b0479SSaeed Bishara return; 211ff7b0479SSaeed Bishara } 212ff7b0479SSaeed Bishara 213ff7b0479SSaeed Bishara config &= ~0x7; 214ff7b0479SSaeed Bishara config |= op_mode; 215ff7b0479SSaeed Bishara __raw_writel(config, XOR_CONFIG(chan)); 216ff7b0479SSaeed Bishara chan->current_type = type; 217ff7b0479SSaeed Bishara } 218ff7b0479SSaeed Bishara 219ff7b0479SSaeed Bishara static void mv_chan_activate(struct mv_xor_chan *chan) 220ff7b0479SSaeed Bishara { 221ff7b0479SSaeed Bishara u32 activation; 222ff7b0479SSaeed Bishara 223ff7b0479SSaeed Bishara dev_dbg(chan->device->common.dev, " activate chan.\n"); 224ff7b0479SSaeed Bishara activation = __raw_readl(XOR_ACTIVATION(chan)); 225ff7b0479SSaeed Bishara activation |= 0x1; 226ff7b0479SSaeed Bishara __raw_writel(activation, XOR_ACTIVATION(chan)); 227ff7b0479SSaeed Bishara } 228ff7b0479SSaeed Bishara 229ff7b0479SSaeed Bishara static char mv_chan_is_busy(struct mv_xor_chan *chan) 230ff7b0479SSaeed Bishara { 231ff7b0479SSaeed Bishara u32 state = __raw_readl(XOR_ACTIVATION(chan)); 232ff7b0479SSaeed Bishara 233ff7b0479SSaeed Bishara state = (state >> 4) & 0x3; 234ff7b0479SSaeed Bishara 235ff7b0479SSaeed Bishara return (state == 1) ? 1 : 0; 236ff7b0479SSaeed Bishara } 237ff7b0479SSaeed Bishara 238ff7b0479SSaeed Bishara static int mv_chan_xor_slot_count(size_t len, int src_cnt) 239ff7b0479SSaeed Bishara { 240ff7b0479SSaeed Bishara return 1; 241ff7b0479SSaeed Bishara } 242ff7b0479SSaeed Bishara 243ff7b0479SSaeed Bishara /** 244ff7b0479SSaeed Bishara * mv_xor_free_slots - flags descriptor slots for reuse 245ff7b0479SSaeed Bishara * @slot: Slot to free 246ff7b0479SSaeed Bishara * Caller must hold &mv_chan->lock while calling this function 247ff7b0479SSaeed Bishara */ 248ff7b0479SSaeed Bishara static void mv_xor_free_slots(struct mv_xor_chan *mv_chan, 249ff7b0479SSaeed Bishara struct mv_xor_desc_slot *slot) 250ff7b0479SSaeed Bishara { 251ff7b0479SSaeed Bishara dev_dbg(mv_chan->device->common.dev, "%s %d slot %p\n", 252ff7b0479SSaeed Bishara __func__, __LINE__, slot); 253ff7b0479SSaeed Bishara 254ff7b0479SSaeed Bishara slot->slots_per_op = 0; 255ff7b0479SSaeed Bishara 256ff7b0479SSaeed Bishara } 257ff7b0479SSaeed Bishara 258ff7b0479SSaeed Bishara /* 259ff7b0479SSaeed Bishara * mv_xor_start_new_chain - program the engine to operate on new chain headed by 260ff7b0479SSaeed Bishara * sw_desc 261ff7b0479SSaeed Bishara * Caller must hold &mv_chan->lock while calling this function 262ff7b0479SSaeed Bishara */ 263ff7b0479SSaeed Bishara static void mv_xor_start_new_chain(struct mv_xor_chan *mv_chan, 264ff7b0479SSaeed Bishara struct mv_xor_desc_slot *sw_desc) 265ff7b0479SSaeed Bishara { 266ff7b0479SSaeed Bishara dev_dbg(mv_chan->device->common.dev, "%s %d: sw_desc %p\n", 267ff7b0479SSaeed Bishara __func__, __LINE__, sw_desc); 268ff7b0479SSaeed Bishara if (sw_desc->type != mv_chan->current_type) 269ff7b0479SSaeed Bishara mv_set_mode(mv_chan, sw_desc->type); 270ff7b0479SSaeed Bishara 271ff7b0479SSaeed Bishara if (sw_desc->type == DMA_MEMSET) { 272ff7b0479SSaeed Bishara /* for memset requests we need to program the engine, no 273ff7b0479SSaeed Bishara * descriptors used. 274ff7b0479SSaeed Bishara */ 275ff7b0479SSaeed Bishara struct mv_xor_desc *hw_desc = sw_desc->hw_desc; 276ff7b0479SSaeed Bishara mv_chan_set_dest_pointer(mv_chan, hw_desc->phy_dest_addr); 277ff7b0479SSaeed Bishara mv_chan_set_block_size(mv_chan, sw_desc->unmap_len); 278ff7b0479SSaeed Bishara mv_chan_set_value(mv_chan, sw_desc->value); 279ff7b0479SSaeed Bishara } else { 280ff7b0479SSaeed Bishara /* set the hardware chain */ 281ff7b0479SSaeed Bishara mv_chan_set_next_descriptor(mv_chan, sw_desc->async_tx.phys); 282ff7b0479SSaeed Bishara } 283ff7b0479SSaeed Bishara mv_chan->pending += sw_desc->slot_cnt; 284ff7b0479SSaeed Bishara mv_xor_issue_pending(&mv_chan->common); 285ff7b0479SSaeed Bishara } 286ff7b0479SSaeed Bishara 287ff7b0479SSaeed Bishara static dma_cookie_t 288ff7b0479SSaeed Bishara mv_xor_run_tx_complete_actions(struct mv_xor_desc_slot *desc, 289ff7b0479SSaeed Bishara struct mv_xor_chan *mv_chan, dma_cookie_t cookie) 290ff7b0479SSaeed Bishara { 291ff7b0479SSaeed Bishara BUG_ON(desc->async_tx.cookie < 0); 292ff7b0479SSaeed Bishara 293ff7b0479SSaeed Bishara if (desc->async_tx.cookie > 0) { 294ff7b0479SSaeed Bishara cookie = desc->async_tx.cookie; 295ff7b0479SSaeed Bishara 296ff7b0479SSaeed Bishara /* call the callback (must not sleep or submit new 297ff7b0479SSaeed Bishara * operations to this channel) 298ff7b0479SSaeed Bishara */ 299ff7b0479SSaeed Bishara if (desc->async_tx.callback) 300ff7b0479SSaeed Bishara desc->async_tx.callback( 301ff7b0479SSaeed Bishara desc->async_tx.callback_param); 302ff7b0479SSaeed Bishara 303ff7b0479SSaeed Bishara /* unmap dma addresses 304ff7b0479SSaeed Bishara * (unmap_single vs unmap_page?) 305ff7b0479SSaeed Bishara */ 306ff7b0479SSaeed Bishara if (desc->group_head && desc->unmap_len) { 307ff7b0479SSaeed Bishara struct mv_xor_desc_slot *unmap = desc->group_head; 308ff7b0479SSaeed Bishara struct device *dev = 309ff7b0479SSaeed Bishara &mv_chan->device->pdev->dev; 310ff7b0479SSaeed Bishara u32 len = unmap->unmap_len; 311ff7b0479SSaeed Bishara u32 src_cnt = unmap->unmap_src_cnt; 312ff7b0479SSaeed Bishara dma_addr_t addr = mv_desc_get_dest_addr(unmap); 313ff7b0479SSaeed Bishara 314ff7b0479SSaeed Bishara dma_unmap_page(dev, addr, len, DMA_FROM_DEVICE); 315ff7b0479SSaeed Bishara while (src_cnt--) { 316ff7b0479SSaeed Bishara addr = mv_desc_get_src_addr(unmap, src_cnt); 317ff7b0479SSaeed Bishara dma_unmap_page(dev, addr, len, DMA_TO_DEVICE); 318ff7b0479SSaeed Bishara } 319ff7b0479SSaeed Bishara desc->group_head = NULL; 320ff7b0479SSaeed Bishara } 321ff7b0479SSaeed Bishara } 322ff7b0479SSaeed Bishara 323ff7b0479SSaeed Bishara /* run dependent operations */ 324ff7b0479SSaeed Bishara async_tx_run_dependencies(&desc->async_tx); 325ff7b0479SSaeed Bishara 326ff7b0479SSaeed Bishara return cookie; 327ff7b0479SSaeed Bishara } 328ff7b0479SSaeed Bishara 329ff7b0479SSaeed Bishara static int 330ff7b0479SSaeed Bishara mv_xor_clean_completed_slots(struct mv_xor_chan *mv_chan) 331ff7b0479SSaeed Bishara { 332ff7b0479SSaeed Bishara struct mv_xor_desc_slot *iter, *_iter; 333ff7b0479SSaeed Bishara 334ff7b0479SSaeed Bishara dev_dbg(mv_chan->device->common.dev, "%s %d\n", __func__, __LINE__); 335ff7b0479SSaeed Bishara list_for_each_entry_safe(iter, _iter, &mv_chan->completed_slots, 336ff7b0479SSaeed Bishara completed_node) { 337ff7b0479SSaeed Bishara 338ff7b0479SSaeed Bishara if (async_tx_test_ack(&iter->async_tx)) { 339ff7b0479SSaeed Bishara list_del(&iter->completed_node); 340ff7b0479SSaeed Bishara mv_xor_free_slots(mv_chan, iter); 341ff7b0479SSaeed Bishara } 342ff7b0479SSaeed Bishara } 343ff7b0479SSaeed Bishara return 0; 344ff7b0479SSaeed Bishara } 345ff7b0479SSaeed Bishara 346ff7b0479SSaeed Bishara static int 347ff7b0479SSaeed Bishara mv_xor_clean_slot(struct mv_xor_desc_slot *desc, 348ff7b0479SSaeed Bishara struct mv_xor_chan *mv_chan) 349ff7b0479SSaeed Bishara { 350ff7b0479SSaeed Bishara dev_dbg(mv_chan->device->common.dev, "%s %d: desc %p flags %d\n", 351ff7b0479SSaeed Bishara __func__, __LINE__, desc, desc->async_tx.flags); 352ff7b0479SSaeed Bishara list_del(&desc->chain_node); 353ff7b0479SSaeed Bishara /* the client is allowed to attach dependent operations 354ff7b0479SSaeed Bishara * until 'ack' is set 355ff7b0479SSaeed Bishara */ 356ff7b0479SSaeed Bishara if (!async_tx_test_ack(&desc->async_tx)) { 357ff7b0479SSaeed Bishara /* move this slot to the completed_slots */ 358ff7b0479SSaeed Bishara list_add_tail(&desc->completed_node, &mv_chan->completed_slots); 359ff7b0479SSaeed Bishara return 0; 360ff7b0479SSaeed Bishara } 361ff7b0479SSaeed Bishara 362ff7b0479SSaeed Bishara mv_xor_free_slots(mv_chan, desc); 363ff7b0479SSaeed Bishara return 0; 364ff7b0479SSaeed Bishara } 365ff7b0479SSaeed Bishara 366ff7b0479SSaeed Bishara static void __mv_xor_slot_cleanup(struct mv_xor_chan *mv_chan) 367ff7b0479SSaeed Bishara { 368ff7b0479SSaeed Bishara struct mv_xor_desc_slot *iter, *_iter; 369ff7b0479SSaeed Bishara dma_cookie_t cookie = 0; 370ff7b0479SSaeed Bishara int busy = mv_chan_is_busy(mv_chan); 371ff7b0479SSaeed Bishara u32 current_desc = mv_chan_get_current_desc(mv_chan); 372ff7b0479SSaeed Bishara int seen_current = 0; 373ff7b0479SSaeed Bishara 374ff7b0479SSaeed Bishara dev_dbg(mv_chan->device->common.dev, "%s %d\n", __func__, __LINE__); 375ff7b0479SSaeed Bishara dev_dbg(mv_chan->device->common.dev, "current_desc %x\n", current_desc); 376ff7b0479SSaeed Bishara mv_xor_clean_completed_slots(mv_chan); 377ff7b0479SSaeed Bishara 378ff7b0479SSaeed Bishara /* free completed slots from the chain starting with 379ff7b0479SSaeed Bishara * the oldest descriptor 380ff7b0479SSaeed Bishara */ 381ff7b0479SSaeed Bishara 382ff7b0479SSaeed Bishara list_for_each_entry_safe(iter, _iter, &mv_chan->chain, 383ff7b0479SSaeed Bishara chain_node) { 384ff7b0479SSaeed Bishara prefetch(_iter); 385ff7b0479SSaeed Bishara prefetch(&_iter->async_tx); 386ff7b0479SSaeed Bishara 387ff7b0479SSaeed Bishara /* do not advance past the current descriptor loaded into the 388ff7b0479SSaeed Bishara * hardware channel, subsequent descriptors are either in 389ff7b0479SSaeed Bishara * process or have not been submitted 390ff7b0479SSaeed Bishara */ 391ff7b0479SSaeed Bishara if (seen_current) 392ff7b0479SSaeed Bishara break; 393ff7b0479SSaeed Bishara 394ff7b0479SSaeed Bishara /* stop the search if we reach the current descriptor and the 395ff7b0479SSaeed Bishara * channel is busy 396ff7b0479SSaeed Bishara */ 397ff7b0479SSaeed Bishara if (iter->async_tx.phys == current_desc) { 398ff7b0479SSaeed Bishara seen_current = 1; 399ff7b0479SSaeed Bishara if (busy) 400ff7b0479SSaeed Bishara break; 401ff7b0479SSaeed Bishara } 402ff7b0479SSaeed Bishara 403ff7b0479SSaeed Bishara cookie = mv_xor_run_tx_complete_actions(iter, mv_chan, cookie); 404ff7b0479SSaeed Bishara 405ff7b0479SSaeed Bishara if (mv_xor_clean_slot(iter, mv_chan)) 406ff7b0479SSaeed Bishara break; 407ff7b0479SSaeed Bishara } 408ff7b0479SSaeed Bishara 409ff7b0479SSaeed Bishara if ((busy == 0) && !list_empty(&mv_chan->chain)) { 410ff7b0479SSaeed Bishara struct mv_xor_desc_slot *chain_head; 411ff7b0479SSaeed Bishara chain_head = list_entry(mv_chan->chain.next, 412ff7b0479SSaeed Bishara struct mv_xor_desc_slot, 413ff7b0479SSaeed Bishara chain_node); 414ff7b0479SSaeed Bishara 415ff7b0479SSaeed Bishara mv_xor_start_new_chain(mv_chan, chain_head); 416ff7b0479SSaeed Bishara } 417ff7b0479SSaeed Bishara 418ff7b0479SSaeed Bishara if (cookie > 0) 419ff7b0479SSaeed Bishara mv_chan->completed_cookie = cookie; 420ff7b0479SSaeed Bishara } 421ff7b0479SSaeed Bishara 422ff7b0479SSaeed Bishara static void 423ff7b0479SSaeed Bishara mv_xor_slot_cleanup(struct mv_xor_chan *mv_chan) 424ff7b0479SSaeed Bishara { 425ff7b0479SSaeed Bishara spin_lock_bh(&mv_chan->lock); 426ff7b0479SSaeed Bishara __mv_xor_slot_cleanup(mv_chan); 427ff7b0479SSaeed Bishara spin_unlock_bh(&mv_chan->lock); 428ff7b0479SSaeed Bishara } 429ff7b0479SSaeed Bishara 430ff7b0479SSaeed Bishara static void mv_xor_tasklet(unsigned long data) 431ff7b0479SSaeed Bishara { 432ff7b0479SSaeed Bishara struct mv_xor_chan *chan = (struct mv_xor_chan *) data; 433ff7b0479SSaeed Bishara __mv_xor_slot_cleanup(chan); 434ff7b0479SSaeed Bishara } 435ff7b0479SSaeed Bishara 436ff7b0479SSaeed Bishara static struct mv_xor_desc_slot * 437ff7b0479SSaeed Bishara mv_xor_alloc_slots(struct mv_xor_chan *mv_chan, int num_slots, 438ff7b0479SSaeed Bishara int slots_per_op) 439ff7b0479SSaeed Bishara { 440ff7b0479SSaeed Bishara struct mv_xor_desc_slot *iter, *_iter, *alloc_start = NULL; 441ff7b0479SSaeed Bishara LIST_HEAD(chain); 442ff7b0479SSaeed Bishara int slots_found, retry = 0; 443ff7b0479SSaeed Bishara 444ff7b0479SSaeed Bishara /* start search from the last allocated descrtiptor 445ff7b0479SSaeed Bishara * if a contiguous allocation can not be found start searching 446ff7b0479SSaeed Bishara * from the beginning of the list 447ff7b0479SSaeed Bishara */ 448ff7b0479SSaeed Bishara retry: 449ff7b0479SSaeed Bishara slots_found = 0; 450ff7b0479SSaeed Bishara if (retry == 0) 451ff7b0479SSaeed Bishara iter = mv_chan->last_used; 452ff7b0479SSaeed Bishara else 453ff7b0479SSaeed Bishara iter = list_entry(&mv_chan->all_slots, 454ff7b0479SSaeed Bishara struct mv_xor_desc_slot, 455ff7b0479SSaeed Bishara slot_node); 456ff7b0479SSaeed Bishara 457ff7b0479SSaeed Bishara list_for_each_entry_safe_continue( 458ff7b0479SSaeed Bishara iter, _iter, &mv_chan->all_slots, slot_node) { 459ff7b0479SSaeed Bishara prefetch(_iter); 460ff7b0479SSaeed Bishara prefetch(&_iter->async_tx); 461ff7b0479SSaeed Bishara if (iter->slots_per_op) { 462ff7b0479SSaeed Bishara /* give up after finding the first busy slot 463ff7b0479SSaeed Bishara * on the second pass through the list 464ff7b0479SSaeed Bishara */ 465ff7b0479SSaeed Bishara if (retry) 466ff7b0479SSaeed Bishara break; 467ff7b0479SSaeed Bishara 468ff7b0479SSaeed Bishara slots_found = 0; 469ff7b0479SSaeed Bishara continue; 470ff7b0479SSaeed Bishara } 471ff7b0479SSaeed Bishara 472ff7b0479SSaeed Bishara /* start the allocation if the slot is correctly aligned */ 473ff7b0479SSaeed Bishara if (!slots_found++) 474ff7b0479SSaeed Bishara alloc_start = iter; 475ff7b0479SSaeed Bishara 476ff7b0479SSaeed Bishara if (slots_found == num_slots) { 477ff7b0479SSaeed Bishara struct mv_xor_desc_slot *alloc_tail = NULL; 478ff7b0479SSaeed Bishara struct mv_xor_desc_slot *last_used = NULL; 479ff7b0479SSaeed Bishara iter = alloc_start; 480ff7b0479SSaeed Bishara while (num_slots) { 481ff7b0479SSaeed Bishara int i; 482ff7b0479SSaeed Bishara 483ff7b0479SSaeed Bishara /* pre-ack all but the last descriptor */ 484ff7b0479SSaeed Bishara async_tx_ack(&iter->async_tx); 485ff7b0479SSaeed Bishara 486ff7b0479SSaeed Bishara list_add_tail(&iter->chain_node, &chain); 487ff7b0479SSaeed Bishara alloc_tail = iter; 488ff7b0479SSaeed Bishara iter->async_tx.cookie = 0; 489ff7b0479SSaeed Bishara iter->slot_cnt = num_slots; 490ff7b0479SSaeed Bishara iter->xor_check_result = NULL; 491ff7b0479SSaeed Bishara for (i = 0; i < slots_per_op; i++) { 492ff7b0479SSaeed Bishara iter->slots_per_op = slots_per_op - i; 493ff7b0479SSaeed Bishara last_used = iter; 494ff7b0479SSaeed Bishara iter = list_entry(iter->slot_node.next, 495ff7b0479SSaeed Bishara struct mv_xor_desc_slot, 496ff7b0479SSaeed Bishara slot_node); 497ff7b0479SSaeed Bishara } 498ff7b0479SSaeed Bishara num_slots -= slots_per_op; 499ff7b0479SSaeed Bishara } 500ff7b0479SSaeed Bishara alloc_tail->group_head = alloc_start; 501ff7b0479SSaeed Bishara alloc_tail->async_tx.cookie = -EBUSY; 502ff7b0479SSaeed Bishara list_splice(&chain, &alloc_tail->async_tx.tx_list); 503ff7b0479SSaeed Bishara mv_chan->last_used = last_used; 504ff7b0479SSaeed Bishara mv_desc_clear_next_desc(alloc_start); 505ff7b0479SSaeed Bishara mv_desc_clear_next_desc(alloc_tail); 506ff7b0479SSaeed Bishara return alloc_tail; 507ff7b0479SSaeed Bishara } 508ff7b0479SSaeed Bishara } 509ff7b0479SSaeed Bishara if (!retry++) 510ff7b0479SSaeed Bishara goto retry; 511ff7b0479SSaeed Bishara 512ff7b0479SSaeed Bishara /* try to free some slots if the allocation fails */ 513ff7b0479SSaeed Bishara tasklet_schedule(&mv_chan->irq_tasklet); 514ff7b0479SSaeed Bishara 515ff7b0479SSaeed Bishara return NULL; 516ff7b0479SSaeed Bishara } 517ff7b0479SSaeed Bishara 518ff7b0479SSaeed Bishara static dma_cookie_t 519ff7b0479SSaeed Bishara mv_desc_assign_cookie(struct mv_xor_chan *mv_chan, 520ff7b0479SSaeed Bishara struct mv_xor_desc_slot *desc) 521ff7b0479SSaeed Bishara { 522ff7b0479SSaeed Bishara dma_cookie_t cookie = mv_chan->common.cookie; 523ff7b0479SSaeed Bishara 524ff7b0479SSaeed Bishara if (++cookie < 0) 525ff7b0479SSaeed Bishara cookie = 1; 526ff7b0479SSaeed Bishara mv_chan->common.cookie = desc->async_tx.cookie = cookie; 527ff7b0479SSaeed Bishara return cookie; 528ff7b0479SSaeed Bishara } 529ff7b0479SSaeed Bishara 530ff7b0479SSaeed Bishara /************************ DMA engine API functions ****************************/ 531ff7b0479SSaeed Bishara static dma_cookie_t 532ff7b0479SSaeed Bishara mv_xor_tx_submit(struct dma_async_tx_descriptor *tx) 533ff7b0479SSaeed Bishara { 534ff7b0479SSaeed Bishara struct mv_xor_desc_slot *sw_desc = to_mv_xor_slot(tx); 535ff7b0479SSaeed Bishara struct mv_xor_chan *mv_chan = to_mv_xor_chan(tx->chan); 536ff7b0479SSaeed Bishara struct mv_xor_desc_slot *grp_start, *old_chain_tail; 537ff7b0479SSaeed Bishara dma_cookie_t cookie; 538ff7b0479SSaeed Bishara int new_hw_chain = 1; 539ff7b0479SSaeed Bishara 540ff7b0479SSaeed Bishara dev_dbg(mv_chan->device->common.dev, 541ff7b0479SSaeed Bishara "%s sw_desc %p: async_tx %p\n", 542ff7b0479SSaeed Bishara __func__, sw_desc, &sw_desc->async_tx); 543ff7b0479SSaeed Bishara 544ff7b0479SSaeed Bishara grp_start = sw_desc->group_head; 545ff7b0479SSaeed Bishara 546ff7b0479SSaeed Bishara spin_lock_bh(&mv_chan->lock); 547ff7b0479SSaeed Bishara cookie = mv_desc_assign_cookie(mv_chan, sw_desc); 548ff7b0479SSaeed Bishara 549ff7b0479SSaeed Bishara if (list_empty(&mv_chan->chain)) 550ff7b0479SSaeed Bishara list_splice_init(&sw_desc->async_tx.tx_list, &mv_chan->chain); 551ff7b0479SSaeed Bishara else { 552ff7b0479SSaeed Bishara new_hw_chain = 0; 553ff7b0479SSaeed Bishara 554ff7b0479SSaeed Bishara old_chain_tail = list_entry(mv_chan->chain.prev, 555ff7b0479SSaeed Bishara struct mv_xor_desc_slot, 556ff7b0479SSaeed Bishara chain_node); 557ff7b0479SSaeed Bishara list_splice_init(&grp_start->async_tx.tx_list, 558ff7b0479SSaeed Bishara &old_chain_tail->chain_node); 559ff7b0479SSaeed Bishara 560ff7b0479SSaeed Bishara if (!mv_can_chain(grp_start)) 561ff7b0479SSaeed Bishara goto submit_done; 562ff7b0479SSaeed Bishara 563ff7b0479SSaeed Bishara dev_dbg(mv_chan->device->common.dev, "Append to last desc %x\n", 564ff7b0479SSaeed Bishara old_chain_tail->async_tx.phys); 565ff7b0479SSaeed Bishara 566ff7b0479SSaeed Bishara /* fix up the hardware chain */ 567ff7b0479SSaeed Bishara mv_desc_set_next_desc(old_chain_tail, grp_start->async_tx.phys); 568ff7b0479SSaeed Bishara 569ff7b0479SSaeed Bishara /* if the channel is not busy */ 570ff7b0479SSaeed Bishara if (!mv_chan_is_busy(mv_chan)) { 571ff7b0479SSaeed Bishara u32 current_desc = mv_chan_get_current_desc(mv_chan); 572ff7b0479SSaeed Bishara /* 573ff7b0479SSaeed Bishara * and the curren desc is the end of the chain before 574ff7b0479SSaeed Bishara * the append, then we need to start the channel 575ff7b0479SSaeed Bishara */ 576ff7b0479SSaeed Bishara if (current_desc == old_chain_tail->async_tx.phys) 577ff7b0479SSaeed Bishara new_hw_chain = 1; 578ff7b0479SSaeed Bishara } 579ff7b0479SSaeed Bishara } 580ff7b0479SSaeed Bishara 581ff7b0479SSaeed Bishara if (new_hw_chain) 582ff7b0479SSaeed Bishara mv_xor_start_new_chain(mv_chan, grp_start); 583ff7b0479SSaeed Bishara 584ff7b0479SSaeed Bishara submit_done: 585ff7b0479SSaeed Bishara spin_unlock_bh(&mv_chan->lock); 586ff7b0479SSaeed Bishara 587ff7b0479SSaeed Bishara return cookie; 588ff7b0479SSaeed Bishara } 589ff7b0479SSaeed Bishara 590ff7b0479SSaeed Bishara /* returns the number of allocated descriptors */ 591ff7b0479SSaeed Bishara static int mv_xor_alloc_chan_resources(struct dma_chan *chan) 592ff7b0479SSaeed Bishara { 593ff7b0479SSaeed Bishara char *hw_desc; 594ff7b0479SSaeed Bishara int idx; 595ff7b0479SSaeed Bishara struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan); 596ff7b0479SSaeed Bishara struct mv_xor_desc_slot *slot = NULL; 597ff7b0479SSaeed Bishara struct mv_xor_platform_data *plat_data = 598ff7b0479SSaeed Bishara mv_chan->device->pdev->dev.platform_data; 599ff7b0479SSaeed Bishara int num_descs_in_pool = plat_data->pool_size/MV_XOR_SLOT_SIZE; 600ff7b0479SSaeed Bishara 601ff7b0479SSaeed Bishara /* Allocate descriptor slots */ 602ff7b0479SSaeed Bishara idx = mv_chan->slots_allocated; 603ff7b0479SSaeed Bishara while (idx < num_descs_in_pool) { 604ff7b0479SSaeed Bishara slot = kzalloc(sizeof(*slot), GFP_KERNEL); 605ff7b0479SSaeed Bishara if (!slot) { 606ff7b0479SSaeed Bishara printk(KERN_INFO "MV XOR Channel only initialized" 607ff7b0479SSaeed Bishara " %d descriptor slots", idx); 608ff7b0479SSaeed Bishara break; 609ff7b0479SSaeed Bishara } 610ff7b0479SSaeed Bishara hw_desc = (char *) mv_chan->device->dma_desc_pool_virt; 611ff7b0479SSaeed Bishara slot->hw_desc = (void *) &hw_desc[idx * MV_XOR_SLOT_SIZE]; 612ff7b0479SSaeed Bishara 613ff7b0479SSaeed Bishara dma_async_tx_descriptor_init(&slot->async_tx, chan); 614ff7b0479SSaeed Bishara slot->async_tx.tx_submit = mv_xor_tx_submit; 615ff7b0479SSaeed Bishara INIT_LIST_HEAD(&slot->chain_node); 616ff7b0479SSaeed Bishara INIT_LIST_HEAD(&slot->slot_node); 617ff7b0479SSaeed Bishara INIT_LIST_HEAD(&slot->async_tx.tx_list); 618ff7b0479SSaeed Bishara hw_desc = (char *) mv_chan->device->dma_desc_pool; 619ff7b0479SSaeed Bishara slot->async_tx.phys = 620ff7b0479SSaeed Bishara (dma_addr_t) &hw_desc[idx * MV_XOR_SLOT_SIZE]; 621ff7b0479SSaeed Bishara slot->idx = idx++; 622ff7b0479SSaeed Bishara 623ff7b0479SSaeed Bishara spin_lock_bh(&mv_chan->lock); 624ff7b0479SSaeed Bishara mv_chan->slots_allocated = idx; 625ff7b0479SSaeed Bishara list_add_tail(&slot->slot_node, &mv_chan->all_slots); 626ff7b0479SSaeed Bishara spin_unlock_bh(&mv_chan->lock); 627ff7b0479SSaeed Bishara } 628ff7b0479SSaeed Bishara 629ff7b0479SSaeed Bishara if (mv_chan->slots_allocated && !mv_chan->last_used) 630ff7b0479SSaeed Bishara mv_chan->last_used = list_entry(mv_chan->all_slots.next, 631ff7b0479SSaeed Bishara struct mv_xor_desc_slot, 632ff7b0479SSaeed Bishara slot_node); 633ff7b0479SSaeed Bishara 634ff7b0479SSaeed Bishara dev_dbg(mv_chan->device->common.dev, 635ff7b0479SSaeed Bishara "allocated %d descriptor slots last_used: %p\n", 636ff7b0479SSaeed Bishara mv_chan->slots_allocated, mv_chan->last_used); 637ff7b0479SSaeed Bishara 638ff7b0479SSaeed Bishara return mv_chan->slots_allocated ? : -ENOMEM; 639ff7b0479SSaeed Bishara } 640ff7b0479SSaeed Bishara 641ff7b0479SSaeed Bishara static struct dma_async_tx_descriptor * 642ff7b0479SSaeed Bishara mv_xor_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, 643ff7b0479SSaeed Bishara size_t len, unsigned long flags) 644ff7b0479SSaeed Bishara { 645ff7b0479SSaeed Bishara struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan); 646ff7b0479SSaeed Bishara struct mv_xor_desc_slot *sw_desc, *grp_start; 647ff7b0479SSaeed Bishara int slot_cnt; 648ff7b0479SSaeed Bishara 649ff7b0479SSaeed Bishara dev_dbg(mv_chan->device->common.dev, 650ff7b0479SSaeed Bishara "%s dest: %x src %x len: %u flags: %ld\n", 651ff7b0479SSaeed Bishara __func__, dest, src, len, flags); 652ff7b0479SSaeed Bishara if (unlikely(len < MV_XOR_MIN_BYTE_COUNT)) 653ff7b0479SSaeed Bishara return NULL; 654ff7b0479SSaeed Bishara 655ff7b0479SSaeed Bishara BUG_ON(unlikely(len > MV_XOR_MAX_BYTE_COUNT)); 656ff7b0479SSaeed Bishara 657ff7b0479SSaeed Bishara spin_lock_bh(&mv_chan->lock); 658ff7b0479SSaeed Bishara slot_cnt = mv_chan_memcpy_slot_count(len); 659ff7b0479SSaeed Bishara sw_desc = mv_xor_alloc_slots(mv_chan, slot_cnt, 1); 660ff7b0479SSaeed Bishara if (sw_desc) { 661ff7b0479SSaeed Bishara sw_desc->type = DMA_MEMCPY; 662ff7b0479SSaeed Bishara sw_desc->async_tx.flags = flags; 663ff7b0479SSaeed Bishara grp_start = sw_desc->group_head; 664ff7b0479SSaeed Bishara mv_desc_init(grp_start, flags); 665ff7b0479SSaeed Bishara mv_desc_set_byte_count(grp_start, len); 666ff7b0479SSaeed Bishara mv_desc_set_dest_addr(sw_desc->group_head, dest); 667ff7b0479SSaeed Bishara mv_desc_set_src_addr(grp_start, 0, src); 668ff7b0479SSaeed Bishara sw_desc->unmap_src_cnt = 1; 669ff7b0479SSaeed Bishara sw_desc->unmap_len = len; 670ff7b0479SSaeed Bishara } 671ff7b0479SSaeed Bishara spin_unlock_bh(&mv_chan->lock); 672ff7b0479SSaeed Bishara 673ff7b0479SSaeed Bishara dev_dbg(mv_chan->device->common.dev, 674ff7b0479SSaeed Bishara "%s sw_desc %p async_tx %p\n", 675ff7b0479SSaeed Bishara __func__, sw_desc, sw_desc ? &sw_desc->async_tx : 0); 676ff7b0479SSaeed Bishara 677ff7b0479SSaeed Bishara return sw_desc ? &sw_desc->async_tx : NULL; 678ff7b0479SSaeed Bishara } 679ff7b0479SSaeed Bishara 680ff7b0479SSaeed Bishara static struct dma_async_tx_descriptor * 681ff7b0479SSaeed Bishara mv_xor_prep_dma_memset(struct dma_chan *chan, dma_addr_t dest, int value, 682ff7b0479SSaeed Bishara size_t len, unsigned long flags) 683ff7b0479SSaeed Bishara { 684ff7b0479SSaeed Bishara struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan); 685ff7b0479SSaeed Bishara struct mv_xor_desc_slot *sw_desc, *grp_start; 686ff7b0479SSaeed Bishara int slot_cnt; 687ff7b0479SSaeed Bishara 688ff7b0479SSaeed Bishara dev_dbg(mv_chan->device->common.dev, 689ff7b0479SSaeed Bishara "%s dest: %x len: %u flags: %ld\n", 690ff7b0479SSaeed Bishara __func__, dest, len, flags); 691ff7b0479SSaeed Bishara if (unlikely(len < MV_XOR_MIN_BYTE_COUNT)) 692ff7b0479SSaeed Bishara return NULL; 693ff7b0479SSaeed Bishara 694ff7b0479SSaeed Bishara BUG_ON(unlikely(len > MV_XOR_MAX_BYTE_COUNT)); 695ff7b0479SSaeed Bishara 696ff7b0479SSaeed Bishara spin_lock_bh(&mv_chan->lock); 697ff7b0479SSaeed Bishara slot_cnt = mv_chan_memset_slot_count(len); 698ff7b0479SSaeed Bishara sw_desc = mv_xor_alloc_slots(mv_chan, slot_cnt, 1); 699ff7b0479SSaeed Bishara if (sw_desc) { 700ff7b0479SSaeed Bishara sw_desc->type = DMA_MEMSET; 701ff7b0479SSaeed Bishara sw_desc->async_tx.flags = flags; 702ff7b0479SSaeed Bishara grp_start = sw_desc->group_head; 703ff7b0479SSaeed Bishara mv_desc_init(grp_start, flags); 704ff7b0479SSaeed Bishara mv_desc_set_byte_count(grp_start, len); 705ff7b0479SSaeed Bishara mv_desc_set_dest_addr(sw_desc->group_head, dest); 706ff7b0479SSaeed Bishara mv_desc_set_block_fill_val(grp_start, value); 707ff7b0479SSaeed Bishara sw_desc->unmap_src_cnt = 1; 708ff7b0479SSaeed Bishara sw_desc->unmap_len = len; 709ff7b0479SSaeed Bishara } 710ff7b0479SSaeed Bishara spin_unlock_bh(&mv_chan->lock); 711ff7b0479SSaeed Bishara dev_dbg(mv_chan->device->common.dev, 712ff7b0479SSaeed Bishara "%s sw_desc %p async_tx %p \n", 713ff7b0479SSaeed Bishara __func__, sw_desc, &sw_desc->async_tx); 714ff7b0479SSaeed Bishara return sw_desc ? &sw_desc->async_tx : NULL; 715ff7b0479SSaeed Bishara } 716ff7b0479SSaeed Bishara 717ff7b0479SSaeed Bishara static struct dma_async_tx_descriptor * 718ff7b0479SSaeed Bishara mv_xor_prep_dma_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src, 719ff7b0479SSaeed Bishara unsigned int src_cnt, size_t len, unsigned long flags) 720ff7b0479SSaeed Bishara { 721ff7b0479SSaeed Bishara struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan); 722ff7b0479SSaeed Bishara struct mv_xor_desc_slot *sw_desc, *grp_start; 723ff7b0479SSaeed Bishara int slot_cnt; 724ff7b0479SSaeed Bishara 725ff7b0479SSaeed Bishara if (unlikely(len < MV_XOR_MIN_BYTE_COUNT)) 726ff7b0479SSaeed Bishara return NULL; 727ff7b0479SSaeed Bishara 728ff7b0479SSaeed Bishara BUG_ON(unlikely(len > MV_XOR_MAX_BYTE_COUNT)); 729ff7b0479SSaeed Bishara 730ff7b0479SSaeed Bishara dev_dbg(mv_chan->device->common.dev, 731ff7b0479SSaeed Bishara "%s src_cnt: %d len: dest %x %u flags: %ld\n", 732ff7b0479SSaeed Bishara __func__, src_cnt, len, dest, flags); 733ff7b0479SSaeed Bishara 734ff7b0479SSaeed Bishara spin_lock_bh(&mv_chan->lock); 735ff7b0479SSaeed Bishara slot_cnt = mv_chan_xor_slot_count(len, src_cnt); 736ff7b0479SSaeed Bishara sw_desc = mv_xor_alloc_slots(mv_chan, slot_cnt, 1); 737ff7b0479SSaeed Bishara if (sw_desc) { 738ff7b0479SSaeed Bishara sw_desc->type = DMA_XOR; 739ff7b0479SSaeed Bishara sw_desc->async_tx.flags = flags; 740ff7b0479SSaeed Bishara grp_start = sw_desc->group_head; 741ff7b0479SSaeed Bishara mv_desc_init(grp_start, flags); 742ff7b0479SSaeed Bishara /* the byte count field is the same as in memcpy desc*/ 743ff7b0479SSaeed Bishara mv_desc_set_byte_count(grp_start, len); 744ff7b0479SSaeed Bishara mv_desc_set_dest_addr(sw_desc->group_head, dest); 745ff7b0479SSaeed Bishara sw_desc->unmap_src_cnt = src_cnt; 746ff7b0479SSaeed Bishara sw_desc->unmap_len = len; 747ff7b0479SSaeed Bishara while (src_cnt--) 748ff7b0479SSaeed Bishara mv_desc_set_src_addr(grp_start, src_cnt, src[src_cnt]); 749ff7b0479SSaeed Bishara } 750ff7b0479SSaeed Bishara spin_unlock_bh(&mv_chan->lock); 751ff7b0479SSaeed Bishara dev_dbg(mv_chan->device->common.dev, 752ff7b0479SSaeed Bishara "%s sw_desc %p async_tx %p \n", 753ff7b0479SSaeed Bishara __func__, sw_desc, &sw_desc->async_tx); 754ff7b0479SSaeed Bishara return sw_desc ? &sw_desc->async_tx : NULL; 755ff7b0479SSaeed Bishara } 756ff7b0479SSaeed Bishara 757ff7b0479SSaeed Bishara static void mv_xor_free_chan_resources(struct dma_chan *chan) 758ff7b0479SSaeed Bishara { 759ff7b0479SSaeed Bishara struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan); 760ff7b0479SSaeed Bishara struct mv_xor_desc_slot *iter, *_iter; 761ff7b0479SSaeed Bishara int in_use_descs = 0; 762ff7b0479SSaeed Bishara 763ff7b0479SSaeed Bishara mv_xor_slot_cleanup(mv_chan); 764ff7b0479SSaeed Bishara 765ff7b0479SSaeed Bishara spin_lock_bh(&mv_chan->lock); 766ff7b0479SSaeed Bishara list_for_each_entry_safe(iter, _iter, &mv_chan->chain, 767ff7b0479SSaeed Bishara chain_node) { 768ff7b0479SSaeed Bishara in_use_descs++; 769ff7b0479SSaeed Bishara list_del(&iter->chain_node); 770ff7b0479SSaeed Bishara } 771ff7b0479SSaeed Bishara list_for_each_entry_safe(iter, _iter, &mv_chan->completed_slots, 772ff7b0479SSaeed Bishara completed_node) { 773ff7b0479SSaeed Bishara in_use_descs++; 774ff7b0479SSaeed Bishara list_del(&iter->completed_node); 775ff7b0479SSaeed Bishara } 776ff7b0479SSaeed Bishara list_for_each_entry_safe_reverse( 777ff7b0479SSaeed Bishara iter, _iter, &mv_chan->all_slots, slot_node) { 778ff7b0479SSaeed Bishara list_del(&iter->slot_node); 779ff7b0479SSaeed Bishara kfree(iter); 780ff7b0479SSaeed Bishara mv_chan->slots_allocated--; 781ff7b0479SSaeed Bishara } 782ff7b0479SSaeed Bishara mv_chan->last_used = NULL; 783ff7b0479SSaeed Bishara 784ff7b0479SSaeed Bishara dev_dbg(mv_chan->device->common.dev, "%s slots_allocated %d\n", 785ff7b0479SSaeed Bishara __func__, mv_chan->slots_allocated); 786ff7b0479SSaeed Bishara spin_unlock_bh(&mv_chan->lock); 787ff7b0479SSaeed Bishara 788ff7b0479SSaeed Bishara if (in_use_descs) 789ff7b0479SSaeed Bishara dev_err(mv_chan->device->common.dev, 790ff7b0479SSaeed Bishara "freeing %d in use descriptors!\n", in_use_descs); 791ff7b0479SSaeed Bishara } 792ff7b0479SSaeed Bishara 793ff7b0479SSaeed Bishara /** 794ff7b0479SSaeed Bishara * mv_xor_is_complete - poll the status of an XOR transaction 795ff7b0479SSaeed Bishara * @chan: XOR channel handle 796ff7b0479SSaeed Bishara * @cookie: XOR transaction identifier 797ff7b0479SSaeed Bishara */ 798ff7b0479SSaeed Bishara static enum dma_status mv_xor_is_complete(struct dma_chan *chan, 799ff7b0479SSaeed Bishara dma_cookie_t cookie, 800ff7b0479SSaeed Bishara dma_cookie_t *done, 801ff7b0479SSaeed Bishara dma_cookie_t *used) 802ff7b0479SSaeed Bishara { 803ff7b0479SSaeed Bishara struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan); 804ff7b0479SSaeed Bishara dma_cookie_t last_used; 805ff7b0479SSaeed Bishara dma_cookie_t last_complete; 806ff7b0479SSaeed Bishara enum dma_status ret; 807ff7b0479SSaeed Bishara 808ff7b0479SSaeed Bishara last_used = chan->cookie; 809ff7b0479SSaeed Bishara last_complete = mv_chan->completed_cookie; 810ff7b0479SSaeed Bishara mv_chan->is_complete_cookie = cookie; 811ff7b0479SSaeed Bishara if (done) 812ff7b0479SSaeed Bishara *done = last_complete; 813ff7b0479SSaeed Bishara if (used) 814ff7b0479SSaeed Bishara *used = last_used; 815ff7b0479SSaeed Bishara 816ff7b0479SSaeed Bishara ret = dma_async_is_complete(cookie, last_complete, last_used); 817ff7b0479SSaeed Bishara if (ret == DMA_SUCCESS) { 818ff7b0479SSaeed Bishara mv_xor_clean_completed_slots(mv_chan); 819ff7b0479SSaeed Bishara return ret; 820ff7b0479SSaeed Bishara } 821ff7b0479SSaeed Bishara mv_xor_slot_cleanup(mv_chan); 822ff7b0479SSaeed Bishara 823ff7b0479SSaeed Bishara last_used = chan->cookie; 824ff7b0479SSaeed Bishara last_complete = mv_chan->completed_cookie; 825ff7b0479SSaeed Bishara 826ff7b0479SSaeed Bishara if (done) 827ff7b0479SSaeed Bishara *done = last_complete; 828ff7b0479SSaeed Bishara if (used) 829ff7b0479SSaeed Bishara *used = last_used; 830ff7b0479SSaeed Bishara 831ff7b0479SSaeed Bishara return dma_async_is_complete(cookie, last_complete, last_used); 832ff7b0479SSaeed Bishara } 833ff7b0479SSaeed Bishara 834ff7b0479SSaeed Bishara static void mv_dump_xor_regs(struct mv_xor_chan *chan) 835ff7b0479SSaeed Bishara { 836ff7b0479SSaeed Bishara u32 val; 837ff7b0479SSaeed Bishara 838ff7b0479SSaeed Bishara val = __raw_readl(XOR_CONFIG(chan)); 839ff7b0479SSaeed Bishara dev_printk(KERN_ERR, chan->device->common.dev, 840ff7b0479SSaeed Bishara "config 0x%08x.\n", val); 841ff7b0479SSaeed Bishara 842ff7b0479SSaeed Bishara val = __raw_readl(XOR_ACTIVATION(chan)); 843ff7b0479SSaeed Bishara dev_printk(KERN_ERR, chan->device->common.dev, 844ff7b0479SSaeed Bishara "activation 0x%08x.\n", val); 845ff7b0479SSaeed Bishara 846ff7b0479SSaeed Bishara val = __raw_readl(XOR_INTR_CAUSE(chan)); 847ff7b0479SSaeed Bishara dev_printk(KERN_ERR, chan->device->common.dev, 848ff7b0479SSaeed Bishara "intr cause 0x%08x.\n", val); 849ff7b0479SSaeed Bishara 850ff7b0479SSaeed Bishara val = __raw_readl(XOR_INTR_MASK(chan)); 851ff7b0479SSaeed Bishara dev_printk(KERN_ERR, chan->device->common.dev, 852ff7b0479SSaeed Bishara "intr mask 0x%08x.\n", val); 853ff7b0479SSaeed Bishara 854ff7b0479SSaeed Bishara val = __raw_readl(XOR_ERROR_CAUSE(chan)); 855ff7b0479SSaeed Bishara dev_printk(KERN_ERR, chan->device->common.dev, 856ff7b0479SSaeed Bishara "error cause 0x%08x.\n", val); 857ff7b0479SSaeed Bishara 858ff7b0479SSaeed Bishara val = __raw_readl(XOR_ERROR_ADDR(chan)); 859ff7b0479SSaeed Bishara dev_printk(KERN_ERR, chan->device->common.dev, 860ff7b0479SSaeed Bishara "error addr 0x%08x.\n", val); 861ff7b0479SSaeed Bishara } 862ff7b0479SSaeed Bishara 863ff7b0479SSaeed Bishara static void mv_xor_err_interrupt_handler(struct mv_xor_chan *chan, 864ff7b0479SSaeed Bishara u32 intr_cause) 865ff7b0479SSaeed Bishara { 866ff7b0479SSaeed Bishara if (intr_cause & (1 << 4)) { 867ff7b0479SSaeed Bishara dev_dbg(chan->device->common.dev, 868ff7b0479SSaeed Bishara "ignore this error\n"); 869ff7b0479SSaeed Bishara return; 870ff7b0479SSaeed Bishara } 871ff7b0479SSaeed Bishara 872ff7b0479SSaeed Bishara dev_printk(KERN_ERR, chan->device->common.dev, 873ff7b0479SSaeed Bishara "error on chan %d. intr cause 0x%08x.\n", 874ff7b0479SSaeed Bishara chan->idx, intr_cause); 875ff7b0479SSaeed Bishara 876ff7b0479SSaeed Bishara mv_dump_xor_regs(chan); 877ff7b0479SSaeed Bishara BUG(); 878ff7b0479SSaeed Bishara } 879ff7b0479SSaeed Bishara 880ff7b0479SSaeed Bishara static irqreturn_t mv_xor_interrupt_handler(int irq, void *data) 881ff7b0479SSaeed Bishara { 882ff7b0479SSaeed Bishara struct mv_xor_chan *chan = data; 883ff7b0479SSaeed Bishara u32 intr_cause = mv_chan_get_intr_cause(chan); 884ff7b0479SSaeed Bishara 885ff7b0479SSaeed Bishara dev_dbg(chan->device->common.dev, "intr cause %x\n", intr_cause); 886ff7b0479SSaeed Bishara 887ff7b0479SSaeed Bishara if (mv_is_err_intr(intr_cause)) 888ff7b0479SSaeed Bishara mv_xor_err_interrupt_handler(chan, intr_cause); 889ff7b0479SSaeed Bishara 890ff7b0479SSaeed Bishara tasklet_schedule(&chan->irq_tasklet); 891ff7b0479SSaeed Bishara 892ff7b0479SSaeed Bishara mv_xor_device_clear_eoc_cause(chan); 893ff7b0479SSaeed Bishara 894ff7b0479SSaeed Bishara return IRQ_HANDLED; 895ff7b0479SSaeed Bishara } 896ff7b0479SSaeed Bishara 897ff7b0479SSaeed Bishara static void mv_xor_issue_pending(struct dma_chan *chan) 898ff7b0479SSaeed Bishara { 899ff7b0479SSaeed Bishara struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan); 900ff7b0479SSaeed Bishara 901ff7b0479SSaeed Bishara if (mv_chan->pending >= MV_XOR_THRESHOLD) { 902ff7b0479SSaeed Bishara mv_chan->pending = 0; 903ff7b0479SSaeed Bishara mv_chan_activate(mv_chan); 904ff7b0479SSaeed Bishara } 905ff7b0479SSaeed Bishara } 906ff7b0479SSaeed Bishara 907ff7b0479SSaeed Bishara /* 908ff7b0479SSaeed Bishara * Perform a transaction to verify the HW works. 909ff7b0479SSaeed Bishara */ 910ff7b0479SSaeed Bishara #define MV_XOR_TEST_SIZE 2000 911ff7b0479SSaeed Bishara 912ff7b0479SSaeed Bishara static int __devinit mv_xor_memcpy_self_test(struct mv_xor_device *device) 913ff7b0479SSaeed Bishara { 914ff7b0479SSaeed Bishara int i; 915ff7b0479SSaeed Bishara void *src, *dest; 916ff7b0479SSaeed Bishara dma_addr_t src_dma, dest_dma; 917ff7b0479SSaeed Bishara struct dma_chan *dma_chan; 918ff7b0479SSaeed Bishara dma_cookie_t cookie; 919ff7b0479SSaeed Bishara struct dma_async_tx_descriptor *tx; 920ff7b0479SSaeed Bishara int err = 0; 921ff7b0479SSaeed Bishara struct mv_xor_chan *mv_chan; 922ff7b0479SSaeed Bishara 923ff7b0479SSaeed Bishara src = kmalloc(sizeof(u8) * MV_XOR_TEST_SIZE, GFP_KERNEL); 924ff7b0479SSaeed Bishara if (!src) 925ff7b0479SSaeed Bishara return -ENOMEM; 926ff7b0479SSaeed Bishara 927ff7b0479SSaeed Bishara dest = kzalloc(sizeof(u8) * MV_XOR_TEST_SIZE, GFP_KERNEL); 928ff7b0479SSaeed Bishara if (!dest) { 929ff7b0479SSaeed Bishara kfree(src); 930ff7b0479SSaeed Bishara return -ENOMEM; 931ff7b0479SSaeed Bishara } 932ff7b0479SSaeed Bishara 933ff7b0479SSaeed Bishara /* Fill in src buffer */ 934ff7b0479SSaeed Bishara for (i = 0; i < MV_XOR_TEST_SIZE; i++) 935ff7b0479SSaeed Bishara ((u8 *) src)[i] = (u8)i; 936ff7b0479SSaeed Bishara 937ff7b0479SSaeed Bishara /* Start copy, using first DMA channel */ 938ff7b0479SSaeed Bishara dma_chan = container_of(device->common.channels.next, 939ff7b0479SSaeed Bishara struct dma_chan, 940ff7b0479SSaeed Bishara device_node); 941ff7b0479SSaeed Bishara if (mv_xor_alloc_chan_resources(dma_chan) < 1) { 942ff7b0479SSaeed Bishara err = -ENODEV; 943ff7b0479SSaeed Bishara goto out; 944ff7b0479SSaeed Bishara } 945ff7b0479SSaeed Bishara 946ff7b0479SSaeed Bishara dest_dma = dma_map_single(dma_chan->device->dev, dest, 947ff7b0479SSaeed Bishara MV_XOR_TEST_SIZE, DMA_FROM_DEVICE); 948ff7b0479SSaeed Bishara 949ff7b0479SSaeed Bishara src_dma = dma_map_single(dma_chan->device->dev, src, 950ff7b0479SSaeed Bishara MV_XOR_TEST_SIZE, DMA_TO_DEVICE); 951ff7b0479SSaeed Bishara 952ff7b0479SSaeed Bishara tx = mv_xor_prep_dma_memcpy(dma_chan, dest_dma, src_dma, 953ff7b0479SSaeed Bishara MV_XOR_TEST_SIZE, 0); 954ff7b0479SSaeed Bishara cookie = mv_xor_tx_submit(tx); 955ff7b0479SSaeed Bishara mv_xor_issue_pending(dma_chan); 956ff7b0479SSaeed Bishara async_tx_ack(tx); 957ff7b0479SSaeed Bishara msleep(1); 958ff7b0479SSaeed Bishara 959ff7b0479SSaeed Bishara if (mv_xor_is_complete(dma_chan, cookie, NULL, NULL) != 960ff7b0479SSaeed Bishara DMA_SUCCESS) { 961ff7b0479SSaeed Bishara dev_printk(KERN_ERR, dma_chan->device->dev, 962ff7b0479SSaeed Bishara "Self-test copy timed out, disabling\n"); 963ff7b0479SSaeed Bishara err = -ENODEV; 964ff7b0479SSaeed Bishara goto free_resources; 965ff7b0479SSaeed Bishara } 966ff7b0479SSaeed Bishara 967ff7b0479SSaeed Bishara mv_chan = to_mv_xor_chan(dma_chan); 968ff7b0479SSaeed Bishara dma_sync_single_for_cpu(&mv_chan->device->pdev->dev, dest_dma, 969ff7b0479SSaeed Bishara MV_XOR_TEST_SIZE, DMA_FROM_DEVICE); 970ff7b0479SSaeed Bishara if (memcmp(src, dest, MV_XOR_TEST_SIZE)) { 971ff7b0479SSaeed Bishara dev_printk(KERN_ERR, dma_chan->device->dev, 972ff7b0479SSaeed Bishara "Self-test copy failed compare, disabling\n"); 973ff7b0479SSaeed Bishara err = -ENODEV; 974ff7b0479SSaeed Bishara goto free_resources; 975ff7b0479SSaeed Bishara } 976ff7b0479SSaeed Bishara 977ff7b0479SSaeed Bishara free_resources: 978ff7b0479SSaeed Bishara mv_xor_free_chan_resources(dma_chan); 979ff7b0479SSaeed Bishara out: 980ff7b0479SSaeed Bishara kfree(src); 981ff7b0479SSaeed Bishara kfree(dest); 982ff7b0479SSaeed Bishara return err; 983ff7b0479SSaeed Bishara } 984ff7b0479SSaeed Bishara 985ff7b0479SSaeed Bishara #define MV_XOR_NUM_SRC_TEST 4 /* must be <= 15 */ 986ff7b0479SSaeed Bishara static int __devinit 987ff7b0479SSaeed Bishara mv_xor_xor_self_test(struct mv_xor_device *device) 988ff7b0479SSaeed Bishara { 989ff7b0479SSaeed Bishara int i, src_idx; 990ff7b0479SSaeed Bishara struct page *dest; 991ff7b0479SSaeed Bishara struct page *xor_srcs[MV_XOR_NUM_SRC_TEST]; 992ff7b0479SSaeed Bishara dma_addr_t dma_srcs[MV_XOR_NUM_SRC_TEST]; 993ff7b0479SSaeed Bishara dma_addr_t dest_dma; 994ff7b0479SSaeed Bishara struct dma_async_tx_descriptor *tx; 995ff7b0479SSaeed Bishara struct dma_chan *dma_chan; 996ff7b0479SSaeed Bishara dma_cookie_t cookie; 997ff7b0479SSaeed Bishara u8 cmp_byte = 0; 998ff7b0479SSaeed Bishara u32 cmp_word; 999ff7b0479SSaeed Bishara int err = 0; 1000ff7b0479SSaeed Bishara struct mv_xor_chan *mv_chan; 1001ff7b0479SSaeed Bishara 1002ff7b0479SSaeed Bishara for (src_idx = 0; src_idx < MV_XOR_NUM_SRC_TEST; src_idx++) { 1003ff7b0479SSaeed Bishara xor_srcs[src_idx] = alloc_page(GFP_KERNEL); 1004ff7b0479SSaeed Bishara if (!xor_srcs[src_idx]) 1005ff7b0479SSaeed Bishara while (src_idx--) { 1006ff7b0479SSaeed Bishara __free_page(xor_srcs[src_idx]); 1007ff7b0479SSaeed Bishara return -ENOMEM; 1008ff7b0479SSaeed Bishara } 1009ff7b0479SSaeed Bishara } 1010ff7b0479SSaeed Bishara 1011ff7b0479SSaeed Bishara dest = alloc_page(GFP_KERNEL); 1012ff7b0479SSaeed Bishara if (!dest) 1013ff7b0479SSaeed Bishara while (src_idx--) { 1014ff7b0479SSaeed Bishara __free_page(xor_srcs[src_idx]); 1015ff7b0479SSaeed Bishara return -ENOMEM; 1016ff7b0479SSaeed Bishara } 1017ff7b0479SSaeed Bishara 1018ff7b0479SSaeed Bishara /* Fill in src buffers */ 1019ff7b0479SSaeed Bishara for (src_idx = 0; src_idx < MV_XOR_NUM_SRC_TEST; src_idx++) { 1020ff7b0479SSaeed Bishara u8 *ptr = page_address(xor_srcs[src_idx]); 1021ff7b0479SSaeed Bishara for (i = 0; i < PAGE_SIZE; i++) 1022ff7b0479SSaeed Bishara ptr[i] = (1 << src_idx); 1023ff7b0479SSaeed Bishara } 1024ff7b0479SSaeed Bishara 1025ff7b0479SSaeed Bishara for (src_idx = 0; src_idx < MV_XOR_NUM_SRC_TEST; src_idx++) 1026ff7b0479SSaeed Bishara cmp_byte ^= (u8) (1 << src_idx); 1027ff7b0479SSaeed Bishara 1028ff7b0479SSaeed Bishara cmp_word = (cmp_byte << 24) | (cmp_byte << 16) | 1029ff7b0479SSaeed Bishara (cmp_byte << 8) | cmp_byte; 1030ff7b0479SSaeed Bishara 1031ff7b0479SSaeed Bishara memset(page_address(dest), 0, PAGE_SIZE); 1032ff7b0479SSaeed Bishara 1033ff7b0479SSaeed Bishara dma_chan = container_of(device->common.channels.next, 1034ff7b0479SSaeed Bishara struct dma_chan, 1035ff7b0479SSaeed Bishara device_node); 1036ff7b0479SSaeed Bishara if (mv_xor_alloc_chan_resources(dma_chan) < 1) { 1037ff7b0479SSaeed Bishara err = -ENODEV; 1038ff7b0479SSaeed Bishara goto out; 1039ff7b0479SSaeed Bishara } 1040ff7b0479SSaeed Bishara 1041ff7b0479SSaeed Bishara /* test xor */ 1042ff7b0479SSaeed Bishara dest_dma = dma_map_page(dma_chan->device->dev, dest, 0, PAGE_SIZE, 1043ff7b0479SSaeed Bishara DMA_FROM_DEVICE); 1044ff7b0479SSaeed Bishara 1045ff7b0479SSaeed Bishara for (i = 0; i < MV_XOR_NUM_SRC_TEST; i++) 1046ff7b0479SSaeed Bishara dma_srcs[i] = dma_map_page(dma_chan->device->dev, xor_srcs[i], 1047ff7b0479SSaeed Bishara 0, PAGE_SIZE, DMA_TO_DEVICE); 1048ff7b0479SSaeed Bishara 1049ff7b0479SSaeed Bishara tx = mv_xor_prep_dma_xor(dma_chan, dest_dma, dma_srcs, 1050ff7b0479SSaeed Bishara MV_XOR_NUM_SRC_TEST, PAGE_SIZE, 0); 1051ff7b0479SSaeed Bishara 1052ff7b0479SSaeed Bishara cookie = mv_xor_tx_submit(tx); 1053ff7b0479SSaeed Bishara mv_xor_issue_pending(dma_chan); 1054ff7b0479SSaeed Bishara async_tx_ack(tx); 1055ff7b0479SSaeed Bishara msleep(8); 1056ff7b0479SSaeed Bishara 1057ff7b0479SSaeed Bishara if (mv_xor_is_complete(dma_chan, cookie, NULL, NULL) != 1058ff7b0479SSaeed Bishara DMA_SUCCESS) { 1059ff7b0479SSaeed Bishara dev_printk(KERN_ERR, dma_chan->device->dev, 1060ff7b0479SSaeed Bishara "Self-test xor timed out, disabling\n"); 1061ff7b0479SSaeed Bishara err = -ENODEV; 1062ff7b0479SSaeed Bishara goto free_resources; 1063ff7b0479SSaeed Bishara } 1064ff7b0479SSaeed Bishara 1065ff7b0479SSaeed Bishara mv_chan = to_mv_xor_chan(dma_chan); 1066ff7b0479SSaeed Bishara dma_sync_single_for_cpu(&mv_chan->device->pdev->dev, dest_dma, 1067ff7b0479SSaeed Bishara PAGE_SIZE, DMA_FROM_DEVICE); 1068ff7b0479SSaeed Bishara for (i = 0; i < (PAGE_SIZE / sizeof(u32)); i++) { 1069ff7b0479SSaeed Bishara u32 *ptr = page_address(dest); 1070ff7b0479SSaeed Bishara if (ptr[i] != cmp_word) { 1071ff7b0479SSaeed Bishara dev_printk(KERN_ERR, dma_chan->device->dev, 1072ff7b0479SSaeed Bishara "Self-test xor failed compare, disabling." 1073ff7b0479SSaeed Bishara " index %d, data %x, expected %x\n", i, 1074ff7b0479SSaeed Bishara ptr[i], cmp_word); 1075ff7b0479SSaeed Bishara err = -ENODEV; 1076ff7b0479SSaeed Bishara goto free_resources; 1077ff7b0479SSaeed Bishara } 1078ff7b0479SSaeed Bishara } 1079ff7b0479SSaeed Bishara 1080ff7b0479SSaeed Bishara free_resources: 1081ff7b0479SSaeed Bishara mv_xor_free_chan_resources(dma_chan); 1082ff7b0479SSaeed Bishara out: 1083ff7b0479SSaeed Bishara src_idx = MV_XOR_NUM_SRC_TEST; 1084ff7b0479SSaeed Bishara while (src_idx--) 1085ff7b0479SSaeed Bishara __free_page(xor_srcs[src_idx]); 1086ff7b0479SSaeed Bishara __free_page(dest); 1087ff7b0479SSaeed Bishara return err; 1088ff7b0479SSaeed Bishara } 1089ff7b0479SSaeed Bishara 1090ff7b0479SSaeed Bishara static int __devexit mv_xor_remove(struct platform_device *dev) 1091ff7b0479SSaeed Bishara { 1092ff7b0479SSaeed Bishara struct mv_xor_device *device = platform_get_drvdata(dev); 1093ff7b0479SSaeed Bishara struct dma_chan *chan, *_chan; 1094ff7b0479SSaeed Bishara struct mv_xor_chan *mv_chan; 1095ff7b0479SSaeed Bishara struct mv_xor_platform_data *plat_data = dev->dev.platform_data; 1096ff7b0479SSaeed Bishara 1097ff7b0479SSaeed Bishara dma_async_device_unregister(&device->common); 1098ff7b0479SSaeed Bishara 1099ff7b0479SSaeed Bishara dma_free_coherent(&dev->dev, plat_data->pool_size, 1100ff7b0479SSaeed Bishara device->dma_desc_pool_virt, device->dma_desc_pool); 1101ff7b0479SSaeed Bishara 1102ff7b0479SSaeed Bishara list_for_each_entry_safe(chan, _chan, &device->common.channels, 1103ff7b0479SSaeed Bishara device_node) { 1104ff7b0479SSaeed Bishara mv_chan = to_mv_xor_chan(chan); 1105ff7b0479SSaeed Bishara list_del(&chan->device_node); 1106ff7b0479SSaeed Bishara } 1107ff7b0479SSaeed Bishara 1108ff7b0479SSaeed Bishara return 0; 1109ff7b0479SSaeed Bishara } 1110ff7b0479SSaeed Bishara 1111ff7b0479SSaeed Bishara static int __devinit mv_xor_probe(struct platform_device *pdev) 1112ff7b0479SSaeed Bishara { 1113ff7b0479SSaeed Bishara int ret = 0; 1114ff7b0479SSaeed Bishara int irq; 1115ff7b0479SSaeed Bishara struct mv_xor_device *adev; 1116ff7b0479SSaeed Bishara struct mv_xor_chan *mv_chan; 1117ff7b0479SSaeed Bishara struct dma_device *dma_dev; 1118ff7b0479SSaeed Bishara struct mv_xor_platform_data *plat_data = pdev->dev.platform_data; 1119ff7b0479SSaeed Bishara 1120ff7b0479SSaeed Bishara 1121ff7b0479SSaeed Bishara adev = devm_kzalloc(&pdev->dev, sizeof(*adev), GFP_KERNEL); 1122ff7b0479SSaeed Bishara if (!adev) 1123ff7b0479SSaeed Bishara return -ENOMEM; 1124ff7b0479SSaeed Bishara 1125ff7b0479SSaeed Bishara dma_dev = &adev->common; 1126ff7b0479SSaeed Bishara 1127ff7b0479SSaeed Bishara /* allocate coherent memory for hardware descriptors 1128ff7b0479SSaeed Bishara * note: writecombine gives slightly better performance, but 1129ff7b0479SSaeed Bishara * requires that we explicitly flush the writes 1130ff7b0479SSaeed Bishara */ 1131ff7b0479SSaeed Bishara adev->dma_desc_pool_virt = dma_alloc_writecombine(&pdev->dev, 1132ff7b0479SSaeed Bishara plat_data->pool_size, 1133ff7b0479SSaeed Bishara &adev->dma_desc_pool, 1134ff7b0479SSaeed Bishara GFP_KERNEL); 1135ff7b0479SSaeed Bishara if (!adev->dma_desc_pool_virt) 1136ff7b0479SSaeed Bishara return -ENOMEM; 1137ff7b0479SSaeed Bishara 1138ff7b0479SSaeed Bishara adev->id = plat_data->hw_id; 1139ff7b0479SSaeed Bishara 1140ff7b0479SSaeed Bishara /* discover transaction capabilites from the platform data */ 1141ff7b0479SSaeed Bishara dma_dev->cap_mask = plat_data->cap_mask; 1142ff7b0479SSaeed Bishara adev->pdev = pdev; 1143ff7b0479SSaeed Bishara platform_set_drvdata(pdev, adev); 1144ff7b0479SSaeed Bishara 1145ff7b0479SSaeed Bishara adev->shared = platform_get_drvdata(plat_data->shared); 1146ff7b0479SSaeed Bishara 1147ff7b0479SSaeed Bishara INIT_LIST_HEAD(&dma_dev->channels); 1148ff7b0479SSaeed Bishara 1149ff7b0479SSaeed Bishara /* set base routines */ 1150ff7b0479SSaeed Bishara dma_dev->device_alloc_chan_resources = mv_xor_alloc_chan_resources; 1151ff7b0479SSaeed Bishara dma_dev->device_free_chan_resources = mv_xor_free_chan_resources; 1152ff7b0479SSaeed Bishara dma_dev->device_is_tx_complete = mv_xor_is_complete; 1153ff7b0479SSaeed Bishara dma_dev->device_issue_pending = mv_xor_issue_pending; 1154ff7b0479SSaeed Bishara dma_dev->dev = &pdev->dev; 1155ff7b0479SSaeed Bishara 1156ff7b0479SSaeed Bishara /* set prep routines based on capability */ 1157ff7b0479SSaeed Bishara if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) 1158ff7b0479SSaeed Bishara dma_dev->device_prep_dma_memcpy = mv_xor_prep_dma_memcpy; 1159ff7b0479SSaeed Bishara if (dma_has_cap(DMA_MEMSET, dma_dev->cap_mask)) 1160ff7b0479SSaeed Bishara dma_dev->device_prep_dma_memset = mv_xor_prep_dma_memset; 1161ff7b0479SSaeed Bishara if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { 1162ff7b0479SSaeed Bishara dma_dev->max_xor = 8; ; 1163ff7b0479SSaeed Bishara dma_dev->device_prep_dma_xor = mv_xor_prep_dma_xor; 1164ff7b0479SSaeed Bishara } 1165ff7b0479SSaeed Bishara 1166ff7b0479SSaeed Bishara mv_chan = devm_kzalloc(&pdev->dev, sizeof(*mv_chan), GFP_KERNEL); 1167ff7b0479SSaeed Bishara if (!mv_chan) { 1168ff7b0479SSaeed Bishara ret = -ENOMEM; 1169ff7b0479SSaeed Bishara goto err_free_dma; 1170ff7b0479SSaeed Bishara } 1171ff7b0479SSaeed Bishara mv_chan->device = adev; 1172ff7b0479SSaeed Bishara mv_chan->idx = plat_data->hw_id; 1173ff7b0479SSaeed Bishara mv_chan->mmr_base = adev->shared->xor_base; 1174ff7b0479SSaeed Bishara 1175ff7b0479SSaeed Bishara if (!mv_chan->mmr_base) { 1176ff7b0479SSaeed Bishara ret = -ENOMEM; 1177ff7b0479SSaeed Bishara goto err_free_dma; 1178ff7b0479SSaeed Bishara } 1179ff7b0479SSaeed Bishara tasklet_init(&mv_chan->irq_tasklet, mv_xor_tasklet, (unsigned long) 1180ff7b0479SSaeed Bishara mv_chan); 1181ff7b0479SSaeed Bishara 1182ff7b0479SSaeed Bishara /* clear errors before enabling interrupts */ 1183ff7b0479SSaeed Bishara mv_xor_device_clear_err_status(mv_chan); 1184ff7b0479SSaeed Bishara 1185ff7b0479SSaeed Bishara irq = platform_get_irq(pdev, 0); 1186ff7b0479SSaeed Bishara if (irq < 0) { 1187ff7b0479SSaeed Bishara ret = irq; 1188ff7b0479SSaeed Bishara goto err_free_dma; 1189ff7b0479SSaeed Bishara } 1190ff7b0479SSaeed Bishara ret = devm_request_irq(&pdev->dev, irq, 1191ff7b0479SSaeed Bishara mv_xor_interrupt_handler, 1192ff7b0479SSaeed Bishara 0, dev_name(&pdev->dev), mv_chan); 1193ff7b0479SSaeed Bishara if (ret) 1194ff7b0479SSaeed Bishara goto err_free_dma; 1195ff7b0479SSaeed Bishara 1196ff7b0479SSaeed Bishara mv_chan_unmask_interrupts(mv_chan); 1197ff7b0479SSaeed Bishara 1198ff7b0479SSaeed Bishara mv_set_mode(mv_chan, DMA_MEMCPY); 1199ff7b0479SSaeed Bishara 1200ff7b0479SSaeed Bishara spin_lock_init(&mv_chan->lock); 1201ff7b0479SSaeed Bishara INIT_LIST_HEAD(&mv_chan->chain); 1202ff7b0479SSaeed Bishara INIT_LIST_HEAD(&mv_chan->completed_slots); 1203ff7b0479SSaeed Bishara INIT_LIST_HEAD(&mv_chan->all_slots); 1204ff7b0479SSaeed Bishara INIT_RCU_HEAD(&mv_chan->common.rcu); 1205ff7b0479SSaeed Bishara mv_chan->common.device = dma_dev; 1206ff7b0479SSaeed Bishara 1207ff7b0479SSaeed Bishara list_add_tail(&mv_chan->common.device_node, &dma_dev->channels); 1208ff7b0479SSaeed Bishara 1209ff7b0479SSaeed Bishara if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) { 1210ff7b0479SSaeed Bishara ret = mv_xor_memcpy_self_test(adev); 1211ff7b0479SSaeed Bishara dev_dbg(&pdev->dev, "memcpy self test returned %d\n", ret); 1212ff7b0479SSaeed Bishara if (ret) 1213ff7b0479SSaeed Bishara goto err_free_dma; 1214ff7b0479SSaeed Bishara } 1215ff7b0479SSaeed Bishara 1216ff7b0479SSaeed Bishara if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { 1217ff7b0479SSaeed Bishara ret = mv_xor_xor_self_test(adev); 1218ff7b0479SSaeed Bishara dev_dbg(&pdev->dev, "xor self test returned %d\n", ret); 1219ff7b0479SSaeed Bishara if (ret) 1220ff7b0479SSaeed Bishara goto err_free_dma; 1221ff7b0479SSaeed Bishara } 1222ff7b0479SSaeed Bishara 1223ff7b0479SSaeed Bishara dev_printk(KERN_INFO, &pdev->dev, "Marvell XOR: " 1224ff7b0479SSaeed Bishara "( %s%s%s%s)\n", 1225ff7b0479SSaeed Bishara dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "", 1226ff7b0479SSaeed Bishara dma_has_cap(DMA_MEMSET, dma_dev->cap_mask) ? "fill " : "", 1227ff7b0479SSaeed Bishara dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "cpy " : "", 1228ff7b0479SSaeed Bishara dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "intr " : ""); 1229ff7b0479SSaeed Bishara 1230ff7b0479SSaeed Bishara dma_async_device_register(dma_dev); 1231ff7b0479SSaeed Bishara goto out; 1232ff7b0479SSaeed Bishara 1233ff7b0479SSaeed Bishara err_free_dma: 1234ff7b0479SSaeed Bishara dma_free_coherent(&adev->pdev->dev, plat_data->pool_size, 1235ff7b0479SSaeed Bishara adev->dma_desc_pool_virt, adev->dma_desc_pool); 1236ff7b0479SSaeed Bishara out: 1237ff7b0479SSaeed Bishara return ret; 1238ff7b0479SSaeed Bishara } 1239ff7b0479SSaeed Bishara 1240ff7b0479SSaeed Bishara static void 1241ff7b0479SSaeed Bishara mv_xor_conf_mbus_windows(struct mv_xor_shared_private *msp, 1242ff7b0479SSaeed Bishara struct mbus_dram_target_info *dram) 1243ff7b0479SSaeed Bishara { 1244ff7b0479SSaeed Bishara void __iomem *base = msp->xor_base; 1245ff7b0479SSaeed Bishara u32 win_enable = 0; 1246ff7b0479SSaeed Bishara int i; 1247ff7b0479SSaeed Bishara 1248ff7b0479SSaeed Bishara for (i = 0; i < 8; i++) { 1249ff7b0479SSaeed Bishara writel(0, base + WINDOW_BASE(i)); 1250ff7b0479SSaeed Bishara writel(0, base + WINDOW_SIZE(i)); 1251ff7b0479SSaeed Bishara if (i < 4) 1252ff7b0479SSaeed Bishara writel(0, base + WINDOW_REMAP_HIGH(i)); 1253ff7b0479SSaeed Bishara } 1254ff7b0479SSaeed Bishara 1255ff7b0479SSaeed Bishara for (i = 0; i < dram->num_cs; i++) { 1256ff7b0479SSaeed Bishara struct mbus_dram_window *cs = dram->cs + i; 1257ff7b0479SSaeed Bishara 1258ff7b0479SSaeed Bishara writel((cs->base & 0xffff0000) | 1259ff7b0479SSaeed Bishara (cs->mbus_attr << 8) | 1260ff7b0479SSaeed Bishara dram->mbus_dram_target_id, base + WINDOW_BASE(i)); 1261ff7b0479SSaeed Bishara writel((cs->size - 1) & 0xffff0000, base + WINDOW_SIZE(i)); 1262ff7b0479SSaeed Bishara 1263ff7b0479SSaeed Bishara win_enable |= (1 << i); 1264ff7b0479SSaeed Bishara win_enable |= 3 << (16 + (2 * i)); 1265ff7b0479SSaeed Bishara } 1266ff7b0479SSaeed Bishara 1267ff7b0479SSaeed Bishara writel(win_enable, base + WINDOW_BAR_ENABLE(0)); 1268ff7b0479SSaeed Bishara writel(win_enable, base + WINDOW_BAR_ENABLE(1)); 1269ff7b0479SSaeed Bishara } 1270ff7b0479SSaeed Bishara 1271ff7b0479SSaeed Bishara static struct platform_driver mv_xor_driver = { 1272ff7b0479SSaeed Bishara .probe = mv_xor_probe, 1273ff7b0479SSaeed Bishara .remove = mv_xor_remove, 1274ff7b0479SSaeed Bishara .driver = { 1275ff7b0479SSaeed Bishara .owner = THIS_MODULE, 1276ff7b0479SSaeed Bishara .name = MV_XOR_NAME, 1277ff7b0479SSaeed Bishara }, 1278ff7b0479SSaeed Bishara }; 1279ff7b0479SSaeed Bishara 1280ff7b0479SSaeed Bishara static int mv_xor_shared_probe(struct platform_device *pdev) 1281ff7b0479SSaeed Bishara { 1282ff7b0479SSaeed Bishara struct mv_xor_platform_shared_data *msd = pdev->dev.platform_data; 1283ff7b0479SSaeed Bishara struct mv_xor_shared_private *msp; 1284ff7b0479SSaeed Bishara struct resource *res; 1285ff7b0479SSaeed Bishara 1286ff7b0479SSaeed Bishara dev_printk(KERN_NOTICE, &pdev->dev, "Marvell shared XOR driver\n"); 1287ff7b0479SSaeed Bishara 1288ff7b0479SSaeed Bishara msp = devm_kzalloc(&pdev->dev, sizeof(*msp), GFP_KERNEL); 1289ff7b0479SSaeed Bishara if (!msp) 1290ff7b0479SSaeed Bishara return -ENOMEM; 1291ff7b0479SSaeed Bishara 1292ff7b0479SSaeed Bishara res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1293ff7b0479SSaeed Bishara if (!res) 1294ff7b0479SSaeed Bishara return -ENODEV; 1295ff7b0479SSaeed Bishara 1296ff7b0479SSaeed Bishara msp->xor_base = devm_ioremap(&pdev->dev, res->start, 1297ff7b0479SSaeed Bishara res->end - res->start + 1); 1298ff7b0479SSaeed Bishara if (!msp->xor_base) 1299ff7b0479SSaeed Bishara return -EBUSY; 1300ff7b0479SSaeed Bishara 1301ff7b0479SSaeed Bishara res = platform_get_resource(pdev, IORESOURCE_MEM, 1); 1302ff7b0479SSaeed Bishara if (!res) 1303ff7b0479SSaeed Bishara return -ENODEV; 1304ff7b0479SSaeed Bishara 1305ff7b0479SSaeed Bishara msp->xor_high_base = devm_ioremap(&pdev->dev, res->start, 1306ff7b0479SSaeed Bishara res->end - res->start + 1); 1307ff7b0479SSaeed Bishara if (!msp->xor_high_base) 1308ff7b0479SSaeed Bishara return -EBUSY; 1309ff7b0479SSaeed Bishara 1310ff7b0479SSaeed Bishara platform_set_drvdata(pdev, msp); 1311ff7b0479SSaeed Bishara 1312ff7b0479SSaeed Bishara /* 1313ff7b0479SSaeed Bishara * (Re-)program MBUS remapping windows if we are asked to. 1314ff7b0479SSaeed Bishara */ 1315ff7b0479SSaeed Bishara if (msd != NULL && msd->dram != NULL) 1316ff7b0479SSaeed Bishara mv_xor_conf_mbus_windows(msp, msd->dram); 1317ff7b0479SSaeed Bishara 1318ff7b0479SSaeed Bishara return 0; 1319ff7b0479SSaeed Bishara } 1320ff7b0479SSaeed Bishara 1321ff7b0479SSaeed Bishara static int mv_xor_shared_remove(struct platform_device *pdev) 1322ff7b0479SSaeed Bishara { 1323ff7b0479SSaeed Bishara return 0; 1324ff7b0479SSaeed Bishara } 1325ff7b0479SSaeed Bishara 1326ff7b0479SSaeed Bishara static struct platform_driver mv_xor_shared_driver = { 1327ff7b0479SSaeed Bishara .probe = mv_xor_shared_probe, 1328ff7b0479SSaeed Bishara .remove = mv_xor_shared_remove, 1329ff7b0479SSaeed Bishara .driver = { 1330ff7b0479SSaeed Bishara .owner = THIS_MODULE, 1331ff7b0479SSaeed Bishara .name = MV_XOR_SHARED_NAME, 1332ff7b0479SSaeed Bishara }, 1333ff7b0479SSaeed Bishara }; 1334ff7b0479SSaeed Bishara 1335ff7b0479SSaeed Bishara 1336ff7b0479SSaeed Bishara static int __init mv_xor_init(void) 1337ff7b0479SSaeed Bishara { 1338ff7b0479SSaeed Bishara int rc; 1339ff7b0479SSaeed Bishara 1340ff7b0479SSaeed Bishara rc = platform_driver_register(&mv_xor_shared_driver); 1341ff7b0479SSaeed Bishara if (!rc) { 1342ff7b0479SSaeed Bishara rc = platform_driver_register(&mv_xor_driver); 1343ff7b0479SSaeed Bishara if (rc) 1344ff7b0479SSaeed Bishara platform_driver_unregister(&mv_xor_shared_driver); 1345ff7b0479SSaeed Bishara } 1346ff7b0479SSaeed Bishara return rc; 1347ff7b0479SSaeed Bishara } 1348ff7b0479SSaeed Bishara module_init(mv_xor_init); 1349ff7b0479SSaeed Bishara 1350ff7b0479SSaeed Bishara /* it's currently unsafe to unload this module */ 1351ff7b0479SSaeed Bishara #if 0 1352ff7b0479SSaeed Bishara static void __exit mv_xor_exit(void) 1353ff7b0479SSaeed Bishara { 1354ff7b0479SSaeed Bishara platform_driver_unregister(&mv_xor_driver); 1355ff7b0479SSaeed Bishara platform_driver_unregister(&mv_xor_shared_driver); 1356ff7b0479SSaeed Bishara return; 1357ff7b0479SSaeed Bishara } 1358ff7b0479SSaeed Bishara 1359ff7b0479SSaeed Bishara module_exit(mv_xor_exit); 1360ff7b0479SSaeed Bishara #endif 1361ff7b0479SSaeed Bishara 1362ff7b0479SSaeed Bishara MODULE_AUTHOR("Saeed Bishara <saeed@marvell.com>"); 1363ff7b0479SSaeed Bishara MODULE_DESCRIPTION("DMA engine driver for Marvell's XOR engine"); 1364ff7b0479SSaeed Bishara MODULE_LICENSE("GPL"); 1365