1 /* 2 * Keystone accumulator queue manager 3 * 4 * Copyright (C) 2014 Texas Instruments Incorporated - http://www.ti.com 5 * Author: Sandeep Nair <sandeep_n@ti.com> 6 * Cyril Chemparathy <cyril@ti.com> 7 * Santosh Shilimkar <santosh.shilimkar@ti.com> 8 * 9 * This program is free software; you can redistribute it and/or 10 * modify it under the terms of the GNU General Public License 11 * version 2 as published by the Free Software Foundation. 12 * 13 * This program is distributed in the hope that it will be useful, but 14 * WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16 * General Public License for more details. 17 */ 18 19 #include <linux/dma-mapping.h> 20 #include <linux/io.h> 21 #include <linux/interrupt.h> 22 #include <linux/module.h> 23 #include <linux/of_address.h> 24 #include <linux/soc/ti/knav_qmss.h> 25 26 #include "knav_qmss.h" 27 28 #define knav_range_offset_to_inst(kdev, range, q) \ 29 (range->queue_base_inst + (q << kdev->inst_shift)) 30 31 static void __knav_acc_notify(struct knav_range_info *range, 32 struct knav_acc_channel *acc) 33 { 34 struct knav_device *kdev = range->kdev; 35 struct knav_queue_inst *inst; 36 int range_base, queue; 37 38 range_base = kdev->base_id + range->queue_base; 39 40 if (range->flags & RANGE_MULTI_QUEUE) { 41 for (queue = 0; queue < range->num_queues; queue++) { 42 inst = knav_range_offset_to_inst(kdev, range, 43 queue); 44 if (inst->notify_needed) { 45 inst->notify_needed = 0; 46 dev_dbg(kdev->dev, "acc-irq: notifying %d\n", 47 range_base + queue); 48 knav_queue_notify(inst); 49 } 50 } 51 } else { 52 queue = acc->channel - range->acc_info.start_channel; 53 inst = knav_range_offset_to_inst(kdev, range, queue); 54 dev_dbg(kdev->dev, "acc-irq: notifying %d\n", 55 range_base + queue); 56 knav_queue_notify(inst); 57 } 58 } 59 60 static int knav_acc_set_notify(struct knav_range_info *range, 61 struct knav_queue_inst *kq, 62 bool enabled) 63 { 64 struct knav_pdsp_info *pdsp = range->acc_info.pdsp; 65 struct knav_device *kdev = range->kdev; 66 u32 mask, offset; 67 68 /* 69 * when enabling, we need to re-trigger an interrupt if we 70 * have descriptors pending 71 */ 72 if (!enabled || atomic_read(&kq->desc_count) <= 0) 73 return 0; 74 75 kq->notify_needed = 1; 76 atomic_inc(&kq->acc->retrigger_count); 77 mask = BIT(kq->acc->channel % 32); 78 offset = ACC_INTD_OFFSET_STATUS(kq->acc->channel); 79 dev_dbg(kdev->dev, "setup-notify: re-triggering irq for %s\n", 80 kq->acc->name); 81 writel_relaxed(mask, pdsp->intd + offset); 82 return 0; 83 } 84 85 static irqreturn_t knav_acc_int_handler(int irq, void *_instdata) 86 { 87 struct knav_acc_channel *acc; 88 struct knav_queue_inst *kq = NULL; 89 struct knav_range_info *range; 90 struct knav_pdsp_info *pdsp; 91 struct knav_acc_info *info; 92 struct knav_device *kdev; 93 94 u32 *list, *list_cpu, val, idx, notifies; 95 int range_base, channel, queue = 0; 96 dma_addr_t list_dma; 97 98 range = _instdata; 99 info = &range->acc_info; 100 kdev = range->kdev; 101 pdsp = range->acc_info.pdsp; 102 acc = range->acc; 103 104 range_base = kdev->base_id + range->queue_base; 105 if ((range->flags & RANGE_MULTI_QUEUE) == 0) { 106 for (queue = 0; queue < range->num_irqs; queue++) 107 if (range->irqs[queue].irq == irq) 108 break; 109 kq = knav_range_offset_to_inst(kdev, range, queue); 110 acc += queue; 111 } 112 113 channel = acc->channel; 114 list_dma = acc->list_dma[acc->list_index]; 115 list_cpu = acc->list_cpu[acc->list_index]; 116 dev_dbg(kdev->dev, "acc-irq: channel %d, list %d, virt %p, dma %pad\n", 117 channel, acc->list_index, list_cpu, &list_dma); 118 if (atomic_read(&acc->retrigger_count)) { 119 atomic_dec(&acc->retrigger_count); 120 __knav_acc_notify(range, acc); 121 writel_relaxed(1, pdsp->intd + ACC_INTD_OFFSET_COUNT(channel)); 122 /* ack the interrupt */ 123 writel_relaxed(ACC_CHANNEL_INT_BASE + channel, 124 pdsp->intd + ACC_INTD_OFFSET_EOI); 125 126 return IRQ_HANDLED; 127 } 128 129 notifies = readl_relaxed(pdsp->intd + ACC_INTD_OFFSET_COUNT(channel)); 130 WARN_ON(!notifies); 131 dma_sync_single_for_cpu(kdev->dev, list_dma, info->list_size, 132 DMA_FROM_DEVICE); 133 134 for (list = list_cpu; list < list_cpu + (info->list_size / sizeof(u32)); 135 list += ACC_LIST_ENTRY_WORDS) { 136 if (ACC_LIST_ENTRY_WORDS == 1) { 137 dev_dbg(kdev->dev, 138 "acc-irq: list %d, entry @%p, %08x\n", 139 acc->list_index, list, list[0]); 140 } else if (ACC_LIST_ENTRY_WORDS == 2) { 141 dev_dbg(kdev->dev, 142 "acc-irq: list %d, entry @%p, %08x %08x\n", 143 acc->list_index, list, list[0], list[1]); 144 } else if (ACC_LIST_ENTRY_WORDS == 4) { 145 dev_dbg(kdev->dev, 146 "acc-irq: list %d, entry @%p, %08x %08x %08x %08x\n", 147 acc->list_index, list, list[0], list[1], 148 list[2], list[3]); 149 } 150 151 val = list[ACC_LIST_ENTRY_DESC_IDX]; 152 if (!val) 153 break; 154 155 if (range->flags & RANGE_MULTI_QUEUE) { 156 queue = list[ACC_LIST_ENTRY_QUEUE_IDX] >> 16; 157 if (queue < range_base || 158 queue >= range_base + range->num_queues) { 159 dev_err(kdev->dev, 160 "bad queue %d, expecting %d-%d\n", 161 queue, range_base, 162 range_base + range->num_queues); 163 break; 164 } 165 queue -= range_base; 166 kq = knav_range_offset_to_inst(kdev, range, 167 queue); 168 } 169 170 if (atomic_inc_return(&kq->desc_count) >= ACC_DESCS_MAX) { 171 atomic_dec(&kq->desc_count); 172 dev_err(kdev->dev, 173 "acc-irq: queue %d full, entry dropped\n", 174 queue + range_base); 175 continue; 176 } 177 178 idx = atomic_inc_return(&kq->desc_tail) & ACC_DESCS_MASK; 179 kq->descs[idx] = val; 180 kq->notify_needed = 1; 181 dev_dbg(kdev->dev, "acc-irq: enqueue %08x at %d, queue %d\n", 182 val, idx, queue + range_base); 183 } 184 185 __knav_acc_notify(range, acc); 186 memset(list_cpu, 0, info->list_size); 187 dma_sync_single_for_device(kdev->dev, list_dma, info->list_size, 188 DMA_TO_DEVICE); 189 190 /* flip to the other list */ 191 acc->list_index ^= 1; 192 193 /* reset the interrupt counter */ 194 writel_relaxed(1, pdsp->intd + ACC_INTD_OFFSET_COUNT(channel)); 195 196 /* ack the interrupt */ 197 writel_relaxed(ACC_CHANNEL_INT_BASE + channel, 198 pdsp->intd + ACC_INTD_OFFSET_EOI); 199 200 return IRQ_HANDLED; 201 } 202 203 static int knav_range_setup_acc_irq(struct knav_range_info *range, 204 int queue, bool enabled) 205 { 206 struct knav_device *kdev = range->kdev; 207 struct knav_acc_channel *acc; 208 unsigned long cpu_map; 209 int ret = 0, irq; 210 u32 old, new; 211 212 if (range->flags & RANGE_MULTI_QUEUE) { 213 acc = range->acc; 214 irq = range->irqs[0].irq; 215 cpu_map = range->irqs[0].cpu_map; 216 } else { 217 acc = range->acc + queue; 218 irq = range->irqs[queue].irq; 219 cpu_map = range->irqs[queue].cpu_map; 220 } 221 222 old = acc->open_mask; 223 if (enabled) 224 new = old | BIT(queue); 225 else 226 new = old & ~BIT(queue); 227 acc->open_mask = new; 228 229 dev_dbg(kdev->dev, 230 "setup-acc-irq: open mask old %08x, new %08x, channel %s\n", 231 old, new, acc->name); 232 233 if (likely(new == old)) 234 return 0; 235 236 if (new && !old) { 237 dev_dbg(kdev->dev, 238 "setup-acc-irq: requesting %s for channel %s\n", 239 acc->name, acc->name); 240 ret = request_irq(irq, knav_acc_int_handler, 0, acc->name, 241 range); 242 if (!ret && cpu_map) { 243 ret = irq_set_affinity_hint(irq, to_cpumask(&cpu_map)); 244 if (ret) { 245 dev_warn(range->kdev->dev, 246 "Failed to set IRQ affinity\n"); 247 return ret; 248 } 249 } 250 } 251 252 if (old && !new) { 253 dev_dbg(kdev->dev, "setup-acc-irq: freeing %s for channel %s\n", 254 acc->name, acc->name); 255 ret = irq_set_affinity_hint(irq, NULL); 256 if (ret) 257 dev_warn(range->kdev->dev, 258 "Failed to set IRQ affinity\n"); 259 free_irq(irq, range); 260 } 261 262 return ret; 263 } 264 265 static const char *knav_acc_result_str(enum knav_acc_result result) 266 { 267 static const char * const result_str[] = { 268 [ACC_RET_IDLE] = "idle", 269 [ACC_RET_SUCCESS] = "success", 270 [ACC_RET_INVALID_COMMAND] = "invalid command", 271 [ACC_RET_INVALID_CHANNEL] = "invalid channel", 272 [ACC_RET_INACTIVE_CHANNEL] = "inactive channel", 273 [ACC_RET_ACTIVE_CHANNEL] = "active channel", 274 [ACC_RET_INVALID_QUEUE] = "invalid queue", 275 [ACC_RET_INVALID_RET] = "invalid return code", 276 }; 277 278 if (result >= ARRAY_SIZE(result_str)) 279 return result_str[ACC_RET_INVALID_RET]; 280 else 281 return result_str[result]; 282 } 283 284 static enum knav_acc_result 285 knav_acc_write(struct knav_device *kdev, struct knav_pdsp_info *pdsp, 286 struct knav_reg_acc_command *cmd) 287 { 288 u32 result; 289 290 dev_dbg(kdev->dev, "acc command %08x %08x %08x %08x %08x\n", 291 cmd->command, cmd->queue_mask, cmd->list_dma, 292 cmd->queue_num, cmd->timer_config); 293 294 writel_relaxed(cmd->timer_config, &pdsp->acc_command->timer_config); 295 writel_relaxed(cmd->queue_num, &pdsp->acc_command->queue_num); 296 writel_relaxed(cmd->list_dma, &pdsp->acc_command->list_dma); 297 writel_relaxed(cmd->queue_mask, &pdsp->acc_command->queue_mask); 298 writel_relaxed(cmd->command, &pdsp->acc_command->command); 299 300 /* wait for the command to clear */ 301 do { 302 result = readl_relaxed(&pdsp->acc_command->command); 303 } while ((result >> 8) & 0xff); 304 305 return (result >> 24) & 0xff; 306 } 307 308 static void knav_acc_setup_cmd(struct knav_device *kdev, 309 struct knav_range_info *range, 310 struct knav_reg_acc_command *cmd, 311 int queue) 312 { 313 struct knav_acc_info *info = &range->acc_info; 314 struct knav_acc_channel *acc; 315 int queue_base; 316 u32 queue_mask; 317 318 if (range->flags & RANGE_MULTI_QUEUE) { 319 acc = range->acc; 320 queue_base = range->queue_base; 321 queue_mask = BIT(range->num_queues) - 1; 322 } else { 323 acc = range->acc + queue; 324 queue_base = range->queue_base + queue; 325 queue_mask = 0; 326 } 327 328 memset(cmd, 0, sizeof(*cmd)); 329 cmd->command = acc->channel; 330 cmd->queue_mask = queue_mask; 331 cmd->list_dma = (u32)acc->list_dma[0]; 332 cmd->queue_num = info->list_entries << 16; 333 cmd->queue_num |= queue_base; 334 335 cmd->timer_config = ACC_LIST_ENTRY_TYPE << 18; 336 if (range->flags & RANGE_MULTI_QUEUE) 337 cmd->timer_config |= ACC_CFG_MULTI_QUEUE; 338 cmd->timer_config |= info->pacing_mode << 16; 339 cmd->timer_config |= info->timer_count; 340 } 341 342 static void knav_acc_stop(struct knav_device *kdev, 343 struct knav_range_info *range, 344 int queue) 345 { 346 struct knav_reg_acc_command cmd; 347 struct knav_acc_channel *acc; 348 enum knav_acc_result result; 349 350 acc = range->acc + queue; 351 352 knav_acc_setup_cmd(kdev, range, &cmd, queue); 353 cmd.command |= ACC_CMD_DISABLE_CHANNEL << 8; 354 result = knav_acc_write(kdev, range->acc_info.pdsp, &cmd); 355 356 dev_dbg(kdev->dev, "stopped acc channel %s, result %s\n", 357 acc->name, knav_acc_result_str(result)); 358 } 359 360 static enum knav_acc_result knav_acc_start(struct knav_device *kdev, 361 struct knav_range_info *range, 362 int queue) 363 { 364 struct knav_reg_acc_command cmd; 365 struct knav_acc_channel *acc; 366 enum knav_acc_result result; 367 368 acc = range->acc + queue; 369 370 knav_acc_setup_cmd(kdev, range, &cmd, queue); 371 cmd.command |= ACC_CMD_ENABLE_CHANNEL << 8; 372 result = knav_acc_write(kdev, range->acc_info.pdsp, &cmd); 373 374 dev_dbg(kdev->dev, "started acc channel %s, result %s\n", 375 acc->name, knav_acc_result_str(result)); 376 377 return result; 378 } 379 380 static int knav_acc_init_range(struct knav_range_info *range) 381 { 382 struct knav_device *kdev = range->kdev; 383 struct knav_acc_channel *acc; 384 enum knav_acc_result result; 385 int queue; 386 387 for (queue = 0; queue < range->num_queues; queue++) { 388 acc = range->acc + queue; 389 390 knav_acc_stop(kdev, range, queue); 391 acc->list_index = 0; 392 result = knav_acc_start(kdev, range, queue); 393 394 if (result != ACC_RET_SUCCESS) 395 return -EIO; 396 397 if (range->flags & RANGE_MULTI_QUEUE) 398 return 0; 399 } 400 return 0; 401 } 402 403 static int knav_acc_init_queue(struct knav_range_info *range, 404 struct knav_queue_inst *kq) 405 { 406 unsigned id = kq->id - range->queue_base; 407 408 kq->descs = devm_kzalloc(range->kdev->dev, 409 ACC_DESCS_MAX * sizeof(u32), GFP_KERNEL); 410 if (!kq->descs) 411 return -ENOMEM; 412 413 kq->acc = range->acc; 414 if ((range->flags & RANGE_MULTI_QUEUE) == 0) 415 kq->acc += id; 416 return 0; 417 } 418 419 static int knav_acc_open_queue(struct knav_range_info *range, 420 struct knav_queue_inst *inst, unsigned flags) 421 { 422 unsigned id = inst->id - range->queue_base; 423 424 return knav_range_setup_acc_irq(range, id, true); 425 } 426 427 static int knav_acc_close_queue(struct knav_range_info *range, 428 struct knav_queue_inst *inst) 429 { 430 unsigned id = inst->id - range->queue_base; 431 432 return knav_range_setup_acc_irq(range, id, false); 433 } 434 435 static int knav_acc_free_range(struct knav_range_info *range) 436 { 437 struct knav_device *kdev = range->kdev; 438 struct knav_acc_channel *acc; 439 struct knav_acc_info *info; 440 int channel, channels; 441 442 info = &range->acc_info; 443 444 if (range->flags & RANGE_MULTI_QUEUE) 445 channels = 1; 446 else 447 channels = range->num_queues; 448 449 for (channel = 0; channel < channels; channel++) { 450 acc = range->acc + channel; 451 if (!acc->list_cpu[0]) 452 continue; 453 dma_unmap_single(kdev->dev, acc->list_dma[0], 454 info->mem_size, DMA_BIDIRECTIONAL); 455 free_pages_exact(acc->list_cpu[0], info->mem_size); 456 } 457 devm_kfree(range->kdev->dev, range->acc); 458 return 0; 459 } 460 461 struct knav_range_ops knav_acc_range_ops = { 462 .set_notify = knav_acc_set_notify, 463 .init_queue = knav_acc_init_queue, 464 .open_queue = knav_acc_open_queue, 465 .close_queue = knav_acc_close_queue, 466 .init_range = knav_acc_init_range, 467 .free_range = knav_acc_free_range, 468 }; 469 470 /** 471 * knav_init_acc_range: Initialise accumulator ranges 472 * 473 * @kdev: qmss device 474 * @node: device node 475 * @range: qmms range information 476 * 477 * Return 0 on success or error 478 */ 479 int knav_init_acc_range(struct knav_device *kdev, 480 struct device_node *node, 481 struct knav_range_info *range) 482 { 483 struct knav_acc_channel *acc; 484 struct knav_pdsp_info *pdsp; 485 struct knav_acc_info *info; 486 int ret, channel, channels; 487 int list_size, mem_size; 488 dma_addr_t list_dma; 489 void *list_mem; 490 u32 config[5]; 491 492 range->flags |= RANGE_HAS_ACCUMULATOR; 493 info = &range->acc_info; 494 495 ret = of_property_read_u32_array(node, "accumulator", config, 5); 496 if (ret) 497 return ret; 498 499 info->pdsp_id = config[0]; 500 info->start_channel = config[1]; 501 info->list_entries = config[2]; 502 info->pacing_mode = config[3]; 503 info->timer_count = config[4] / ACC_DEFAULT_PERIOD; 504 505 if (info->start_channel > ACC_MAX_CHANNEL) { 506 dev_err(kdev->dev, "channel %d invalid for range %s\n", 507 info->start_channel, range->name); 508 return -EINVAL; 509 } 510 511 if (info->pacing_mode > 3) { 512 dev_err(kdev->dev, "pacing mode %d invalid for range %s\n", 513 info->pacing_mode, range->name); 514 return -EINVAL; 515 } 516 517 pdsp = knav_find_pdsp(kdev, info->pdsp_id); 518 if (!pdsp) { 519 dev_err(kdev->dev, "pdsp id %d not found for range %s\n", 520 info->pdsp_id, range->name); 521 return -EINVAL; 522 } 523 524 if (!pdsp->started) { 525 dev_err(kdev->dev, "pdsp id %d not started for range %s\n", 526 info->pdsp_id, range->name); 527 return -ENODEV; 528 } 529 530 info->pdsp = pdsp; 531 channels = range->num_queues; 532 if (of_get_property(node, "multi-queue", NULL)) { 533 range->flags |= RANGE_MULTI_QUEUE; 534 channels = 1; 535 if (range->queue_base & (32 - 1)) { 536 dev_err(kdev->dev, 537 "misaligned multi-queue accumulator range %s\n", 538 range->name); 539 return -EINVAL; 540 } 541 if (range->num_queues > 32) { 542 dev_err(kdev->dev, 543 "too many queues in accumulator range %s\n", 544 range->name); 545 return -EINVAL; 546 } 547 } 548 549 /* figure out list size */ 550 list_size = info->list_entries; 551 list_size *= ACC_LIST_ENTRY_WORDS * sizeof(u32); 552 info->list_size = list_size; 553 mem_size = PAGE_ALIGN(list_size * 2); 554 info->mem_size = mem_size; 555 range->acc = devm_kzalloc(kdev->dev, channels * sizeof(*range->acc), 556 GFP_KERNEL); 557 if (!range->acc) 558 return -ENOMEM; 559 560 for (channel = 0; channel < channels; channel++) { 561 acc = range->acc + channel; 562 acc->channel = info->start_channel + channel; 563 564 /* allocate memory for the two lists */ 565 list_mem = alloc_pages_exact(mem_size, GFP_KERNEL | GFP_DMA); 566 if (!list_mem) 567 return -ENOMEM; 568 569 list_dma = dma_map_single(kdev->dev, list_mem, mem_size, 570 DMA_BIDIRECTIONAL); 571 if (dma_mapping_error(kdev->dev, list_dma)) { 572 free_pages_exact(list_mem, mem_size); 573 return -ENOMEM; 574 } 575 576 memset(list_mem, 0, mem_size); 577 dma_sync_single_for_device(kdev->dev, list_dma, mem_size, 578 DMA_TO_DEVICE); 579 scnprintf(acc->name, sizeof(acc->name), "hwqueue-acc-%d", 580 acc->channel); 581 acc->list_cpu[0] = list_mem; 582 acc->list_cpu[1] = list_mem + list_size; 583 acc->list_dma[0] = list_dma; 584 acc->list_dma[1] = list_dma + list_size; 585 dev_dbg(kdev->dev, "%s: channel %d, dma %pad, virt %8p\n", 586 acc->name, acc->channel, &list_dma, list_mem); 587 } 588 589 range->ops = &knav_acc_range_ops; 590 return 0; 591 } 592 EXPORT_SYMBOL_GPL(knav_init_acc_range); 593