1 /* Copyright 2008 - 2016 Freescale Semiconductor, Inc. 2 * 3 * Redistribution and use in source and binary forms, with or without 4 * modification, are permitted provided that the following conditions are met: 5 * * Redistributions of source code must retain the above copyright 6 * notice, this list of conditions and the following disclaimer. 7 * * Redistributions in binary form must reproduce the above copyright 8 * notice, this list of conditions and the following disclaimer in the 9 * documentation and/or other materials provided with the distribution. 10 * * Neither the name of Freescale Semiconductor nor the 11 * names of its contributors may be used to endorse or promote products 12 * derived from this software without specific prior written permission. 13 * 14 * ALTERNATIVELY, this software may be distributed under the terms of the 15 * GNU General Public License ("GPL") as published by the Free Software 16 * Foundation, either version 2 of that License or (at your option) any 17 * later version. 18 * 19 * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY 20 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 21 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 22 * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY 23 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 24 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 25 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 26 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 28 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 */ 30 31 #include "qman_priv.h" 32 33 u16 qman_ip_rev; 34 EXPORT_SYMBOL(qman_ip_rev); 35 u16 qm_channel_pool1 = QMAN_CHANNEL_POOL1; 36 EXPORT_SYMBOL(qm_channel_pool1); 37 38 /* Register offsets */ 39 #define REG_QCSP_LIO_CFG(n) (0x0000 + ((n) * 0x10)) 40 #define REG_QCSP_IO_CFG(n) (0x0004 + ((n) * 0x10)) 41 #define REG_QCSP_DD_CFG(n) (0x000c + ((n) * 0x10)) 42 #define REG_DD_CFG 0x0200 43 #define REG_DCP_CFG(n) (0x0300 + ((n) * 0x10)) 44 #define REG_DCP_DD_CFG(n) (0x0304 + ((n) * 0x10)) 45 #define REG_DCP_DLM_AVG(n) (0x030c + ((n) * 0x10)) 46 #define REG_PFDR_FPC 0x0400 47 #define REG_PFDR_FP_HEAD 0x0404 48 #define REG_PFDR_FP_TAIL 0x0408 49 #define REG_PFDR_FP_LWIT 0x0410 50 #define REG_PFDR_CFG 0x0414 51 #define REG_SFDR_CFG 0x0500 52 #define REG_SFDR_IN_USE 0x0504 53 #define REG_WQ_CS_CFG(n) (0x0600 + ((n) * 0x04)) 54 #define REG_WQ_DEF_ENC_WQID 0x0630 55 #define REG_WQ_SC_DD_CFG(n) (0x640 + ((n) * 0x04)) 56 #define REG_WQ_PC_DD_CFG(n) (0x680 + ((n) * 0x04)) 57 #define REG_WQ_DC0_DD_CFG(n) (0x6c0 + ((n) * 0x04)) 58 #define REG_WQ_DC1_DD_CFG(n) (0x700 + ((n) * 0x04)) 59 #define REG_WQ_DCn_DD_CFG(n) (0x6c0 + ((n) * 0x40)) /* n=2,3 */ 60 #define REG_CM_CFG 0x0800 61 #define REG_ECSR 0x0a00 62 #define REG_ECIR 0x0a04 63 #define REG_EADR 0x0a08 64 #define REG_ECIR2 0x0a0c 65 #define REG_EDATA(n) (0x0a10 + ((n) * 0x04)) 66 #define REG_SBEC(n) (0x0a80 + ((n) * 0x04)) 67 #define REG_MCR 0x0b00 68 #define REG_MCP(n) (0x0b04 + ((n) * 0x04)) 69 #define REG_MISC_CFG 0x0be0 70 #define REG_HID_CFG 0x0bf0 71 #define REG_IDLE_STAT 0x0bf4 72 #define REG_IP_REV_1 0x0bf8 73 #define REG_IP_REV_2 0x0bfc 74 #define REG_FQD_BARE 0x0c00 75 #define REG_PFDR_BARE 0x0c20 76 #define REG_offset_BAR 0x0004 /* relative to REG_[FQD|PFDR]_BARE */ 77 #define REG_offset_AR 0x0010 /* relative to REG_[FQD|PFDR]_BARE */ 78 #define REG_QCSP_BARE 0x0c80 79 #define REG_QCSP_BAR 0x0c84 80 #define REG_CI_SCHED_CFG 0x0d00 81 #define REG_SRCIDR 0x0d04 82 #define REG_LIODNR 0x0d08 83 #define REG_CI_RLM_AVG 0x0d14 84 #define REG_ERR_ISR 0x0e00 85 #define REG_ERR_IER 0x0e04 86 #define REG_REV3_QCSP_LIO_CFG(n) (0x1000 + ((n) * 0x10)) 87 #define REG_REV3_QCSP_IO_CFG(n) (0x1004 + ((n) * 0x10)) 88 #define REG_REV3_QCSP_DD_CFG(n) (0x100c + ((n) * 0x10)) 89 90 /* Assists for QMAN_MCR */ 91 #define MCR_INIT_PFDR 0x01000000 92 #define MCR_get_rslt(v) (u8)((v) >> 24) 93 #define MCR_rslt_idle(r) (!(r) || ((r) >= 0xf0)) 94 #define MCR_rslt_ok(r) ((r) == 0xf0) 95 #define MCR_rslt_eaccess(r) ((r) == 0xf8) 96 #define MCR_rslt_inval(r) ((r) == 0xff) 97 98 /* 99 * Corenet initiator settings. Stash request queues are 4-deep to match cores 100 * ability to snarf. Stash priority is 3, other priorities are 2. 101 */ 102 #define QM_CI_SCHED_CFG_SRCCIV 4 103 #define QM_CI_SCHED_CFG_SRQ_W 3 104 #define QM_CI_SCHED_CFG_RW_W 2 105 #define QM_CI_SCHED_CFG_BMAN_W 2 106 /* write SRCCIV enable */ 107 #define QM_CI_SCHED_CFG_SRCCIV_EN BIT(31) 108 109 /* Follows WQ_CS_CFG0-5 */ 110 enum qm_wq_class { 111 qm_wq_portal = 0, 112 qm_wq_pool = 1, 113 qm_wq_fman0 = 2, 114 qm_wq_fman1 = 3, 115 qm_wq_caam = 4, 116 qm_wq_pme = 5, 117 qm_wq_first = qm_wq_portal, 118 qm_wq_last = qm_wq_pme 119 }; 120 121 /* Follows FQD_[BARE|BAR|AR] and PFDR_[BARE|BAR|AR] */ 122 enum qm_memory { 123 qm_memory_fqd, 124 qm_memory_pfdr 125 }; 126 127 /* Used by all error interrupt registers except 'inhibit' */ 128 #define QM_EIRQ_CIDE 0x20000000 /* Corenet Initiator Data Error */ 129 #define QM_EIRQ_CTDE 0x10000000 /* Corenet Target Data Error */ 130 #define QM_EIRQ_CITT 0x08000000 /* Corenet Invalid Target Transaction */ 131 #define QM_EIRQ_PLWI 0x04000000 /* PFDR Low Watermark */ 132 #define QM_EIRQ_MBEI 0x02000000 /* Multi-bit ECC Error */ 133 #define QM_EIRQ_SBEI 0x01000000 /* Single-bit ECC Error */ 134 #define QM_EIRQ_PEBI 0x00800000 /* PFDR Enqueues Blocked Interrupt */ 135 #define QM_EIRQ_IFSI 0x00020000 /* Invalid FQ Flow Control State */ 136 #define QM_EIRQ_ICVI 0x00010000 /* Invalid Command Verb */ 137 #define QM_EIRQ_IDDI 0x00000800 /* Invalid Dequeue (Direct-connect) */ 138 #define QM_EIRQ_IDFI 0x00000400 /* Invalid Dequeue FQ */ 139 #define QM_EIRQ_IDSI 0x00000200 /* Invalid Dequeue Source */ 140 #define QM_EIRQ_IDQI 0x00000100 /* Invalid Dequeue Queue */ 141 #define QM_EIRQ_IECE 0x00000010 /* Invalid Enqueue Configuration */ 142 #define QM_EIRQ_IEOI 0x00000008 /* Invalid Enqueue Overflow */ 143 #define QM_EIRQ_IESI 0x00000004 /* Invalid Enqueue State */ 144 #define QM_EIRQ_IECI 0x00000002 /* Invalid Enqueue Channel */ 145 #define QM_EIRQ_IEQI 0x00000001 /* Invalid Enqueue Queue */ 146 147 /* QMAN_ECIR valid error bit */ 148 #define PORTAL_ECSR_ERR (QM_EIRQ_IEQI | QM_EIRQ_IESI | QM_EIRQ_IEOI | \ 149 QM_EIRQ_IDQI | QM_EIRQ_IDSI | QM_EIRQ_IDFI | \ 150 QM_EIRQ_IDDI | QM_EIRQ_ICVI | QM_EIRQ_IFSI) 151 #define FQID_ECSR_ERR (QM_EIRQ_IEQI | QM_EIRQ_IECI | QM_EIRQ_IESI | \ 152 QM_EIRQ_IEOI | QM_EIRQ_IDQI | QM_EIRQ_IDFI | \ 153 QM_EIRQ_IFSI) 154 155 struct qm_ecir { 156 u32 info; /* res[30-31], ptyp[29], pnum[24-28], fqid[0-23] */ 157 }; 158 159 static bool qm_ecir_is_dcp(const struct qm_ecir *p) 160 { 161 return p->info & BIT(29); 162 } 163 164 static int qm_ecir_get_pnum(const struct qm_ecir *p) 165 { 166 return (p->info >> 24) & 0x1f; 167 } 168 169 static int qm_ecir_get_fqid(const struct qm_ecir *p) 170 { 171 return p->info & (BIT(24) - 1); 172 } 173 174 struct qm_ecir2 { 175 u32 info; /* ptyp[31], res[10-30], pnum[0-9] */ 176 }; 177 178 static bool qm_ecir2_is_dcp(const struct qm_ecir2 *p) 179 { 180 return p->info & BIT(31); 181 } 182 183 static int qm_ecir2_get_pnum(const struct qm_ecir2 *p) 184 { 185 return p->info & (BIT(10) - 1); 186 } 187 188 struct qm_eadr { 189 u32 info; /* memid[24-27], eadr[0-11] */ 190 /* v3: memid[24-28], eadr[0-15] */ 191 }; 192 193 static int qm_eadr_get_memid(const struct qm_eadr *p) 194 { 195 return (p->info >> 24) & 0xf; 196 } 197 198 static int qm_eadr_get_eadr(const struct qm_eadr *p) 199 { 200 return p->info & (BIT(12) - 1); 201 } 202 203 static int qm_eadr_v3_get_memid(const struct qm_eadr *p) 204 { 205 return (p->info >> 24) & 0x1f; 206 } 207 208 static int qm_eadr_v3_get_eadr(const struct qm_eadr *p) 209 { 210 return p->info & (BIT(16) - 1); 211 } 212 213 struct qman_hwerr_txt { 214 u32 mask; 215 const char *txt; 216 }; 217 218 219 static const struct qman_hwerr_txt qman_hwerr_txts[] = { 220 { QM_EIRQ_CIDE, "Corenet Initiator Data Error" }, 221 { QM_EIRQ_CTDE, "Corenet Target Data Error" }, 222 { QM_EIRQ_CITT, "Corenet Invalid Target Transaction" }, 223 { QM_EIRQ_PLWI, "PFDR Low Watermark" }, 224 { QM_EIRQ_MBEI, "Multi-bit ECC Error" }, 225 { QM_EIRQ_SBEI, "Single-bit ECC Error" }, 226 { QM_EIRQ_PEBI, "PFDR Enqueues Blocked Interrupt" }, 227 { QM_EIRQ_ICVI, "Invalid Command Verb" }, 228 { QM_EIRQ_IFSI, "Invalid Flow Control State" }, 229 { QM_EIRQ_IDDI, "Invalid Dequeue (Direct-connect)" }, 230 { QM_EIRQ_IDFI, "Invalid Dequeue FQ" }, 231 { QM_EIRQ_IDSI, "Invalid Dequeue Source" }, 232 { QM_EIRQ_IDQI, "Invalid Dequeue Queue" }, 233 { QM_EIRQ_IECE, "Invalid Enqueue Configuration" }, 234 { QM_EIRQ_IEOI, "Invalid Enqueue Overflow" }, 235 { QM_EIRQ_IESI, "Invalid Enqueue State" }, 236 { QM_EIRQ_IECI, "Invalid Enqueue Channel" }, 237 { QM_EIRQ_IEQI, "Invalid Enqueue Queue" }, 238 }; 239 240 struct qman_error_info_mdata { 241 u16 addr_mask; 242 u16 bits; 243 const char *txt; 244 }; 245 246 static const struct qman_error_info_mdata error_mdata[] = { 247 { 0x01FF, 24, "FQD cache tag memory 0" }, 248 { 0x01FF, 24, "FQD cache tag memory 1" }, 249 { 0x01FF, 24, "FQD cache tag memory 2" }, 250 { 0x01FF, 24, "FQD cache tag memory 3" }, 251 { 0x0FFF, 512, "FQD cache memory" }, 252 { 0x07FF, 128, "SFDR memory" }, 253 { 0x01FF, 72, "WQ context memory" }, 254 { 0x00FF, 240, "CGR memory" }, 255 { 0x00FF, 302, "Internal Order Restoration List memory" }, 256 { 0x01FF, 256, "SW portal ring memory" }, 257 }; 258 259 #define QMAN_ERRS_TO_DISABLE (QM_EIRQ_PLWI | QM_EIRQ_PEBI) 260 261 /* 262 * TODO: unimplemented registers 263 * 264 * Keeping a list here of QMan registers I have not yet covered; 265 * QCSP_DD_IHRSR, QCSP_DD_IHRFR, QCSP_DD_HASR, 266 * DCP_DD_IHRSR, DCP_DD_IHRFR, DCP_DD_HASR, CM_CFG, 267 * QMAN_EECC, QMAN_SBET, QMAN_EINJ, QMAN_SBEC0-12 268 */ 269 270 /* Pointer to the start of the QMan's CCSR space */ 271 static u32 __iomem *qm_ccsr_start; 272 /* A SDQCR mask comprising all the available/visible pool channels */ 273 static u32 qm_pools_sdqcr; 274 275 static inline u32 qm_ccsr_in(u32 offset) 276 { 277 return ioread32be(qm_ccsr_start + offset/4); 278 } 279 280 static inline void qm_ccsr_out(u32 offset, u32 val) 281 { 282 iowrite32be(val, qm_ccsr_start + offset/4); 283 } 284 285 u32 qm_get_pools_sdqcr(void) 286 { 287 return qm_pools_sdqcr; 288 } 289 290 enum qm_dc_portal { 291 qm_dc_portal_fman0 = 0, 292 qm_dc_portal_fman1 = 1 293 }; 294 295 static void qm_set_dc(enum qm_dc_portal portal, int ed, u8 sernd) 296 { 297 DPAA_ASSERT(!ed || portal == qm_dc_portal_fman0 || 298 portal == qm_dc_portal_fman1); 299 if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) 300 qm_ccsr_out(REG_DCP_CFG(portal), 301 (ed ? 0x1000 : 0) | (sernd & 0x3ff)); 302 else 303 qm_ccsr_out(REG_DCP_CFG(portal), 304 (ed ? 0x100 : 0) | (sernd & 0x1f)); 305 } 306 307 static void qm_set_wq_scheduling(enum qm_wq_class wq_class, 308 u8 cs_elev, u8 csw2, u8 csw3, u8 csw4, 309 u8 csw5, u8 csw6, u8 csw7) 310 { 311 qm_ccsr_out(REG_WQ_CS_CFG(wq_class), ((cs_elev & 0xff) << 24) | 312 ((csw2 & 0x7) << 20) | ((csw3 & 0x7) << 16) | 313 ((csw4 & 0x7) << 12) | ((csw5 & 0x7) << 8) | 314 ((csw6 & 0x7) << 4) | (csw7 & 0x7)); 315 } 316 317 static void qm_set_hid(void) 318 { 319 qm_ccsr_out(REG_HID_CFG, 0); 320 } 321 322 static void qm_set_corenet_initiator(void) 323 { 324 qm_ccsr_out(REG_CI_SCHED_CFG, QM_CI_SCHED_CFG_SRCCIV_EN | 325 (QM_CI_SCHED_CFG_SRCCIV << 24) | 326 (QM_CI_SCHED_CFG_SRQ_W << 8) | 327 (QM_CI_SCHED_CFG_RW_W << 4) | 328 QM_CI_SCHED_CFG_BMAN_W); 329 } 330 331 static void qm_get_version(u16 *id, u8 *major, u8 *minor) 332 { 333 u32 v = qm_ccsr_in(REG_IP_REV_1); 334 *id = (v >> 16); 335 *major = (v >> 8) & 0xff; 336 *minor = v & 0xff; 337 } 338 339 #define PFDR_AR_EN BIT(31) 340 static void qm_set_memory(enum qm_memory memory, u64 ba, u32 size) 341 { 342 u32 offset = (memory == qm_memory_fqd) ? REG_FQD_BARE : REG_PFDR_BARE; 343 u32 exp = ilog2(size); 344 345 /* choke if size isn't within range */ 346 DPAA_ASSERT((size >= 4096) && (size <= 1024*1024*1024) && 347 is_power_of_2(size)); 348 /* choke if 'ba' has lower-alignment than 'size' */ 349 DPAA_ASSERT(!(ba & (size - 1))); 350 qm_ccsr_out(offset, upper_32_bits(ba)); 351 qm_ccsr_out(offset + REG_offset_BAR, lower_32_bits(ba)); 352 qm_ccsr_out(offset + REG_offset_AR, PFDR_AR_EN | (exp - 1)); 353 } 354 355 static void qm_set_pfdr_threshold(u32 th, u8 k) 356 { 357 qm_ccsr_out(REG_PFDR_FP_LWIT, th & 0xffffff); 358 qm_ccsr_out(REG_PFDR_CFG, k); 359 } 360 361 static void qm_set_sfdr_threshold(u16 th) 362 { 363 qm_ccsr_out(REG_SFDR_CFG, th & 0x3ff); 364 } 365 366 static int qm_init_pfdr(struct device *dev, u32 pfdr_start, u32 num) 367 { 368 u8 rslt = MCR_get_rslt(qm_ccsr_in(REG_MCR)); 369 370 DPAA_ASSERT(pfdr_start && !(pfdr_start & 7) && !(num & 7) && num); 371 /* Make sure the command interface is 'idle' */ 372 if (!MCR_rslt_idle(rslt)) { 373 dev_crit(dev, "QMAN_MCR isn't idle"); 374 WARN_ON(1); 375 } 376 377 /* Write the MCR command params then the verb */ 378 qm_ccsr_out(REG_MCP(0), pfdr_start); 379 /* 380 * TODO: remove this - it's a workaround for a model bug that is 381 * corrected in more recent versions. We use the workaround until 382 * everyone has upgraded. 383 */ 384 qm_ccsr_out(REG_MCP(1), pfdr_start + num - 16); 385 dma_wmb(); 386 qm_ccsr_out(REG_MCR, MCR_INIT_PFDR); 387 /* Poll for the result */ 388 do { 389 rslt = MCR_get_rslt(qm_ccsr_in(REG_MCR)); 390 } while (!MCR_rslt_idle(rslt)); 391 if (MCR_rslt_ok(rslt)) 392 return 0; 393 if (MCR_rslt_eaccess(rslt)) 394 return -EACCES; 395 if (MCR_rslt_inval(rslt)) 396 return -EINVAL; 397 dev_crit(dev, "Unexpected result from MCR_INIT_PFDR: %02x\n", rslt); 398 return -ENODEV; 399 } 400 401 /* 402 * Ideally we would use the DMA API to turn rmem->base into a DMA address 403 * (especially if iommu translations ever get involved). Unfortunately, the 404 * DMA API currently does not allow mapping anything that is not backed with 405 * a struct page. 406 */ 407 static dma_addr_t fqd_a, pfdr_a; 408 static size_t fqd_sz, pfdr_sz; 409 410 static int qman_fqd(struct reserved_mem *rmem) 411 { 412 fqd_a = rmem->base; 413 fqd_sz = rmem->size; 414 415 WARN_ON(!(fqd_a && fqd_sz)); 416 417 return 0; 418 } 419 RESERVEDMEM_OF_DECLARE(qman_fqd, "fsl,qman-fqd", qman_fqd); 420 421 static int qman_pfdr(struct reserved_mem *rmem) 422 { 423 pfdr_a = rmem->base; 424 pfdr_sz = rmem->size; 425 426 WARN_ON(!(pfdr_a && pfdr_sz)); 427 428 return 0; 429 } 430 RESERVEDMEM_OF_DECLARE(qman_pfdr, "fsl,qman-pfdr", qman_pfdr); 431 432 static unsigned int qm_get_fqid_maxcnt(void) 433 { 434 return fqd_sz / 64; 435 } 436 437 /* 438 * Flush this memory range from data cache so that QMAN originated 439 * transactions for this memory region could be marked non-coherent. 440 */ 441 static int zero_priv_mem(struct device *dev, struct device_node *node, 442 phys_addr_t addr, size_t sz) 443 { 444 /* map as cacheable, non-guarded */ 445 void __iomem *tmpp = ioremap_prot(addr, sz, 0); 446 447 if (!tmpp) 448 return -ENOMEM; 449 450 memset_io(tmpp, 0, sz); 451 flush_dcache_range((unsigned long)tmpp, 452 (unsigned long)tmpp + sz); 453 iounmap(tmpp); 454 455 return 0; 456 } 457 458 static void log_edata_bits(struct device *dev, u32 bit_count) 459 { 460 u32 i, j, mask = 0xffffffff; 461 462 dev_warn(dev, "ErrInt, EDATA:\n"); 463 i = bit_count / 32; 464 if (bit_count % 32) { 465 i++; 466 mask = ~(mask << bit_count % 32); 467 } 468 j = 16 - i; 469 dev_warn(dev, " 0x%08x\n", qm_ccsr_in(REG_EDATA(j)) & mask); 470 j++; 471 for (; j < 16; j++) 472 dev_warn(dev, " 0x%08x\n", qm_ccsr_in(REG_EDATA(j))); 473 } 474 475 static void log_additional_error_info(struct device *dev, u32 isr_val, 476 u32 ecsr_val) 477 { 478 struct qm_ecir ecir_val; 479 struct qm_eadr eadr_val; 480 int memid; 481 482 ecir_val.info = qm_ccsr_in(REG_ECIR); 483 /* Is portal info valid */ 484 if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) { 485 struct qm_ecir2 ecir2_val; 486 487 ecir2_val.info = qm_ccsr_in(REG_ECIR2); 488 if (ecsr_val & PORTAL_ECSR_ERR) { 489 dev_warn(dev, "ErrInt: %s id %d\n", 490 qm_ecir2_is_dcp(&ecir2_val) ? "DCP" : "SWP", 491 qm_ecir2_get_pnum(&ecir2_val)); 492 } 493 if (ecsr_val & (FQID_ECSR_ERR | QM_EIRQ_IECE)) 494 dev_warn(dev, "ErrInt: ecir.fqid 0x%x\n", 495 qm_ecir_get_fqid(&ecir_val)); 496 497 if (ecsr_val & (QM_EIRQ_SBEI|QM_EIRQ_MBEI)) { 498 eadr_val.info = qm_ccsr_in(REG_EADR); 499 memid = qm_eadr_v3_get_memid(&eadr_val); 500 dev_warn(dev, "ErrInt: EADR Memory: %s, 0x%x\n", 501 error_mdata[memid].txt, 502 error_mdata[memid].addr_mask 503 & qm_eadr_v3_get_eadr(&eadr_val)); 504 log_edata_bits(dev, error_mdata[memid].bits); 505 } 506 } else { 507 if (ecsr_val & PORTAL_ECSR_ERR) { 508 dev_warn(dev, "ErrInt: %s id %d\n", 509 qm_ecir_is_dcp(&ecir_val) ? "DCP" : "SWP", 510 qm_ecir_get_pnum(&ecir_val)); 511 } 512 if (ecsr_val & FQID_ECSR_ERR) 513 dev_warn(dev, "ErrInt: ecir.fqid 0x%x\n", 514 qm_ecir_get_fqid(&ecir_val)); 515 516 if (ecsr_val & (QM_EIRQ_SBEI|QM_EIRQ_MBEI)) { 517 eadr_val.info = qm_ccsr_in(REG_EADR); 518 memid = qm_eadr_get_memid(&eadr_val); 519 dev_warn(dev, "ErrInt: EADR Memory: %s, 0x%x\n", 520 error_mdata[memid].txt, 521 error_mdata[memid].addr_mask 522 & qm_eadr_get_eadr(&eadr_val)); 523 log_edata_bits(dev, error_mdata[memid].bits); 524 } 525 } 526 } 527 528 static irqreturn_t qman_isr(int irq, void *ptr) 529 { 530 u32 isr_val, ier_val, ecsr_val, isr_mask, i; 531 struct device *dev = ptr; 532 533 ier_val = qm_ccsr_in(REG_ERR_IER); 534 isr_val = qm_ccsr_in(REG_ERR_ISR); 535 ecsr_val = qm_ccsr_in(REG_ECSR); 536 isr_mask = isr_val & ier_val; 537 538 if (!isr_mask) 539 return IRQ_NONE; 540 541 for (i = 0; i < ARRAY_SIZE(qman_hwerr_txts); i++) { 542 if (qman_hwerr_txts[i].mask & isr_mask) { 543 dev_err_ratelimited(dev, "ErrInt: %s\n", 544 qman_hwerr_txts[i].txt); 545 if (qman_hwerr_txts[i].mask & ecsr_val) { 546 log_additional_error_info(dev, isr_mask, 547 ecsr_val); 548 /* Re-arm error capture registers */ 549 qm_ccsr_out(REG_ECSR, ecsr_val); 550 } 551 if (qman_hwerr_txts[i].mask & QMAN_ERRS_TO_DISABLE) { 552 dev_dbg(dev, "Disabling error 0x%x\n", 553 qman_hwerr_txts[i].mask); 554 ier_val &= ~qman_hwerr_txts[i].mask; 555 qm_ccsr_out(REG_ERR_IER, ier_val); 556 } 557 } 558 } 559 qm_ccsr_out(REG_ERR_ISR, isr_val); 560 561 return IRQ_HANDLED; 562 } 563 564 static int qman_init_ccsr(struct device *dev) 565 { 566 int i, err; 567 568 /* FQD memory */ 569 qm_set_memory(qm_memory_fqd, fqd_a, fqd_sz); 570 /* PFDR memory */ 571 qm_set_memory(qm_memory_pfdr, pfdr_a, pfdr_sz); 572 err = qm_init_pfdr(dev, 8, pfdr_sz / 64 - 8); 573 if (err) 574 return err; 575 /* thresholds */ 576 qm_set_pfdr_threshold(512, 64); 577 qm_set_sfdr_threshold(128); 578 /* clear stale PEBI bit from interrupt status register */ 579 qm_ccsr_out(REG_ERR_ISR, QM_EIRQ_PEBI); 580 /* corenet initiator settings */ 581 qm_set_corenet_initiator(); 582 /* HID settings */ 583 qm_set_hid(); 584 /* Set scheduling weights to defaults */ 585 for (i = qm_wq_first; i <= qm_wq_last; i++) 586 qm_set_wq_scheduling(i, 0, 0, 0, 0, 0, 0, 0); 587 /* We are not prepared to accept ERNs for hardware enqueues */ 588 qm_set_dc(qm_dc_portal_fman0, 1, 0); 589 qm_set_dc(qm_dc_portal_fman1, 1, 0); 590 return 0; 591 } 592 593 #define LIO_CFG_LIODN_MASK 0x0fff0000 594 void qman_liodn_fixup(u16 channel) 595 { 596 static int done; 597 static u32 liodn_offset; 598 u32 before, after; 599 int idx = channel - QM_CHANNEL_SWPORTAL0; 600 601 if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) 602 before = qm_ccsr_in(REG_REV3_QCSP_LIO_CFG(idx)); 603 else 604 before = qm_ccsr_in(REG_QCSP_LIO_CFG(idx)); 605 if (!done) { 606 liodn_offset = before & LIO_CFG_LIODN_MASK; 607 done = 1; 608 return; 609 } 610 after = (before & (~LIO_CFG_LIODN_MASK)) | liodn_offset; 611 if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) 612 qm_ccsr_out(REG_REV3_QCSP_LIO_CFG(idx), after); 613 else 614 qm_ccsr_out(REG_QCSP_LIO_CFG(idx), after); 615 } 616 617 #define IO_CFG_SDEST_MASK 0x00ff0000 618 void qman_set_sdest(u16 channel, unsigned int cpu_idx) 619 { 620 int idx = channel - QM_CHANNEL_SWPORTAL0; 621 u32 before, after; 622 623 if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) { 624 before = qm_ccsr_in(REG_REV3_QCSP_IO_CFG(idx)); 625 /* Each pair of vcpu share the same SRQ(SDEST) */ 626 cpu_idx /= 2; 627 after = (before & (~IO_CFG_SDEST_MASK)) | (cpu_idx << 16); 628 qm_ccsr_out(REG_REV3_QCSP_IO_CFG(idx), after); 629 } else { 630 before = qm_ccsr_in(REG_QCSP_IO_CFG(idx)); 631 after = (before & (~IO_CFG_SDEST_MASK)) | (cpu_idx << 16); 632 qm_ccsr_out(REG_QCSP_IO_CFG(idx), after); 633 } 634 } 635 636 static int qman_resource_init(struct device *dev) 637 { 638 int pool_chan_num, cgrid_num; 639 int ret, i; 640 641 switch (qman_ip_rev >> 8) { 642 case 1: 643 pool_chan_num = 15; 644 cgrid_num = 256; 645 break; 646 case 2: 647 pool_chan_num = 3; 648 cgrid_num = 64; 649 break; 650 case 3: 651 pool_chan_num = 15; 652 cgrid_num = 256; 653 break; 654 default: 655 return -ENODEV; 656 } 657 658 ret = gen_pool_add(qm_qpalloc, qm_channel_pool1 | DPAA_GENALLOC_OFF, 659 pool_chan_num, -1); 660 if (ret) { 661 dev_err(dev, "Failed to seed pool channels (%d)\n", ret); 662 return ret; 663 } 664 665 ret = gen_pool_add(qm_cgralloc, DPAA_GENALLOC_OFF, cgrid_num, -1); 666 if (ret) { 667 dev_err(dev, "Failed to seed CGRID range (%d)\n", ret); 668 return ret; 669 } 670 671 /* parse pool channels into the SDQCR mask */ 672 for (i = 0; i < cgrid_num; i++) 673 qm_pools_sdqcr |= QM_SDQCR_CHANNELS_POOL_CONV(i); 674 675 ret = gen_pool_add(qm_fqalloc, QM_FQID_RANGE_START | DPAA_GENALLOC_OFF, 676 qm_get_fqid_maxcnt() - QM_FQID_RANGE_START, -1); 677 if (ret) { 678 dev_err(dev, "Failed to seed FQID range (%d)\n", ret); 679 return ret; 680 } 681 682 return 0; 683 } 684 685 static int fsl_qman_probe(struct platform_device *pdev) 686 { 687 struct device *dev = &pdev->dev; 688 struct device_node *node = dev->of_node; 689 struct resource *res; 690 int ret, err_irq; 691 u16 id; 692 u8 major, minor; 693 694 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 695 if (!res) { 696 dev_err(dev, "Can't get %s property 'IORESOURCE_MEM'\n", 697 node->full_name); 698 return -ENXIO; 699 } 700 qm_ccsr_start = devm_ioremap(dev, res->start, resource_size(res)); 701 if (!qm_ccsr_start) 702 return -ENXIO; 703 704 qm_get_version(&id, &major, &minor); 705 if (major == 1 && minor == 0) { 706 dev_err(dev, "Rev1.0 on P4080 rev1 is not supported!\n"); 707 return -ENODEV; 708 } else if (major == 1 && minor == 1) 709 qman_ip_rev = QMAN_REV11; 710 else if (major == 1 && minor == 2) 711 qman_ip_rev = QMAN_REV12; 712 else if (major == 2 && minor == 0) 713 qman_ip_rev = QMAN_REV20; 714 else if (major == 3 && minor == 0) 715 qman_ip_rev = QMAN_REV30; 716 else if (major == 3 && minor == 1) 717 qman_ip_rev = QMAN_REV31; 718 else { 719 dev_err(dev, "Unknown QMan version\n"); 720 return -ENODEV; 721 } 722 723 if ((qman_ip_rev & 0xff00) >= QMAN_REV30) 724 qm_channel_pool1 = QMAN_CHANNEL_POOL1_REV3; 725 726 ret = zero_priv_mem(dev, node, fqd_a, fqd_sz); 727 WARN_ON(ret); 728 if (ret) 729 return -ENODEV; 730 731 ret = qman_init_ccsr(dev); 732 if (ret) { 733 dev_err(dev, "CCSR setup failed\n"); 734 return ret; 735 } 736 737 err_irq = platform_get_irq(pdev, 0); 738 if (err_irq <= 0) { 739 dev_info(dev, "Can't get %s property 'interrupts'\n", 740 node->full_name); 741 return -ENODEV; 742 } 743 ret = devm_request_irq(dev, err_irq, qman_isr, IRQF_SHARED, "qman-err", 744 dev); 745 if (ret) { 746 dev_err(dev, "devm_request_irq() failed %d for '%s'\n", 747 ret, node->full_name); 748 return ret; 749 } 750 751 /* 752 * Write-to-clear any stale bits, (eg. starvation being asserted prior 753 * to resource allocation during driver init). 754 */ 755 qm_ccsr_out(REG_ERR_ISR, 0xffffffff); 756 /* Enable Error Interrupts */ 757 qm_ccsr_out(REG_ERR_IER, 0xffffffff); 758 759 qm_fqalloc = devm_gen_pool_create(dev, 0, -1, "qman-fqalloc"); 760 if (IS_ERR(qm_fqalloc)) { 761 ret = PTR_ERR(qm_fqalloc); 762 dev_err(dev, "qman-fqalloc pool init failed (%d)\n", ret); 763 return ret; 764 } 765 766 qm_qpalloc = devm_gen_pool_create(dev, 0, -1, "qman-qpalloc"); 767 if (IS_ERR(qm_qpalloc)) { 768 ret = PTR_ERR(qm_qpalloc); 769 dev_err(dev, "qman-qpalloc pool init failed (%d)\n", ret); 770 return ret; 771 } 772 773 qm_cgralloc = devm_gen_pool_create(dev, 0, -1, "qman-cgralloc"); 774 if (IS_ERR(qm_cgralloc)) { 775 ret = PTR_ERR(qm_cgralloc); 776 dev_err(dev, "qman-cgralloc pool init failed (%d)\n", ret); 777 return ret; 778 } 779 780 ret = qman_resource_init(dev); 781 if (ret) 782 return ret; 783 784 ret = qman_alloc_fq_table(qm_get_fqid_maxcnt()); 785 if (ret) 786 return ret; 787 788 ret = qman_wq_alloc(); 789 if (ret) 790 return ret; 791 792 return 0; 793 } 794 795 static const struct of_device_id fsl_qman_ids[] = { 796 { 797 .compatible = "fsl,qman", 798 }, 799 {} 800 }; 801 802 static struct platform_driver fsl_qman_driver = { 803 .driver = { 804 .name = KBUILD_MODNAME, 805 .of_match_table = fsl_qman_ids, 806 .suppress_bind_attrs = true, 807 }, 808 .probe = fsl_qman_probe, 809 }; 810 811 builtin_platform_driver(fsl_qman_driver); 812