1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) 2 /* 3 * Copyright (C) 2014-2016 Freescale Semiconductor, Inc. 4 * Copyright 2016-2019 NXP 5 * 6 */ 7 8 #include <asm/cacheflush.h> 9 #include <linux/io.h> 10 #include <linux/slab.h> 11 #include <linux/spinlock.h> 12 #include <soc/fsl/dpaa2-global.h> 13 14 #include "qbman-portal.h" 15 16 /* All QBMan command and result structures use this "valid bit" encoding */ 17 #define QB_VALID_BIT ((u32)0x80) 18 19 /* QBMan portal management command codes */ 20 #define QBMAN_MC_ACQUIRE 0x30 21 #define QBMAN_WQCHAN_CONFIGURE 0x46 22 23 /* CINH register offsets */ 24 #define QBMAN_CINH_SWP_EQCR_PI 0x800 25 #define QBMAN_CINH_SWP_EQCR_CI 0x840 26 #define QBMAN_CINH_SWP_EQAR 0x8c0 27 #define QBMAN_CINH_SWP_CR_RT 0x900 28 #define QBMAN_CINH_SWP_VDQCR_RT 0x940 29 #define QBMAN_CINH_SWP_EQCR_AM_RT 0x980 30 #define QBMAN_CINH_SWP_RCR_AM_RT 0x9c0 31 #define QBMAN_CINH_SWP_DQPI 0xa00 32 #define QBMAN_CINH_SWP_DCAP 0xac0 33 #define QBMAN_CINH_SWP_SDQCR 0xb00 34 #define QBMAN_CINH_SWP_EQCR_AM_RT2 0xb40 35 #define QBMAN_CINH_SWP_RCR_PI 0xc00 36 #define QBMAN_CINH_SWP_RAR 0xcc0 37 #define QBMAN_CINH_SWP_ISR 0xe00 38 #define QBMAN_CINH_SWP_IER 0xe40 39 #define QBMAN_CINH_SWP_ISDR 0xe80 40 #define QBMAN_CINH_SWP_IIR 0xec0 41 42 /* CENA register offsets */ 43 #define QBMAN_CENA_SWP_EQCR(n) (0x000 + ((u32)(n) << 6)) 44 #define QBMAN_CENA_SWP_DQRR(n) (0x200 + ((u32)(n) << 6)) 45 #define QBMAN_CENA_SWP_RCR(n) (0x400 + ((u32)(n) << 6)) 46 #define QBMAN_CENA_SWP_CR 0x600 47 #define QBMAN_CENA_SWP_RR(vb) (0x700 + ((u32)(vb) >> 1)) 48 #define QBMAN_CENA_SWP_VDQCR 0x780 49 #define QBMAN_CENA_SWP_EQCR_CI 0x840 50 #define QBMAN_CENA_SWP_EQCR_CI_MEMBACK 0x1840 51 52 /* CENA register offsets in memory-backed mode */ 53 #define QBMAN_CENA_SWP_DQRR_MEM(n) (0x800 + ((u32)(n) << 6)) 54 #define QBMAN_CENA_SWP_RCR_MEM(n) (0x1400 + ((u32)(n) << 6)) 55 #define QBMAN_CENA_SWP_CR_MEM 0x1600 56 #define QBMAN_CENA_SWP_RR_MEM 0x1680 57 #define QBMAN_CENA_SWP_VDQCR_MEM 0x1780 58 59 /* Reverse mapping of QBMAN_CENA_SWP_DQRR() */ 60 #define QBMAN_IDX_FROM_DQRR(p) (((unsigned long)(p) & 0x1ff) >> 6) 61 62 /* Define token used to determine if response written to memory is valid */ 63 #define QMAN_DQ_TOKEN_VALID 1 64 65 /* SDQCR attribute codes */ 66 #define QB_SDQCR_FC_SHIFT 29 67 #define QB_SDQCR_FC_MASK 0x1 68 #define QB_SDQCR_DCT_SHIFT 24 69 #define QB_SDQCR_DCT_MASK 0x3 70 #define QB_SDQCR_TOK_SHIFT 16 71 #define QB_SDQCR_TOK_MASK 0xff 72 #define QB_SDQCR_SRC_SHIFT 0 73 #define QB_SDQCR_SRC_MASK 0xffff 74 75 /* opaque token for static dequeues */ 76 #define QMAN_SDQCR_TOKEN 0xbb 77 78 #define QBMAN_EQCR_DCA_IDXMASK 0x0f 79 #define QBMAN_ENQUEUE_FLAG_DCA (1ULL << 31) 80 81 #define EQ_DESC_SIZE_WITHOUT_FD 29 82 #define EQ_DESC_SIZE_FD_START 32 83 84 enum qbman_sdqcr_dct { 85 qbman_sdqcr_dct_null = 0, 86 qbman_sdqcr_dct_prio_ics, 87 qbman_sdqcr_dct_active_ics, 88 qbman_sdqcr_dct_active 89 }; 90 91 enum qbman_sdqcr_fc { 92 qbman_sdqcr_fc_one = 0, 93 qbman_sdqcr_fc_up_to_3 = 1 94 }; 95 96 /* Internal Function declaration */ 97 static int qbman_swp_enqueue_direct(struct qbman_swp *s, 98 const struct qbman_eq_desc *d, 99 const struct dpaa2_fd *fd); 100 static int qbman_swp_enqueue_mem_back(struct qbman_swp *s, 101 const struct qbman_eq_desc *d, 102 const struct dpaa2_fd *fd); 103 static int qbman_swp_enqueue_multiple_direct(struct qbman_swp *s, 104 const struct qbman_eq_desc *d, 105 const struct dpaa2_fd *fd, 106 uint32_t *flags, 107 int num_frames); 108 static int qbman_swp_enqueue_multiple_mem_back(struct qbman_swp *s, 109 const struct qbman_eq_desc *d, 110 const struct dpaa2_fd *fd, 111 uint32_t *flags, 112 int num_frames); 113 static int 114 qbman_swp_enqueue_multiple_desc_direct(struct qbman_swp *s, 115 const struct qbman_eq_desc *d, 116 const struct dpaa2_fd *fd, 117 int num_frames); 118 static 119 int qbman_swp_enqueue_multiple_desc_mem_back(struct qbman_swp *s, 120 const struct qbman_eq_desc *d, 121 const struct dpaa2_fd *fd, 122 int num_frames); 123 static int qbman_swp_pull_direct(struct qbman_swp *s, 124 struct qbman_pull_desc *d); 125 static int qbman_swp_pull_mem_back(struct qbman_swp *s, 126 struct qbman_pull_desc *d); 127 128 const struct dpaa2_dq *qbman_swp_dqrr_next_direct(struct qbman_swp *s); 129 const struct dpaa2_dq *qbman_swp_dqrr_next_mem_back(struct qbman_swp *s); 130 131 static int qbman_swp_release_direct(struct qbman_swp *s, 132 const struct qbman_release_desc *d, 133 const u64 *buffers, 134 unsigned int num_buffers); 135 static int qbman_swp_release_mem_back(struct qbman_swp *s, 136 const struct qbman_release_desc *d, 137 const u64 *buffers, 138 unsigned int num_buffers); 139 140 /* Function pointers */ 141 int (*qbman_swp_enqueue_ptr)(struct qbman_swp *s, 142 const struct qbman_eq_desc *d, 143 const struct dpaa2_fd *fd) 144 = qbman_swp_enqueue_direct; 145 146 int (*qbman_swp_enqueue_multiple_ptr)(struct qbman_swp *s, 147 const struct qbman_eq_desc *d, 148 const struct dpaa2_fd *fd, 149 uint32_t *flags, 150 int num_frames) 151 = qbman_swp_enqueue_multiple_direct; 152 153 int 154 (*qbman_swp_enqueue_multiple_desc_ptr)(struct qbman_swp *s, 155 const struct qbman_eq_desc *d, 156 const struct dpaa2_fd *fd, 157 int num_frames) 158 = qbman_swp_enqueue_multiple_desc_direct; 159 160 int (*qbman_swp_pull_ptr)(struct qbman_swp *s, struct qbman_pull_desc *d) 161 = qbman_swp_pull_direct; 162 163 const struct dpaa2_dq *(*qbman_swp_dqrr_next_ptr)(struct qbman_swp *s) 164 = qbman_swp_dqrr_next_direct; 165 166 int (*qbman_swp_release_ptr)(struct qbman_swp *s, 167 const struct qbman_release_desc *d, 168 const u64 *buffers, 169 unsigned int num_buffers) 170 = qbman_swp_release_direct; 171 172 /* Portal Access */ 173 174 static inline u32 qbman_read_register(struct qbman_swp *p, u32 offset) 175 { 176 return readl_relaxed(p->addr_cinh + offset); 177 } 178 179 static inline void qbman_write_register(struct qbman_swp *p, u32 offset, 180 u32 value) 181 { 182 writel_relaxed(value, p->addr_cinh + offset); 183 } 184 185 static inline void *qbman_get_cmd(struct qbman_swp *p, u32 offset) 186 { 187 return p->addr_cena + offset; 188 } 189 190 #define QBMAN_CINH_SWP_CFG 0xd00 191 192 #define SWP_CFG_DQRR_MF_SHIFT 20 193 #define SWP_CFG_EST_SHIFT 16 194 #define SWP_CFG_CPBS_SHIFT 15 195 #define SWP_CFG_WN_SHIFT 14 196 #define SWP_CFG_RPM_SHIFT 12 197 #define SWP_CFG_DCM_SHIFT 10 198 #define SWP_CFG_EPM_SHIFT 8 199 #define SWP_CFG_VPM_SHIFT 7 200 #define SWP_CFG_CPM_SHIFT 6 201 #define SWP_CFG_SD_SHIFT 5 202 #define SWP_CFG_SP_SHIFT 4 203 #define SWP_CFG_SE_SHIFT 3 204 #define SWP_CFG_DP_SHIFT 2 205 #define SWP_CFG_DE_SHIFT 1 206 #define SWP_CFG_EP_SHIFT 0 207 208 static inline u32 qbman_set_swp_cfg(u8 max_fill, u8 wn, u8 est, u8 rpm, u8 dcm, 209 u8 epm, int sd, int sp, int se, 210 int dp, int de, int ep) 211 { 212 return (max_fill << SWP_CFG_DQRR_MF_SHIFT | 213 est << SWP_CFG_EST_SHIFT | 214 wn << SWP_CFG_WN_SHIFT | 215 rpm << SWP_CFG_RPM_SHIFT | 216 dcm << SWP_CFG_DCM_SHIFT | 217 epm << SWP_CFG_EPM_SHIFT | 218 sd << SWP_CFG_SD_SHIFT | 219 sp << SWP_CFG_SP_SHIFT | 220 se << SWP_CFG_SE_SHIFT | 221 dp << SWP_CFG_DP_SHIFT | 222 de << SWP_CFG_DE_SHIFT | 223 ep << SWP_CFG_EP_SHIFT); 224 } 225 226 #define QMAN_RT_MODE 0x00000100 227 228 static inline u8 qm_cyc_diff(u8 ringsize, u8 first, u8 last) 229 { 230 /* 'first' is included, 'last' is excluded */ 231 if (first <= last) 232 return last - first; 233 else 234 return (2 * ringsize) - (first - last); 235 } 236 237 /** 238 * qbman_swp_init() - Create a functional object representing the given 239 * QBMan portal descriptor. 240 * @d: the given qbman swp descriptor 241 * 242 * Return qbman_swp portal for success, NULL if the object cannot 243 * be created. 244 */ 245 struct qbman_swp *qbman_swp_init(const struct qbman_swp_desc *d) 246 { 247 struct qbman_swp *p = kzalloc(sizeof(*p), GFP_KERNEL); 248 u32 reg; 249 u32 mask_size; 250 u32 eqcr_pi; 251 252 if (!p) 253 return NULL; 254 255 spin_lock_init(&p->access_spinlock); 256 257 p->desc = d; 258 p->mc.valid_bit = QB_VALID_BIT; 259 p->sdq = 0; 260 p->sdq |= qbman_sdqcr_dct_prio_ics << QB_SDQCR_DCT_SHIFT; 261 p->sdq |= qbman_sdqcr_fc_up_to_3 << QB_SDQCR_FC_SHIFT; 262 p->sdq |= QMAN_SDQCR_TOKEN << QB_SDQCR_TOK_SHIFT; 263 if ((p->desc->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000) 264 p->mr.valid_bit = QB_VALID_BIT; 265 266 atomic_set(&p->vdq.available, 1); 267 p->vdq.valid_bit = QB_VALID_BIT; 268 p->dqrr.next_idx = 0; 269 p->dqrr.valid_bit = QB_VALID_BIT; 270 271 if ((p->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_4100) { 272 p->dqrr.dqrr_size = 4; 273 p->dqrr.reset_bug = 1; 274 } else { 275 p->dqrr.dqrr_size = 8; 276 p->dqrr.reset_bug = 0; 277 } 278 279 p->addr_cena = d->cena_bar; 280 p->addr_cinh = d->cinh_bar; 281 282 if ((p->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000) { 283 284 reg = qbman_set_swp_cfg(p->dqrr.dqrr_size, 285 1, /* Writes Non-cacheable */ 286 0, /* EQCR_CI stashing threshold */ 287 3, /* RPM: RCR in array mode */ 288 2, /* DCM: Discrete consumption ack */ 289 2, /* EPM: EQCR in ring mode */ 290 1, /* mem stashing drop enable enable */ 291 1, /* mem stashing priority enable */ 292 1, /* mem stashing enable */ 293 1, /* dequeue stashing priority enable */ 294 0, /* dequeue stashing enable enable */ 295 0); /* EQCR_CI stashing priority enable */ 296 } else { 297 memset(p->addr_cena, 0, 64 * 1024); 298 reg = qbman_set_swp_cfg(p->dqrr.dqrr_size, 299 1, /* Writes Non-cacheable */ 300 1, /* EQCR_CI stashing threshold */ 301 3, /* RPM: RCR in array mode */ 302 2, /* DCM: Discrete consumption ack */ 303 0, /* EPM: EQCR in ring mode */ 304 1, /* mem stashing drop enable */ 305 1, /* mem stashing priority enable */ 306 1, /* mem stashing enable */ 307 1, /* dequeue stashing priority enable */ 308 0, /* dequeue stashing enable */ 309 0); /* EQCR_CI stashing priority enable */ 310 reg |= 1 << SWP_CFG_CPBS_SHIFT | /* memory-backed mode */ 311 1 << SWP_CFG_VPM_SHIFT | /* VDQCR read triggered mode */ 312 1 << SWP_CFG_CPM_SHIFT; /* CR read triggered mode */ 313 } 314 315 qbman_write_register(p, QBMAN_CINH_SWP_CFG, reg); 316 reg = qbman_read_register(p, QBMAN_CINH_SWP_CFG); 317 if (!reg) { 318 pr_err("qbman: the portal is not enabled!\n"); 319 kfree(p); 320 return NULL; 321 } 322 323 if ((p->desc->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000) { 324 qbman_write_register(p, QBMAN_CINH_SWP_EQCR_PI, QMAN_RT_MODE); 325 qbman_write_register(p, QBMAN_CINH_SWP_RCR_PI, QMAN_RT_MODE); 326 } 327 /* 328 * SDQCR needs to be initialized to 0 when no channels are 329 * being dequeued from or else the QMan HW will indicate an 330 * error. The values that were calculated above will be 331 * applied when dequeues from a specific channel are enabled. 332 */ 333 qbman_write_register(p, QBMAN_CINH_SWP_SDQCR, 0); 334 335 p->eqcr.pi_ring_size = 8; 336 if ((p->desc->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000) { 337 p->eqcr.pi_ring_size = 32; 338 qbman_swp_enqueue_ptr = 339 qbman_swp_enqueue_mem_back; 340 qbman_swp_enqueue_multiple_ptr = 341 qbman_swp_enqueue_multiple_mem_back; 342 qbman_swp_enqueue_multiple_desc_ptr = 343 qbman_swp_enqueue_multiple_desc_mem_back; 344 qbman_swp_pull_ptr = qbman_swp_pull_mem_back; 345 qbman_swp_dqrr_next_ptr = qbman_swp_dqrr_next_mem_back; 346 qbman_swp_release_ptr = qbman_swp_release_mem_back; 347 } 348 349 for (mask_size = p->eqcr.pi_ring_size; mask_size > 0; mask_size >>= 1) 350 p->eqcr.pi_ci_mask = (p->eqcr.pi_ci_mask << 1) + 1; 351 eqcr_pi = qbman_read_register(p, QBMAN_CINH_SWP_EQCR_PI); 352 p->eqcr.pi = eqcr_pi & p->eqcr.pi_ci_mask; 353 p->eqcr.pi_vb = eqcr_pi & QB_VALID_BIT; 354 p->eqcr.ci = qbman_read_register(p, QBMAN_CINH_SWP_EQCR_CI) 355 & p->eqcr.pi_ci_mask; 356 p->eqcr.available = p->eqcr.pi_ring_size; 357 358 return p; 359 } 360 361 /** 362 * qbman_swp_finish() - Create and destroy a functional object representing 363 * the given QBMan portal descriptor. 364 * @p: the qbman_swp object to be destroyed 365 */ 366 void qbman_swp_finish(struct qbman_swp *p) 367 { 368 kfree(p); 369 } 370 371 /** 372 * qbman_swp_interrupt_read_status() 373 * @p: the given software portal 374 * 375 * Return the value in the SWP_ISR register. 376 */ 377 u32 qbman_swp_interrupt_read_status(struct qbman_swp *p) 378 { 379 return qbman_read_register(p, QBMAN_CINH_SWP_ISR); 380 } 381 382 /** 383 * qbman_swp_interrupt_clear_status() 384 * @p: the given software portal 385 * @mask: The mask to clear in SWP_ISR register 386 */ 387 void qbman_swp_interrupt_clear_status(struct qbman_swp *p, u32 mask) 388 { 389 qbman_write_register(p, QBMAN_CINH_SWP_ISR, mask); 390 } 391 392 /** 393 * qbman_swp_interrupt_get_trigger() - read interrupt enable register 394 * @p: the given software portal 395 * 396 * Return the value in the SWP_IER register. 397 */ 398 u32 qbman_swp_interrupt_get_trigger(struct qbman_swp *p) 399 { 400 return qbman_read_register(p, QBMAN_CINH_SWP_IER); 401 } 402 403 /** 404 * qbman_swp_interrupt_set_trigger() - enable interrupts for a swp 405 * @p: the given software portal 406 * @mask: The mask of bits to enable in SWP_IER 407 */ 408 void qbman_swp_interrupt_set_trigger(struct qbman_swp *p, u32 mask) 409 { 410 qbman_write_register(p, QBMAN_CINH_SWP_IER, mask); 411 } 412 413 /** 414 * qbman_swp_interrupt_get_inhibit() - read interrupt mask register 415 * @p: the given software portal object 416 * 417 * Return the value in the SWP_IIR register. 418 */ 419 int qbman_swp_interrupt_get_inhibit(struct qbman_swp *p) 420 { 421 return qbman_read_register(p, QBMAN_CINH_SWP_IIR); 422 } 423 424 /** 425 * qbman_swp_interrupt_set_inhibit() - write interrupt mask register 426 * @p: the given software portal object 427 * @mask: The mask to set in SWP_IIR register 428 */ 429 void qbman_swp_interrupt_set_inhibit(struct qbman_swp *p, int inhibit) 430 { 431 qbman_write_register(p, QBMAN_CINH_SWP_IIR, inhibit ? 0xffffffff : 0); 432 } 433 434 /* 435 * Different management commands all use this common base layer of code to issue 436 * commands and poll for results. 437 */ 438 439 /* 440 * Returns a pointer to where the caller should fill in their management command 441 * (caller should ignore the verb byte) 442 */ 443 void *qbman_swp_mc_start(struct qbman_swp *p) 444 { 445 if ((p->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000) 446 return qbman_get_cmd(p, QBMAN_CENA_SWP_CR); 447 else 448 return qbman_get_cmd(p, QBMAN_CENA_SWP_CR_MEM); 449 } 450 451 /* 452 * Commits merges in the caller-supplied command verb (which should not include 453 * the valid-bit) and submits the command to hardware 454 */ 455 void qbman_swp_mc_submit(struct qbman_swp *p, void *cmd, u8 cmd_verb) 456 { 457 u8 *v = cmd; 458 459 if ((p->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000) { 460 dma_wmb(); 461 *v = cmd_verb | p->mc.valid_bit; 462 } else { 463 *v = cmd_verb | p->mc.valid_bit; 464 dma_wmb(); 465 qbman_write_register(p, QBMAN_CINH_SWP_CR_RT, QMAN_RT_MODE); 466 } 467 } 468 469 /* 470 * Checks for a completed response (returns non-NULL if only if the response 471 * is complete). 472 */ 473 void *qbman_swp_mc_result(struct qbman_swp *p) 474 { 475 u32 *ret, verb; 476 477 if ((p->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000) { 478 ret = qbman_get_cmd(p, QBMAN_CENA_SWP_RR(p->mc.valid_bit)); 479 /* Remove the valid-bit - command completed if the rest 480 * is non-zero. 481 */ 482 verb = ret[0] & ~QB_VALID_BIT; 483 if (!verb) 484 return NULL; 485 p->mc.valid_bit ^= QB_VALID_BIT; 486 } else { 487 ret = qbman_get_cmd(p, QBMAN_CENA_SWP_RR_MEM); 488 /* Command completed if the valid bit is toggled */ 489 if (p->mr.valid_bit != (ret[0] & QB_VALID_BIT)) 490 return NULL; 491 /* Command completed if the rest is non-zero */ 492 verb = ret[0] & ~QB_VALID_BIT; 493 if (!verb) 494 return NULL; 495 p->mr.valid_bit ^= QB_VALID_BIT; 496 } 497 498 return ret; 499 } 500 501 #define QB_ENQUEUE_CMD_OPTIONS_SHIFT 0 502 enum qb_enqueue_commands { 503 enqueue_empty = 0, 504 enqueue_response_always = 1, 505 enqueue_rejects_to_fq = 2 506 }; 507 508 #define QB_ENQUEUE_CMD_ORP_ENABLE_SHIFT 2 509 #define QB_ENQUEUE_CMD_IRQ_ON_DISPATCH_SHIFT 3 510 #define QB_ENQUEUE_CMD_TARGET_TYPE_SHIFT 4 511 #define QB_ENQUEUE_CMD_DCA_EN_SHIFT 7 512 513 /** 514 * qbman_eq_desc_clear() - Clear the contents of a descriptor to 515 * default/starting state. 516 */ 517 void qbman_eq_desc_clear(struct qbman_eq_desc *d) 518 { 519 memset(d, 0, sizeof(*d)); 520 } 521 522 /** 523 * qbman_eq_desc_set_no_orp() - Set enqueue descriptor without orp 524 * @d: the enqueue descriptor. 525 * @response_success: 1 = enqueue with response always; 0 = enqueue with 526 * rejections returned on a FQ. 527 */ 528 void qbman_eq_desc_set_no_orp(struct qbman_eq_desc *d, int respond_success) 529 { 530 d->verb &= ~(1 << QB_ENQUEUE_CMD_ORP_ENABLE_SHIFT); 531 if (respond_success) 532 d->verb |= enqueue_response_always; 533 else 534 d->verb |= enqueue_rejects_to_fq; 535 } 536 537 /* 538 * Exactly one of the following descriptor "targets" should be set. (Calling any 539 * one of these will replace the effect of any prior call to one of these.) 540 * -enqueue to a frame queue 541 * -enqueue to a queuing destination 542 */ 543 544 /** 545 * qbman_eq_desc_set_fq() - set the FQ for the enqueue command 546 * @d: the enqueue descriptor 547 * @fqid: the id of the frame queue to be enqueued 548 */ 549 void qbman_eq_desc_set_fq(struct qbman_eq_desc *d, u32 fqid) 550 { 551 d->verb &= ~(1 << QB_ENQUEUE_CMD_TARGET_TYPE_SHIFT); 552 d->tgtid = cpu_to_le32(fqid); 553 } 554 555 /** 556 * qbman_eq_desc_set_qd() - Set Queuing Destination for the enqueue command 557 * @d: the enqueue descriptor 558 * @qdid: the id of the queuing destination to be enqueued 559 * @qd_bin: the queuing destination bin 560 * @qd_prio: the queuing destination priority 561 */ 562 void qbman_eq_desc_set_qd(struct qbman_eq_desc *d, u32 qdid, 563 u32 qd_bin, u32 qd_prio) 564 { 565 d->verb |= 1 << QB_ENQUEUE_CMD_TARGET_TYPE_SHIFT; 566 d->tgtid = cpu_to_le32(qdid); 567 d->qdbin = cpu_to_le16(qd_bin); 568 d->qpri = qd_prio; 569 } 570 571 #define EQAR_IDX(eqar) ((eqar) & 0x7) 572 #define EQAR_VB(eqar) ((eqar) & 0x80) 573 #define EQAR_SUCCESS(eqar) ((eqar) & 0x100) 574 575 static inline void qbman_write_eqcr_am_rt_register(struct qbman_swp *p, 576 u8 idx) 577 { 578 if (idx < 16) 579 qbman_write_register(p, QBMAN_CINH_SWP_EQCR_AM_RT + idx * 4, 580 QMAN_RT_MODE); 581 else 582 qbman_write_register(p, QBMAN_CINH_SWP_EQCR_AM_RT2 + 583 (idx - 16) * 4, 584 QMAN_RT_MODE); 585 } 586 587 #define QB_RT_BIT ((u32)0x100) 588 /** 589 * qbman_swp_enqueue_direct() - Issue an enqueue command 590 * @s: the software portal used for enqueue 591 * @d: the enqueue descriptor 592 * @fd: the frame descriptor to be enqueued 593 * 594 * Please note that 'fd' should only be NULL if the "action" of the 595 * descriptor is "orp_hole" or "orp_nesn". 596 * 597 * Return 0 for successful enqueue, -EBUSY if the EQCR is not ready. 598 */ 599 static 600 int qbman_swp_enqueue_direct(struct qbman_swp *s, 601 const struct qbman_eq_desc *d, 602 const struct dpaa2_fd *fd) 603 { 604 int flags = 0; 605 int ret = qbman_swp_enqueue_multiple_direct(s, d, fd, &flags, 1); 606 607 if (ret >= 0) 608 ret = 0; 609 else 610 ret = -EBUSY; 611 return ret; 612 } 613 614 /** 615 * qbman_swp_enqueue_mem_back() - Issue an enqueue command 616 * @s: the software portal used for enqueue 617 * @d: the enqueue descriptor 618 * @fd: the frame descriptor to be enqueued 619 * 620 * Please note that 'fd' should only be NULL if the "action" of the 621 * descriptor is "orp_hole" or "orp_nesn". 622 * 623 * Return 0 for successful enqueue, -EBUSY if the EQCR is not ready. 624 */ 625 static 626 int qbman_swp_enqueue_mem_back(struct qbman_swp *s, 627 const struct qbman_eq_desc *d, 628 const struct dpaa2_fd *fd) 629 { 630 int flags = 0; 631 int ret = qbman_swp_enqueue_multiple_mem_back(s, d, fd, &flags, 1); 632 633 if (ret >= 0) 634 ret = 0; 635 else 636 ret = -EBUSY; 637 return ret; 638 } 639 640 /** 641 * qbman_swp_enqueue_multiple_direct() - Issue a multi enqueue command 642 * using one enqueue descriptor 643 * @s: the software portal used for enqueue 644 * @d: the enqueue descriptor 645 * @fd: table pointer of frame descriptor table to be enqueued 646 * @flags: table pointer of QBMAN_ENQUEUE_FLAG_DCA flags, not used if NULL 647 * @num_frames: number of fd to be enqueued 648 * 649 * Return the number of fd enqueued, or a negative error number. 650 */ 651 static 652 int qbman_swp_enqueue_multiple_direct(struct qbman_swp *s, 653 const struct qbman_eq_desc *d, 654 const struct dpaa2_fd *fd, 655 uint32_t *flags, 656 int num_frames) 657 { 658 uint32_t *p = NULL; 659 const uint32_t *cl = (uint32_t *)d; 660 uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask; 661 int i, num_enqueued = 0; 662 uint64_t addr_cena; 663 664 spin_lock(&s->access_spinlock); 665 half_mask = (s->eqcr.pi_ci_mask>>1); 666 full_mask = s->eqcr.pi_ci_mask; 667 668 if (!s->eqcr.available) { 669 eqcr_ci = s->eqcr.ci; 670 p = s->addr_cena + QBMAN_CENA_SWP_EQCR_CI; 671 s->eqcr.ci = qbman_read_register(s, QBMAN_CINH_SWP_EQCR_CI); 672 673 s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size, 674 eqcr_ci, s->eqcr.ci); 675 if (!s->eqcr.available) { 676 spin_unlock(&s->access_spinlock); 677 return 0; 678 } 679 } 680 681 eqcr_pi = s->eqcr.pi; 682 num_enqueued = (s->eqcr.available < num_frames) ? 683 s->eqcr.available : num_frames; 684 s->eqcr.available -= num_enqueued; 685 /* Fill in the EQCR ring */ 686 for (i = 0; i < num_enqueued; i++) { 687 p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); 688 /* Skip copying the verb */ 689 memcpy(&p[1], &cl[1], EQ_DESC_SIZE_WITHOUT_FD - 1); 690 memcpy(&p[EQ_DESC_SIZE_FD_START/sizeof(uint32_t)], 691 &fd[i], sizeof(*fd)); 692 eqcr_pi++; 693 } 694 695 dma_wmb(); 696 697 /* Set the verb byte, have to substitute in the valid-bit */ 698 eqcr_pi = s->eqcr.pi; 699 for (i = 0; i < num_enqueued; i++) { 700 p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); 701 p[0] = cl[0] | s->eqcr.pi_vb; 702 if (flags && (flags[i] & QBMAN_ENQUEUE_FLAG_DCA)) { 703 struct qbman_eq_desc *d = (struct qbman_eq_desc *)p; 704 705 d->dca = (1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT) | 706 ((flags[i]) & QBMAN_EQCR_DCA_IDXMASK); 707 } 708 eqcr_pi++; 709 if (!(eqcr_pi & half_mask)) 710 s->eqcr.pi_vb ^= QB_VALID_BIT; 711 } 712 713 /* Flush all the cacheline without load/store in between */ 714 eqcr_pi = s->eqcr.pi; 715 addr_cena = (size_t)s->addr_cena; 716 for (i = 0; i < num_enqueued; i++) 717 eqcr_pi++; 718 s->eqcr.pi = eqcr_pi & full_mask; 719 spin_unlock(&s->access_spinlock); 720 721 return num_enqueued; 722 } 723 724 /** 725 * qbman_swp_enqueue_multiple_mem_back() - Issue a multi enqueue command 726 * using one enqueue descriptor 727 * @s: the software portal used for enqueue 728 * @d: the enqueue descriptor 729 * @fd: table pointer of frame descriptor table to be enqueued 730 * @flags: table pointer of QBMAN_ENQUEUE_FLAG_DCA flags, not used if NULL 731 * @num_frames: number of fd to be enqueued 732 * 733 * Return the number of fd enqueued, or a negative error number. 734 */ 735 static 736 int qbman_swp_enqueue_multiple_mem_back(struct qbman_swp *s, 737 const struct qbman_eq_desc *d, 738 const struct dpaa2_fd *fd, 739 uint32_t *flags, 740 int num_frames) 741 { 742 uint32_t *p = NULL; 743 const uint32_t *cl = (uint32_t *)(d); 744 uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask; 745 int i, num_enqueued = 0; 746 unsigned long irq_flags; 747 748 spin_lock(&s->access_spinlock); 749 local_irq_save(irq_flags); 750 751 half_mask = (s->eqcr.pi_ci_mask>>1); 752 full_mask = s->eqcr.pi_ci_mask; 753 if (!s->eqcr.available) { 754 eqcr_ci = s->eqcr.ci; 755 p = s->addr_cena + QBMAN_CENA_SWP_EQCR_CI_MEMBACK; 756 s->eqcr.ci = *p & full_mask; 757 s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size, 758 eqcr_ci, s->eqcr.ci); 759 if (!s->eqcr.available) { 760 local_irq_restore(irq_flags); 761 spin_unlock(&s->access_spinlock); 762 return 0; 763 } 764 } 765 766 eqcr_pi = s->eqcr.pi; 767 num_enqueued = (s->eqcr.available < num_frames) ? 768 s->eqcr.available : num_frames; 769 s->eqcr.available -= num_enqueued; 770 /* Fill in the EQCR ring */ 771 for (i = 0; i < num_enqueued; i++) { 772 p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); 773 /* Skip copying the verb */ 774 memcpy(&p[1], &cl[1], EQ_DESC_SIZE_WITHOUT_FD - 1); 775 memcpy(&p[EQ_DESC_SIZE_FD_START/sizeof(uint32_t)], 776 &fd[i], sizeof(*fd)); 777 eqcr_pi++; 778 } 779 780 /* Set the verb byte, have to substitute in the valid-bit */ 781 eqcr_pi = s->eqcr.pi; 782 for (i = 0; i < num_enqueued; i++) { 783 p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); 784 p[0] = cl[0] | s->eqcr.pi_vb; 785 if (flags && (flags[i] & QBMAN_ENQUEUE_FLAG_DCA)) { 786 struct qbman_eq_desc *d = (struct qbman_eq_desc *)p; 787 788 d->dca = (1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT) | 789 ((flags[i]) & QBMAN_EQCR_DCA_IDXMASK); 790 } 791 eqcr_pi++; 792 if (!(eqcr_pi & half_mask)) 793 s->eqcr.pi_vb ^= QB_VALID_BIT; 794 } 795 s->eqcr.pi = eqcr_pi & full_mask; 796 797 dma_wmb(); 798 qbman_write_register(s, QBMAN_CINH_SWP_EQCR_PI, 799 (QB_RT_BIT)|(s->eqcr.pi)|s->eqcr.pi_vb); 800 local_irq_restore(irq_flags); 801 spin_unlock(&s->access_spinlock); 802 803 return num_enqueued; 804 } 805 806 /** 807 * qbman_swp_enqueue_multiple_desc_direct() - Issue a multi enqueue command 808 * using multiple enqueue descriptor 809 * @s: the software portal used for enqueue 810 * @d: table of minimal enqueue descriptor 811 * @fd: table pointer of frame descriptor table to be enqueued 812 * @num_frames: number of fd to be enqueued 813 * 814 * Return the number of fd enqueued, or a negative error number. 815 */ 816 static 817 int qbman_swp_enqueue_multiple_desc_direct(struct qbman_swp *s, 818 const struct qbman_eq_desc *d, 819 const struct dpaa2_fd *fd, 820 int num_frames) 821 { 822 uint32_t *p; 823 const uint32_t *cl; 824 uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask; 825 int i, num_enqueued = 0; 826 827 half_mask = (s->eqcr.pi_ci_mask>>1); 828 full_mask = s->eqcr.pi_ci_mask; 829 if (!s->eqcr.available) { 830 eqcr_ci = s->eqcr.ci; 831 p = s->addr_cena + QBMAN_CENA_SWP_EQCR_CI; 832 s->eqcr.ci = qbman_read_register(s, QBMAN_CINH_SWP_EQCR_CI); 833 s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size, 834 eqcr_ci, s->eqcr.ci); 835 if (!s->eqcr.available) 836 return 0; 837 } 838 839 eqcr_pi = s->eqcr.pi; 840 num_enqueued = (s->eqcr.available < num_frames) ? 841 s->eqcr.available : num_frames; 842 s->eqcr.available -= num_enqueued; 843 /* Fill in the EQCR ring */ 844 for (i = 0; i < num_enqueued; i++) { 845 p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); 846 cl = (uint32_t *)(&d[i]); 847 /* Skip copying the verb */ 848 memcpy(&p[1], &cl[1], EQ_DESC_SIZE_WITHOUT_FD - 1); 849 memcpy(&p[EQ_DESC_SIZE_FD_START/sizeof(uint32_t)], 850 &fd[i], sizeof(*fd)); 851 eqcr_pi++; 852 } 853 854 dma_wmb(); 855 856 /* Set the verb byte, have to substitute in the valid-bit */ 857 eqcr_pi = s->eqcr.pi; 858 for (i = 0; i < num_enqueued; i++) { 859 p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); 860 cl = (uint32_t *)(&d[i]); 861 p[0] = cl[0] | s->eqcr.pi_vb; 862 eqcr_pi++; 863 if (!(eqcr_pi & half_mask)) 864 s->eqcr.pi_vb ^= QB_VALID_BIT; 865 } 866 867 /* Flush all the cacheline without load/store in between */ 868 eqcr_pi = s->eqcr.pi; 869 for (i = 0; i < num_enqueued; i++) 870 eqcr_pi++; 871 s->eqcr.pi = eqcr_pi & full_mask; 872 873 return num_enqueued; 874 } 875 876 /** 877 * qbman_swp_enqueue_multiple_desc_mem_back() - Issue a multi enqueue command 878 * using multiple enqueue descriptor 879 * @s: the software portal used for enqueue 880 * @d: table of minimal enqueue descriptor 881 * @fd: table pointer of frame descriptor table to be enqueued 882 * @num_frames: number of fd to be enqueued 883 * 884 * Return the number of fd enqueued, or a negative error number. 885 */ 886 static 887 int qbman_swp_enqueue_multiple_desc_mem_back(struct qbman_swp *s, 888 const struct qbman_eq_desc *d, 889 const struct dpaa2_fd *fd, 890 int num_frames) 891 { 892 uint32_t *p; 893 const uint32_t *cl; 894 uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask; 895 int i, num_enqueued = 0; 896 897 half_mask = (s->eqcr.pi_ci_mask>>1); 898 full_mask = s->eqcr.pi_ci_mask; 899 if (!s->eqcr.available) { 900 eqcr_ci = s->eqcr.ci; 901 p = s->addr_cena + QBMAN_CENA_SWP_EQCR_CI_MEMBACK; 902 s->eqcr.ci = *p & full_mask; 903 s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size, 904 eqcr_ci, s->eqcr.ci); 905 if (!s->eqcr.available) 906 return 0; 907 } 908 909 eqcr_pi = s->eqcr.pi; 910 num_enqueued = (s->eqcr.available < num_frames) ? 911 s->eqcr.available : num_frames; 912 s->eqcr.available -= num_enqueued; 913 /* Fill in the EQCR ring */ 914 for (i = 0; i < num_enqueued; i++) { 915 p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); 916 cl = (uint32_t *)(&d[i]); 917 /* Skip copying the verb */ 918 memcpy(&p[1], &cl[1], EQ_DESC_SIZE_WITHOUT_FD - 1); 919 memcpy(&p[EQ_DESC_SIZE_FD_START/sizeof(uint32_t)], 920 &fd[i], sizeof(*fd)); 921 eqcr_pi++; 922 } 923 924 /* Set the verb byte, have to substitute in the valid-bit */ 925 eqcr_pi = s->eqcr.pi; 926 for (i = 0; i < num_enqueued; i++) { 927 p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); 928 cl = (uint32_t *)(&d[i]); 929 p[0] = cl[0] | s->eqcr.pi_vb; 930 eqcr_pi++; 931 if (!(eqcr_pi & half_mask)) 932 s->eqcr.pi_vb ^= QB_VALID_BIT; 933 } 934 935 s->eqcr.pi = eqcr_pi & full_mask; 936 937 dma_wmb(); 938 qbman_write_register(s, QBMAN_CINH_SWP_EQCR_PI, 939 (QB_RT_BIT)|(s->eqcr.pi)|s->eqcr.pi_vb); 940 941 return num_enqueued; 942 } 943 944 /* Static (push) dequeue */ 945 946 /** 947 * qbman_swp_push_get() - Get the push dequeue setup 948 * @p: the software portal object 949 * @channel_idx: the channel index to query 950 * @enabled: returned boolean to show whether the push dequeue is enabled 951 * for the given channel 952 */ 953 void qbman_swp_push_get(struct qbman_swp *s, u8 channel_idx, int *enabled) 954 { 955 u16 src = (s->sdq >> QB_SDQCR_SRC_SHIFT) & QB_SDQCR_SRC_MASK; 956 957 WARN_ON(channel_idx > 15); 958 *enabled = src | (1 << channel_idx); 959 } 960 961 /** 962 * qbman_swp_push_set() - Enable or disable push dequeue 963 * @p: the software portal object 964 * @channel_idx: the channel index (0 to 15) 965 * @enable: enable or disable push dequeue 966 */ 967 void qbman_swp_push_set(struct qbman_swp *s, u8 channel_idx, int enable) 968 { 969 u16 dqsrc; 970 971 WARN_ON(channel_idx > 15); 972 if (enable) 973 s->sdq |= 1 << channel_idx; 974 else 975 s->sdq &= ~(1 << channel_idx); 976 977 /* Read make the complete src map. If no channels are enabled 978 * the SDQCR must be 0 or else QMan will assert errors 979 */ 980 dqsrc = (s->sdq >> QB_SDQCR_SRC_SHIFT) & QB_SDQCR_SRC_MASK; 981 if (dqsrc != 0) 982 qbman_write_register(s, QBMAN_CINH_SWP_SDQCR, s->sdq); 983 else 984 qbman_write_register(s, QBMAN_CINH_SWP_SDQCR, 0); 985 } 986 987 #define QB_VDQCR_VERB_DCT_SHIFT 0 988 #define QB_VDQCR_VERB_DT_SHIFT 2 989 #define QB_VDQCR_VERB_RLS_SHIFT 4 990 #define QB_VDQCR_VERB_WAE_SHIFT 5 991 992 enum qb_pull_dt_e { 993 qb_pull_dt_channel, 994 qb_pull_dt_workqueue, 995 qb_pull_dt_framequeue 996 }; 997 998 /** 999 * qbman_pull_desc_clear() - Clear the contents of a descriptor to 1000 * default/starting state 1001 * @d: the pull dequeue descriptor to be cleared 1002 */ 1003 void qbman_pull_desc_clear(struct qbman_pull_desc *d) 1004 { 1005 memset(d, 0, sizeof(*d)); 1006 } 1007 1008 /** 1009 * qbman_pull_desc_set_storage()- Set the pull dequeue storage 1010 * @d: the pull dequeue descriptor to be set 1011 * @storage: the pointer of the memory to store the dequeue result 1012 * @storage_phys: the physical address of the storage memory 1013 * @stash: to indicate whether write allocate is enabled 1014 * 1015 * If not called, or if called with 'storage' as NULL, the result pull dequeues 1016 * will produce results to DQRR. If 'storage' is non-NULL, then results are 1017 * produced to the given memory location (using the DMA address which 1018 * the caller provides in 'storage_phys'), and 'stash' controls whether or not 1019 * those writes to main-memory express a cache-warming attribute. 1020 */ 1021 void qbman_pull_desc_set_storage(struct qbman_pull_desc *d, 1022 struct dpaa2_dq *storage, 1023 dma_addr_t storage_phys, 1024 int stash) 1025 { 1026 /* save the virtual address */ 1027 d->rsp_addr_virt = (u64)(uintptr_t)storage; 1028 1029 if (!storage) { 1030 d->verb &= ~(1 << QB_VDQCR_VERB_RLS_SHIFT); 1031 return; 1032 } 1033 d->verb |= 1 << QB_VDQCR_VERB_RLS_SHIFT; 1034 if (stash) 1035 d->verb |= 1 << QB_VDQCR_VERB_WAE_SHIFT; 1036 else 1037 d->verb &= ~(1 << QB_VDQCR_VERB_WAE_SHIFT); 1038 1039 d->rsp_addr = cpu_to_le64(storage_phys); 1040 } 1041 1042 /** 1043 * qbman_pull_desc_set_numframes() - Set the number of frames to be dequeued 1044 * @d: the pull dequeue descriptor to be set 1045 * @numframes: number of frames to be set, must be between 1 and 16, inclusive 1046 */ 1047 void qbman_pull_desc_set_numframes(struct qbman_pull_desc *d, u8 numframes) 1048 { 1049 d->numf = numframes - 1; 1050 } 1051 1052 /* 1053 * Exactly one of the following descriptor "actions" should be set. (Calling any 1054 * one of these will replace the effect of any prior call to one of these.) 1055 * - pull dequeue from the given frame queue (FQ) 1056 * - pull dequeue from any FQ in the given work queue (WQ) 1057 * - pull dequeue from any FQ in any WQ in the given channel 1058 */ 1059 1060 /** 1061 * qbman_pull_desc_set_fq() - Set fqid from which the dequeue command dequeues 1062 * @fqid: the frame queue index of the given FQ 1063 */ 1064 void qbman_pull_desc_set_fq(struct qbman_pull_desc *d, u32 fqid) 1065 { 1066 d->verb |= 1 << QB_VDQCR_VERB_DCT_SHIFT; 1067 d->verb |= qb_pull_dt_framequeue << QB_VDQCR_VERB_DT_SHIFT; 1068 d->dq_src = cpu_to_le32(fqid); 1069 } 1070 1071 /** 1072 * qbman_pull_desc_set_wq() - Set wqid from which the dequeue command dequeues 1073 * @wqid: composed of channel id and wqid within the channel 1074 * @dct: the dequeue command type 1075 */ 1076 void qbman_pull_desc_set_wq(struct qbman_pull_desc *d, u32 wqid, 1077 enum qbman_pull_type_e dct) 1078 { 1079 d->verb |= dct << QB_VDQCR_VERB_DCT_SHIFT; 1080 d->verb |= qb_pull_dt_workqueue << QB_VDQCR_VERB_DT_SHIFT; 1081 d->dq_src = cpu_to_le32(wqid); 1082 } 1083 1084 /** 1085 * qbman_pull_desc_set_channel() - Set channelid from which the dequeue command 1086 * dequeues 1087 * @chid: the channel id to be dequeued 1088 * @dct: the dequeue command type 1089 */ 1090 void qbman_pull_desc_set_channel(struct qbman_pull_desc *d, u32 chid, 1091 enum qbman_pull_type_e dct) 1092 { 1093 d->verb |= dct << QB_VDQCR_VERB_DCT_SHIFT; 1094 d->verb |= qb_pull_dt_channel << QB_VDQCR_VERB_DT_SHIFT; 1095 d->dq_src = cpu_to_le32(chid); 1096 } 1097 1098 /** 1099 * qbman_swp_pull_direct() - Issue the pull dequeue command 1100 * @s: the software portal object 1101 * @d: the software portal descriptor which has been configured with 1102 * the set of qbman_pull_desc_set_*() calls 1103 * 1104 * Return 0 for success, and -EBUSY if the software portal is not ready 1105 * to do pull dequeue. 1106 */ 1107 static 1108 int qbman_swp_pull_direct(struct qbman_swp *s, struct qbman_pull_desc *d) 1109 { 1110 struct qbman_pull_desc *p; 1111 1112 if (!atomic_dec_and_test(&s->vdq.available)) { 1113 atomic_inc(&s->vdq.available); 1114 return -EBUSY; 1115 } 1116 s->vdq.storage = (void *)(uintptr_t)d->rsp_addr_virt; 1117 if ((s->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000) 1118 p = qbman_get_cmd(s, QBMAN_CENA_SWP_VDQCR); 1119 else 1120 p = qbman_get_cmd(s, QBMAN_CENA_SWP_VDQCR_MEM); 1121 p->numf = d->numf; 1122 p->tok = QMAN_DQ_TOKEN_VALID; 1123 p->dq_src = d->dq_src; 1124 p->rsp_addr = d->rsp_addr; 1125 p->rsp_addr_virt = d->rsp_addr_virt; 1126 dma_wmb(); 1127 /* Set the verb byte, have to substitute in the valid-bit */ 1128 p->verb = d->verb | s->vdq.valid_bit; 1129 s->vdq.valid_bit ^= QB_VALID_BIT; 1130 1131 return 0; 1132 } 1133 1134 /** 1135 * qbman_swp_pull_mem_back() - Issue the pull dequeue command 1136 * @s: the software portal object 1137 * @d: the software portal descriptor which has been configured with 1138 * the set of qbman_pull_desc_set_*() calls 1139 * 1140 * Return 0 for success, and -EBUSY if the software portal is not ready 1141 * to do pull dequeue. 1142 */ 1143 static 1144 int qbman_swp_pull_mem_back(struct qbman_swp *s, struct qbman_pull_desc *d) 1145 { 1146 struct qbman_pull_desc *p; 1147 1148 if (!atomic_dec_and_test(&s->vdq.available)) { 1149 atomic_inc(&s->vdq.available); 1150 return -EBUSY; 1151 } 1152 s->vdq.storage = (void *)(uintptr_t)d->rsp_addr_virt; 1153 if ((s->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000) 1154 p = qbman_get_cmd(s, QBMAN_CENA_SWP_VDQCR); 1155 else 1156 p = qbman_get_cmd(s, QBMAN_CENA_SWP_VDQCR_MEM); 1157 p->numf = d->numf; 1158 p->tok = QMAN_DQ_TOKEN_VALID; 1159 p->dq_src = d->dq_src; 1160 p->rsp_addr = d->rsp_addr; 1161 p->rsp_addr_virt = d->rsp_addr_virt; 1162 1163 /* Set the verb byte, have to substitute in the valid-bit */ 1164 p->verb = d->verb | s->vdq.valid_bit; 1165 s->vdq.valid_bit ^= QB_VALID_BIT; 1166 dma_wmb(); 1167 qbman_write_register(s, QBMAN_CINH_SWP_VDQCR_RT, QMAN_RT_MODE); 1168 1169 return 0; 1170 } 1171 1172 #define QMAN_DQRR_PI_MASK 0xf 1173 1174 /** 1175 * qbman_swp_dqrr_next_direct() - Get an valid DQRR entry 1176 * @s: the software portal object 1177 * 1178 * Return NULL if there are no unconsumed DQRR entries. Return a DQRR entry 1179 * only once, so repeated calls can return a sequence of DQRR entries, without 1180 * requiring they be consumed immediately or in any particular order. 1181 */ 1182 const struct dpaa2_dq *qbman_swp_dqrr_next_direct(struct qbman_swp *s) 1183 { 1184 u32 verb; 1185 u32 response_verb; 1186 u32 flags; 1187 struct dpaa2_dq *p; 1188 1189 /* Before using valid-bit to detect if something is there, we have to 1190 * handle the case of the DQRR reset bug... 1191 */ 1192 if (unlikely(s->dqrr.reset_bug)) { 1193 /* 1194 * We pick up new entries by cache-inhibited producer index, 1195 * which means that a non-coherent mapping would require us to 1196 * invalidate and read *only* once that PI has indicated that 1197 * there's an entry here. The first trip around the DQRR ring 1198 * will be much less efficient than all subsequent trips around 1199 * it... 1200 */ 1201 u8 pi = qbman_read_register(s, QBMAN_CINH_SWP_DQPI) & 1202 QMAN_DQRR_PI_MASK; 1203 1204 /* there are new entries if pi != next_idx */ 1205 if (pi == s->dqrr.next_idx) 1206 return NULL; 1207 1208 /* 1209 * if next_idx is/was the last ring index, and 'pi' is 1210 * different, we can disable the workaround as all the ring 1211 * entries have now been DMA'd to so valid-bit checking is 1212 * repaired. Note: this logic needs to be based on next_idx 1213 * (which increments one at a time), rather than on pi (which 1214 * can burst and wrap-around between our snapshots of it). 1215 */ 1216 if (s->dqrr.next_idx == (s->dqrr.dqrr_size - 1)) { 1217 pr_debug("next_idx=%d, pi=%d, clear reset bug\n", 1218 s->dqrr.next_idx, pi); 1219 s->dqrr.reset_bug = 0; 1220 } 1221 prefetch(qbman_get_cmd(s, 1222 QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx))); 1223 } 1224 1225 p = qbman_get_cmd(s, QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)); 1226 verb = p->dq.verb; 1227 1228 /* 1229 * If the valid-bit isn't of the expected polarity, nothing there. Note, 1230 * in the DQRR reset bug workaround, we shouldn't need to skip these 1231 * check, because we've already determined that a new entry is available 1232 * and we've invalidated the cacheline before reading it, so the 1233 * valid-bit behaviour is repaired and should tell us what we already 1234 * knew from reading PI. 1235 */ 1236 if ((verb & QB_VALID_BIT) != s->dqrr.valid_bit) { 1237 prefetch(qbman_get_cmd(s, 1238 QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx))); 1239 return NULL; 1240 } 1241 /* 1242 * There's something there. Move "next_idx" attention to the next ring 1243 * entry (and prefetch it) before returning what we found. 1244 */ 1245 s->dqrr.next_idx++; 1246 s->dqrr.next_idx &= s->dqrr.dqrr_size - 1; /* Wrap around */ 1247 if (!s->dqrr.next_idx) 1248 s->dqrr.valid_bit ^= QB_VALID_BIT; 1249 1250 /* 1251 * If this is the final response to a volatile dequeue command 1252 * indicate that the vdq is available 1253 */ 1254 flags = p->dq.stat; 1255 response_verb = verb & QBMAN_RESULT_MASK; 1256 if ((response_verb == QBMAN_RESULT_DQ) && 1257 (flags & DPAA2_DQ_STAT_VOLATILE) && 1258 (flags & DPAA2_DQ_STAT_EXPIRED)) 1259 atomic_inc(&s->vdq.available); 1260 1261 prefetch(qbman_get_cmd(s, QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx))); 1262 1263 return p; 1264 } 1265 1266 /** 1267 * qbman_swp_dqrr_next_mem_back() - Get an valid DQRR entry 1268 * @s: the software portal object 1269 * 1270 * Return NULL if there are no unconsumed DQRR entries. Return a DQRR entry 1271 * only once, so repeated calls can return a sequence of DQRR entries, without 1272 * requiring they be consumed immediately or in any particular order. 1273 */ 1274 const struct dpaa2_dq *qbman_swp_dqrr_next_mem_back(struct qbman_swp *s) 1275 { 1276 u32 verb; 1277 u32 response_verb; 1278 u32 flags; 1279 struct dpaa2_dq *p; 1280 1281 /* Before using valid-bit to detect if something is there, we have to 1282 * handle the case of the DQRR reset bug... 1283 */ 1284 if (unlikely(s->dqrr.reset_bug)) { 1285 /* 1286 * We pick up new entries by cache-inhibited producer index, 1287 * which means that a non-coherent mapping would require us to 1288 * invalidate and read *only* once that PI has indicated that 1289 * there's an entry here. The first trip around the DQRR ring 1290 * will be much less efficient than all subsequent trips around 1291 * it... 1292 */ 1293 u8 pi = qbman_read_register(s, QBMAN_CINH_SWP_DQPI) & 1294 QMAN_DQRR_PI_MASK; 1295 1296 /* there are new entries if pi != next_idx */ 1297 if (pi == s->dqrr.next_idx) 1298 return NULL; 1299 1300 /* 1301 * if next_idx is/was the last ring index, and 'pi' is 1302 * different, we can disable the workaround as all the ring 1303 * entries have now been DMA'd to so valid-bit checking is 1304 * repaired. Note: this logic needs to be based on next_idx 1305 * (which increments one at a time), rather than on pi (which 1306 * can burst and wrap-around between our snapshots of it). 1307 */ 1308 if (s->dqrr.next_idx == (s->dqrr.dqrr_size - 1)) { 1309 pr_debug("next_idx=%d, pi=%d, clear reset bug\n", 1310 s->dqrr.next_idx, pi); 1311 s->dqrr.reset_bug = 0; 1312 } 1313 prefetch(qbman_get_cmd(s, 1314 QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx))); 1315 } 1316 1317 p = qbman_get_cmd(s, QBMAN_CENA_SWP_DQRR_MEM(s->dqrr.next_idx)); 1318 verb = p->dq.verb; 1319 1320 /* 1321 * If the valid-bit isn't of the expected polarity, nothing there. Note, 1322 * in the DQRR reset bug workaround, we shouldn't need to skip these 1323 * check, because we've already determined that a new entry is available 1324 * and we've invalidated the cacheline before reading it, so the 1325 * valid-bit behaviour is repaired and should tell us what we already 1326 * knew from reading PI. 1327 */ 1328 if ((verb & QB_VALID_BIT) != s->dqrr.valid_bit) { 1329 prefetch(qbman_get_cmd(s, 1330 QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx))); 1331 return NULL; 1332 } 1333 /* 1334 * There's something there. Move "next_idx" attention to the next ring 1335 * entry (and prefetch it) before returning what we found. 1336 */ 1337 s->dqrr.next_idx++; 1338 s->dqrr.next_idx &= s->dqrr.dqrr_size - 1; /* Wrap around */ 1339 if (!s->dqrr.next_idx) 1340 s->dqrr.valid_bit ^= QB_VALID_BIT; 1341 1342 /* 1343 * If this is the final response to a volatile dequeue command 1344 * indicate that the vdq is available 1345 */ 1346 flags = p->dq.stat; 1347 response_verb = verb & QBMAN_RESULT_MASK; 1348 if ((response_verb == QBMAN_RESULT_DQ) && 1349 (flags & DPAA2_DQ_STAT_VOLATILE) && 1350 (flags & DPAA2_DQ_STAT_EXPIRED)) 1351 atomic_inc(&s->vdq.available); 1352 1353 prefetch(qbman_get_cmd(s, QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx))); 1354 1355 return p; 1356 } 1357 1358 /** 1359 * qbman_swp_dqrr_consume() - Consume DQRR entries previously returned from 1360 * qbman_swp_dqrr_next(). 1361 * @s: the software portal object 1362 * @dq: the DQRR entry to be consumed 1363 */ 1364 void qbman_swp_dqrr_consume(struct qbman_swp *s, const struct dpaa2_dq *dq) 1365 { 1366 qbman_write_register(s, QBMAN_CINH_SWP_DCAP, QBMAN_IDX_FROM_DQRR(dq)); 1367 } 1368 1369 /** 1370 * qbman_result_has_new_result() - Check and get the dequeue response from the 1371 * dq storage memory set in pull dequeue command 1372 * @s: the software portal object 1373 * @dq: the dequeue result read from the memory 1374 * 1375 * Return 1 for getting a valid dequeue result, or 0 for not getting a valid 1376 * dequeue result. 1377 * 1378 * Only used for user-provided storage of dequeue results, not DQRR. For 1379 * efficiency purposes, the driver will perform any required endianness 1380 * conversion to ensure that the user's dequeue result storage is in host-endian 1381 * format. As such, once the user has called qbman_result_has_new_result() and 1382 * been returned a valid dequeue result, they should not call it again on 1383 * the same memory location (except of course if another dequeue command has 1384 * been executed to produce a new result to that location). 1385 */ 1386 int qbman_result_has_new_result(struct qbman_swp *s, const struct dpaa2_dq *dq) 1387 { 1388 if (dq->dq.tok != QMAN_DQ_TOKEN_VALID) 1389 return 0; 1390 1391 /* 1392 * Set token to be 0 so we will detect change back to 1 1393 * next time the looping is traversed. Const is cast away here 1394 * as we want users to treat the dequeue responses as read only. 1395 */ 1396 ((struct dpaa2_dq *)dq)->dq.tok = 0; 1397 1398 /* 1399 * Determine whether VDQCR is available based on whether the 1400 * current result is sitting in the first storage location of 1401 * the busy command. 1402 */ 1403 if (s->vdq.storage == dq) { 1404 s->vdq.storage = NULL; 1405 atomic_inc(&s->vdq.available); 1406 } 1407 1408 return 1; 1409 } 1410 1411 /** 1412 * qbman_release_desc_clear() - Clear the contents of a descriptor to 1413 * default/starting state. 1414 */ 1415 void qbman_release_desc_clear(struct qbman_release_desc *d) 1416 { 1417 memset(d, 0, sizeof(*d)); 1418 d->verb = 1 << 5; /* Release Command Valid */ 1419 } 1420 1421 /** 1422 * qbman_release_desc_set_bpid() - Set the ID of the buffer pool to release to 1423 */ 1424 void qbman_release_desc_set_bpid(struct qbman_release_desc *d, u16 bpid) 1425 { 1426 d->bpid = cpu_to_le16(bpid); 1427 } 1428 1429 /** 1430 * qbman_release_desc_set_rcdi() - Determines whether or not the portal's RCDI 1431 * interrupt source should be asserted after the release command is completed. 1432 */ 1433 void qbman_release_desc_set_rcdi(struct qbman_release_desc *d, int enable) 1434 { 1435 if (enable) 1436 d->verb |= 1 << 6; 1437 else 1438 d->verb &= ~(1 << 6); 1439 } 1440 1441 #define RAR_IDX(rar) ((rar) & 0x7) 1442 #define RAR_VB(rar) ((rar) & 0x80) 1443 #define RAR_SUCCESS(rar) ((rar) & 0x100) 1444 1445 /** 1446 * qbman_swp_release_direct() - Issue a buffer release command 1447 * @s: the software portal object 1448 * @d: the release descriptor 1449 * @buffers: a pointer pointing to the buffer address to be released 1450 * @num_buffers: number of buffers to be released, must be less than 8 1451 * 1452 * Return 0 for success, -EBUSY if the release command ring is not ready. 1453 */ 1454 int qbman_swp_release_direct(struct qbman_swp *s, 1455 const struct qbman_release_desc *d, 1456 const u64 *buffers, unsigned int num_buffers) 1457 { 1458 int i; 1459 struct qbman_release_desc *p; 1460 u32 rar; 1461 1462 if (!num_buffers || (num_buffers > 7)) 1463 return -EINVAL; 1464 1465 rar = qbman_read_register(s, QBMAN_CINH_SWP_RAR); 1466 if (!RAR_SUCCESS(rar)) 1467 return -EBUSY; 1468 1469 /* Start the release command */ 1470 p = qbman_get_cmd(s, QBMAN_CENA_SWP_RCR(RAR_IDX(rar))); 1471 1472 /* Copy the caller's buffer pointers to the command */ 1473 for (i = 0; i < num_buffers; i++) 1474 p->buf[i] = cpu_to_le64(buffers[i]); 1475 p->bpid = d->bpid; 1476 1477 /* 1478 * Set the verb byte, have to substitute in the valid-bit 1479 * and the number of buffers. 1480 */ 1481 dma_wmb(); 1482 p->verb = d->verb | RAR_VB(rar) | num_buffers; 1483 1484 return 0; 1485 } 1486 1487 /** 1488 * qbman_swp_release_mem_back() - Issue a buffer release command 1489 * @s: the software portal object 1490 * @d: the release descriptor 1491 * @buffers: a pointer pointing to the buffer address to be released 1492 * @num_buffers: number of buffers to be released, must be less than 8 1493 * 1494 * Return 0 for success, -EBUSY if the release command ring is not ready. 1495 */ 1496 int qbman_swp_release_mem_back(struct qbman_swp *s, 1497 const struct qbman_release_desc *d, 1498 const u64 *buffers, unsigned int num_buffers) 1499 { 1500 int i; 1501 struct qbman_release_desc *p; 1502 u32 rar; 1503 1504 if (!num_buffers || (num_buffers > 7)) 1505 return -EINVAL; 1506 1507 rar = qbman_read_register(s, QBMAN_CINH_SWP_RAR); 1508 if (!RAR_SUCCESS(rar)) 1509 return -EBUSY; 1510 1511 /* Start the release command */ 1512 p = qbman_get_cmd(s, QBMAN_CENA_SWP_RCR_MEM(RAR_IDX(rar))); 1513 1514 /* Copy the caller's buffer pointers to the command */ 1515 for (i = 0; i < num_buffers; i++) 1516 p->buf[i] = cpu_to_le64(buffers[i]); 1517 p->bpid = d->bpid; 1518 1519 p->verb = d->verb | RAR_VB(rar) | num_buffers; 1520 dma_wmb(); 1521 qbman_write_register(s, QBMAN_CINH_SWP_RCR_AM_RT + 1522 RAR_IDX(rar) * 4, QMAN_RT_MODE); 1523 1524 return 0; 1525 } 1526 1527 struct qbman_acquire_desc { 1528 u8 verb; 1529 u8 reserved; 1530 __le16 bpid; 1531 u8 num; 1532 u8 reserved2[59]; 1533 }; 1534 1535 struct qbman_acquire_rslt { 1536 u8 verb; 1537 u8 rslt; 1538 __le16 reserved; 1539 u8 num; 1540 u8 reserved2[3]; 1541 __le64 buf[7]; 1542 }; 1543 1544 /** 1545 * qbman_swp_acquire() - Issue a buffer acquire command 1546 * @s: the software portal object 1547 * @bpid: the buffer pool index 1548 * @buffers: a pointer pointing to the acquired buffer addresses 1549 * @num_buffers: number of buffers to be acquired, must be less than 8 1550 * 1551 * Return 0 for success, or negative error code if the acquire command 1552 * fails. 1553 */ 1554 int qbman_swp_acquire(struct qbman_swp *s, u16 bpid, u64 *buffers, 1555 unsigned int num_buffers) 1556 { 1557 struct qbman_acquire_desc *p; 1558 struct qbman_acquire_rslt *r; 1559 int i; 1560 1561 if (!num_buffers || (num_buffers > 7)) 1562 return -EINVAL; 1563 1564 /* Start the management command */ 1565 p = qbman_swp_mc_start(s); 1566 1567 if (!p) 1568 return -EBUSY; 1569 1570 /* Encode the caller-provided attributes */ 1571 p->bpid = cpu_to_le16(bpid); 1572 p->num = num_buffers; 1573 1574 /* Complete the management command */ 1575 r = qbman_swp_mc_complete(s, p, QBMAN_MC_ACQUIRE); 1576 if (unlikely(!r)) { 1577 pr_err("qbman: acquire from BPID %d failed, no response\n", 1578 bpid); 1579 return -EIO; 1580 } 1581 1582 /* Decode the outcome */ 1583 WARN_ON((r->verb & 0x7f) != QBMAN_MC_ACQUIRE); 1584 1585 /* Determine success or failure */ 1586 if (unlikely(r->rslt != QBMAN_MC_RSLT_OK)) { 1587 pr_err("qbman: acquire from BPID 0x%x failed, code=0x%02x\n", 1588 bpid, r->rslt); 1589 return -EIO; 1590 } 1591 1592 WARN_ON(r->num > num_buffers); 1593 1594 /* Copy the acquired buffers to the caller's array */ 1595 for (i = 0; i < r->num; i++) 1596 buffers[i] = le64_to_cpu(r->buf[i]); 1597 1598 return (int)r->num; 1599 } 1600 1601 struct qbman_alt_fq_state_desc { 1602 u8 verb; 1603 u8 reserved[3]; 1604 __le32 fqid; 1605 u8 reserved2[56]; 1606 }; 1607 1608 struct qbman_alt_fq_state_rslt { 1609 u8 verb; 1610 u8 rslt; 1611 u8 reserved[62]; 1612 }; 1613 1614 #define ALT_FQ_FQID_MASK 0x00FFFFFF 1615 1616 int qbman_swp_alt_fq_state(struct qbman_swp *s, u32 fqid, 1617 u8 alt_fq_verb) 1618 { 1619 struct qbman_alt_fq_state_desc *p; 1620 struct qbman_alt_fq_state_rslt *r; 1621 1622 /* Start the management command */ 1623 p = qbman_swp_mc_start(s); 1624 if (!p) 1625 return -EBUSY; 1626 1627 p->fqid = cpu_to_le32(fqid & ALT_FQ_FQID_MASK); 1628 1629 /* Complete the management command */ 1630 r = qbman_swp_mc_complete(s, p, alt_fq_verb); 1631 if (unlikely(!r)) { 1632 pr_err("qbman: mgmt cmd failed, no response (verb=0x%x)\n", 1633 alt_fq_verb); 1634 return -EIO; 1635 } 1636 1637 /* Decode the outcome */ 1638 WARN_ON((r->verb & QBMAN_RESULT_MASK) != alt_fq_verb); 1639 1640 /* Determine success or failure */ 1641 if (unlikely(r->rslt != QBMAN_MC_RSLT_OK)) { 1642 pr_err("qbman: ALT FQID %d failed: verb = 0x%08x code = 0x%02x\n", 1643 fqid, r->verb, r->rslt); 1644 return -EIO; 1645 } 1646 1647 return 0; 1648 } 1649 1650 struct qbman_cdan_ctrl_desc { 1651 u8 verb; 1652 u8 reserved; 1653 __le16 ch; 1654 u8 we; 1655 u8 ctrl; 1656 __le16 reserved2; 1657 __le64 cdan_ctx; 1658 u8 reserved3[48]; 1659 1660 }; 1661 1662 struct qbman_cdan_ctrl_rslt { 1663 u8 verb; 1664 u8 rslt; 1665 __le16 ch; 1666 u8 reserved[60]; 1667 }; 1668 1669 int qbman_swp_CDAN_set(struct qbman_swp *s, u16 channelid, 1670 u8 we_mask, u8 cdan_en, 1671 u64 ctx) 1672 { 1673 struct qbman_cdan_ctrl_desc *p = NULL; 1674 struct qbman_cdan_ctrl_rslt *r = NULL; 1675 1676 /* Start the management command */ 1677 p = qbman_swp_mc_start(s); 1678 if (!p) 1679 return -EBUSY; 1680 1681 /* Encode the caller-provided attributes */ 1682 p->ch = cpu_to_le16(channelid); 1683 p->we = we_mask; 1684 if (cdan_en) 1685 p->ctrl = 1; 1686 else 1687 p->ctrl = 0; 1688 p->cdan_ctx = cpu_to_le64(ctx); 1689 1690 /* Complete the management command */ 1691 r = qbman_swp_mc_complete(s, p, QBMAN_WQCHAN_CONFIGURE); 1692 if (unlikely(!r)) { 1693 pr_err("qbman: wqchan config failed, no response\n"); 1694 return -EIO; 1695 } 1696 1697 WARN_ON((r->verb & 0x7f) != QBMAN_WQCHAN_CONFIGURE); 1698 1699 /* Determine success or failure */ 1700 if (unlikely(r->rslt != QBMAN_MC_RSLT_OK)) { 1701 pr_err("qbman: CDAN cQID %d failed: code = 0x%02x\n", 1702 channelid, r->rslt); 1703 return -EIO; 1704 } 1705 1706 return 0; 1707 } 1708 1709 #define QBMAN_RESPONSE_VERB_MASK 0x7f 1710 #define QBMAN_FQ_QUERY_NP 0x45 1711 #define QBMAN_BP_QUERY 0x32 1712 1713 struct qbman_fq_query_desc { 1714 u8 verb; 1715 u8 reserved[3]; 1716 __le32 fqid; 1717 u8 reserved2[56]; 1718 }; 1719 1720 int qbman_fq_query_state(struct qbman_swp *s, u32 fqid, 1721 struct qbman_fq_query_np_rslt *r) 1722 { 1723 struct qbman_fq_query_desc *p; 1724 void *resp; 1725 1726 p = (struct qbman_fq_query_desc *)qbman_swp_mc_start(s); 1727 if (!p) 1728 return -EBUSY; 1729 1730 /* FQID is a 24 bit value */ 1731 p->fqid = cpu_to_le32(fqid & 0x00FFFFFF); 1732 resp = qbman_swp_mc_complete(s, p, QBMAN_FQ_QUERY_NP); 1733 if (!resp) { 1734 pr_err("qbman: Query FQID %d NP fields failed, no response\n", 1735 fqid); 1736 return -EIO; 1737 } 1738 *r = *(struct qbman_fq_query_np_rslt *)resp; 1739 /* Decode the outcome */ 1740 WARN_ON((r->verb & QBMAN_RESPONSE_VERB_MASK) != QBMAN_FQ_QUERY_NP); 1741 1742 /* Determine success or failure */ 1743 if (r->rslt != QBMAN_MC_RSLT_OK) { 1744 pr_err("Query NP fields of FQID 0x%x failed, code=0x%02x\n", 1745 p->fqid, r->rslt); 1746 return -EIO; 1747 } 1748 1749 return 0; 1750 } 1751 1752 u32 qbman_fq_state_frame_count(const struct qbman_fq_query_np_rslt *r) 1753 { 1754 return (le32_to_cpu(r->frm_cnt) & 0x00FFFFFF); 1755 } 1756 1757 u32 qbman_fq_state_byte_count(const struct qbman_fq_query_np_rslt *r) 1758 { 1759 return le32_to_cpu(r->byte_cnt); 1760 } 1761 1762 struct qbman_bp_query_desc { 1763 u8 verb; 1764 u8 reserved; 1765 __le16 bpid; 1766 u8 reserved2[60]; 1767 }; 1768 1769 int qbman_bp_query(struct qbman_swp *s, u16 bpid, 1770 struct qbman_bp_query_rslt *r) 1771 { 1772 struct qbman_bp_query_desc *p; 1773 void *resp; 1774 1775 p = (struct qbman_bp_query_desc *)qbman_swp_mc_start(s); 1776 if (!p) 1777 return -EBUSY; 1778 1779 p->bpid = cpu_to_le16(bpid); 1780 resp = qbman_swp_mc_complete(s, p, QBMAN_BP_QUERY); 1781 if (!resp) { 1782 pr_err("qbman: Query BPID %d fields failed, no response\n", 1783 bpid); 1784 return -EIO; 1785 } 1786 *r = *(struct qbman_bp_query_rslt *)resp; 1787 /* Decode the outcome */ 1788 WARN_ON((r->verb & QBMAN_RESPONSE_VERB_MASK) != QBMAN_BP_QUERY); 1789 1790 /* Determine success or failure */ 1791 if (r->rslt != QBMAN_MC_RSLT_OK) { 1792 pr_err("Query fields of BPID 0x%x failed, code=0x%02x\n", 1793 bpid, r->rslt); 1794 return -EIO; 1795 } 1796 1797 return 0; 1798 } 1799 1800 u32 qbman_bp_info_num_free_bufs(struct qbman_bp_query_rslt *a) 1801 { 1802 return le32_to_cpu(a->fill); 1803 } 1804