1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 /* Copyright (C) 2018 Netronome Systems, Inc. */ 3 4 #include <linux/bitops.h> 5 #include <linux/kernel.h> 6 #include <linux/log2.h> 7 8 #include "../nfpcore/nfp_cpp.h" 9 #include "../nfpcore/nfp_nffw.h" 10 #include "../nfp_app.h" 11 #include "../nfp_abi.h" 12 #include "../nfp_main.h" 13 #include "../nfp_net.h" 14 #include "main.h" 15 16 #define NFP_NUM_PRIOS_SYM_NAME "_abi_pci_dscp_num_prio_%u" 17 #define NFP_NUM_BANDS_SYM_NAME "_abi_pci_dscp_num_band_%u" 18 #define NFP_ACT_MASK_SYM_NAME "_abi_nfd_out_q_actions_%u" 19 20 #define NFP_QLVL_SYM_NAME "_abi_nfd_out_q_lvls_%u%s" 21 #define NFP_QLVL_STRIDE 16 22 #define NFP_QLVL_BLOG_BYTES 0 23 #define NFP_QLVL_BLOG_PKTS 4 24 #define NFP_QLVL_THRS 8 25 #define NFP_QLVL_ACT 12 26 27 #define NFP_QMSTAT_SYM_NAME "_abi_nfdqm%u_stats%s" 28 #define NFP_QMSTAT_STRIDE 32 29 #define NFP_QMSTAT_NON_STO 0 30 #define NFP_QMSTAT_STO 8 31 #define NFP_QMSTAT_DROP 16 32 #define NFP_QMSTAT_ECN 24 33 34 #define NFP_Q_STAT_SYM_NAME "_abi_nfd_rxq_stats%u%s" 35 #define NFP_Q_STAT_STRIDE 16 36 #define NFP_Q_STAT_PKTS 0 37 #define NFP_Q_STAT_BYTES 8 38 39 #define NFP_NET_ABM_MBOX_CMD NFP_NET_CFG_MBOX_SIMPLE_CMD 40 #define NFP_NET_ABM_MBOX_RET NFP_NET_CFG_MBOX_SIMPLE_RET 41 #define NFP_NET_ABM_MBOX_DATALEN NFP_NET_CFG_MBOX_SIMPLE_VAL 42 #define NFP_NET_ABM_MBOX_RESERVED (NFP_NET_CFG_MBOX_SIMPLE_VAL + 4) 43 #define NFP_NET_ABM_MBOX_DATA (NFP_NET_CFG_MBOX_SIMPLE_VAL + 8) 44 45 static int 46 nfp_abm_ctrl_stat(struct nfp_abm_link *alink, const struct nfp_rtsym *sym, 47 unsigned int stride, unsigned int offset, unsigned int band, 48 unsigned int queue, bool is_u64, u64 *res) 49 { 50 struct nfp_cpp *cpp = alink->abm->app->cpp; 51 u64 val, sym_offset; 52 unsigned int qid; 53 u32 val32; 54 int err; 55 56 qid = band * NFP_NET_MAX_RX_RINGS + alink->queue_base + queue; 57 58 sym_offset = qid * stride + offset; 59 if (is_u64) 60 err = __nfp_rtsym_readq(cpp, sym, 3, 0, sym_offset, &val); 61 else 62 err = __nfp_rtsym_readl(cpp, sym, 3, 0, sym_offset, &val32); 63 if (err) { 64 nfp_err(cpp, "RED offload reading stat failed on vNIC %d band %d queue %d (+ %d)\n", 65 alink->id, band, queue, alink->queue_base); 66 return err; 67 } 68 69 *res = is_u64 ? val : val32; 70 return 0; 71 } 72 73 int __nfp_abm_ctrl_set_q_lvl(struct nfp_abm *abm, unsigned int id, u32 val) 74 { 75 struct nfp_cpp *cpp = abm->app->cpp; 76 u64 sym_offset; 77 int err; 78 79 __clear_bit(id, abm->threshold_undef); 80 if (abm->thresholds[id] == val) 81 return 0; 82 83 sym_offset = id * NFP_QLVL_STRIDE + NFP_QLVL_THRS; 84 err = __nfp_rtsym_writel(cpp, abm->q_lvls, 4, 0, sym_offset, val); 85 if (err) { 86 nfp_err(cpp, 87 "RED offload setting level failed on subqueue %d\n", 88 id); 89 return err; 90 } 91 92 abm->thresholds[id] = val; 93 return 0; 94 } 95 96 int nfp_abm_ctrl_set_q_lvl(struct nfp_abm_link *alink, unsigned int band, 97 unsigned int queue, u32 val) 98 { 99 unsigned int threshold; 100 101 threshold = band * NFP_NET_MAX_RX_RINGS + alink->queue_base + queue; 102 103 return __nfp_abm_ctrl_set_q_lvl(alink->abm, threshold, val); 104 } 105 106 int __nfp_abm_ctrl_set_q_act(struct nfp_abm *abm, unsigned int id, 107 enum nfp_abm_q_action act) 108 { 109 struct nfp_cpp *cpp = abm->app->cpp; 110 u64 sym_offset; 111 int err; 112 113 if (abm->actions[id] == act) 114 return 0; 115 116 sym_offset = id * NFP_QLVL_STRIDE + NFP_QLVL_ACT; 117 err = __nfp_rtsym_writel(cpp, abm->q_lvls, 4, 0, sym_offset, act); 118 if (err) { 119 nfp_err(cpp, 120 "RED offload setting action failed on subqueue %d\n", 121 id); 122 return err; 123 } 124 125 abm->actions[id] = act; 126 return 0; 127 } 128 129 int nfp_abm_ctrl_set_q_act(struct nfp_abm_link *alink, unsigned int band, 130 unsigned int queue, enum nfp_abm_q_action act) 131 { 132 unsigned int qid; 133 134 qid = band * NFP_NET_MAX_RX_RINGS + alink->queue_base + queue; 135 136 return __nfp_abm_ctrl_set_q_act(alink->abm, qid, act); 137 } 138 139 u64 nfp_abm_ctrl_stat_non_sto(struct nfp_abm_link *alink, unsigned int queue) 140 { 141 unsigned int band; 142 u64 val, sum = 0; 143 144 for (band = 0; band < alink->abm->num_bands; band++) { 145 if (nfp_abm_ctrl_stat(alink, alink->abm->qm_stats, 146 NFP_QMSTAT_STRIDE, NFP_QMSTAT_NON_STO, 147 band, queue, true, &val)) 148 return 0; 149 sum += val; 150 } 151 152 return sum; 153 } 154 155 u64 nfp_abm_ctrl_stat_sto(struct nfp_abm_link *alink, unsigned int queue) 156 { 157 unsigned int band; 158 u64 val, sum = 0; 159 160 for (band = 0; band < alink->abm->num_bands; band++) { 161 if (nfp_abm_ctrl_stat(alink, alink->abm->qm_stats, 162 NFP_QMSTAT_STRIDE, NFP_QMSTAT_STO, 163 band, queue, true, &val)) 164 return 0; 165 sum += val; 166 } 167 168 return sum; 169 } 170 171 static int 172 nfp_abm_ctrl_stat_basic(struct nfp_abm_link *alink, unsigned int band, 173 unsigned int queue, unsigned int off, u64 *val) 174 { 175 if (!nfp_abm_has_prio(alink->abm)) { 176 if (!band) { 177 unsigned int id = alink->queue_base + queue; 178 179 *val = nn_readq(alink->vnic, 180 NFP_NET_CFG_RXR_STATS(id) + off); 181 } else { 182 *val = 0; 183 } 184 185 return 0; 186 } else { 187 return nfp_abm_ctrl_stat(alink, alink->abm->q_stats, 188 NFP_Q_STAT_STRIDE, off, band, queue, 189 true, val); 190 } 191 } 192 193 int nfp_abm_ctrl_read_q_stats(struct nfp_abm_link *alink, unsigned int band, 194 unsigned int queue, struct nfp_alink_stats *stats) 195 { 196 int err; 197 198 err = nfp_abm_ctrl_stat_basic(alink, band, queue, NFP_Q_STAT_PKTS, 199 &stats->tx_pkts); 200 if (err) 201 return err; 202 203 err = nfp_abm_ctrl_stat_basic(alink, band, queue, NFP_Q_STAT_BYTES, 204 &stats->tx_bytes); 205 if (err) 206 return err; 207 208 err = nfp_abm_ctrl_stat(alink, alink->abm->q_lvls, NFP_QLVL_STRIDE, 209 NFP_QLVL_BLOG_BYTES, band, queue, false, 210 &stats->backlog_bytes); 211 if (err) 212 return err; 213 214 err = nfp_abm_ctrl_stat(alink, alink->abm->q_lvls, 215 NFP_QLVL_STRIDE, NFP_QLVL_BLOG_PKTS, 216 band, queue, false, &stats->backlog_pkts); 217 if (err) 218 return err; 219 220 err = nfp_abm_ctrl_stat(alink, alink->abm->qm_stats, 221 NFP_QMSTAT_STRIDE, NFP_QMSTAT_DROP, 222 band, queue, true, &stats->drops); 223 if (err) 224 return err; 225 226 return nfp_abm_ctrl_stat(alink, alink->abm->qm_stats, 227 NFP_QMSTAT_STRIDE, NFP_QMSTAT_ECN, 228 band, queue, true, &stats->overlimits); 229 } 230 231 int nfp_abm_ctrl_read_q_xstats(struct nfp_abm_link *alink, 232 unsigned int band, unsigned int queue, 233 struct nfp_alink_xstats *xstats) 234 { 235 int err; 236 237 err = nfp_abm_ctrl_stat(alink, alink->abm->qm_stats, 238 NFP_QMSTAT_STRIDE, NFP_QMSTAT_DROP, 239 band, queue, true, &xstats->pdrop); 240 if (err) 241 return err; 242 243 return nfp_abm_ctrl_stat(alink, alink->abm->qm_stats, 244 NFP_QMSTAT_STRIDE, NFP_QMSTAT_ECN, 245 band, queue, true, &xstats->ecn_marked); 246 } 247 248 int nfp_abm_ctrl_qm_enable(struct nfp_abm *abm) 249 { 250 return nfp_mbox_cmd(abm->app->pf, NFP_MBOX_PCIE_ABM_ENABLE, 251 NULL, 0, NULL, 0); 252 } 253 254 int nfp_abm_ctrl_qm_disable(struct nfp_abm *abm) 255 { 256 return nfp_mbox_cmd(abm->app->pf, NFP_MBOX_PCIE_ABM_DISABLE, 257 NULL, 0, NULL, 0); 258 } 259 260 int nfp_abm_ctrl_prio_map_update(struct nfp_abm_link *alink, u32 *packed) 261 { 262 struct nfp_net *nn = alink->vnic; 263 unsigned int i; 264 int err; 265 266 /* Write data_len and wipe reserved */ 267 nn_writeq(nn, nn->tlv_caps.mbox_off + NFP_NET_ABM_MBOX_DATALEN, 268 alink->abm->prio_map_len); 269 270 for (i = 0; i < alink->abm->prio_map_len; i += sizeof(u32)) 271 nn_writel(nn, nn->tlv_caps.mbox_off + NFP_NET_ABM_MBOX_DATA + i, 272 packed[i / sizeof(u32)]); 273 274 err = nfp_net_reconfig_mbox(nn, 275 NFP_NET_CFG_MBOX_CMD_PCI_DSCP_PRIOMAP_SET); 276 if (err) 277 nfp_err(alink->abm->app->cpp, 278 "setting DSCP -> VQ map failed with error %d\n", err); 279 return err; 280 } 281 282 static int nfp_abm_ctrl_prio_check_params(struct nfp_abm_link *alink) 283 { 284 struct nfp_abm *abm = alink->abm; 285 struct nfp_net *nn = alink->vnic; 286 unsigned int min_mbox_sz; 287 288 if (!nfp_abm_has_prio(alink->abm)) 289 return 0; 290 291 min_mbox_sz = NFP_NET_ABM_MBOX_DATA + alink->abm->prio_map_len; 292 if (nn->tlv_caps.mbox_len < min_mbox_sz) { 293 nfp_err(abm->app->pf->cpp, "vNIC mailbox too small for prio offload: %u, need: %u\n", 294 nn->tlv_caps.mbox_len, min_mbox_sz); 295 return -EINVAL; 296 } 297 298 return 0; 299 } 300 301 int nfp_abm_ctrl_read_params(struct nfp_abm_link *alink) 302 { 303 alink->queue_base = nn_readl(alink->vnic, NFP_NET_CFG_START_RXQ); 304 alink->queue_base /= alink->vnic->stride_rx; 305 306 return nfp_abm_ctrl_prio_check_params(alink); 307 } 308 309 static unsigned int nfp_abm_ctrl_prio_map_size(struct nfp_abm *abm) 310 { 311 unsigned int size; 312 313 size = roundup_pow_of_two(order_base_2(abm->num_bands)); 314 size = DIV_ROUND_UP(size * abm->num_prios, BITS_PER_BYTE); 315 size = round_up(size, sizeof(u32)); 316 317 return size; 318 } 319 320 static const struct nfp_rtsym * 321 nfp_abm_ctrl_find_rtsym(struct nfp_pf *pf, const char *name, unsigned int size) 322 { 323 const struct nfp_rtsym *sym; 324 325 sym = nfp_rtsym_lookup(pf->rtbl, name); 326 if (!sym) { 327 nfp_err(pf->cpp, "Symbol '%s' not found\n", name); 328 return ERR_PTR(-ENOENT); 329 } 330 if (nfp_rtsym_size(sym) != size) { 331 nfp_err(pf->cpp, 332 "Symbol '%s' wrong size: expected %u got %llu\n", 333 name, size, nfp_rtsym_size(sym)); 334 return ERR_PTR(-EINVAL); 335 } 336 337 return sym; 338 } 339 340 static const struct nfp_rtsym * 341 nfp_abm_ctrl_find_q_rtsym(struct nfp_abm *abm, const char *name_fmt, 342 size_t size) 343 { 344 char pf_symbol[64]; 345 346 size = array3_size(size, abm->num_bands, NFP_NET_MAX_RX_RINGS); 347 snprintf(pf_symbol, sizeof(pf_symbol), name_fmt, 348 abm->pf_id, nfp_abm_has_prio(abm) ? "_per_band" : ""); 349 350 return nfp_abm_ctrl_find_rtsym(abm->app->pf, pf_symbol, size); 351 } 352 353 int nfp_abm_ctrl_find_addrs(struct nfp_abm *abm) 354 { 355 struct nfp_pf *pf = abm->app->pf; 356 const struct nfp_rtsym *sym; 357 int res; 358 359 abm->pf_id = nfp_cppcore_pcie_unit(pf->cpp); 360 361 /* Read count of prios and prio bands */ 362 res = nfp_pf_rtsym_read_optional(pf, NFP_NUM_BANDS_SYM_NAME, 1); 363 if (res < 0) 364 return res; 365 abm->num_bands = res; 366 367 res = nfp_pf_rtsym_read_optional(pf, NFP_NUM_PRIOS_SYM_NAME, 1); 368 if (res < 0) 369 return res; 370 abm->num_prios = res; 371 372 /* Read available actions */ 373 res = nfp_pf_rtsym_read_optional(pf, NFP_ACT_MASK_SYM_NAME, 374 BIT(NFP_ABM_ACT_MARK_DROP)); 375 if (res < 0) 376 return res; 377 abm->action_mask = res; 378 379 abm->prio_map_len = nfp_abm_ctrl_prio_map_size(abm); 380 abm->dscp_mask = GENMASK(7, 8 - order_base_2(abm->num_prios)); 381 382 /* Check values are sane, U16_MAX is arbitrarily chosen as max */ 383 if (!is_power_of_2(abm->num_bands) || !is_power_of_2(abm->num_prios) || 384 abm->num_bands > U16_MAX || abm->num_prios > U16_MAX || 385 (abm->num_bands == 1) != (abm->num_prios == 1)) { 386 nfp_err(pf->cpp, 387 "invalid priomap description num bands: %u and num prios: %u\n", 388 abm->num_bands, abm->num_prios); 389 return -EINVAL; 390 } 391 392 /* Find level and stat symbols */ 393 sym = nfp_abm_ctrl_find_q_rtsym(abm, NFP_QLVL_SYM_NAME, 394 NFP_QLVL_STRIDE); 395 if (IS_ERR(sym)) 396 return PTR_ERR(sym); 397 abm->q_lvls = sym; 398 399 sym = nfp_abm_ctrl_find_q_rtsym(abm, NFP_QMSTAT_SYM_NAME, 400 NFP_QMSTAT_STRIDE); 401 if (IS_ERR(sym)) 402 return PTR_ERR(sym); 403 abm->qm_stats = sym; 404 405 if (nfp_abm_has_prio(abm)) { 406 sym = nfp_abm_ctrl_find_q_rtsym(abm, NFP_Q_STAT_SYM_NAME, 407 NFP_Q_STAT_STRIDE); 408 if (IS_ERR(sym)) 409 return PTR_ERR(sym); 410 abm->q_stats = sym; 411 } 412 413 return 0; 414 } 415