1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 /* Copyright (C) 2018 Netronome Systems, Inc. */ 3 4 #include <linux/bitops.h> 5 #include <linux/kernel.h> 6 #include <linux/log2.h> 7 8 #include "../nfpcore/nfp_cpp.h" 9 #include "../nfpcore/nfp_nffw.h" 10 #include "../nfp_app.h" 11 #include "../nfp_abi.h" 12 #include "../nfp_main.h" 13 #include "../nfp_net.h" 14 #include "main.h" 15 16 #define NFP_NUM_PRIOS_SYM_NAME "_abi_pci_dscp_num_prio_%u" 17 #define NFP_NUM_BANDS_SYM_NAME "_abi_pci_dscp_num_band_%u" 18 19 #define NFP_QLVL_SYM_NAME "_abi_nfd_out_q_lvls_%u%s" 20 #define NFP_QLVL_STRIDE 16 21 #define NFP_QLVL_BLOG_BYTES 0 22 #define NFP_QLVL_BLOG_PKTS 4 23 #define NFP_QLVL_THRS 8 24 25 #define NFP_QMSTAT_SYM_NAME "_abi_nfdqm%u_stats%s" 26 #define NFP_QMSTAT_STRIDE 32 27 #define NFP_QMSTAT_NON_STO 0 28 #define NFP_QMSTAT_STO 8 29 #define NFP_QMSTAT_DROP 16 30 #define NFP_QMSTAT_ECN 24 31 32 #define NFP_Q_STAT_SYM_NAME "_abi_nfd_rxq_stats%u%s" 33 #define NFP_Q_STAT_STRIDE 16 34 #define NFP_Q_STAT_PKTS 0 35 #define NFP_Q_STAT_BYTES 8 36 37 #define NFP_NET_ABM_MBOX_CMD NFP_NET_CFG_MBOX_SIMPLE_CMD 38 #define NFP_NET_ABM_MBOX_RET NFP_NET_CFG_MBOX_SIMPLE_RET 39 #define NFP_NET_ABM_MBOX_DATALEN NFP_NET_CFG_MBOX_SIMPLE_VAL 40 #define NFP_NET_ABM_MBOX_RESERVED (NFP_NET_CFG_MBOX_SIMPLE_VAL + 4) 41 #define NFP_NET_ABM_MBOX_DATA (NFP_NET_CFG_MBOX_SIMPLE_VAL + 8) 42 43 static int 44 nfp_abm_ctrl_stat(struct nfp_abm_link *alink, const struct nfp_rtsym *sym, 45 unsigned int stride, unsigned int offset, unsigned int band, 46 unsigned int queue, bool is_u64, u64 *res) 47 { 48 struct nfp_cpp *cpp = alink->abm->app->cpp; 49 u64 val, sym_offset; 50 unsigned int qid; 51 u32 val32; 52 int err; 53 54 qid = band * NFP_NET_MAX_RX_RINGS + alink->queue_base + queue; 55 56 sym_offset = qid * stride + offset; 57 if (is_u64) 58 err = __nfp_rtsym_readq(cpp, sym, 3, 0, sym_offset, &val); 59 else 60 err = __nfp_rtsym_readl(cpp, sym, 3, 0, sym_offset, &val32); 61 if (err) { 62 nfp_err(cpp, "RED offload reading stat failed on vNIC %d band %d queue %d (+ %d)\n", 63 alink->id, band, queue, alink->queue_base); 64 return err; 65 } 66 67 *res = is_u64 ? val : val32; 68 return 0; 69 } 70 71 int __nfp_abm_ctrl_set_q_lvl(struct nfp_abm *abm, unsigned int id, u32 val) 72 { 73 struct nfp_cpp *cpp = abm->app->cpp; 74 u64 sym_offset; 75 int err; 76 77 __clear_bit(id, abm->threshold_undef); 78 if (abm->thresholds[id] == val) 79 return 0; 80 81 sym_offset = id * NFP_QLVL_STRIDE + NFP_QLVL_THRS; 82 err = __nfp_rtsym_writel(cpp, abm->q_lvls, 4, 0, sym_offset, val); 83 if (err) { 84 nfp_err(cpp, 85 "RED offload setting level failed on subqueue %d\n", 86 id); 87 return err; 88 } 89 90 abm->thresholds[id] = val; 91 return 0; 92 } 93 94 int nfp_abm_ctrl_set_q_lvl(struct nfp_abm_link *alink, unsigned int band, 95 unsigned int queue, u32 val) 96 { 97 unsigned int threshold; 98 99 threshold = band * NFP_NET_MAX_RX_RINGS + alink->queue_base + queue; 100 101 return __nfp_abm_ctrl_set_q_lvl(alink->abm, threshold, val); 102 } 103 104 u64 nfp_abm_ctrl_stat_non_sto(struct nfp_abm_link *alink, unsigned int queue) 105 { 106 unsigned int band; 107 u64 val, sum = 0; 108 109 for (band = 0; band < alink->abm->num_bands; band++) { 110 if (nfp_abm_ctrl_stat(alink, alink->abm->qm_stats, 111 NFP_QMSTAT_STRIDE, NFP_QMSTAT_NON_STO, 112 band, queue, true, &val)) 113 return 0; 114 sum += val; 115 } 116 117 return sum; 118 } 119 120 u64 nfp_abm_ctrl_stat_sto(struct nfp_abm_link *alink, unsigned int queue) 121 { 122 unsigned int band; 123 u64 val, sum = 0; 124 125 for (band = 0; band < alink->abm->num_bands; band++) { 126 if (nfp_abm_ctrl_stat(alink, alink->abm->qm_stats, 127 NFP_QMSTAT_STRIDE, NFP_QMSTAT_STO, 128 band, queue, true, &val)) 129 return 0; 130 sum += val; 131 } 132 133 return sum; 134 } 135 136 static int 137 nfp_abm_ctrl_stat_basic(struct nfp_abm_link *alink, unsigned int band, 138 unsigned int queue, unsigned int off, u64 *val) 139 { 140 if (!nfp_abm_has_prio(alink->abm)) { 141 if (!band) { 142 unsigned int id = alink->queue_base + queue; 143 144 *val = nn_readq(alink->vnic, 145 NFP_NET_CFG_RXR_STATS(id) + off); 146 } else { 147 *val = 0; 148 } 149 150 return 0; 151 } else { 152 return nfp_abm_ctrl_stat(alink, alink->abm->q_stats, 153 NFP_Q_STAT_STRIDE, off, band, queue, 154 true, val); 155 } 156 } 157 158 int nfp_abm_ctrl_read_q_stats(struct nfp_abm_link *alink, unsigned int band, 159 unsigned int queue, struct nfp_alink_stats *stats) 160 { 161 int err; 162 163 err = nfp_abm_ctrl_stat_basic(alink, band, queue, NFP_Q_STAT_PKTS, 164 &stats->tx_pkts); 165 if (err) 166 return err; 167 168 err = nfp_abm_ctrl_stat_basic(alink, band, queue, NFP_Q_STAT_BYTES, 169 &stats->tx_bytes); 170 if (err) 171 return err; 172 173 err = nfp_abm_ctrl_stat(alink, alink->abm->q_lvls, NFP_QLVL_STRIDE, 174 NFP_QLVL_BLOG_BYTES, band, queue, false, 175 &stats->backlog_bytes); 176 if (err) 177 return err; 178 179 err = nfp_abm_ctrl_stat(alink, alink->abm->q_lvls, 180 NFP_QLVL_STRIDE, NFP_QLVL_BLOG_PKTS, 181 band, queue, false, &stats->backlog_pkts); 182 if (err) 183 return err; 184 185 err = nfp_abm_ctrl_stat(alink, alink->abm->qm_stats, 186 NFP_QMSTAT_STRIDE, NFP_QMSTAT_DROP, 187 band, queue, true, &stats->drops); 188 if (err) 189 return err; 190 191 return nfp_abm_ctrl_stat(alink, alink->abm->qm_stats, 192 NFP_QMSTAT_STRIDE, NFP_QMSTAT_ECN, 193 band, queue, true, &stats->overlimits); 194 } 195 196 int nfp_abm_ctrl_read_q_xstats(struct nfp_abm_link *alink, 197 unsigned int band, unsigned int queue, 198 struct nfp_alink_xstats *xstats) 199 { 200 int err; 201 202 err = nfp_abm_ctrl_stat(alink, alink->abm->qm_stats, 203 NFP_QMSTAT_STRIDE, NFP_QMSTAT_DROP, 204 band, queue, true, &xstats->pdrop); 205 if (err) 206 return err; 207 208 return nfp_abm_ctrl_stat(alink, alink->abm->qm_stats, 209 NFP_QMSTAT_STRIDE, NFP_QMSTAT_ECN, 210 band, queue, true, &xstats->ecn_marked); 211 } 212 213 int nfp_abm_ctrl_qm_enable(struct nfp_abm *abm) 214 { 215 return nfp_mbox_cmd(abm->app->pf, NFP_MBOX_PCIE_ABM_ENABLE, 216 NULL, 0, NULL, 0); 217 } 218 219 int nfp_abm_ctrl_qm_disable(struct nfp_abm *abm) 220 { 221 return nfp_mbox_cmd(abm->app->pf, NFP_MBOX_PCIE_ABM_DISABLE, 222 NULL, 0, NULL, 0); 223 } 224 225 int nfp_abm_ctrl_prio_map_update(struct nfp_abm_link *alink, u32 *packed) 226 { 227 struct nfp_net *nn = alink->vnic; 228 unsigned int i; 229 int err; 230 231 /* Write data_len and wipe reserved */ 232 nn_writeq(nn, nn->tlv_caps.mbox_off + NFP_NET_ABM_MBOX_DATALEN, 233 alink->abm->prio_map_len); 234 235 for (i = 0; i < alink->abm->prio_map_len; i += sizeof(u32)) 236 nn_writel(nn, nn->tlv_caps.mbox_off + NFP_NET_ABM_MBOX_DATA + i, 237 packed[i / sizeof(u32)]); 238 239 err = nfp_net_reconfig_mbox(nn, 240 NFP_NET_CFG_MBOX_CMD_PCI_DSCP_PRIOMAP_SET); 241 if (err) 242 nfp_err(alink->abm->app->cpp, 243 "setting DSCP -> VQ map failed with error %d\n", err); 244 return err; 245 } 246 247 static int nfp_abm_ctrl_prio_check_params(struct nfp_abm_link *alink) 248 { 249 struct nfp_abm *abm = alink->abm; 250 struct nfp_net *nn = alink->vnic; 251 unsigned int min_mbox_sz; 252 253 if (!nfp_abm_has_prio(alink->abm)) 254 return 0; 255 256 min_mbox_sz = NFP_NET_ABM_MBOX_DATA + alink->abm->prio_map_len; 257 if (nn->tlv_caps.mbox_len < min_mbox_sz) { 258 nfp_err(abm->app->pf->cpp, "vNIC mailbox too small for prio offload: %u, need: %u\n", 259 nn->tlv_caps.mbox_len, min_mbox_sz); 260 return -EINVAL; 261 } 262 263 return 0; 264 } 265 266 int nfp_abm_ctrl_read_params(struct nfp_abm_link *alink) 267 { 268 alink->queue_base = nn_readl(alink->vnic, NFP_NET_CFG_START_RXQ); 269 alink->queue_base /= alink->vnic->stride_rx; 270 271 return nfp_abm_ctrl_prio_check_params(alink); 272 } 273 274 static unsigned int nfp_abm_ctrl_prio_map_size(struct nfp_abm *abm) 275 { 276 unsigned int size; 277 278 size = roundup_pow_of_two(order_base_2(abm->num_bands)); 279 size = DIV_ROUND_UP(size * abm->num_prios, BITS_PER_BYTE); 280 size = round_up(size, sizeof(u32)); 281 282 return size; 283 } 284 285 static const struct nfp_rtsym * 286 nfp_abm_ctrl_find_rtsym(struct nfp_pf *pf, const char *name, unsigned int size) 287 { 288 const struct nfp_rtsym *sym; 289 290 sym = nfp_rtsym_lookup(pf->rtbl, name); 291 if (!sym) { 292 nfp_err(pf->cpp, "Symbol '%s' not found\n", name); 293 return ERR_PTR(-ENOENT); 294 } 295 if (nfp_rtsym_size(sym) != size) { 296 nfp_err(pf->cpp, 297 "Symbol '%s' wrong size: expected %u got %llu\n", 298 name, size, nfp_rtsym_size(sym)); 299 return ERR_PTR(-EINVAL); 300 } 301 302 return sym; 303 } 304 305 static const struct nfp_rtsym * 306 nfp_abm_ctrl_find_q_rtsym(struct nfp_abm *abm, const char *name_fmt, 307 size_t size) 308 { 309 char pf_symbol[64]; 310 311 size = array3_size(size, abm->num_bands, NFP_NET_MAX_RX_RINGS); 312 snprintf(pf_symbol, sizeof(pf_symbol), name_fmt, 313 abm->pf_id, nfp_abm_has_prio(abm) ? "_per_band" : ""); 314 315 return nfp_abm_ctrl_find_rtsym(abm->app->pf, pf_symbol, size); 316 } 317 318 int nfp_abm_ctrl_find_addrs(struct nfp_abm *abm) 319 { 320 struct nfp_pf *pf = abm->app->pf; 321 const struct nfp_rtsym *sym; 322 int res; 323 324 abm->pf_id = nfp_cppcore_pcie_unit(pf->cpp); 325 326 /* Read count of prios and prio bands */ 327 res = nfp_pf_rtsym_read_optional(pf, NFP_NUM_BANDS_SYM_NAME, 1); 328 if (res < 0) 329 return res; 330 abm->num_bands = res; 331 332 res = nfp_pf_rtsym_read_optional(pf, NFP_NUM_PRIOS_SYM_NAME, 1); 333 if (res < 0) 334 return res; 335 abm->num_prios = res; 336 337 abm->prio_map_len = nfp_abm_ctrl_prio_map_size(abm); 338 339 /* Check values are sane, U16_MAX is arbitrarily chosen as max */ 340 if (!is_power_of_2(abm->num_bands) || !is_power_of_2(abm->num_prios) || 341 abm->num_bands > U16_MAX || abm->num_prios > U16_MAX || 342 (abm->num_bands == 1) != (abm->num_prios == 1)) { 343 nfp_err(pf->cpp, 344 "invalid priomap description num bands: %u and num prios: %u\n", 345 abm->num_bands, abm->num_prios); 346 return -EINVAL; 347 } 348 349 /* Find level and stat symbols */ 350 sym = nfp_abm_ctrl_find_q_rtsym(abm, NFP_QLVL_SYM_NAME, 351 NFP_QLVL_STRIDE); 352 if (IS_ERR(sym)) 353 return PTR_ERR(sym); 354 abm->q_lvls = sym; 355 356 sym = nfp_abm_ctrl_find_q_rtsym(abm, NFP_QMSTAT_SYM_NAME, 357 NFP_QMSTAT_STRIDE); 358 if (IS_ERR(sym)) 359 return PTR_ERR(sym); 360 abm->qm_stats = sym; 361 362 if (nfp_abm_has_prio(abm)) { 363 sym = nfp_abm_ctrl_find_q_rtsym(abm, NFP_Q_STAT_SYM_NAME, 364 NFP_Q_STAT_STRIDE); 365 if (IS_ERR(sym)) 366 return PTR_ERR(sym); 367 abm->q_stats = sym; 368 } 369 370 return 0; 371 } 372