15a2cc190SJeff Kirsher /*
25a2cc190SJeff Kirsher * Copyright (c) 2007 Mellanox Technologies. All rights reserved.
35a2cc190SJeff Kirsher *
45a2cc190SJeff Kirsher * This software is available to you under a choice of one of two
55a2cc190SJeff Kirsher * licenses. You may choose to be licensed under the terms of the GNU
65a2cc190SJeff Kirsher * General Public License (GPL) Version 2, available from the file
75a2cc190SJeff Kirsher * COPYING in the main directory of this source tree, or the
85a2cc190SJeff Kirsher * OpenIB.org BSD license below:
95a2cc190SJeff Kirsher *
105a2cc190SJeff Kirsher * Redistribution and use in source and binary forms, with or
115a2cc190SJeff Kirsher * without modification, are permitted provided that the following
125a2cc190SJeff Kirsher * conditions are met:
135a2cc190SJeff Kirsher *
145a2cc190SJeff Kirsher * - Redistributions of source code must retain the above
155a2cc190SJeff Kirsher * copyright notice, this list of conditions and the following
165a2cc190SJeff Kirsher * disclaimer.
175a2cc190SJeff Kirsher *
185a2cc190SJeff Kirsher * - Redistributions in binary form must reproduce the above
195a2cc190SJeff Kirsher * copyright notice, this list of conditions and the following
205a2cc190SJeff Kirsher * disclaimer in the documentation and/or other materials
215a2cc190SJeff Kirsher * provided with the distribution.
225a2cc190SJeff Kirsher *
235a2cc190SJeff Kirsher * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
245a2cc190SJeff Kirsher * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
255a2cc190SJeff Kirsher * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
265a2cc190SJeff Kirsher * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
275a2cc190SJeff Kirsher * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
285a2cc190SJeff Kirsher * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
295a2cc190SJeff Kirsher * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
305a2cc190SJeff Kirsher * SOFTWARE.
315a2cc190SJeff Kirsher *
325a2cc190SJeff Kirsher */
335a2cc190SJeff Kirsher
345a2cc190SJeff Kirsher #include <linux/mlx4/cq.h>
355a2cc190SJeff Kirsher #include <linux/mlx4/qp.h>
365a2cc190SJeff Kirsher #include <linux/mlx4/cmd.h>
375a2cc190SJeff Kirsher
385a2cc190SJeff Kirsher #include "mlx4_en.h"
395a2cc190SJeff Kirsher
mlx4_en_cq_event(struct mlx4_cq * cq,enum mlx4_event event)405a2cc190SJeff Kirsher static void mlx4_en_cq_event(struct mlx4_cq *cq, enum mlx4_event event)
415a2cc190SJeff Kirsher {
425a2cc190SJeff Kirsher return;
435a2cc190SJeff Kirsher }
445a2cc190SJeff Kirsher
455a2cc190SJeff Kirsher
mlx4_en_create_cq(struct mlx4_en_priv * priv,struct mlx4_en_cq ** pcq,int entries,int ring,enum cq_type mode,int node)465a2cc190SJeff Kirsher int mlx4_en_create_cq(struct mlx4_en_priv *priv,
4741d942d5SEugenia Emantayev struct mlx4_en_cq **pcq,
48163561a4SEugenia Emantayev int entries, int ring, enum cq_type mode,
49163561a4SEugenia Emantayev int node)
505a2cc190SJeff Kirsher {
515a2cc190SJeff Kirsher struct mlx4_en_dev *mdev = priv->mdev;
5241d942d5SEugenia Emantayev struct mlx4_en_cq *cq;
535a2cc190SJeff Kirsher int err;
545a2cc190SJeff Kirsher
55163561a4SEugenia Emantayev cq = kzalloc_node(sizeof(*cq), GFP_KERNEL, node);
56163561a4SEugenia Emantayev if (!cq) {
5741d942d5SEugenia Emantayev en_err(priv, "Failed to allocate CQ structure\n");
5841d942d5SEugenia Emantayev return -ENOMEM;
5941d942d5SEugenia Emantayev }
6041d942d5SEugenia Emantayev
615a2cc190SJeff Kirsher cq->size = entries;
6208ff3235SOr Gerlitz cq->buf_size = cq->size * mdev->dev->caps.cqe_size;
635a2cc190SJeff Kirsher
645a2cc190SJeff Kirsher cq->ring = ring;
65ccc109b8STariq Toukan cq->type = mode;
66c66fa19cSMatan Barak cq->vector = mdev->dev->caps.num_comp_vectors;
675a2cc190SJeff Kirsher
68163561a4SEugenia Emantayev /* Allocate HW buffers on provided NUMA node.
69163561a4SEugenia Emantayev * dev->numa_node is used in mtt range allocation flow.
70163561a4SEugenia Emantayev */
71872bf2fbSYishai Hadas set_dev_node(&mdev->dev->persist->pdev->dev, node);
725a2cc190SJeff Kirsher err = mlx4_alloc_hwq_res(mdev->dev, &cq->wqres,
7373898db0SHaggai Abramovsky cq->buf_size);
74872bf2fbSYishai Hadas set_dev_node(&mdev->dev->persist->pdev->dev, mdev->dev->numa_node);
755a2cc190SJeff Kirsher if (err)
7641d942d5SEugenia Emantayev goto err_cq;
775a2cc190SJeff Kirsher
7841d942d5SEugenia Emantayev cq->buf = (struct mlx4_cqe *)cq->wqres.buf.direct.buf;
7941d942d5SEugenia Emantayev *pcq = cq;
8041d942d5SEugenia Emantayev
8141d942d5SEugenia Emantayev return 0;
8241d942d5SEugenia Emantayev
8341d942d5SEugenia Emantayev err_cq:
8441d942d5SEugenia Emantayev kfree(cq);
8541d942d5SEugenia Emantayev *pcq = NULL;
865a2cc190SJeff Kirsher return err;
875a2cc190SJeff Kirsher }
885a2cc190SJeff Kirsher
mlx4_en_activate_cq(struct mlx4_en_priv * priv,struct mlx4_en_cq * cq,int cq_idx)8976532d0cSAlexander Guller int mlx4_en_activate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq,
9076532d0cSAlexander Guller int cq_idx)
915a2cc190SJeff Kirsher {
925a2cc190SJeff Kirsher struct mlx4_en_dev *mdev = priv->mdev;
9380a62deeSThomas Gleixner int irq, err = 0;
94ec693d47SAmir Vadai int timestamp_en = 0;
95c66fa19cSMatan Barak bool assigned_eq = false;
965a2cc190SJeff Kirsher
975a2cc190SJeff Kirsher cq->dev = mdev->pndev[priv->port];
985a2cc190SJeff Kirsher cq->mcq.set_ci_db = cq->wqres.db.db;
995a2cc190SJeff Kirsher cq->mcq.arm_db = cq->wqres.db.db + 1;
1005a2cc190SJeff Kirsher *cq->mcq.set_ci_db = 0;
1015a2cc190SJeff Kirsher *cq->mcq.arm_db = 0;
1025a2cc190SJeff Kirsher memset(cq->buf, 0, cq->buf_size);
1035a2cc190SJeff Kirsher
104ccc109b8STariq Toukan if (cq->type == RX) {
105c66fa19cSMatan Barak if (!mlx4_is_eq_vector_valid(mdev->dev, priv->port,
106c66fa19cSMatan Barak cq->vector)) {
107de161803SIdo Shamay cq->vector = cpumask_first(priv->rx_ring[cq->ring]->affinity_mask);
108c66fa19cSMatan Barak
109c66fa19cSMatan Barak err = mlx4_assign_eq(mdev->dev, priv->port,
110c66fa19cSMatan Barak &cq->vector);
111c66fa19cSMatan Barak if (err) {
112b0f64463SCarol L Soto mlx4_err(mdev, "Failed assigning an EQ to CQ vector %d\n",
113b0f64463SCarol L Soto cq->vector);
114c66fa19cSMatan Barak goto free_eq;
1155a2cc190SJeff Kirsher }
11635f6f453SAmir Vadai
117c66fa19cSMatan Barak assigned_eq = true;
1185a2cc190SJeff Kirsher }
11980a62deeSThomas Gleixner irq = mlx4_eq_get_irq(mdev->dev, cq->vector);
120197d2370SThomas Gleixner cq->aff_mask = irq_get_effective_affinity_mask(irq);
1215a2cc190SJeff Kirsher } else {
12276532d0cSAlexander Guller /* For TX we use the same irq per
12376532d0cSAlexander Guller ring we assigned for the RX */
12476532d0cSAlexander Guller struct mlx4_en_cq *rx_cq;
12576532d0cSAlexander Guller
12676532d0cSAlexander Guller cq_idx = cq_idx % priv->rx_ring_num;
12741d942d5SEugenia Emantayev rx_cq = priv->rx_cq[cq_idx];
12876532d0cSAlexander Guller cq->vector = rx_cq->vector;
1295a2cc190SJeff Kirsher }
1305a2cc190SJeff Kirsher
131ccc109b8STariq Toukan if (cq->type == RX)
13241d942d5SEugenia Emantayev cq->size = priv->rx_ring[cq->ring]->actual_size;
1335a2cc190SJeff Kirsher
134ccc109b8STariq Toukan if ((cq->type != RX && priv->hwtstamp_config.tx_type) ||
135ccc109b8STariq Toukan (cq->type == RX && priv->hwtstamp_config.rx_filter))
136ec693d47SAmir Vadai timestamp_en = 1;
137ec693d47SAmir Vadai
138f3301870SMoshe Shemesh cq->mcq.usage = MLX4_RES_USAGE_DRIVER;
139ec693d47SAmir Vadai err = mlx4_cq_alloc(mdev->dev, cq->size, &cq->wqres.mtt,
140ec693d47SAmir Vadai &mdev->priv_uar, cq->wqres.db.dma, &cq->mcq,
141e4567897SDaniel Jurgens cq->vector, 0, timestamp_en, &cq->wqres.buf, false);
1425a2cc190SJeff Kirsher if (err)
143c66fa19cSMatan Barak goto free_eq;
1445a2cc190SJeff Kirsher
1455a2cc190SJeff Kirsher cq->mcq.event = mlx4_en_cq_event;
1465a2cc190SJeff Kirsher
1476c78511bSTariq Toukan switch (cq->type) {
1486c78511bSTariq Toukan case TX:
1496c78511bSTariq Toukan cq->mcq.comp = mlx4_en_tx_irq;
15016d083e2SJakub Kicinski netif_napi_add_tx(cq->dev, &cq->napi, mlx4_en_poll_tx_cq);
1510276a330SEugenia Emantayev napi_enable(&cq->napi);
1526c78511bSTariq Toukan break;
1536c78511bSTariq Toukan case RX:
1546c78511bSTariq Toukan cq->mcq.comp = mlx4_en_rx_irq;
155*b48b89f9SJakub Kicinski netif_napi_add(cq->dev, &cq->napi, mlx4_en_poll_rx_cq);
1566c78511bSTariq Toukan napi_enable(&cq->napi);
1576c78511bSTariq Toukan break;
1586c78511bSTariq Toukan case TX_XDP:
1596c78511bSTariq Toukan /* nothing regarding napi, it's shared with rx ring */
1606c78511bSTariq Toukan cq->xdp_busy = false;
1616c78511bSTariq Toukan break;
1626c78511bSTariq Toukan }
1630276a330SEugenia Emantayev
1645a2cc190SJeff Kirsher return 0;
165c66fa19cSMatan Barak
166c66fa19cSMatan Barak free_eq:
167c66fa19cSMatan Barak if (assigned_eq)
168c66fa19cSMatan Barak mlx4_release_eq(mdev->dev, cq->vector);
169c66fa19cSMatan Barak cq->vector = mdev->dev->caps.num_comp_vectors;
170c66fa19cSMatan Barak return err;
1715a2cc190SJeff Kirsher }
1725a2cc190SJeff Kirsher
mlx4_en_destroy_cq(struct mlx4_en_priv * priv,struct mlx4_en_cq ** pcq)17341d942d5SEugenia Emantayev void mlx4_en_destroy_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq **pcq)
1745a2cc190SJeff Kirsher {
1755a2cc190SJeff Kirsher struct mlx4_en_dev *mdev = priv->mdev;
17641d942d5SEugenia Emantayev struct mlx4_en_cq *cq = *pcq;
1775a2cc190SJeff Kirsher
1785a2cc190SJeff Kirsher mlx4_free_hwq_res(mdev->dev, &cq->wqres, cq->buf_size);
179c66fa19cSMatan Barak if (mlx4_is_eq_vector_valid(mdev->dev, priv->port, cq->vector) &&
180ccc109b8STariq Toukan cq->type == RX)
1815a2cc190SJeff Kirsher mlx4_release_eq(priv->mdev->dev, cq->vector);
182cd3109d2SYevgeny Petrilin cq->vector = 0;
1835a2cc190SJeff Kirsher cq->buf_size = 0;
1845a2cc190SJeff Kirsher cq->buf = NULL;
18541d942d5SEugenia Emantayev kfree(cq);
18641d942d5SEugenia Emantayev *pcq = NULL;
1875a2cc190SJeff Kirsher }
1885a2cc190SJeff Kirsher
mlx4_en_deactivate_cq(struct mlx4_en_priv * priv,struct mlx4_en_cq * cq)1895a2cc190SJeff Kirsher void mlx4_en_deactivate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq)
1905a2cc190SJeff Kirsher {
1916c78511bSTariq Toukan if (cq->type != TX_XDP) {
1925a2cc190SJeff Kirsher napi_disable(&cq->napi);
1930276a330SEugenia Emantayev netif_napi_del(&cq->napi);
1946c78511bSTariq Toukan }
1955a2cc190SJeff Kirsher
196e22979d9SYevgeny Petrilin mlx4_cq_free(priv->mdev->dev, &cq->mcq);
1975a2cc190SJeff Kirsher }
1985a2cc190SJeff Kirsher
1995a2cc190SJeff Kirsher /* Set rx cq moderation parameters */
mlx4_en_set_cq_moder(struct mlx4_en_priv * priv,struct mlx4_en_cq * cq)2005a2cc190SJeff Kirsher int mlx4_en_set_cq_moder(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq)
2015a2cc190SJeff Kirsher {
2025a2cc190SJeff Kirsher return mlx4_cq_modify(priv->mdev->dev, &cq->mcq,
2035a2cc190SJeff Kirsher cq->moder_cnt, cq->moder_time);
2045a2cc190SJeff Kirsher }
2055a2cc190SJeff Kirsher
mlx4_en_arm_cq(struct mlx4_en_priv * priv,struct mlx4_en_cq * cq)206f3eebe88SZhu Yanjun void mlx4_en_arm_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq)
2075a2cc190SJeff Kirsher {
2085a2cc190SJeff Kirsher mlx4_cq_arm(&cq->mcq, MLX4_CQ_DB_REQ_NOT, priv->mdev->uar_map,
2095a2cc190SJeff Kirsher &priv->mdev->uar_lock);
2105a2cc190SJeff Kirsher }
2115a2cc190SJeff Kirsher
2125a2cc190SJeff Kirsher
213