18e99ea8dSJohannes Berg // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
28e99ea8dSJohannes Berg /*
38e99ea8dSJohannes Berg * Copyright (C) 2017 Intel Deutschland GmbH
48e99ea8dSJohannes Berg * Copyright (C) 2018-2020 Intel Corporation
58e99ea8dSJohannes Berg */
66ffe5de3SSara Sharon #include <net/tso.h>
7f890269bSLuca Coelho #include <linux/tcp.h>
86b35ff91SSara Sharon
96b35ff91SSara Sharon #include "iwl-debug.h"
106b35ff91SSara Sharon #include "iwl-csr.h"
116b35ff91SSara Sharon #include "iwl-io.h"
126b35ff91SSara Sharon #include "internal.h"
13d172a5efSJohannes Berg #include "fw/api/tx.h"
140cd1ad2dSMordechay Goodstein #include "queue/tx.h"
156b35ff91SSara Sharon
16ca60da2eSSara Sharon /*************** HOST COMMAND QUEUE FUNCTIONS *****/
17ca60da2eSSara Sharon
18ca60da2eSSara Sharon /*
19ca60da2eSSara Sharon * iwl_pcie_gen2_enqueue_hcmd - enqueue a uCode command
20ca60da2eSSara Sharon * @priv: device private data point
21ca60da2eSSara Sharon * @cmd: a pointer to the ucode command structure
22ca60da2eSSara Sharon *
23ca60da2eSSara Sharon * The function returns < 0 values to indicate the operation
24ca60da2eSSara Sharon * failed. On success, it returns the index (>= 0) of command in the
25ca60da2eSSara Sharon * command queue.
26ca60da2eSSara Sharon */
iwl_pcie_gen2_enqueue_hcmd(struct iwl_trans * trans,struct iwl_host_cmd * cmd)2713f028b4SMordechay Goodstein int iwl_pcie_gen2_enqueue_hcmd(struct iwl_trans *trans,
28ca60da2eSSara Sharon struct iwl_host_cmd *cmd)
29ca60da2eSSara Sharon {
30ca60da2eSSara Sharon struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
314f4822b7SMordechay Goodstein struct iwl_txq *txq = trans->txqs.txq[trans->txqs.cmd.q_id];
32ca60da2eSSara Sharon struct iwl_device_cmd *out_cmd;
33ca60da2eSSara Sharon struct iwl_cmd_meta *out_meta;
34ca60da2eSSara Sharon void *dup_buf = NULL;
35ca60da2eSSara Sharon dma_addr_t phys_addr;
363506832bSShaul Triebitz int i, cmd_pos, idx;
37ca60da2eSSara Sharon u16 copy_size, cmd_size, tb0_size;
38ca60da2eSSara Sharon bool had_nocopy = false;
39ca60da2eSSara Sharon u8 group_id = iwl_cmd_groupid(cmd->id);
40ca60da2eSSara Sharon const u8 *cmddata[IWL_MAX_CMD_TBS_PER_TFD];
41ca60da2eSSara Sharon u16 cmdlen[IWL_MAX_CMD_TBS_PER_TFD];
423506832bSShaul Triebitz struct iwl_tfh_tfd *tfd;
43e7020bb0SJiri Kosina unsigned long flags;
44ca60da2eSSara Sharon
45ca60da2eSSara Sharon copy_size = sizeof(struct iwl_cmd_header_wide);
46ca60da2eSSara Sharon cmd_size = sizeof(struct iwl_cmd_header_wide);
47ca60da2eSSara Sharon
48ca60da2eSSara Sharon for (i = 0; i < IWL_MAX_CMD_TBS_PER_TFD; i++) {
49ca60da2eSSara Sharon cmddata[i] = cmd->data[i];
50ca60da2eSSara Sharon cmdlen[i] = cmd->len[i];
51ca60da2eSSara Sharon
52ca60da2eSSara Sharon if (!cmd->len[i])
53ca60da2eSSara Sharon continue;
54ca60da2eSSara Sharon
55ca60da2eSSara Sharon /* need at least IWL_FIRST_TB_SIZE copied */
56ca60da2eSSara Sharon if (copy_size < IWL_FIRST_TB_SIZE) {
57ca60da2eSSara Sharon int copy = IWL_FIRST_TB_SIZE - copy_size;
58ca60da2eSSara Sharon
59ca60da2eSSara Sharon if (copy > cmdlen[i])
60ca60da2eSSara Sharon copy = cmdlen[i];
61ca60da2eSSara Sharon cmdlen[i] -= copy;
62ca60da2eSSara Sharon cmddata[i] += copy;
63ca60da2eSSara Sharon copy_size += copy;
64ca60da2eSSara Sharon }
65ca60da2eSSara Sharon
66ca60da2eSSara Sharon if (cmd->dataflags[i] & IWL_HCMD_DFL_NOCOPY) {
67ca60da2eSSara Sharon had_nocopy = true;
68ca60da2eSSara Sharon if (WARN_ON(cmd->dataflags[i] & IWL_HCMD_DFL_DUP)) {
69ca60da2eSSara Sharon idx = -EINVAL;
70ca60da2eSSara Sharon goto free_dup_buf;
71ca60da2eSSara Sharon }
72ca60da2eSSara Sharon } else if (cmd->dataflags[i] & IWL_HCMD_DFL_DUP) {
73ca60da2eSSara Sharon /*
74ca60da2eSSara Sharon * This is also a chunk that isn't copied
75ca60da2eSSara Sharon * to the static buffer so set had_nocopy.
76ca60da2eSSara Sharon */
77ca60da2eSSara Sharon had_nocopy = true;
78ca60da2eSSara Sharon
79ca60da2eSSara Sharon /* only allowed once */
80ca60da2eSSara Sharon if (WARN_ON(dup_buf)) {
81ca60da2eSSara Sharon idx = -EINVAL;
82ca60da2eSSara Sharon goto free_dup_buf;
83ca60da2eSSara Sharon }
84ca60da2eSSara Sharon
85ca60da2eSSara Sharon dup_buf = kmemdup(cmddata[i], cmdlen[i],
86ca60da2eSSara Sharon GFP_ATOMIC);
87ca60da2eSSara Sharon if (!dup_buf)
88ca60da2eSSara Sharon return -ENOMEM;
89ca60da2eSSara Sharon } else {
90ca60da2eSSara Sharon /* NOCOPY must not be followed by normal! */
91ca60da2eSSara Sharon if (WARN_ON(had_nocopy)) {
92ca60da2eSSara Sharon idx = -EINVAL;
93ca60da2eSSara Sharon goto free_dup_buf;
94ca60da2eSSara Sharon }
95ca60da2eSSara Sharon copy_size += cmdlen[i];
96ca60da2eSSara Sharon }
97ca60da2eSSara Sharon cmd_size += cmd->len[i];
98ca60da2eSSara Sharon }
99ca60da2eSSara Sharon
100ca60da2eSSara Sharon /*
101ca60da2eSSara Sharon * If any of the command structures end up being larger than the
102ca60da2eSSara Sharon * TFD_MAX_PAYLOAD_SIZE and they aren't dynamically allocated into
103ca60da2eSSara Sharon * separate TFDs, then we will need to increase the size of the buffers
104ca60da2eSSara Sharon */
105ca60da2eSSara Sharon if (WARN(copy_size > TFD_MAX_PAYLOAD_SIZE,
106ca60da2eSSara Sharon "Command %s (%#x) is too large (%d bytes)\n",
107ca60da2eSSara Sharon iwl_get_cmd_string(trans, cmd->id), cmd->id, copy_size)) {
108ca60da2eSSara Sharon idx = -EINVAL;
109ca60da2eSSara Sharon goto free_dup_buf;
110ca60da2eSSara Sharon }
111ca60da2eSSara Sharon
112e7020bb0SJiri Kosina spin_lock_irqsave(&txq->lock, flags);
113ca60da2eSSara Sharon
1140cd1ad2dSMordechay Goodstein idx = iwl_txq_get_cmd_index(txq, txq->write_ptr);
1150cd1ad2dSMordechay Goodstein tfd = iwl_txq_get_tfd(trans, txq, txq->write_ptr);
1163506832bSShaul Triebitz memset(tfd, 0, sizeof(*tfd));
1173506832bSShaul Triebitz
1180cd1ad2dSMordechay Goodstein if (iwl_txq_space(trans, txq) < ((cmd->flags & CMD_ASYNC) ? 2 : 1)) {
119e7020bb0SJiri Kosina spin_unlock_irqrestore(&txq->lock, flags);
120ca60da2eSSara Sharon
121ca60da2eSSara Sharon IWL_ERR(trans, "No space in command queue\n");
122ca60da2eSSara Sharon iwl_op_mode_cmd_queue_full(trans->op_mode);
123ca60da2eSSara Sharon idx = -ENOSPC;
124ca60da2eSSara Sharon goto free_dup_buf;
125ca60da2eSSara Sharon }
126ca60da2eSSara Sharon
127ca60da2eSSara Sharon out_cmd = txq->entries[idx].cmd;
128ca60da2eSSara Sharon out_meta = &txq->entries[idx].meta;
129ca60da2eSSara Sharon
130ca60da2eSSara Sharon /* re-initialize to NULL */
131ca60da2eSSara Sharon memset(out_meta, 0, sizeof(*out_meta));
132ca60da2eSSara Sharon if (cmd->flags & CMD_WANT_SKB)
133ca60da2eSSara Sharon out_meta->source = cmd;
134ca60da2eSSara Sharon
135ca60da2eSSara Sharon /* set up the header */
136ca60da2eSSara Sharon out_cmd->hdr_wide.cmd = iwl_cmd_opcode(cmd->id);
137ca60da2eSSara Sharon out_cmd->hdr_wide.group_id = group_id;
138ca60da2eSSara Sharon out_cmd->hdr_wide.version = iwl_cmd_version(cmd->id);
139ca60da2eSSara Sharon out_cmd->hdr_wide.length =
140ca60da2eSSara Sharon cpu_to_le16(cmd_size - sizeof(struct iwl_cmd_header_wide));
141ca60da2eSSara Sharon out_cmd->hdr_wide.reserved = 0;
142ca60da2eSSara Sharon out_cmd->hdr_wide.sequence =
1434f4822b7SMordechay Goodstein cpu_to_le16(QUEUE_TO_SEQ(trans->txqs.cmd.q_id) |
144ca60da2eSSara Sharon INDEX_TO_SEQ(txq->write_ptr));
145ca60da2eSSara Sharon
146ca60da2eSSara Sharon cmd_pos = sizeof(struct iwl_cmd_header_wide);
147ca60da2eSSara Sharon copy_size = sizeof(struct iwl_cmd_header_wide);
148ca60da2eSSara Sharon
149ca60da2eSSara Sharon /* and copy the data that needs to be copied */
150ca60da2eSSara Sharon for (i = 0; i < IWL_MAX_CMD_TBS_PER_TFD; i++) {
151ca60da2eSSara Sharon int copy;
152ca60da2eSSara Sharon
153ca60da2eSSara Sharon if (!cmd->len[i])
154ca60da2eSSara Sharon continue;
155ca60da2eSSara Sharon
156ca60da2eSSara Sharon /* copy everything if not nocopy/dup */
157ca60da2eSSara Sharon if (!(cmd->dataflags[i] & (IWL_HCMD_DFL_NOCOPY |
158ca60da2eSSara Sharon IWL_HCMD_DFL_DUP))) {
159ca60da2eSSara Sharon copy = cmd->len[i];
160ca60da2eSSara Sharon
161ca60da2eSSara Sharon memcpy((u8 *)out_cmd + cmd_pos, cmd->data[i], copy);
162ca60da2eSSara Sharon cmd_pos += copy;
163ca60da2eSSara Sharon copy_size += copy;
164ca60da2eSSara Sharon continue;
165ca60da2eSSara Sharon }
166ca60da2eSSara Sharon
167ca60da2eSSara Sharon /*
168ca60da2eSSara Sharon * Otherwise we need at least IWL_FIRST_TB_SIZE copied
169ca60da2eSSara Sharon * in total (for bi-directional DMA), but copy up to what
170ca60da2eSSara Sharon * we can fit into the payload for debug dump purposes.
171ca60da2eSSara Sharon */
172ca60da2eSSara Sharon copy = min_t(int, TFD_MAX_PAYLOAD_SIZE - cmd_pos, cmd->len[i]);
173ca60da2eSSara Sharon
174ca60da2eSSara Sharon memcpy((u8 *)out_cmd + cmd_pos, cmd->data[i], copy);
175ca60da2eSSara Sharon cmd_pos += copy;
176ca60da2eSSara Sharon
177ca60da2eSSara Sharon /* However, treat copy_size the proper way, we need it below */
178ca60da2eSSara Sharon if (copy_size < IWL_FIRST_TB_SIZE) {
179ca60da2eSSara Sharon copy = IWL_FIRST_TB_SIZE - copy_size;
180ca60da2eSSara Sharon
181ca60da2eSSara Sharon if (copy > cmd->len[i])
182ca60da2eSSara Sharon copy = cmd->len[i];
183ca60da2eSSara Sharon copy_size += copy;
184ca60da2eSSara Sharon }
185ca60da2eSSara Sharon }
186ca60da2eSSara Sharon
187ca60da2eSSara Sharon IWL_DEBUG_HC(trans,
188ca60da2eSSara Sharon "Sending command %s (%.2x.%.2x), seq: 0x%04X, %d bytes at %d[%d]:%d\n",
189ca60da2eSSara Sharon iwl_get_cmd_string(trans, cmd->id), group_id,
190ca60da2eSSara Sharon out_cmd->hdr.cmd, le16_to_cpu(out_cmd->hdr.sequence),
1914f4822b7SMordechay Goodstein cmd_size, txq->write_ptr, idx, trans->txqs.cmd.q_id);
192ca60da2eSSara Sharon
193ca60da2eSSara Sharon /* start the TFD with the minimum copy bytes */
194ca60da2eSSara Sharon tb0_size = min_t(int, copy_size, IWL_FIRST_TB_SIZE);
19529cf396fSLuca Coelho memcpy(&txq->first_tb_bufs[idx], out_cmd, tb0_size);
1960cd1ad2dSMordechay Goodstein iwl_txq_gen2_set_tb(trans, tfd, iwl_txq_get_first_tb_dma(txq, idx),
197ca60da2eSSara Sharon tb0_size);
198ca60da2eSSara Sharon
199ca60da2eSSara Sharon /* map first command fragment, if any remains */
200ca60da2eSSara Sharon if (copy_size > tb0_size) {
201ca60da2eSSara Sharon phys_addr = dma_map_single(trans->dev,
20229cf396fSLuca Coelho (u8 *)out_cmd + tb0_size,
203ca60da2eSSara Sharon copy_size - tb0_size,
204ca60da2eSSara Sharon DMA_TO_DEVICE);
205ca60da2eSSara Sharon if (dma_mapping_error(trans->dev, phys_addr)) {
206ca60da2eSSara Sharon idx = -ENOMEM;
2070cd1ad2dSMordechay Goodstein iwl_txq_gen2_tfd_unmap(trans, out_meta, tfd);
208ca60da2eSSara Sharon goto out;
209ca60da2eSSara Sharon }
2100cd1ad2dSMordechay Goodstein iwl_txq_gen2_set_tb(trans, tfd, phys_addr,
211ca60da2eSSara Sharon copy_size - tb0_size);
212ca60da2eSSara Sharon }
213ca60da2eSSara Sharon
214ca60da2eSSara Sharon /* map the remaining (adjusted) nocopy/dup fragments */
215ca60da2eSSara Sharon for (i = 0; i < IWL_MAX_CMD_TBS_PER_TFD; i++) {
216*0301bcd5SBjoern A. Zeeb void *data = (void *)(uintptr_t)cmddata[i];
217ca60da2eSSara Sharon
218ca60da2eSSara Sharon if (!cmdlen[i])
219ca60da2eSSara Sharon continue;
220ca60da2eSSara Sharon if (!(cmd->dataflags[i] & (IWL_HCMD_DFL_NOCOPY |
221ca60da2eSSara Sharon IWL_HCMD_DFL_DUP)))
222ca60da2eSSara Sharon continue;
223ca60da2eSSara Sharon if (cmd->dataflags[i] & IWL_HCMD_DFL_DUP)
224ca60da2eSSara Sharon data = dup_buf;
225*0301bcd5SBjoern A. Zeeb phys_addr = dma_map_single(trans->dev, data,
226ca60da2eSSara Sharon cmdlen[i], DMA_TO_DEVICE);
227ca60da2eSSara Sharon if (dma_mapping_error(trans->dev, phys_addr)) {
228ca60da2eSSara Sharon idx = -ENOMEM;
2290cd1ad2dSMordechay Goodstein iwl_txq_gen2_tfd_unmap(trans, out_meta, tfd);
230ca60da2eSSara Sharon goto out;
231ca60da2eSSara Sharon }
2320cd1ad2dSMordechay Goodstein iwl_txq_gen2_set_tb(trans, tfd, phys_addr, cmdlen[i]);
233ca60da2eSSara Sharon }
234ca60da2eSSara Sharon
235ca60da2eSSara Sharon BUILD_BUG_ON(IWL_TFH_NUM_TBS > sizeof(out_meta->tbs) * BITS_PER_BYTE);
236ca60da2eSSara Sharon out_meta->flags = cmd->flags;
237ca60da2eSSara Sharon if (WARN_ON_ONCE(txq->entries[idx].free_buf))
238453431a5SWaiman Long kfree_sensitive(txq->entries[idx].free_buf);
239ca60da2eSSara Sharon txq->entries[idx].free_buf = dup_buf;
240ca60da2eSSara Sharon
241ca60da2eSSara Sharon trace_iwlwifi_dev_hcmd(trans->dev, cmd, cmd_size, &out_cmd->hdr_wide);
242ca60da2eSSara Sharon
243ca60da2eSSara Sharon /* start timer if queue currently empty */
244ca60da2eSSara Sharon if (txq->read_ptr == txq->write_ptr && txq->wd_timeout)
245ca60da2eSSara Sharon mod_timer(&txq->stuck_timer, jiffies + txq->wd_timeout);
246ca60da2eSSara Sharon
247874020f8SJohannes Berg spin_lock(&trans_pcie->reg_lock);
248ca60da2eSSara Sharon /* Increment and update queue's write index */
2490cd1ad2dSMordechay Goodstein txq->write_ptr = iwl_txq_inc_wrap(trans, txq->write_ptr);
2500cd1ad2dSMordechay Goodstein iwl_txq_inc_wr_ptr(trans, txq);
251874020f8SJohannes Berg spin_unlock(&trans_pcie->reg_lock);
252ca60da2eSSara Sharon
253ca60da2eSSara Sharon out:
254e7020bb0SJiri Kosina spin_unlock_irqrestore(&txq->lock, flags);
255ca60da2eSSara Sharon free_dup_buf:
256ca60da2eSSara Sharon if (idx < 0)
257ca60da2eSSara Sharon kfree(dup_buf);
258ca60da2eSSara Sharon return idx;
259ca60da2eSSara Sharon }
260