102a19356SDavid Howells // SPDX-License-Identifier: GPL-2.0-or-later
202a19356SDavid Howells /* RxRPC Tx data buffering.
302a19356SDavid Howells *
402a19356SDavid Howells * Copyright (C) 2022 Red Hat, Inc. All Rights Reserved.
502a19356SDavid Howells * Written by David Howells (dhowells@redhat.com)
602a19356SDavid Howells */
702a19356SDavid Howells
802a19356SDavid Howells #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
902a19356SDavid Howells
1002a19356SDavid Howells #include <linux/slab.h>
1102a19356SDavid Howells #include "ar-internal.h"
1202a19356SDavid Howells
1302a19356SDavid Howells static atomic_t rxrpc_txbuf_debug_ids;
1402a19356SDavid Howells atomic_t rxrpc_nr_txbuf;
1502a19356SDavid Howells
1602a19356SDavid Howells /*
1702a19356SDavid Howells * Allocate and partially initialise an I/O request structure.
1802a19356SDavid Howells */
rxrpc_alloc_txbuf(struct rxrpc_call * call,u8 packet_type,gfp_t gfp)1902a19356SDavid Howells struct rxrpc_txbuf *rxrpc_alloc_txbuf(struct rxrpc_call *call, u8 packet_type,
2002a19356SDavid Howells gfp_t gfp)
2102a19356SDavid Howells {
2202a19356SDavid Howells struct rxrpc_txbuf *txb;
2302a19356SDavid Howells
2402a19356SDavid Howells txb = kmalloc(sizeof(*txb), gfp);
2502a19356SDavid Howells if (txb) {
2602a19356SDavid Howells INIT_LIST_HEAD(&txb->call_link);
2702a19356SDavid Howells INIT_LIST_HEAD(&txb->tx_link);
2802a19356SDavid Howells refcount_set(&txb->ref, 1);
2902a19356SDavid Howells txb->call_debug_id = call->debug_id;
3002a19356SDavid Howells txb->debug_id = atomic_inc_return(&rxrpc_txbuf_debug_ids);
3102a19356SDavid Howells txb->space = sizeof(txb->data);
3202a19356SDavid Howells txb->len = 0;
3302a19356SDavid Howells txb->offset = 0;
3402a19356SDavid Howells txb->flags = 0;
3572f0c6fbSDavid Howells txb->ack_why = 0;
36cf37b598SDavid Howells txb->seq = call->tx_prepared + 1;
3702a19356SDavid Howells txb->wire.epoch = htonl(call->conn->proto.epoch);
3802a19356SDavid Howells txb->wire.cid = htonl(call->cid);
3902a19356SDavid Howells txb->wire.callNumber = htonl(call->call_id);
4002a19356SDavid Howells txb->wire.seq = htonl(txb->seq);
4102a19356SDavid Howells txb->wire.type = packet_type;
4202a19356SDavid Howells txb->wire.flags = call->conn->out_clientflag;
4302a19356SDavid Howells txb->wire.userStatus = 0;
4402a19356SDavid Howells txb->wire.securityIndex = call->security_ix;
4502a19356SDavid Howells txb->wire._rsvd = 0;
46f3441d41SDavid Howells txb->wire.serviceId = htons(call->dest_srx.srx_service);
4702a19356SDavid Howells
4802a19356SDavid Howells trace_rxrpc_txbuf(txb->debug_id,
4902a19356SDavid Howells txb->call_debug_id, txb->seq, 1,
5002a19356SDavid Howells packet_type == RXRPC_PACKET_TYPE_DATA ?
5102a19356SDavid Howells rxrpc_txbuf_alloc_data :
5202a19356SDavid Howells rxrpc_txbuf_alloc_ack);
5302a19356SDavid Howells atomic_inc(&rxrpc_nr_txbuf);
5402a19356SDavid Howells }
5502a19356SDavid Howells
5602a19356SDavid Howells return txb;
5702a19356SDavid Howells }
5802a19356SDavid Howells
rxrpc_get_txbuf(struct rxrpc_txbuf * txb,enum rxrpc_txbuf_trace what)5902a19356SDavid Howells void rxrpc_get_txbuf(struct rxrpc_txbuf *txb, enum rxrpc_txbuf_trace what)
6002a19356SDavid Howells {
6102a19356SDavid Howells int r;
6202a19356SDavid Howells
6302a19356SDavid Howells __refcount_inc(&txb->ref, &r);
6402a19356SDavid Howells trace_rxrpc_txbuf(txb->debug_id, txb->call_debug_id, txb->seq, r + 1, what);
6502a19356SDavid Howells }
6602a19356SDavid Howells
rxrpc_see_txbuf(struct rxrpc_txbuf * txb,enum rxrpc_txbuf_trace what)6702a19356SDavid Howells void rxrpc_see_txbuf(struct rxrpc_txbuf *txb, enum rxrpc_txbuf_trace what)
6802a19356SDavid Howells {
6902a19356SDavid Howells int r = refcount_read(&txb->ref);
7002a19356SDavid Howells
7102a19356SDavid Howells trace_rxrpc_txbuf(txb->debug_id, txb->call_debug_id, txb->seq, r, what);
7202a19356SDavid Howells }
7302a19356SDavid Howells
rxrpc_free_txbuf(struct rcu_head * rcu)7402a19356SDavid Howells static void rxrpc_free_txbuf(struct rcu_head *rcu)
7502a19356SDavid Howells {
7602a19356SDavid Howells struct rxrpc_txbuf *txb = container_of(rcu, struct rxrpc_txbuf, rcu);
7702a19356SDavid Howells
7802a19356SDavid Howells trace_rxrpc_txbuf(txb->debug_id, txb->call_debug_id, txb->seq, 0,
7902a19356SDavid Howells rxrpc_txbuf_free);
8002a19356SDavid Howells kfree(txb);
8102a19356SDavid Howells atomic_dec(&rxrpc_nr_txbuf);
8202a19356SDavid Howells }
8302a19356SDavid Howells
rxrpc_put_txbuf(struct rxrpc_txbuf * txb,enum rxrpc_txbuf_trace what)8402a19356SDavid Howells void rxrpc_put_txbuf(struct rxrpc_txbuf *txb, enum rxrpc_txbuf_trace what)
8502a19356SDavid Howells {
8602a19356SDavid Howells unsigned int debug_id, call_debug_id;
8702a19356SDavid Howells rxrpc_seq_t seq;
8802a19356SDavid Howells bool dead;
8902a19356SDavid Howells int r;
9002a19356SDavid Howells
9102a19356SDavid Howells if (txb) {
9202a19356SDavid Howells debug_id = txb->debug_id;
9302a19356SDavid Howells call_debug_id = txb->call_debug_id;
9402a19356SDavid Howells seq = txb->seq;
9502a19356SDavid Howells dead = __refcount_dec_and_test(&txb->ref, &r);
9602a19356SDavid Howells trace_rxrpc_txbuf(debug_id, call_debug_id, seq, r - 1, what);
9702a19356SDavid Howells if (dead)
9802a19356SDavid Howells call_rcu(&txb->rcu, rxrpc_free_txbuf);
9902a19356SDavid Howells }
10002a19356SDavid Howells }
101a4ea4c47SDavid Howells
102a4ea4c47SDavid Howells /*
103a4ea4c47SDavid Howells * Shrink the transmit buffer.
104a4ea4c47SDavid Howells */
rxrpc_shrink_call_tx_buffer(struct rxrpc_call * call)105a4ea4c47SDavid Howells void rxrpc_shrink_call_tx_buffer(struct rxrpc_call *call)
106a4ea4c47SDavid Howells {
107a4ea4c47SDavid Howells struct rxrpc_txbuf *txb;
108a4ea4c47SDavid Howells rxrpc_seq_t hard_ack = smp_load_acquire(&call->acks_hard_ack);
109cf37b598SDavid Howells bool wake = false;
110a4ea4c47SDavid Howells
111a4ea4c47SDavid Howells _enter("%x/%x/%x", call->tx_bottom, call->acks_hard_ack, call->tx_top);
112a4ea4c47SDavid Howells
113*b30d61f4SDavid Howells while ((txb = list_first_entry_or_null(&call->tx_buffer,
114*b30d61f4SDavid Howells struct rxrpc_txbuf, call_link))) {
115a4ea4c47SDavid Howells hard_ack = smp_load_acquire(&call->acks_hard_ack);
116a4ea4c47SDavid Howells if (before(hard_ack, txb->seq))
117a4ea4c47SDavid Howells break;
118a4ea4c47SDavid Howells
1193feda9d6SDavid Howells if (txb->seq != call->tx_bottom + 1)
1203feda9d6SDavid Howells rxrpc_see_txbuf(txb, rxrpc_txbuf_see_out_of_step);
121a4ea4c47SDavid Howells ASSERTCMP(txb->seq, ==, call->tx_bottom + 1);
122cf37b598SDavid Howells smp_store_release(&call->tx_bottom, call->tx_bottom + 1);
123a4ea4c47SDavid Howells list_del_rcu(&txb->call_link);
124a4ea4c47SDavid Howells
125a4ea4c47SDavid Howells trace_rxrpc_txqueue(call, rxrpc_txqueue_dequeue);
126a4ea4c47SDavid Howells
127a4ea4c47SDavid Howells rxrpc_put_txbuf(txb, rxrpc_txbuf_put_rotated);
128cf37b598SDavid Howells if (after(call->acks_hard_ack, call->tx_bottom + 128))
129cf37b598SDavid Howells wake = true;
130a4ea4c47SDavid Howells }
131a4ea4c47SDavid Howells
132cf37b598SDavid Howells if (wake)
133cf37b598SDavid Howells wake_up(&call->waitq);
134a4ea4c47SDavid Howells }
135