1*38c8a9a5SSteve French // SPDX-License-Identifier: GPL-2.0-or-later 2*38c8a9a5SSteve French /* 3*38c8a9a5SSteve French * Copyright (C) 2017, Microsoft Corporation. 4*38c8a9a5SSteve French * 5*38c8a9a5SSteve French * Author(s): Long Li <longli@microsoft.com> 6*38c8a9a5SSteve French */ 7*38c8a9a5SSteve French #include <linux/module.h> 8*38c8a9a5SSteve French #include <linux/highmem.h> 9*38c8a9a5SSteve French #include "smbdirect.h" 10*38c8a9a5SSteve French #include "cifs_debug.h" 11*38c8a9a5SSteve French #include "cifsproto.h" 12*38c8a9a5SSteve French #include "smb2proto.h" 13*38c8a9a5SSteve French 14*38c8a9a5SSteve French static struct smbd_response *get_empty_queue_buffer( 15*38c8a9a5SSteve French struct smbd_connection *info); 16*38c8a9a5SSteve French static struct smbd_response *get_receive_buffer( 17*38c8a9a5SSteve French struct smbd_connection *info); 18*38c8a9a5SSteve French static void put_receive_buffer( 19*38c8a9a5SSteve French struct smbd_connection *info, 20*38c8a9a5SSteve French struct smbd_response *response); 21*38c8a9a5SSteve French static int allocate_receive_buffers(struct smbd_connection *info, int num_buf); 22*38c8a9a5SSteve French static void destroy_receive_buffers(struct smbd_connection *info); 23*38c8a9a5SSteve French 24*38c8a9a5SSteve French static void put_empty_packet( 25*38c8a9a5SSteve French struct smbd_connection *info, struct smbd_response *response); 26*38c8a9a5SSteve French static void enqueue_reassembly( 27*38c8a9a5SSteve French struct smbd_connection *info, 28*38c8a9a5SSteve French struct smbd_response *response, int data_length); 29*38c8a9a5SSteve French static struct smbd_response *_get_first_reassembly( 30*38c8a9a5SSteve French struct smbd_connection *info); 31*38c8a9a5SSteve French 32*38c8a9a5SSteve French static int smbd_post_recv( 33*38c8a9a5SSteve French struct smbd_connection *info, 34*38c8a9a5SSteve French struct smbd_response *response); 35*38c8a9a5SSteve French 36*38c8a9a5SSteve French static int smbd_post_send_empty(struct smbd_connection *info); 37*38c8a9a5SSteve French 38*38c8a9a5SSteve French static void destroy_mr_list(struct smbd_connection *info); 39*38c8a9a5SSteve French static int allocate_mr_list(struct smbd_connection *info); 40*38c8a9a5SSteve French 41*38c8a9a5SSteve French struct smb_extract_to_rdma { 42*38c8a9a5SSteve French struct ib_sge *sge; 43*38c8a9a5SSteve French unsigned int nr_sge; 44*38c8a9a5SSteve French unsigned int max_sge; 45*38c8a9a5SSteve French struct ib_device *device; 46*38c8a9a5SSteve French u32 local_dma_lkey; 47*38c8a9a5SSteve French enum dma_data_direction direction; 48*38c8a9a5SSteve French }; 49*38c8a9a5SSteve French static ssize_t smb_extract_iter_to_rdma(struct iov_iter *iter, size_t len, 50*38c8a9a5SSteve French struct smb_extract_to_rdma *rdma); 51*38c8a9a5SSteve French 52*38c8a9a5SSteve French /* SMBD version number */ 53*38c8a9a5SSteve French #define SMBD_V1 0x0100 54*38c8a9a5SSteve French 55*38c8a9a5SSteve French /* Port numbers for SMBD transport */ 56*38c8a9a5SSteve French #define SMB_PORT 445 57*38c8a9a5SSteve French #define SMBD_PORT 5445 58*38c8a9a5SSteve French 59*38c8a9a5SSteve French /* Address lookup and resolve timeout in ms */ 60*38c8a9a5SSteve French #define RDMA_RESOLVE_TIMEOUT 5000 61*38c8a9a5SSteve French 62*38c8a9a5SSteve French /* SMBD negotiation timeout in seconds */ 63*38c8a9a5SSteve French #define SMBD_NEGOTIATE_TIMEOUT 120 64*38c8a9a5SSteve French 65*38c8a9a5SSteve French /* SMBD minimum receive size and fragmented sized defined in [MS-SMBD] */ 66*38c8a9a5SSteve French #define SMBD_MIN_RECEIVE_SIZE 128 67*38c8a9a5SSteve French #define SMBD_MIN_FRAGMENTED_SIZE 131072 68*38c8a9a5SSteve French 69*38c8a9a5SSteve French /* 70*38c8a9a5SSteve French * Default maximum number of RDMA read/write outstanding on this connection 71*38c8a9a5SSteve French * This value is possibly decreased during QP creation on hardware limit 72*38c8a9a5SSteve French */ 73*38c8a9a5SSteve French #define SMBD_CM_RESPONDER_RESOURCES 32 74*38c8a9a5SSteve French 75*38c8a9a5SSteve French /* Maximum number of retries on data transfer operations */ 76*38c8a9a5SSteve French #define SMBD_CM_RETRY 6 77*38c8a9a5SSteve French /* No need to retry on Receiver Not Ready since SMBD manages credits */ 78*38c8a9a5SSteve French #define SMBD_CM_RNR_RETRY 0 79*38c8a9a5SSteve French 80*38c8a9a5SSteve French /* 81*38c8a9a5SSteve French * User configurable initial values per SMBD transport connection 82*38c8a9a5SSteve French * as defined in [MS-SMBD] 3.1.1.1 83*38c8a9a5SSteve French * Those may change after a SMBD negotiation 84*38c8a9a5SSteve French */ 85*38c8a9a5SSteve French /* The local peer's maximum number of credits to grant to the peer */ 86*38c8a9a5SSteve French int smbd_receive_credit_max = 255; 87*38c8a9a5SSteve French 88*38c8a9a5SSteve French /* The remote peer's credit request of local peer */ 89*38c8a9a5SSteve French int smbd_send_credit_target = 255; 90*38c8a9a5SSteve French 91*38c8a9a5SSteve French /* The maximum single message size can be sent to remote peer */ 92*38c8a9a5SSteve French int smbd_max_send_size = 1364; 93*38c8a9a5SSteve French 94*38c8a9a5SSteve French /* The maximum fragmented upper-layer payload receive size supported */ 95*38c8a9a5SSteve French int smbd_max_fragmented_recv_size = 1024 * 1024; 96*38c8a9a5SSteve French 97*38c8a9a5SSteve French /* The maximum single-message size which can be received */ 98*38c8a9a5SSteve French int smbd_max_receive_size = 1364; 99*38c8a9a5SSteve French 100*38c8a9a5SSteve French /* The timeout to initiate send of a keepalive message on idle */ 101*38c8a9a5SSteve French int smbd_keep_alive_interval = 120; 102*38c8a9a5SSteve French 103*38c8a9a5SSteve French /* 104*38c8a9a5SSteve French * User configurable initial values for RDMA transport 105*38c8a9a5SSteve French * The actual values used may be lower and are limited to hardware capabilities 106*38c8a9a5SSteve French */ 107*38c8a9a5SSteve French /* Default maximum number of pages in a single RDMA write/read */ 108*38c8a9a5SSteve French int smbd_max_frmr_depth = 2048; 109*38c8a9a5SSteve French 110*38c8a9a5SSteve French /* If payload is less than this byte, use RDMA send/recv not read/write */ 111*38c8a9a5SSteve French int rdma_readwrite_threshold = 4096; 112*38c8a9a5SSteve French 113*38c8a9a5SSteve French /* Transport logging functions 114*38c8a9a5SSteve French * Logging are defined as classes. They can be OR'ed to define the actual 115*38c8a9a5SSteve French * logging level via module parameter smbd_logging_class 116*38c8a9a5SSteve French * e.g. cifs.smbd_logging_class=0xa0 will log all log_rdma_recv() and 117*38c8a9a5SSteve French * log_rdma_event() 118*38c8a9a5SSteve French */ 119*38c8a9a5SSteve French #define LOG_OUTGOING 0x1 120*38c8a9a5SSteve French #define LOG_INCOMING 0x2 121*38c8a9a5SSteve French #define LOG_READ 0x4 122*38c8a9a5SSteve French #define LOG_WRITE 0x8 123*38c8a9a5SSteve French #define LOG_RDMA_SEND 0x10 124*38c8a9a5SSteve French #define LOG_RDMA_RECV 0x20 125*38c8a9a5SSteve French #define LOG_KEEP_ALIVE 0x40 126*38c8a9a5SSteve French #define LOG_RDMA_EVENT 0x80 127*38c8a9a5SSteve French #define LOG_RDMA_MR 0x100 128*38c8a9a5SSteve French static unsigned int smbd_logging_class; 129*38c8a9a5SSteve French module_param(smbd_logging_class, uint, 0644); 130*38c8a9a5SSteve French MODULE_PARM_DESC(smbd_logging_class, 131*38c8a9a5SSteve French "Logging class for SMBD transport 0x0 to 0x100"); 132*38c8a9a5SSteve French 133*38c8a9a5SSteve French #define ERR 0x0 134*38c8a9a5SSteve French #define INFO 0x1 135*38c8a9a5SSteve French static unsigned int smbd_logging_level = ERR; 136*38c8a9a5SSteve French module_param(smbd_logging_level, uint, 0644); 137*38c8a9a5SSteve French MODULE_PARM_DESC(smbd_logging_level, 138*38c8a9a5SSteve French "Logging level for SMBD transport, 0 (default): error, 1: info"); 139*38c8a9a5SSteve French 140*38c8a9a5SSteve French #define log_rdma(level, class, fmt, args...) \ 141*38c8a9a5SSteve French do { \ 142*38c8a9a5SSteve French if (level <= smbd_logging_level || class & smbd_logging_class) \ 143*38c8a9a5SSteve French cifs_dbg(VFS, "%s:%d " fmt, __func__, __LINE__, ##args);\ 144*38c8a9a5SSteve French } while (0) 145*38c8a9a5SSteve French 146*38c8a9a5SSteve French #define log_outgoing(level, fmt, args...) \ 147*38c8a9a5SSteve French log_rdma(level, LOG_OUTGOING, fmt, ##args) 148*38c8a9a5SSteve French #define log_incoming(level, fmt, args...) \ 149*38c8a9a5SSteve French log_rdma(level, LOG_INCOMING, fmt, ##args) 150*38c8a9a5SSteve French #define log_read(level, fmt, args...) log_rdma(level, LOG_READ, fmt, ##args) 151*38c8a9a5SSteve French #define log_write(level, fmt, args...) log_rdma(level, LOG_WRITE, fmt, ##args) 152*38c8a9a5SSteve French #define log_rdma_send(level, fmt, args...) \ 153*38c8a9a5SSteve French log_rdma(level, LOG_RDMA_SEND, fmt, ##args) 154*38c8a9a5SSteve French #define log_rdma_recv(level, fmt, args...) \ 155*38c8a9a5SSteve French log_rdma(level, LOG_RDMA_RECV, fmt, ##args) 156*38c8a9a5SSteve French #define log_keep_alive(level, fmt, args...) \ 157*38c8a9a5SSteve French log_rdma(level, LOG_KEEP_ALIVE, fmt, ##args) 158*38c8a9a5SSteve French #define log_rdma_event(level, fmt, args...) \ 159*38c8a9a5SSteve French log_rdma(level, LOG_RDMA_EVENT, fmt, ##args) 160*38c8a9a5SSteve French #define log_rdma_mr(level, fmt, args...) \ 161*38c8a9a5SSteve French log_rdma(level, LOG_RDMA_MR, fmt, ##args) 162*38c8a9a5SSteve French 163*38c8a9a5SSteve French static void smbd_disconnect_rdma_work(struct work_struct *work) 164*38c8a9a5SSteve French { 165*38c8a9a5SSteve French struct smbd_connection *info = 166*38c8a9a5SSteve French container_of(work, struct smbd_connection, disconnect_work); 167*38c8a9a5SSteve French 168*38c8a9a5SSteve French if (info->transport_status == SMBD_CONNECTED) { 169*38c8a9a5SSteve French info->transport_status = SMBD_DISCONNECTING; 170*38c8a9a5SSteve French rdma_disconnect(info->id); 171*38c8a9a5SSteve French } 172*38c8a9a5SSteve French } 173*38c8a9a5SSteve French 174*38c8a9a5SSteve French static void smbd_disconnect_rdma_connection(struct smbd_connection *info) 175*38c8a9a5SSteve French { 176*38c8a9a5SSteve French queue_work(info->workqueue, &info->disconnect_work); 177*38c8a9a5SSteve French } 178*38c8a9a5SSteve French 179*38c8a9a5SSteve French /* Upcall from RDMA CM */ 180*38c8a9a5SSteve French static int smbd_conn_upcall( 181*38c8a9a5SSteve French struct rdma_cm_id *id, struct rdma_cm_event *event) 182*38c8a9a5SSteve French { 183*38c8a9a5SSteve French struct smbd_connection *info = id->context; 184*38c8a9a5SSteve French 185*38c8a9a5SSteve French log_rdma_event(INFO, "event=%d status=%d\n", 186*38c8a9a5SSteve French event->event, event->status); 187*38c8a9a5SSteve French 188*38c8a9a5SSteve French switch (event->event) { 189*38c8a9a5SSteve French case RDMA_CM_EVENT_ADDR_RESOLVED: 190*38c8a9a5SSteve French case RDMA_CM_EVENT_ROUTE_RESOLVED: 191*38c8a9a5SSteve French info->ri_rc = 0; 192*38c8a9a5SSteve French complete(&info->ri_done); 193*38c8a9a5SSteve French break; 194*38c8a9a5SSteve French 195*38c8a9a5SSteve French case RDMA_CM_EVENT_ADDR_ERROR: 196*38c8a9a5SSteve French info->ri_rc = -EHOSTUNREACH; 197*38c8a9a5SSteve French complete(&info->ri_done); 198*38c8a9a5SSteve French break; 199*38c8a9a5SSteve French 200*38c8a9a5SSteve French case RDMA_CM_EVENT_ROUTE_ERROR: 201*38c8a9a5SSteve French info->ri_rc = -ENETUNREACH; 202*38c8a9a5SSteve French complete(&info->ri_done); 203*38c8a9a5SSteve French break; 204*38c8a9a5SSteve French 205*38c8a9a5SSteve French case RDMA_CM_EVENT_ESTABLISHED: 206*38c8a9a5SSteve French log_rdma_event(INFO, "connected event=%d\n", event->event); 207*38c8a9a5SSteve French info->transport_status = SMBD_CONNECTED; 208*38c8a9a5SSteve French wake_up_interruptible(&info->conn_wait); 209*38c8a9a5SSteve French break; 210*38c8a9a5SSteve French 211*38c8a9a5SSteve French case RDMA_CM_EVENT_CONNECT_ERROR: 212*38c8a9a5SSteve French case RDMA_CM_EVENT_UNREACHABLE: 213*38c8a9a5SSteve French case RDMA_CM_EVENT_REJECTED: 214*38c8a9a5SSteve French log_rdma_event(INFO, "connecting failed event=%d\n", event->event); 215*38c8a9a5SSteve French info->transport_status = SMBD_DISCONNECTED; 216*38c8a9a5SSteve French wake_up_interruptible(&info->conn_wait); 217*38c8a9a5SSteve French break; 218*38c8a9a5SSteve French 219*38c8a9a5SSteve French case RDMA_CM_EVENT_DEVICE_REMOVAL: 220*38c8a9a5SSteve French case RDMA_CM_EVENT_DISCONNECTED: 221*38c8a9a5SSteve French /* This happenes when we fail the negotiation */ 222*38c8a9a5SSteve French if (info->transport_status == SMBD_NEGOTIATE_FAILED) { 223*38c8a9a5SSteve French info->transport_status = SMBD_DISCONNECTED; 224*38c8a9a5SSteve French wake_up(&info->conn_wait); 225*38c8a9a5SSteve French break; 226*38c8a9a5SSteve French } 227*38c8a9a5SSteve French 228*38c8a9a5SSteve French info->transport_status = SMBD_DISCONNECTED; 229*38c8a9a5SSteve French wake_up_interruptible(&info->disconn_wait); 230*38c8a9a5SSteve French wake_up_interruptible(&info->wait_reassembly_queue); 231*38c8a9a5SSteve French wake_up_interruptible_all(&info->wait_send_queue); 232*38c8a9a5SSteve French break; 233*38c8a9a5SSteve French 234*38c8a9a5SSteve French default: 235*38c8a9a5SSteve French break; 236*38c8a9a5SSteve French } 237*38c8a9a5SSteve French 238*38c8a9a5SSteve French return 0; 239*38c8a9a5SSteve French } 240*38c8a9a5SSteve French 241*38c8a9a5SSteve French /* Upcall from RDMA QP */ 242*38c8a9a5SSteve French static void 243*38c8a9a5SSteve French smbd_qp_async_error_upcall(struct ib_event *event, void *context) 244*38c8a9a5SSteve French { 245*38c8a9a5SSteve French struct smbd_connection *info = context; 246*38c8a9a5SSteve French 247*38c8a9a5SSteve French log_rdma_event(ERR, "%s on device %s info %p\n", 248*38c8a9a5SSteve French ib_event_msg(event->event), event->device->name, info); 249*38c8a9a5SSteve French 250*38c8a9a5SSteve French switch (event->event) { 251*38c8a9a5SSteve French case IB_EVENT_CQ_ERR: 252*38c8a9a5SSteve French case IB_EVENT_QP_FATAL: 253*38c8a9a5SSteve French smbd_disconnect_rdma_connection(info); 254*38c8a9a5SSteve French break; 255*38c8a9a5SSteve French 256*38c8a9a5SSteve French default: 257*38c8a9a5SSteve French break; 258*38c8a9a5SSteve French } 259*38c8a9a5SSteve French } 260*38c8a9a5SSteve French 261*38c8a9a5SSteve French static inline void *smbd_request_payload(struct smbd_request *request) 262*38c8a9a5SSteve French { 263*38c8a9a5SSteve French return (void *)request->packet; 264*38c8a9a5SSteve French } 265*38c8a9a5SSteve French 266*38c8a9a5SSteve French static inline void *smbd_response_payload(struct smbd_response *response) 267*38c8a9a5SSteve French { 268*38c8a9a5SSteve French return (void *)response->packet; 269*38c8a9a5SSteve French } 270*38c8a9a5SSteve French 271*38c8a9a5SSteve French /* Called when a RDMA send is done */ 272*38c8a9a5SSteve French static void send_done(struct ib_cq *cq, struct ib_wc *wc) 273*38c8a9a5SSteve French { 274*38c8a9a5SSteve French int i; 275*38c8a9a5SSteve French struct smbd_request *request = 276*38c8a9a5SSteve French container_of(wc->wr_cqe, struct smbd_request, cqe); 277*38c8a9a5SSteve French 278*38c8a9a5SSteve French log_rdma_send(INFO, "smbd_request 0x%p completed wc->status=%d\n", 279*38c8a9a5SSteve French request, wc->status); 280*38c8a9a5SSteve French 281*38c8a9a5SSteve French if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_SEND) { 282*38c8a9a5SSteve French log_rdma_send(ERR, "wc->status=%d wc->opcode=%d\n", 283*38c8a9a5SSteve French wc->status, wc->opcode); 284*38c8a9a5SSteve French smbd_disconnect_rdma_connection(request->info); 285*38c8a9a5SSteve French } 286*38c8a9a5SSteve French 287*38c8a9a5SSteve French for (i = 0; i < request->num_sge; i++) 288*38c8a9a5SSteve French ib_dma_unmap_single(request->info->id->device, 289*38c8a9a5SSteve French request->sge[i].addr, 290*38c8a9a5SSteve French request->sge[i].length, 291*38c8a9a5SSteve French DMA_TO_DEVICE); 292*38c8a9a5SSteve French 293*38c8a9a5SSteve French if (atomic_dec_and_test(&request->info->send_pending)) 294*38c8a9a5SSteve French wake_up(&request->info->wait_send_pending); 295*38c8a9a5SSteve French 296*38c8a9a5SSteve French wake_up(&request->info->wait_post_send); 297*38c8a9a5SSteve French 298*38c8a9a5SSteve French mempool_free(request, request->info->request_mempool); 299*38c8a9a5SSteve French } 300*38c8a9a5SSteve French 301*38c8a9a5SSteve French static void dump_smbd_negotiate_resp(struct smbd_negotiate_resp *resp) 302*38c8a9a5SSteve French { 303*38c8a9a5SSteve French log_rdma_event(INFO, "resp message min_version %u max_version %u negotiated_version %u credits_requested %u credits_granted %u status %u max_readwrite_size %u preferred_send_size %u max_receive_size %u max_fragmented_size %u\n", 304*38c8a9a5SSteve French resp->min_version, resp->max_version, 305*38c8a9a5SSteve French resp->negotiated_version, resp->credits_requested, 306*38c8a9a5SSteve French resp->credits_granted, resp->status, 307*38c8a9a5SSteve French resp->max_readwrite_size, resp->preferred_send_size, 308*38c8a9a5SSteve French resp->max_receive_size, resp->max_fragmented_size); 309*38c8a9a5SSteve French } 310*38c8a9a5SSteve French 311*38c8a9a5SSteve French /* 312*38c8a9a5SSteve French * Process a negotiation response message, according to [MS-SMBD]3.1.5.7 313*38c8a9a5SSteve French * response, packet_length: the negotiation response message 314*38c8a9a5SSteve French * return value: true if negotiation is a success, false if failed 315*38c8a9a5SSteve French */ 316*38c8a9a5SSteve French static bool process_negotiation_response( 317*38c8a9a5SSteve French struct smbd_response *response, int packet_length) 318*38c8a9a5SSteve French { 319*38c8a9a5SSteve French struct smbd_connection *info = response->info; 320*38c8a9a5SSteve French struct smbd_negotiate_resp *packet = smbd_response_payload(response); 321*38c8a9a5SSteve French 322*38c8a9a5SSteve French if (packet_length < sizeof(struct smbd_negotiate_resp)) { 323*38c8a9a5SSteve French log_rdma_event(ERR, 324*38c8a9a5SSteve French "error: packet_length=%d\n", packet_length); 325*38c8a9a5SSteve French return false; 326*38c8a9a5SSteve French } 327*38c8a9a5SSteve French 328*38c8a9a5SSteve French if (le16_to_cpu(packet->negotiated_version) != SMBD_V1) { 329*38c8a9a5SSteve French log_rdma_event(ERR, "error: negotiated_version=%x\n", 330*38c8a9a5SSteve French le16_to_cpu(packet->negotiated_version)); 331*38c8a9a5SSteve French return false; 332*38c8a9a5SSteve French } 333*38c8a9a5SSteve French info->protocol = le16_to_cpu(packet->negotiated_version); 334*38c8a9a5SSteve French 335*38c8a9a5SSteve French if (packet->credits_requested == 0) { 336*38c8a9a5SSteve French log_rdma_event(ERR, "error: credits_requested==0\n"); 337*38c8a9a5SSteve French return false; 338*38c8a9a5SSteve French } 339*38c8a9a5SSteve French info->receive_credit_target = le16_to_cpu(packet->credits_requested); 340*38c8a9a5SSteve French 341*38c8a9a5SSteve French if (packet->credits_granted == 0) { 342*38c8a9a5SSteve French log_rdma_event(ERR, "error: credits_granted==0\n"); 343*38c8a9a5SSteve French return false; 344*38c8a9a5SSteve French } 345*38c8a9a5SSteve French atomic_set(&info->send_credits, le16_to_cpu(packet->credits_granted)); 346*38c8a9a5SSteve French 347*38c8a9a5SSteve French atomic_set(&info->receive_credits, 0); 348*38c8a9a5SSteve French 349*38c8a9a5SSteve French if (le32_to_cpu(packet->preferred_send_size) > info->max_receive_size) { 350*38c8a9a5SSteve French log_rdma_event(ERR, "error: preferred_send_size=%d\n", 351*38c8a9a5SSteve French le32_to_cpu(packet->preferred_send_size)); 352*38c8a9a5SSteve French return false; 353*38c8a9a5SSteve French } 354*38c8a9a5SSteve French info->max_receive_size = le32_to_cpu(packet->preferred_send_size); 355*38c8a9a5SSteve French 356*38c8a9a5SSteve French if (le32_to_cpu(packet->max_receive_size) < SMBD_MIN_RECEIVE_SIZE) { 357*38c8a9a5SSteve French log_rdma_event(ERR, "error: max_receive_size=%d\n", 358*38c8a9a5SSteve French le32_to_cpu(packet->max_receive_size)); 359*38c8a9a5SSteve French return false; 360*38c8a9a5SSteve French } 361*38c8a9a5SSteve French info->max_send_size = min_t(int, info->max_send_size, 362*38c8a9a5SSteve French le32_to_cpu(packet->max_receive_size)); 363*38c8a9a5SSteve French 364*38c8a9a5SSteve French if (le32_to_cpu(packet->max_fragmented_size) < 365*38c8a9a5SSteve French SMBD_MIN_FRAGMENTED_SIZE) { 366*38c8a9a5SSteve French log_rdma_event(ERR, "error: max_fragmented_size=%d\n", 367*38c8a9a5SSteve French le32_to_cpu(packet->max_fragmented_size)); 368*38c8a9a5SSteve French return false; 369*38c8a9a5SSteve French } 370*38c8a9a5SSteve French info->max_fragmented_send_size = 371*38c8a9a5SSteve French le32_to_cpu(packet->max_fragmented_size); 372*38c8a9a5SSteve French info->rdma_readwrite_threshold = 373*38c8a9a5SSteve French rdma_readwrite_threshold > info->max_fragmented_send_size ? 374*38c8a9a5SSteve French info->max_fragmented_send_size : 375*38c8a9a5SSteve French rdma_readwrite_threshold; 376*38c8a9a5SSteve French 377*38c8a9a5SSteve French 378*38c8a9a5SSteve French info->max_readwrite_size = min_t(u32, 379*38c8a9a5SSteve French le32_to_cpu(packet->max_readwrite_size), 380*38c8a9a5SSteve French info->max_frmr_depth * PAGE_SIZE); 381*38c8a9a5SSteve French info->max_frmr_depth = info->max_readwrite_size / PAGE_SIZE; 382*38c8a9a5SSteve French 383*38c8a9a5SSteve French return true; 384*38c8a9a5SSteve French } 385*38c8a9a5SSteve French 386*38c8a9a5SSteve French static void smbd_post_send_credits(struct work_struct *work) 387*38c8a9a5SSteve French { 388*38c8a9a5SSteve French int ret = 0; 389*38c8a9a5SSteve French int use_receive_queue = 1; 390*38c8a9a5SSteve French int rc; 391*38c8a9a5SSteve French struct smbd_response *response; 392*38c8a9a5SSteve French struct smbd_connection *info = 393*38c8a9a5SSteve French container_of(work, struct smbd_connection, 394*38c8a9a5SSteve French post_send_credits_work); 395*38c8a9a5SSteve French 396*38c8a9a5SSteve French if (info->transport_status != SMBD_CONNECTED) { 397*38c8a9a5SSteve French wake_up(&info->wait_receive_queues); 398*38c8a9a5SSteve French return; 399*38c8a9a5SSteve French } 400*38c8a9a5SSteve French 401*38c8a9a5SSteve French if (info->receive_credit_target > 402*38c8a9a5SSteve French atomic_read(&info->receive_credits)) { 403*38c8a9a5SSteve French while (true) { 404*38c8a9a5SSteve French if (use_receive_queue) 405*38c8a9a5SSteve French response = get_receive_buffer(info); 406*38c8a9a5SSteve French else 407*38c8a9a5SSteve French response = get_empty_queue_buffer(info); 408*38c8a9a5SSteve French if (!response) { 409*38c8a9a5SSteve French /* now switch to emtpy packet queue */ 410*38c8a9a5SSteve French if (use_receive_queue) { 411*38c8a9a5SSteve French use_receive_queue = 0; 412*38c8a9a5SSteve French continue; 413*38c8a9a5SSteve French } else 414*38c8a9a5SSteve French break; 415*38c8a9a5SSteve French } 416*38c8a9a5SSteve French 417*38c8a9a5SSteve French response->type = SMBD_TRANSFER_DATA; 418*38c8a9a5SSteve French response->first_segment = false; 419*38c8a9a5SSteve French rc = smbd_post_recv(info, response); 420*38c8a9a5SSteve French if (rc) { 421*38c8a9a5SSteve French log_rdma_recv(ERR, 422*38c8a9a5SSteve French "post_recv failed rc=%d\n", rc); 423*38c8a9a5SSteve French put_receive_buffer(info, response); 424*38c8a9a5SSteve French break; 425*38c8a9a5SSteve French } 426*38c8a9a5SSteve French 427*38c8a9a5SSteve French ret++; 428*38c8a9a5SSteve French } 429*38c8a9a5SSteve French } 430*38c8a9a5SSteve French 431*38c8a9a5SSteve French spin_lock(&info->lock_new_credits_offered); 432*38c8a9a5SSteve French info->new_credits_offered += ret; 433*38c8a9a5SSteve French spin_unlock(&info->lock_new_credits_offered); 434*38c8a9a5SSteve French 435*38c8a9a5SSteve French /* Promptly send an immediate packet as defined in [MS-SMBD] 3.1.1.1 */ 436*38c8a9a5SSteve French info->send_immediate = true; 437*38c8a9a5SSteve French if (atomic_read(&info->receive_credits) < 438*38c8a9a5SSteve French info->receive_credit_target - 1) { 439*38c8a9a5SSteve French if (info->keep_alive_requested == KEEP_ALIVE_PENDING || 440*38c8a9a5SSteve French info->send_immediate) { 441*38c8a9a5SSteve French log_keep_alive(INFO, "send an empty message\n"); 442*38c8a9a5SSteve French smbd_post_send_empty(info); 443*38c8a9a5SSteve French } 444*38c8a9a5SSteve French } 445*38c8a9a5SSteve French } 446*38c8a9a5SSteve French 447*38c8a9a5SSteve French /* Called from softirq, when recv is done */ 448*38c8a9a5SSteve French static void recv_done(struct ib_cq *cq, struct ib_wc *wc) 449*38c8a9a5SSteve French { 450*38c8a9a5SSteve French struct smbd_data_transfer *data_transfer; 451*38c8a9a5SSteve French struct smbd_response *response = 452*38c8a9a5SSteve French container_of(wc->wr_cqe, struct smbd_response, cqe); 453*38c8a9a5SSteve French struct smbd_connection *info = response->info; 454*38c8a9a5SSteve French int data_length = 0; 455*38c8a9a5SSteve French 456*38c8a9a5SSteve French log_rdma_recv(INFO, "response=0x%p type=%d wc status=%d wc opcode %d byte_len=%d pkey_index=%u\n", 457*38c8a9a5SSteve French response, response->type, wc->status, wc->opcode, 458*38c8a9a5SSteve French wc->byte_len, wc->pkey_index); 459*38c8a9a5SSteve French 460*38c8a9a5SSteve French if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_RECV) { 461*38c8a9a5SSteve French log_rdma_recv(INFO, "wc->status=%d opcode=%d\n", 462*38c8a9a5SSteve French wc->status, wc->opcode); 463*38c8a9a5SSteve French smbd_disconnect_rdma_connection(info); 464*38c8a9a5SSteve French goto error; 465*38c8a9a5SSteve French } 466*38c8a9a5SSteve French 467*38c8a9a5SSteve French ib_dma_sync_single_for_cpu( 468*38c8a9a5SSteve French wc->qp->device, 469*38c8a9a5SSteve French response->sge.addr, 470*38c8a9a5SSteve French response->sge.length, 471*38c8a9a5SSteve French DMA_FROM_DEVICE); 472*38c8a9a5SSteve French 473*38c8a9a5SSteve French switch (response->type) { 474*38c8a9a5SSteve French /* SMBD negotiation response */ 475*38c8a9a5SSteve French case SMBD_NEGOTIATE_RESP: 476*38c8a9a5SSteve French dump_smbd_negotiate_resp(smbd_response_payload(response)); 477*38c8a9a5SSteve French info->full_packet_received = true; 478*38c8a9a5SSteve French info->negotiate_done = 479*38c8a9a5SSteve French process_negotiation_response(response, wc->byte_len); 480*38c8a9a5SSteve French complete(&info->negotiate_completion); 481*38c8a9a5SSteve French break; 482*38c8a9a5SSteve French 483*38c8a9a5SSteve French /* SMBD data transfer packet */ 484*38c8a9a5SSteve French case SMBD_TRANSFER_DATA: 485*38c8a9a5SSteve French data_transfer = smbd_response_payload(response); 486*38c8a9a5SSteve French data_length = le32_to_cpu(data_transfer->data_length); 487*38c8a9a5SSteve French 488*38c8a9a5SSteve French /* 489*38c8a9a5SSteve French * If this is a packet with data playload place the data in 490*38c8a9a5SSteve French * reassembly queue and wake up the reading thread 491*38c8a9a5SSteve French */ 492*38c8a9a5SSteve French if (data_length) { 493*38c8a9a5SSteve French if (info->full_packet_received) 494*38c8a9a5SSteve French response->first_segment = true; 495*38c8a9a5SSteve French 496*38c8a9a5SSteve French if (le32_to_cpu(data_transfer->remaining_data_length)) 497*38c8a9a5SSteve French info->full_packet_received = false; 498*38c8a9a5SSteve French else 499*38c8a9a5SSteve French info->full_packet_received = true; 500*38c8a9a5SSteve French 501*38c8a9a5SSteve French enqueue_reassembly( 502*38c8a9a5SSteve French info, 503*38c8a9a5SSteve French response, 504*38c8a9a5SSteve French data_length); 505*38c8a9a5SSteve French } else 506*38c8a9a5SSteve French put_empty_packet(info, response); 507*38c8a9a5SSteve French 508*38c8a9a5SSteve French if (data_length) 509*38c8a9a5SSteve French wake_up_interruptible(&info->wait_reassembly_queue); 510*38c8a9a5SSteve French 511*38c8a9a5SSteve French atomic_dec(&info->receive_credits); 512*38c8a9a5SSteve French info->receive_credit_target = 513*38c8a9a5SSteve French le16_to_cpu(data_transfer->credits_requested); 514*38c8a9a5SSteve French if (le16_to_cpu(data_transfer->credits_granted)) { 515*38c8a9a5SSteve French atomic_add(le16_to_cpu(data_transfer->credits_granted), 516*38c8a9a5SSteve French &info->send_credits); 517*38c8a9a5SSteve French /* 518*38c8a9a5SSteve French * We have new send credits granted from remote peer 519*38c8a9a5SSteve French * If any sender is waiting for credits, unblock it 520*38c8a9a5SSteve French */ 521*38c8a9a5SSteve French wake_up_interruptible(&info->wait_send_queue); 522*38c8a9a5SSteve French } 523*38c8a9a5SSteve French 524*38c8a9a5SSteve French log_incoming(INFO, "data flags %d data_offset %d data_length %d remaining_data_length %d\n", 525*38c8a9a5SSteve French le16_to_cpu(data_transfer->flags), 526*38c8a9a5SSteve French le32_to_cpu(data_transfer->data_offset), 527*38c8a9a5SSteve French le32_to_cpu(data_transfer->data_length), 528*38c8a9a5SSteve French le32_to_cpu(data_transfer->remaining_data_length)); 529*38c8a9a5SSteve French 530*38c8a9a5SSteve French /* Send a KEEP_ALIVE response right away if requested */ 531*38c8a9a5SSteve French info->keep_alive_requested = KEEP_ALIVE_NONE; 532*38c8a9a5SSteve French if (le16_to_cpu(data_transfer->flags) & 533*38c8a9a5SSteve French SMB_DIRECT_RESPONSE_REQUESTED) { 534*38c8a9a5SSteve French info->keep_alive_requested = KEEP_ALIVE_PENDING; 535*38c8a9a5SSteve French } 536*38c8a9a5SSteve French 537*38c8a9a5SSteve French return; 538*38c8a9a5SSteve French 539*38c8a9a5SSteve French default: 540*38c8a9a5SSteve French log_rdma_recv(ERR, 541*38c8a9a5SSteve French "unexpected response type=%d\n", response->type); 542*38c8a9a5SSteve French } 543*38c8a9a5SSteve French 544*38c8a9a5SSteve French error: 545*38c8a9a5SSteve French put_receive_buffer(info, response); 546*38c8a9a5SSteve French } 547*38c8a9a5SSteve French 548*38c8a9a5SSteve French static struct rdma_cm_id *smbd_create_id( 549*38c8a9a5SSteve French struct smbd_connection *info, 550*38c8a9a5SSteve French struct sockaddr *dstaddr, int port) 551*38c8a9a5SSteve French { 552*38c8a9a5SSteve French struct rdma_cm_id *id; 553*38c8a9a5SSteve French int rc; 554*38c8a9a5SSteve French __be16 *sport; 555*38c8a9a5SSteve French 556*38c8a9a5SSteve French id = rdma_create_id(&init_net, smbd_conn_upcall, info, 557*38c8a9a5SSteve French RDMA_PS_TCP, IB_QPT_RC); 558*38c8a9a5SSteve French if (IS_ERR(id)) { 559*38c8a9a5SSteve French rc = PTR_ERR(id); 560*38c8a9a5SSteve French log_rdma_event(ERR, "rdma_create_id() failed %i\n", rc); 561*38c8a9a5SSteve French return id; 562*38c8a9a5SSteve French } 563*38c8a9a5SSteve French 564*38c8a9a5SSteve French if (dstaddr->sa_family == AF_INET6) 565*38c8a9a5SSteve French sport = &((struct sockaddr_in6 *)dstaddr)->sin6_port; 566*38c8a9a5SSteve French else 567*38c8a9a5SSteve French sport = &((struct sockaddr_in *)dstaddr)->sin_port; 568*38c8a9a5SSteve French 569*38c8a9a5SSteve French *sport = htons(port); 570*38c8a9a5SSteve French 571*38c8a9a5SSteve French init_completion(&info->ri_done); 572*38c8a9a5SSteve French info->ri_rc = -ETIMEDOUT; 573*38c8a9a5SSteve French 574*38c8a9a5SSteve French rc = rdma_resolve_addr(id, NULL, (struct sockaddr *)dstaddr, 575*38c8a9a5SSteve French RDMA_RESOLVE_TIMEOUT); 576*38c8a9a5SSteve French if (rc) { 577*38c8a9a5SSteve French log_rdma_event(ERR, "rdma_resolve_addr() failed %i\n", rc); 578*38c8a9a5SSteve French goto out; 579*38c8a9a5SSteve French } 580*38c8a9a5SSteve French rc = wait_for_completion_interruptible_timeout( 581*38c8a9a5SSteve French &info->ri_done, msecs_to_jiffies(RDMA_RESOLVE_TIMEOUT)); 582*38c8a9a5SSteve French /* e.g. if interrupted returns -ERESTARTSYS */ 583*38c8a9a5SSteve French if (rc < 0) { 584*38c8a9a5SSteve French log_rdma_event(ERR, "rdma_resolve_addr timeout rc: %i\n", rc); 585*38c8a9a5SSteve French goto out; 586*38c8a9a5SSteve French } 587*38c8a9a5SSteve French rc = info->ri_rc; 588*38c8a9a5SSteve French if (rc) { 589*38c8a9a5SSteve French log_rdma_event(ERR, "rdma_resolve_addr() completed %i\n", rc); 590*38c8a9a5SSteve French goto out; 591*38c8a9a5SSteve French } 592*38c8a9a5SSteve French 593*38c8a9a5SSteve French info->ri_rc = -ETIMEDOUT; 594*38c8a9a5SSteve French rc = rdma_resolve_route(id, RDMA_RESOLVE_TIMEOUT); 595*38c8a9a5SSteve French if (rc) { 596*38c8a9a5SSteve French log_rdma_event(ERR, "rdma_resolve_route() failed %i\n", rc); 597*38c8a9a5SSteve French goto out; 598*38c8a9a5SSteve French } 599*38c8a9a5SSteve French rc = wait_for_completion_interruptible_timeout( 600*38c8a9a5SSteve French &info->ri_done, msecs_to_jiffies(RDMA_RESOLVE_TIMEOUT)); 601*38c8a9a5SSteve French /* e.g. if interrupted returns -ERESTARTSYS */ 602*38c8a9a5SSteve French if (rc < 0) { 603*38c8a9a5SSteve French log_rdma_event(ERR, "rdma_resolve_addr timeout rc: %i\n", rc); 604*38c8a9a5SSteve French goto out; 605*38c8a9a5SSteve French } 606*38c8a9a5SSteve French rc = info->ri_rc; 607*38c8a9a5SSteve French if (rc) { 608*38c8a9a5SSteve French log_rdma_event(ERR, "rdma_resolve_route() completed %i\n", rc); 609*38c8a9a5SSteve French goto out; 610*38c8a9a5SSteve French } 611*38c8a9a5SSteve French 612*38c8a9a5SSteve French return id; 613*38c8a9a5SSteve French 614*38c8a9a5SSteve French out: 615*38c8a9a5SSteve French rdma_destroy_id(id); 616*38c8a9a5SSteve French return ERR_PTR(rc); 617*38c8a9a5SSteve French } 618*38c8a9a5SSteve French 619*38c8a9a5SSteve French /* 620*38c8a9a5SSteve French * Test if FRWR (Fast Registration Work Requests) is supported on the device 621*38c8a9a5SSteve French * This implementation requries FRWR on RDMA read/write 622*38c8a9a5SSteve French * return value: true if it is supported 623*38c8a9a5SSteve French */ 624*38c8a9a5SSteve French static bool frwr_is_supported(struct ib_device_attr *attrs) 625*38c8a9a5SSteve French { 626*38c8a9a5SSteve French if (!(attrs->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS)) 627*38c8a9a5SSteve French return false; 628*38c8a9a5SSteve French if (attrs->max_fast_reg_page_list_len == 0) 629*38c8a9a5SSteve French return false; 630*38c8a9a5SSteve French return true; 631*38c8a9a5SSteve French } 632*38c8a9a5SSteve French 633*38c8a9a5SSteve French static int smbd_ia_open( 634*38c8a9a5SSteve French struct smbd_connection *info, 635*38c8a9a5SSteve French struct sockaddr *dstaddr, int port) 636*38c8a9a5SSteve French { 637*38c8a9a5SSteve French int rc; 638*38c8a9a5SSteve French 639*38c8a9a5SSteve French info->id = smbd_create_id(info, dstaddr, port); 640*38c8a9a5SSteve French if (IS_ERR(info->id)) { 641*38c8a9a5SSteve French rc = PTR_ERR(info->id); 642*38c8a9a5SSteve French goto out1; 643*38c8a9a5SSteve French } 644*38c8a9a5SSteve French 645*38c8a9a5SSteve French if (!frwr_is_supported(&info->id->device->attrs)) { 646*38c8a9a5SSteve French log_rdma_event(ERR, "Fast Registration Work Requests (FRWR) is not supported\n"); 647*38c8a9a5SSteve French log_rdma_event(ERR, "Device capability flags = %llx max_fast_reg_page_list_len = %u\n", 648*38c8a9a5SSteve French info->id->device->attrs.device_cap_flags, 649*38c8a9a5SSteve French info->id->device->attrs.max_fast_reg_page_list_len); 650*38c8a9a5SSteve French rc = -EPROTONOSUPPORT; 651*38c8a9a5SSteve French goto out2; 652*38c8a9a5SSteve French } 653*38c8a9a5SSteve French info->max_frmr_depth = min_t(int, 654*38c8a9a5SSteve French smbd_max_frmr_depth, 655*38c8a9a5SSteve French info->id->device->attrs.max_fast_reg_page_list_len); 656*38c8a9a5SSteve French info->mr_type = IB_MR_TYPE_MEM_REG; 657*38c8a9a5SSteve French if (info->id->device->attrs.kernel_cap_flags & IBK_SG_GAPS_REG) 658*38c8a9a5SSteve French info->mr_type = IB_MR_TYPE_SG_GAPS; 659*38c8a9a5SSteve French 660*38c8a9a5SSteve French info->pd = ib_alloc_pd(info->id->device, 0); 661*38c8a9a5SSteve French if (IS_ERR(info->pd)) { 662*38c8a9a5SSteve French rc = PTR_ERR(info->pd); 663*38c8a9a5SSteve French log_rdma_event(ERR, "ib_alloc_pd() returned %d\n", rc); 664*38c8a9a5SSteve French goto out2; 665*38c8a9a5SSteve French } 666*38c8a9a5SSteve French 667*38c8a9a5SSteve French return 0; 668*38c8a9a5SSteve French 669*38c8a9a5SSteve French out2: 670*38c8a9a5SSteve French rdma_destroy_id(info->id); 671*38c8a9a5SSteve French info->id = NULL; 672*38c8a9a5SSteve French 673*38c8a9a5SSteve French out1: 674*38c8a9a5SSteve French return rc; 675*38c8a9a5SSteve French } 676*38c8a9a5SSteve French 677*38c8a9a5SSteve French /* 678*38c8a9a5SSteve French * Send a negotiation request message to the peer 679*38c8a9a5SSteve French * The negotiation procedure is in [MS-SMBD] 3.1.5.2 and 3.1.5.3 680*38c8a9a5SSteve French * After negotiation, the transport is connected and ready for 681*38c8a9a5SSteve French * carrying upper layer SMB payload 682*38c8a9a5SSteve French */ 683*38c8a9a5SSteve French static int smbd_post_send_negotiate_req(struct smbd_connection *info) 684*38c8a9a5SSteve French { 685*38c8a9a5SSteve French struct ib_send_wr send_wr; 686*38c8a9a5SSteve French int rc = -ENOMEM; 687*38c8a9a5SSteve French struct smbd_request *request; 688*38c8a9a5SSteve French struct smbd_negotiate_req *packet; 689*38c8a9a5SSteve French 690*38c8a9a5SSteve French request = mempool_alloc(info->request_mempool, GFP_KERNEL); 691*38c8a9a5SSteve French if (!request) 692*38c8a9a5SSteve French return rc; 693*38c8a9a5SSteve French 694*38c8a9a5SSteve French request->info = info; 695*38c8a9a5SSteve French 696*38c8a9a5SSteve French packet = smbd_request_payload(request); 697*38c8a9a5SSteve French packet->min_version = cpu_to_le16(SMBD_V1); 698*38c8a9a5SSteve French packet->max_version = cpu_to_le16(SMBD_V1); 699*38c8a9a5SSteve French packet->reserved = 0; 700*38c8a9a5SSteve French packet->credits_requested = cpu_to_le16(info->send_credit_target); 701*38c8a9a5SSteve French packet->preferred_send_size = cpu_to_le32(info->max_send_size); 702*38c8a9a5SSteve French packet->max_receive_size = cpu_to_le32(info->max_receive_size); 703*38c8a9a5SSteve French packet->max_fragmented_size = 704*38c8a9a5SSteve French cpu_to_le32(info->max_fragmented_recv_size); 705*38c8a9a5SSteve French 706*38c8a9a5SSteve French request->num_sge = 1; 707*38c8a9a5SSteve French request->sge[0].addr = ib_dma_map_single( 708*38c8a9a5SSteve French info->id->device, (void *)packet, 709*38c8a9a5SSteve French sizeof(*packet), DMA_TO_DEVICE); 710*38c8a9a5SSteve French if (ib_dma_mapping_error(info->id->device, request->sge[0].addr)) { 711*38c8a9a5SSteve French rc = -EIO; 712*38c8a9a5SSteve French goto dma_mapping_failed; 713*38c8a9a5SSteve French } 714*38c8a9a5SSteve French 715*38c8a9a5SSteve French request->sge[0].length = sizeof(*packet); 716*38c8a9a5SSteve French request->sge[0].lkey = info->pd->local_dma_lkey; 717*38c8a9a5SSteve French 718*38c8a9a5SSteve French ib_dma_sync_single_for_device( 719*38c8a9a5SSteve French info->id->device, request->sge[0].addr, 720*38c8a9a5SSteve French request->sge[0].length, DMA_TO_DEVICE); 721*38c8a9a5SSteve French 722*38c8a9a5SSteve French request->cqe.done = send_done; 723*38c8a9a5SSteve French 724*38c8a9a5SSteve French send_wr.next = NULL; 725*38c8a9a5SSteve French send_wr.wr_cqe = &request->cqe; 726*38c8a9a5SSteve French send_wr.sg_list = request->sge; 727*38c8a9a5SSteve French send_wr.num_sge = request->num_sge; 728*38c8a9a5SSteve French send_wr.opcode = IB_WR_SEND; 729*38c8a9a5SSteve French send_wr.send_flags = IB_SEND_SIGNALED; 730*38c8a9a5SSteve French 731*38c8a9a5SSteve French log_rdma_send(INFO, "sge addr=0x%llx length=%u lkey=0x%x\n", 732*38c8a9a5SSteve French request->sge[0].addr, 733*38c8a9a5SSteve French request->sge[0].length, request->sge[0].lkey); 734*38c8a9a5SSteve French 735*38c8a9a5SSteve French atomic_inc(&info->send_pending); 736*38c8a9a5SSteve French rc = ib_post_send(info->id->qp, &send_wr, NULL); 737*38c8a9a5SSteve French if (!rc) 738*38c8a9a5SSteve French return 0; 739*38c8a9a5SSteve French 740*38c8a9a5SSteve French /* if we reach here, post send failed */ 741*38c8a9a5SSteve French log_rdma_send(ERR, "ib_post_send failed rc=%d\n", rc); 742*38c8a9a5SSteve French atomic_dec(&info->send_pending); 743*38c8a9a5SSteve French ib_dma_unmap_single(info->id->device, request->sge[0].addr, 744*38c8a9a5SSteve French request->sge[0].length, DMA_TO_DEVICE); 745*38c8a9a5SSteve French 746*38c8a9a5SSteve French smbd_disconnect_rdma_connection(info); 747*38c8a9a5SSteve French 748*38c8a9a5SSteve French dma_mapping_failed: 749*38c8a9a5SSteve French mempool_free(request, info->request_mempool); 750*38c8a9a5SSteve French return rc; 751*38c8a9a5SSteve French } 752*38c8a9a5SSteve French 753*38c8a9a5SSteve French /* 754*38c8a9a5SSteve French * Extend the credits to remote peer 755*38c8a9a5SSteve French * This implements [MS-SMBD] 3.1.5.9 756*38c8a9a5SSteve French * The idea is that we should extend credits to remote peer as quickly as 757*38c8a9a5SSteve French * it's allowed, to maintain data flow. We allocate as much receive 758*38c8a9a5SSteve French * buffer as possible, and extend the receive credits to remote peer 759*38c8a9a5SSteve French * return value: the new credtis being granted. 760*38c8a9a5SSteve French */ 761*38c8a9a5SSteve French static int manage_credits_prior_sending(struct smbd_connection *info) 762*38c8a9a5SSteve French { 763*38c8a9a5SSteve French int new_credits; 764*38c8a9a5SSteve French 765*38c8a9a5SSteve French spin_lock(&info->lock_new_credits_offered); 766*38c8a9a5SSteve French new_credits = info->new_credits_offered; 767*38c8a9a5SSteve French info->new_credits_offered = 0; 768*38c8a9a5SSteve French spin_unlock(&info->lock_new_credits_offered); 769*38c8a9a5SSteve French 770*38c8a9a5SSteve French return new_credits; 771*38c8a9a5SSteve French } 772*38c8a9a5SSteve French 773*38c8a9a5SSteve French /* 774*38c8a9a5SSteve French * Check if we need to send a KEEP_ALIVE message 775*38c8a9a5SSteve French * The idle connection timer triggers a KEEP_ALIVE message when expires 776*38c8a9a5SSteve French * SMB_DIRECT_RESPONSE_REQUESTED is set in the message flag to have peer send 777*38c8a9a5SSteve French * back a response. 778*38c8a9a5SSteve French * return value: 779*38c8a9a5SSteve French * 1 if SMB_DIRECT_RESPONSE_REQUESTED needs to be set 780*38c8a9a5SSteve French * 0: otherwise 781*38c8a9a5SSteve French */ 782*38c8a9a5SSteve French static int manage_keep_alive_before_sending(struct smbd_connection *info) 783*38c8a9a5SSteve French { 784*38c8a9a5SSteve French if (info->keep_alive_requested == KEEP_ALIVE_PENDING) { 785*38c8a9a5SSteve French info->keep_alive_requested = KEEP_ALIVE_SENT; 786*38c8a9a5SSteve French return 1; 787*38c8a9a5SSteve French } 788*38c8a9a5SSteve French return 0; 789*38c8a9a5SSteve French } 790*38c8a9a5SSteve French 791*38c8a9a5SSteve French /* Post the send request */ 792*38c8a9a5SSteve French static int smbd_post_send(struct smbd_connection *info, 793*38c8a9a5SSteve French struct smbd_request *request) 794*38c8a9a5SSteve French { 795*38c8a9a5SSteve French struct ib_send_wr send_wr; 796*38c8a9a5SSteve French int rc, i; 797*38c8a9a5SSteve French 798*38c8a9a5SSteve French for (i = 0; i < request->num_sge; i++) { 799*38c8a9a5SSteve French log_rdma_send(INFO, 800*38c8a9a5SSteve French "rdma_request sge[%d] addr=0x%llx length=%u\n", 801*38c8a9a5SSteve French i, request->sge[i].addr, request->sge[i].length); 802*38c8a9a5SSteve French ib_dma_sync_single_for_device( 803*38c8a9a5SSteve French info->id->device, 804*38c8a9a5SSteve French request->sge[i].addr, 805*38c8a9a5SSteve French request->sge[i].length, 806*38c8a9a5SSteve French DMA_TO_DEVICE); 807*38c8a9a5SSteve French } 808*38c8a9a5SSteve French 809*38c8a9a5SSteve French request->cqe.done = send_done; 810*38c8a9a5SSteve French 811*38c8a9a5SSteve French send_wr.next = NULL; 812*38c8a9a5SSteve French send_wr.wr_cqe = &request->cqe; 813*38c8a9a5SSteve French send_wr.sg_list = request->sge; 814*38c8a9a5SSteve French send_wr.num_sge = request->num_sge; 815*38c8a9a5SSteve French send_wr.opcode = IB_WR_SEND; 816*38c8a9a5SSteve French send_wr.send_flags = IB_SEND_SIGNALED; 817*38c8a9a5SSteve French 818*38c8a9a5SSteve French rc = ib_post_send(info->id->qp, &send_wr, NULL); 819*38c8a9a5SSteve French if (rc) { 820*38c8a9a5SSteve French log_rdma_send(ERR, "ib_post_send failed rc=%d\n", rc); 821*38c8a9a5SSteve French smbd_disconnect_rdma_connection(info); 822*38c8a9a5SSteve French rc = -EAGAIN; 823*38c8a9a5SSteve French } else 824*38c8a9a5SSteve French /* Reset timer for idle connection after packet is sent */ 825*38c8a9a5SSteve French mod_delayed_work(info->workqueue, &info->idle_timer_work, 826*38c8a9a5SSteve French info->keep_alive_interval*HZ); 827*38c8a9a5SSteve French 828*38c8a9a5SSteve French return rc; 829*38c8a9a5SSteve French } 830*38c8a9a5SSteve French 831*38c8a9a5SSteve French static int smbd_post_send_iter(struct smbd_connection *info, 832*38c8a9a5SSteve French struct iov_iter *iter, 833*38c8a9a5SSteve French int *_remaining_data_length) 834*38c8a9a5SSteve French { 835*38c8a9a5SSteve French int i, rc; 836*38c8a9a5SSteve French int header_length; 837*38c8a9a5SSteve French int data_length; 838*38c8a9a5SSteve French struct smbd_request *request; 839*38c8a9a5SSteve French struct smbd_data_transfer *packet; 840*38c8a9a5SSteve French int new_credits = 0; 841*38c8a9a5SSteve French 842*38c8a9a5SSteve French wait_credit: 843*38c8a9a5SSteve French /* Wait for send credits. A SMBD packet needs one credit */ 844*38c8a9a5SSteve French rc = wait_event_interruptible(info->wait_send_queue, 845*38c8a9a5SSteve French atomic_read(&info->send_credits) > 0 || 846*38c8a9a5SSteve French info->transport_status != SMBD_CONNECTED); 847*38c8a9a5SSteve French if (rc) 848*38c8a9a5SSteve French goto err_wait_credit; 849*38c8a9a5SSteve French 850*38c8a9a5SSteve French if (info->transport_status != SMBD_CONNECTED) { 851*38c8a9a5SSteve French log_outgoing(ERR, "disconnected not sending on wait_credit\n"); 852*38c8a9a5SSteve French rc = -EAGAIN; 853*38c8a9a5SSteve French goto err_wait_credit; 854*38c8a9a5SSteve French } 855*38c8a9a5SSteve French if (unlikely(atomic_dec_return(&info->send_credits) < 0)) { 856*38c8a9a5SSteve French atomic_inc(&info->send_credits); 857*38c8a9a5SSteve French goto wait_credit; 858*38c8a9a5SSteve French } 859*38c8a9a5SSteve French 860*38c8a9a5SSteve French wait_send_queue: 861*38c8a9a5SSteve French wait_event(info->wait_post_send, 862*38c8a9a5SSteve French atomic_read(&info->send_pending) < info->send_credit_target || 863*38c8a9a5SSteve French info->transport_status != SMBD_CONNECTED); 864*38c8a9a5SSteve French 865*38c8a9a5SSteve French if (info->transport_status != SMBD_CONNECTED) { 866*38c8a9a5SSteve French log_outgoing(ERR, "disconnected not sending on wait_send_queue\n"); 867*38c8a9a5SSteve French rc = -EAGAIN; 868*38c8a9a5SSteve French goto err_wait_send_queue; 869*38c8a9a5SSteve French } 870*38c8a9a5SSteve French 871*38c8a9a5SSteve French if (unlikely(atomic_inc_return(&info->send_pending) > 872*38c8a9a5SSteve French info->send_credit_target)) { 873*38c8a9a5SSteve French atomic_dec(&info->send_pending); 874*38c8a9a5SSteve French goto wait_send_queue; 875*38c8a9a5SSteve French } 876*38c8a9a5SSteve French 877*38c8a9a5SSteve French request = mempool_alloc(info->request_mempool, GFP_KERNEL); 878*38c8a9a5SSteve French if (!request) { 879*38c8a9a5SSteve French rc = -ENOMEM; 880*38c8a9a5SSteve French goto err_alloc; 881*38c8a9a5SSteve French } 882*38c8a9a5SSteve French 883*38c8a9a5SSteve French request->info = info; 884*38c8a9a5SSteve French memset(request->sge, 0, sizeof(request->sge)); 885*38c8a9a5SSteve French 886*38c8a9a5SSteve French /* Fill in the data payload to find out how much data we can add */ 887*38c8a9a5SSteve French if (iter) { 888*38c8a9a5SSteve French struct smb_extract_to_rdma extract = { 889*38c8a9a5SSteve French .nr_sge = 1, 890*38c8a9a5SSteve French .max_sge = SMBDIRECT_MAX_SEND_SGE, 891*38c8a9a5SSteve French .sge = request->sge, 892*38c8a9a5SSteve French .device = info->id->device, 893*38c8a9a5SSteve French .local_dma_lkey = info->pd->local_dma_lkey, 894*38c8a9a5SSteve French .direction = DMA_TO_DEVICE, 895*38c8a9a5SSteve French }; 896*38c8a9a5SSteve French 897*38c8a9a5SSteve French rc = smb_extract_iter_to_rdma(iter, *_remaining_data_length, 898*38c8a9a5SSteve French &extract); 899*38c8a9a5SSteve French if (rc < 0) 900*38c8a9a5SSteve French goto err_dma; 901*38c8a9a5SSteve French data_length = rc; 902*38c8a9a5SSteve French request->num_sge = extract.nr_sge; 903*38c8a9a5SSteve French *_remaining_data_length -= data_length; 904*38c8a9a5SSteve French } else { 905*38c8a9a5SSteve French data_length = 0; 906*38c8a9a5SSteve French request->num_sge = 1; 907*38c8a9a5SSteve French } 908*38c8a9a5SSteve French 909*38c8a9a5SSteve French /* Fill in the packet header */ 910*38c8a9a5SSteve French packet = smbd_request_payload(request); 911*38c8a9a5SSteve French packet->credits_requested = cpu_to_le16(info->send_credit_target); 912*38c8a9a5SSteve French 913*38c8a9a5SSteve French new_credits = manage_credits_prior_sending(info); 914*38c8a9a5SSteve French atomic_add(new_credits, &info->receive_credits); 915*38c8a9a5SSteve French packet->credits_granted = cpu_to_le16(new_credits); 916*38c8a9a5SSteve French 917*38c8a9a5SSteve French info->send_immediate = false; 918*38c8a9a5SSteve French 919*38c8a9a5SSteve French packet->flags = 0; 920*38c8a9a5SSteve French if (manage_keep_alive_before_sending(info)) 921*38c8a9a5SSteve French packet->flags |= cpu_to_le16(SMB_DIRECT_RESPONSE_REQUESTED); 922*38c8a9a5SSteve French 923*38c8a9a5SSteve French packet->reserved = 0; 924*38c8a9a5SSteve French if (!data_length) 925*38c8a9a5SSteve French packet->data_offset = 0; 926*38c8a9a5SSteve French else 927*38c8a9a5SSteve French packet->data_offset = cpu_to_le32(24); 928*38c8a9a5SSteve French packet->data_length = cpu_to_le32(data_length); 929*38c8a9a5SSteve French packet->remaining_data_length = cpu_to_le32(*_remaining_data_length); 930*38c8a9a5SSteve French packet->padding = 0; 931*38c8a9a5SSteve French 932*38c8a9a5SSteve French log_outgoing(INFO, "credits_requested=%d credits_granted=%d data_offset=%d data_length=%d remaining_data_length=%d\n", 933*38c8a9a5SSteve French le16_to_cpu(packet->credits_requested), 934*38c8a9a5SSteve French le16_to_cpu(packet->credits_granted), 935*38c8a9a5SSteve French le32_to_cpu(packet->data_offset), 936*38c8a9a5SSteve French le32_to_cpu(packet->data_length), 937*38c8a9a5SSteve French le32_to_cpu(packet->remaining_data_length)); 938*38c8a9a5SSteve French 939*38c8a9a5SSteve French /* Map the packet to DMA */ 940*38c8a9a5SSteve French header_length = sizeof(struct smbd_data_transfer); 941*38c8a9a5SSteve French /* If this is a packet without payload, don't send padding */ 942*38c8a9a5SSteve French if (!data_length) 943*38c8a9a5SSteve French header_length = offsetof(struct smbd_data_transfer, padding); 944*38c8a9a5SSteve French 945*38c8a9a5SSteve French request->sge[0].addr = ib_dma_map_single(info->id->device, 946*38c8a9a5SSteve French (void *)packet, 947*38c8a9a5SSteve French header_length, 948*38c8a9a5SSteve French DMA_TO_DEVICE); 949*38c8a9a5SSteve French if (ib_dma_mapping_error(info->id->device, request->sge[0].addr)) { 950*38c8a9a5SSteve French rc = -EIO; 951*38c8a9a5SSteve French request->sge[0].addr = 0; 952*38c8a9a5SSteve French goto err_dma; 953*38c8a9a5SSteve French } 954*38c8a9a5SSteve French 955*38c8a9a5SSteve French request->sge[0].length = header_length; 956*38c8a9a5SSteve French request->sge[0].lkey = info->pd->local_dma_lkey; 957*38c8a9a5SSteve French 958*38c8a9a5SSteve French rc = smbd_post_send(info, request); 959*38c8a9a5SSteve French if (!rc) 960*38c8a9a5SSteve French return 0; 961*38c8a9a5SSteve French 962*38c8a9a5SSteve French err_dma: 963*38c8a9a5SSteve French for (i = 0; i < request->num_sge; i++) 964*38c8a9a5SSteve French if (request->sge[i].addr) 965*38c8a9a5SSteve French ib_dma_unmap_single(info->id->device, 966*38c8a9a5SSteve French request->sge[i].addr, 967*38c8a9a5SSteve French request->sge[i].length, 968*38c8a9a5SSteve French DMA_TO_DEVICE); 969*38c8a9a5SSteve French mempool_free(request, info->request_mempool); 970*38c8a9a5SSteve French 971*38c8a9a5SSteve French /* roll back receive credits and credits to be offered */ 972*38c8a9a5SSteve French spin_lock(&info->lock_new_credits_offered); 973*38c8a9a5SSteve French info->new_credits_offered += new_credits; 974*38c8a9a5SSteve French spin_unlock(&info->lock_new_credits_offered); 975*38c8a9a5SSteve French atomic_sub(new_credits, &info->receive_credits); 976*38c8a9a5SSteve French 977*38c8a9a5SSteve French err_alloc: 978*38c8a9a5SSteve French if (atomic_dec_and_test(&info->send_pending)) 979*38c8a9a5SSteve French wake_up(&info->wait_send_pending); 980*38c8a9a5SSteve French 981*38c8a9a5SSteve French err_wait_send_queue: 982*38c8a9a5SSteve French /* roll back send credits and pending */ 983*38c8a9a5SSteve French atomic_inc(&info->send_credits); 984*38c8a9a5SSteve French 985*38c8a9a5SSteve French err_wait_credit: 986*38c8a9a5SSteve French return rc; 987*38c8a9a5SSteve French } 988*38c8a9a5SSteve French 989*38c8a9a5SSteve French /* 990*38c8a9a5SSteve French * Send an empty message 991*38c8a9a5SSteve French * Empty message is used to extend credits to peer to for keep live 992*38c8a9a5SSteve French * while there is no upper layer payload to send at the time 993*38c8a9a5SSteve French */ 994*38c8a9a5SSteve French static int smbd_post_send_empty(struct smbd_connection *info) 995*38c8a9a5SSteve French { 996*38c8a9a5SSteve French int remaining_data_length = 0; 997*38c8a9a5SSteve French 998*38c8a9a5SSteve French info->count_send_empty++; 999*38c8a9a5SSteve French return smbd_post_send_iter(info, NULL, &remaining_data_length); 1000*38c8a9a5SSteve French } 1001*38c8a9a5SSteve French 1002*38c8a9a5SSteve French /* 1003*38c8a9a5SSteve French * Post a receive request to the transport 1004*38c8a9a5SSteve French * The remote peer can only send data when a receive request is posted 1005*38c8a9a5SSteve French * The interaction is controlled by send/receive credit system 1006*38c8a9a5SSteve French */ 1007*38c8a9a5SSteve French static int smbd_post_recv( 1008*38c8a9a5SSteve French struct smbd_connection *info, struct smbd_response *response) 1009*38c8a9a5SSteve French { 1010*38c8a9a5SSteve French struct ib_recv_wr recv_wr; 1011*38c8a9a5SSteve French int rc = -EIO; 1012*38c8a9a5SSteve French 1013*38c8a9a5SSteve French response->sge.addr = ib_dma_map_single( 1014*38c8a9a5SSteve French info->id->device, response->packet, 1015*38c8a9a5SSteve French info->max_receive_size, DMA_FROM_DEVICE); 1016*38c8a9a5SSteve French if (ib_dma_mapping_error(info->id->device, response->sge.addr)) 1017*38c8a9a5SSteve French return rc; 1018*38c8a9a5SSteve French 1019*38c8a9a5SSteve French response->sge.length = info->max_receive_size; 1020*38c8a9a5SSteve French response->sge.lkey = info->pd->local_dma_lkey; 1021*38c8a9a5SSteve French 1022*38c8a9a5SSteve French response->cqe.done = recv_done; 1023*38c8a9a5SSteve French 1024*38c8a9a5SSteve French recv_wr.wr_cqe = &response->cqe; 1025*38c8a9a5SSteve French recv_wr.next = NULL; 1026*38c8a9a5SSteve French recv_wr.sg_list = &response->sge; 1027*38c8a9a5SSteve French recv_wr.num_sge = 1; 1028*38c8a9a5SSteve French 1029*38c8a9a5SSteve French rc = ib_post_recv(info->id->qp, &recv_wr, NULL); 1030*38c8a9a5SSteve French if (rc) { 1031*38c8a9a5SSteve French ib_dma_unmap_single(info->id->device, response->sge.addr, 1032*38c8a9a5SSteve French response->sge.length, DMA_FROM_DEVICE); 1033*38c8a9a5SSteve French smbd_disconnect_rdma_connection(info); 1034*38c8a9a5SSteve French log_rdma_recv(ERR, "ib_post_recv failed rc=%d\n", rc); 1035*38c8a9a5SSteve French } 1036*38c8a9a5SSteve French 1037*38c8a9a5SSteve French return rc; 1038*38c8a9a5SSteve French } 1039*38c8a9a5SSteve French 1040*38c8a9a5SSteve French /* Perform SMBD negotiate according to [MS-SMBD] 3.1.5.2 */ 1041*38c8a9a5SSteve French static int smbd_negotiate(struct smbd_connection *info) 1042*38c8a9a5SSteve French { 1043*38c8a9a5SSteve French int rc; 1044*38c8a9a5SSteve French struct smbd_response *response = get_receive_buffer(info); 1045*38c8a9a5SSteve French 1046*38c8a9a5SSteve French response->type = SMBD_NEGOTIATE_RESP; 1047*38c8a9a5SSteve French rc = smbd_post_recv(info, response); 1048*38c8a9a5SSteve French log_rdma_event(INFO, "smbd_post_recv rc=%d iov.addr=0x%llx iov.length=%u iov.lkey=0x%x\n", 1049*38c8a9a5SSteve French rc, response->sge.addr, 1050*38c8a9a5SSteve French response->sge.length, response->sge.lkey); 1051*38c8a9a5SSteve French if (rc) 1052*38c8a9a5SSteve French return rc; 1053*38c8a9a5SSteve French 1054*38c8a9a5SSteve French init_completion(&info->negotiate_completion); 1055*38c8a9a5SSteve French info->negotiate_done = false; 1056*38c8a9a5SSteve French rc = smbd_post_send_negotiate_req(info); 1057*38c8a9a5SSteve French if (rc) 1058*38c8a9a5SSteve French return rc; 1059*38c8a9a5SSteve French 1060*38c8a9a5SSteve French rc = wait_for_completion_interruptible_timeout( 1061*38c8a9a5SSteve French &info->negotiate_completion, SMBD_NEGOTIATE_TIMEOUT * HZ); 1062*38c8a9a5SSteve French log_rdma_event(INFO, "wait_for_completion_timeout rc=%d\n", rc); 1063*38c8a9a5SSteve French 1064*38c8a9a5SSteve French if (info->negotiate_done) 1065*38c8a9a5SSteve French return 0; 1066*38c8a9a5SSteve French 1067*38c8a9a5SSteve French if (rc == 0) 1068*38c8a9a5SSteve French rc = -ETIMEDOUT; 1069*38c8a9a5SSteve French else if (rc == -ERESTARTSYS) 1070*38c8a9a5SSteve French rc = -EINTR; 1071*38c8a9a5SSteve French else 1072*38c8a9a5SSteve French rc = -ENOTCONN; 1073*38c8a9a5SSteve French 1074*38c8a9a5SSteve French return rc; 1075*38c8a9a5SSteve French } 1076*38c8a9a5SSteve French 1077*38c8a9a5SSteve French static void put_empty_packet( 1078*38c8a9a5SSteve French struct smbd_connection *info, struct smbd_response *response) 1079*38c8a9a5SSteve French { 1080*38c8a9a5SSteve French spin_lock(&info->empty_packet_queue_lock); 1081*38c8a9a5SSteve French list_add_tail(&response->list, &info->empty_packet_queue); 1082*38c8a9a5SSteve French info->count_empty_packet_queue++; 1083*38c8a9a5SSteve French spin_unlock(&info->empty_packet_queue_lock); 1084*38c8a9a5SSteve French 1085*38c8a9a5SSteve French queue_work(info->workqueue, &info->post_send_credits_work); 1086*38c8a9a5SSteve French } 1087*38c8a9a5SSteve French 1088*38c8a9a5SSteve French /* 1089*38c8a9a5SSteve French * Implement Connection.FragmentReassemblyBuffer defined in [MS-SMBD] 3.1.1.1 1090*38c8a9a5SSteve French * This is a queue for reassembling upper layer payload and present to upper 1091*38c8a9a5SSteve French * layer. All the inncoming payload go to the reassembly queue, regardless of 1092*38c8a9a5SSteve French * if reassembly is required. The uuper layer code reads from the queue for all 1093*38c8a9a5SSteve French * incoming payloads. 1094*38c8a9a5SSteve French * Put a received packet to the reassembly queue 1095*38c8a9a5SSteve French * response: the packet received 1096*38c8a9a5SSteve French * data_length: the size of payload in this packet 1097*38c8a9a5SSteve French */ 1098*38c8a9a5SSteve French static void enqueue_reassembly( 1099*38c8a9a5SSteve French struct smbd_connection *info, 1100*38c8a9a5SSteve French struct smbd_response *response, 1101*38c8a9a5SSteve French int data_length) 1102*38c8a9a5SSteve French { 1103*38c8a9a5SSteve French spin_lock(&info->reassembly_queue_lock); 1104*38c8a9a5SSteve French list_add_tail(&response->list, &info->reassembly_queue); 1105*38c8a9a5SSteve French info->reassembly_queue_length++; 1106*38c8a9a5SSteve French /* 1107*38c8a9a5SSteve French * Make sure reassembly_data_length is updated after list and 1108*38c8a9a5SSteve French * reassembly_queue_length are updated. On the dequeue side 1109*38c8a9a5SSteve French * reassembly_data_length is checked without a lock to determine 1110*38c8a9a5SSteve French * if reassembly_queue_length and list is up to date 1111*38c8a9a5SSteve French */ 1112*38c8a9a5SSteve French virt_wmb(); 1113*38c8a9a5SSteve French info->reassembly_data_length += data_length; 1114*38c8a9a5SSteve French spin_unlock(&info->reassembly_queue_lock); 1115*38c8a9a5SSteve French info->count_reassembly_queue++; 1116*38c8a9a5SSteve French info->count_enqueue_reassembly_queue++; 1117*38c8a9a5SSteve French } 1118*38c8a9a5SSteve French 1119*38c8a9a5SSteve French /* 1120*38c8a9a5SSteve French * Get the first entry at the front of reassembly queue 1121*38c8a9a5SSteve French * Caller is responsible for locking 1122*38c8a9a5SSteve French * return value: the first entry if any, NULL if queue is empty 1123*38c8a9a5SSteve French */ 1124*38c8a9a5SSteve French static struct smbd_response *_get_first_reassembly(struct smbd_connection *info) 1125*38c8a9a5SSteve French { 1126*38c8a9a5SSteve French struct smbd_response *ret = NULL; 1127*38c8a9a5SSteve French 1128*38c8a9a5SSteve French if (!list_empty(&info->reassembly_queue)) { 1129*38c8a9a5SSteve French ret = list_first_entry( 1130*38c8a9a5SSteve French &info->reassembly_queue, 1131*38c8a9a5SSteve French struct smbd_response, list); 1132*38c8a9a5SSteve French } 1133*38c8a9a5SSteve French return ret; 1134*38c8a9a5SSteve French } 1135*38c8a9a5SSteve French 1136*38c8a9a5SSteve French static struct smbd_response *get_empty_queue_buffer( 1137*38c8a9a5SSteve French struct smbd_connection *info) 1138*38c8a9a5SSteve French { 1139*38c8a9a5SSteve French struct smbd_response *ret = NULL; 1140*38c8a9a5SSteve French unsigned long flags; 1141*38c8a9a5SSteve French 1142*38c8a9a5SSteve French spin_lock_irqsave(&info->empty_packet_queue_lock, flags); 1143*38c8a9a5SSteve French if (!list_empty(&info->empty_packet_queue)) { 1144*38c8a9a5SSteve French ret = list_first_entry( 1145*38c8a9a5SSteve French &info->empty_packet_queue, 1146*38c8a9a5SSteve French struct smbd_response, list); 1147*38c8a9a5SSteve French list_del(&ret->list); 1148*38c8a9a5SSteve French info->count_empty_packet_queue--; 1149*38c8a9a5SSteve French } 1150*38c8a9a5SSteve French spin_unlock_irqrestore(&info->empty_packet_queue_lock, flags); 1151*38c8a9a5SSteve French 1152*38c8a9a5SSteve French return ret; 1153*38c8a9a5SSteve French } 1154*38c8a9a5SSteve French 1155*38c8a9a5SSteve French /* 1156*38c8a9a5SSteve French * Get a receive buffer 1157*38c8a9a5SSteve French * For each remote send, we need to post a receive. The receive buffers are 1158*38c8a9a5SSteve French * pre-allocated in advance. 1159*38c8a9a5SSteve French * return value: the receive buffer, NULL if none is available 1160*38c8a9a5SSteve French */ 1161*38c8a9a5SSteve French static struct smbd_response *get_receive_buffer(struct smbd_connection *info) 1162*38c8a9a5SSteve French { 1163*38c8a9a5SSteve French struct smbd_response *ret = NULL; 1164*38c8a9a5SSteve French unsigned long flags; 1165*38c8a9a5SSteve French 1166*38c8a9a5SSteve French spin_lock_irqsave(&info->receive_queue_lock, flags); 1167*38c8a9a5SSteve French if (!list_empty(&info->receive_queue)) { 1168*38c8a9a5SSteve French ret = list_first_entry( 1169*38c8a9a5SSteve French &info->receive_queue, 1170*38c8a9a5SSteve French struct smbd_response, list); 1171*38c8a9a5SSteve French list_del(&ret->list); 1172*38c8a9a5SSteve French info->count_receive_queue--; 1173*38c8a9a5SSteve French info->count_get_receive_buffer++; 1174*38c8a9a5SSteve French } 1175*38c8a9a5SSteve French spin_unlock_irqrestore(&info->receive_queue_lock, flags); 1176*38c8a9a5SSteve French 1177*38c8a9a5SSteve French return ret; 1178*38c8a9a5SSteve French } 1179*38c8a9a5SSteve French 1180*38c8a9a5SSteve French /* 1181*38c8a9a5SSteve French * Return a receive buffer 1182*38c8a9a5SSteve French * Upon returning of a receive buffer, we can post new receive and extend 1183*38c8a9a5SSteve French * more receive credits to remote peer. This is done immediately after a 1184*38c8a9a5SSteve French * receive buffer is returned. 1185*38c8a9a5SSteve French */ 1186*38c8a9a5SSteve French static void put_receive_buffer( 1187*38c8a9a5SSteve French struct smbd_connection *info, struct smbd_response *response) 1188*38c8a9a5SSteve French { 1189*38c8a9a5SSteve French unsigned long flags; 1190*38c8a9a5SSteve French 1191*38c8a9a5SSteve French ib_dma_unmap_single(info->id->device, response->sge.addr, 1192*38c8a9a5SSteve French response->sge.length, DMA_FROM_DEVICE); 1193*38c8a9a5SSteve French 1194*38c8a9a5SSteve French spin_lock_irqsave(&info->receive_queue_lock, flags); 1195*38c8a9a5SSteve French list_add_tail(&response->list, &info->receive_queue); 1196*38c8a9a5SSteve French info->count_receive_queue++; 1197*38c8a9a5SSteve French info->count_put_receive_buffer++; 1198*38c8a9a5SSteve French spin_unlock_irqrestore(&info->receive_queue_lock, flags); 1199*38c8a9a5SSteve French 1200*38c8a9a5SSteve French queue_work(info->workqueue, &info->post_send_credits_work); 1201*38c8a9a5SSteve French } 1202*38c8a9a5SSteve French 1203*38c8a9a5SSteve French /* Preallocate all receive buffer on transport establishment */ 1204*38c8a9a5SSteve French static int allocate_receive_buffers(struct smbd_connection *info, int num_buf) 1205*38c8a9a5SSteve French { 1206*38c8a9a5SSteve French int i; 1207*38c8a9a5SSteve French struct smbd_response *response; 1208*38c8a9a5SSteve French 1209*38c8a9a5SSteve French INIT_LIST_HEAD(&info->reassembly_queue); 1210*38c8a9a5SSteve French spin_lock_init(&info->reassembly_queue_lock); 1211*38c8a9a5SSteve French info->reassembly_data_length = 0; 1212*38c8a9a5SSteve French info->reassembly_queue_length = 0; 1213*38c8a9a5SSteve French 1214*38c8a9a5SSteve French INIT_LIST_HEAD(&info->receive_queue); 1215*38c8a9a5SSteve French spin_lock_init(&info->receive_queue_lock); 1216*38c8a9a5SSteve French info->count_receive_queue = 0; 1217*38c8a9a5SSteve French 1218*38c8a9a5SSteve French INIT_LIST_HEAD(&info->empty_packet_queue); 1219*38c8a9a5SSteve French spin_lock_init(&info->empty_packet_queue_lock); 1220*38c8a9a5SSteve French info->count_empty_packet_queue = 0; 1221*38c8a9a5SSteve French 1222*38c8a9a5SSteve French init_waitqueue_head(&info->wait_receive_queues); 1223*38c8a9a5SSteve French 1224*38c8a9a5SSteve French for (i = 0; i < num_buf; i++) { 1225*38c8a9a5SSteve French response = mempool_alloc(info->response_mempool, GFP_KERNEL); 1226*38c8a9a5SSteve French if (!response) 1227*38c8a9a5SSteve French goto allocate_failed; 1228*38c8a9a5SSteve French 1229*38c8a9a5SSteve French response->info = info; 1230*38c8a9a5SSteve French list_add_tail(&response->list, &info->receive_queue); 1231*38c8a9a5SSteve French info->count_receive_queue++; 1232*38c8a9a5SSteve French } 1233*38c8a9a5SSteve French 1234*38c8a9a5SSteve French return 0; 1235*38c8a9a5SSteve French 1236*38c8a9a5SSteve French allocate_failed: 1237*38c8a9a5SSteve French while (!list_empty(&info->receive_queue)) { 1238*38c8a9a5SSteve French response = list_first_entry( 1239*38c8a9a5SSteve French &info->receive_queue, 1240*38c8a9a5SSteve French struct smbd_response, list); 1241*38c8a9a5SSteve French list_del(&response->list); 1242*38c8a9a5SSteve French info->count_receive_queue--; 1243*38c8a9a5SSteve French 1244*38c8a9a5SSteve French mempool_free(response, info->response_mempool); 1245*38c8a9a5SSteve French } 1246*38c8a9a5SSteve French return -ENOMEM; 1247*38c8a9a5SSteve French } 1248*38c8a9a5SSteve French 1249*38c8a9a5SSteve French static void destroy_receive_buffers(struct smbd_connection *info) 1250*38c8a9a5SSteve French { 1251*38c8a9a5SSteve French struct smbd_response *response; 1252*38c8a9a5SSteve French 1253*38c8a9a5SSteve French while ((response = get_receive_buffer(info))) 1254*38c8a9a5SSteve French mempool_free(response, info->response_mempool); 1255*38c8a9a5SSteve French 1256*38c8a9a5SSteve French while ((response = get_empty_queue_buffer(info))) 1257*38c8a9a5SSteve French mempool_free(response, info->response_mempool); 1258*38c8a9a5SSteve French } 1259*38c8a9a5SSteve French 1260*38c8a9a5SSteve French /* Implement idle connection timer [MS-SMBD] 3.1.6.2 */ 1261*38c8a9a5SSteve French static void idle_connection_timer(struct work_struct *work) 1262*38c8a9a5SSteve French { 1263*38c8a9a5SSteve French struct smbd_connection *info = container_of( 1264*38c8a9a5SSteve French work, struct smbd_connection, 1265*38c8a9a5SSteve French idle_timer_work.work); 1266*38c8a9a5SSteve French 1267*38c8a9a5SSteve French if (info->keep_alive_requested != KEEP_ALIVE_NONE) { 1268*38c8a9a5SSteve French log_keep_alive(ERR, 1269*38c8a9a5SSteve French "error status info->keep_alive_requested=%d\n", 1270*38c8a9a5SSteve French info->keep_alive_requested); 1271*38c8a9a5SSteve French smbd_disconnect_rdma_connection(info); 1272*38c8a9a5SSteve French return; 1273*38c8a9a5SSteve French } 1274*38c8a9a5SSteve French 1275*38c8a9a5SSteve French log_keep_alive(INFO, "about to send an empty idle message\n"); 1276*38c8a9a5SSteve French smbd_post_send_empty(info); 1277*38c8a9a5SSteve French 1278*38c8a9a5SSteve French /* Setup the next idle timeout work */ 1279*38c8a9a5SSteve French queue_delayed_work(info->workqueue, &info->idle_timer_work, 1280*38c8a9a5SSteve French info->keep_alive_interval*HZ); 1281*38c8a9a5SSteve French } 1282*38c8a9a5SSteve French 1283*38c8a9a5SSteve French /* 1284*38c8a9a5SSteve French * Destroy the transport and related RDMA and memory resources 1285*38c8a9a5SSteve French * Need to go through all the pending counters and make sure on one is using 1286*38c8a9a5SSteve French * the transport while it is destroyed 1287*38c8a9a5SSteve French */ 1288*38c8a9a5SSteve French void smbd_destroy(struct TCP_Server_Info *server) 1289*38c8a9a5SSteve French { 1290*38c8a9a5SSteve French struct smbd_connection *info = server->smbd_conn; 1291*38c8a9a5SSteve French struct smbd_response *response; 1292*38c8a9a5SSteve French unsigned long flags; 1293*38c8a9a5SSteve French 1294*38c8a9a5SSteve French if (!info) { 1295*38c8a9a5SSteve French log_rdma_event(INFO, "rdma session already destroyed\n"); 1296*38c8a9a5SSteve French return; 1297*38c8a9a5SSteve French } 1298*38c8a9a5SSteve French 1299*38c8a9a5SSteve French log_rdma_event(INFO, "destroying rdma session\n"); 1300*38c8a9a5SSteve French if (info->transport_status != SMBD_DISCONNECTED) { 1301*38c8a9a5SSteve French rdma_disconnect(server->smbd_conn->id); 1302*38c8a9a5SSteve French log_rdma_event(INFO, "wait for transport being disconnected\n"); 1303*38c8a9a5SSteve French wait_event_interruptible( 1304*38c8a9a5SSteve French info->disconn_wait, 1305*38c8a9a5SSteve French info->transport_status == SMBD_DISCONNECTED); 1306*38c8a9a5SSteve French } 1307*38c8a9a5SSteve French 1308*38c8a9a5SSteve French log_rdma_event(INFO, "destroying qp\n"); 1309*38c8a9a5SSteve French ib_drain_qp(info->id->qp); 1310*38c8a9a5SSteve French rdma_destroy_qp(info->id); 1311*38c8a9a5SSteve French 1312*38c8a9a5SSteve French log_rdma_event(INFO, "cancelling idle timer\n"); 1313*38c8a9a5SSteve French cancel_delayed_work_sync(&info->idle_timer_work); 1314*38c8a9a5SSteve French 1315*38c8a9a5SSteve French log_rdma_event(INFO, "wait for all send posted to IB to finish\n"); 1316*38c8a9a5SSteve French wait_event(info->wait_send_pending, 1317*38c8a9a5SSteve French atomic_read(&info->send_pending) == 0); 1318*38c8a9a5SSteve French 1319*38c8a9a5SSteve French /* It's not possible for upper layer to get to reassembly */ 1320*38c8a9a5SSteve French log_rdma_event(INFO, "drain the reassembly queue\n"); 1321*38c8a9a5SSteve French do { 1322*38c8a9a5SSteve French spin_lock_irqsave(&info->reassembly_queue_lock, flags); 1323*38c8a9a5SSteve French response = _get_first_reassembly(info); 1324*38c8a9a5SSteve French if (response) { 1325*38c8a9a5SSteve French list_del(&response->list); 1326*38c8a9a5SSteve French spin_unlock_irqrestore( 1327*38c8a9a5SSteve French &info->reassembly_queue_lock, flags); 1328*38c8a9a5SSteve French put_receive_buffer(info, response); 1329*38c8a9a5SSteve French } else 1330*38c8a9a5SSteve French spin_unlock_irqrestore( 1331*38c8a9a5SSteve French &info->reassembly_queue_lock, flags); 1332*38c8a9a5SSteve French } while (response); 1333*38c8a9a5SSteve French info->reassembly_data_length = 0; 1334*38c8a9a5SSteve French 1335*38c8a9a5SSteve French log_rdma_event(INFO, "free receive buffers\n"); 1336*38c8a9a5SSteve French wait_event(info->wait_receive_queues, 1337*38c8a9a5SSteve French info->count_receive_queue + info->count_empty_packet_queue 1338*38c8a9a5SSteve French == info->receive_credit_max); 1339*38c8a9a5SSteve French destroy_receive_buffers(info); 1340*38c8a9a5SSteve French 1341*38c8a9a5SSteve French /* 1342*38c8a9a5SSteve French * For performance reasons, memory registration and deregistration 1343*38c8a9a5SSteve French * are not locked by srv_mutex. It is possible some processes are 1344*38c8a9a5SSteve French * blocked on transport srv_mutex while holding memory registration. 1345*38c8a9a5SSteve French * Release the transport srv_mutex to allow them to hit the failure 1346*38c8a9a5SSteve French * path when sending data, and then release memory registartions. 1347*38c8a9a5SSteve French */ 1348*38c8a9a5SSteve French log_rdma_event(INFO, "freeing mr list\n"); 1349*38c8a9a5SSteve French wake_up_interruptible_all(&info->wait_mr); 1350*38c8a9a5SSteve French while (atomic_read(&info->mr_used_count)) { 1351*38c8a9a5SSteve French cifs_server_unlock(server); 1352*38c8a9a5SSteve French msleep(1000); 1353*38c8a9a5SSteve French cifs_server_lock(server); 1354*38c8a9a5SSteve French } 1355*38c8a9a5SSteve French destroy_mr_list(info); 1356*38c8a9a5SSteve French 1357*38c8a9a5SSteve French ib_free_cq(info->send_cq); 1358*38c8a9a5SSteve French ib_free_cq(info->recv_cq); 1359*38c8a9a5SSteve French ib_dealloc_pd(info->pd); 1360*38c8a9a5SSteve French rdma_destroy_id(info->id); 1361*38c8a9a5SSteve French 1362*38c8a9a5SSteve French /* free mempools */ 1363*38c8a9a5SSteve French mempool_destroy(info->request_mempool); 1364*38c8a9a5SSteve French kmem_cache_destroy(info->request_cache); 1365*38c8a9a5SSteve French 1366*38c8a9a5SSteve French mempool_destroy(info->response_mempool); 1367*38c8a9a5SSteve French kmem_cache_destroy(info->response_cache); 1368*38c8a9a5SSteve French 1369*38c8a9a5SSteve French info->transport_status = SMBD_DESTROYED; 1370*38c8a9a5SSteve French 1371*38c8a9a5SSteve French destroy_workqueue(info->workqueue); 1372*38c8a9a5SSteve French log_rdma_event(INFO, "rdma session destroyed\n"); 1373*38c8a9a5SSteve French kfree(info); 1374*38c8a9a5SSteve French server->smbd_conn = NULL; 1375*38c8a9a5SSteve French } 1376*38c8a9a5SSteve French 1377*38c8a9a5SSteve French /* 1378*38c8a9a5SSteve French * Reconnect this SMBD connection, called from upper layer 1379*38c8a9a5SSteve French * return value: 0 on success, or actual error code 1380*38c8a9a5SSteve French */ 1381*38c8a9a5SSteve French int smbd_reconnect(struct TCP_Server_Info *server) 1382*38c8a9a5SSteve French { 1383*38c8a9a5SSteve French log_rdma_event(INFO, "reconnecting rdma session\n"); 1384*38c8a9a5SSteve French 1385*38c8a9a5SSteve French if (!server->smbd_conn) { 1386*38c8a9a5SSteve French log_rdma_event(INFO, "rdma session already destroyed\n"); 1387*38c8a9a5SSteve French goto create_conn; 1388*38c8a9a5SSteve French } 1389*38c8a9a5SSteve French 1390*38c8a9a5SSteve French /* 1391*38c8a9a5SSteve French * This is possible if transport is disconnected and we haven't received 1392*38c8a9a5SSteve French * notification from RDMA, but upper layer has detected timeout 1393*38c8a9a5SSteve French */ 1394*38c8a9a5SSteve French if (server->smbd_conn->transport_status == SMBD_CONNECTED) { 1395*38c8a9a5SSteve French log_rdma_event(INFO, "disconnecting transport\n"); 1396*38c8a9a5SSteve French smbd_destroy(server); 1397*38c8a9a5SSteve French } 1398*38c8a9a5SSteve French 1399*38c8a9a5SSteve French create_conn: 1400*38c8a9a5SSteve French log_rdma_event(INFO, "creating rdma session\n"); 1401*38c8a9a5SSteve French server->smbd_conn = smbd_get_connection( 1402*38c8a9a5SSteve French server, (struct sockaddr *) &server->dstaddr); 1403*38c8a9a5SSteve French 1404*38c8a9a5SSteve French if (server->smbd_conn) 1405*38c8a9a5SSteve French cifs_dbg(VFS, "RDMA transport re-established\n"); 1406*38c8a9a5SSteve French 1407*38c8a9a5SSteve French return server->smbd_conn ? 0 : -ENOENT; 1408*38c8a9a5SSteve French } 1409*38c8a9a5SSteve French 1410*38c8a9a5SSteve French static void destroy_caches_and_workqueue(struct smbd_connection *info) 1411*38c8a9a5SSteve French { 1412*38c8a9a5SSteve French destroy_receive_buffers(info); 1413*38c8a9a5SSteve French destroy_workqueue(info->workqueue); 1414*38c8a9a5SSteve French mempool_destroy(info->response_mempool); 1415*38c8a9a5SSteve French kmem_cache_destroy(info->response_cache); 1416*38c8a9a5SSteve French mempool_destroy(info->request_mempool); 1417*38c8a9a5SSteve French kmem_cache_destroy(info->request_cache); 1418*38c8a9a5SSteve French } 1419*38c8a9a5SSteve French 1420*38c8a9a5SSteve French #define MAX_NAME_LEN 80 1421*38c8a9a5SSteve French static int allocate_caches_and_workqueue(struct smbd_connection *info) 1422*38c8a9a5SSteve French { 1423*38c8a9a5SSteve French char name[MAX_NAME_LEN]; 1424*38c8a9a5SSteve French int rc; 1425*38c8a9a5SSteve French 1426*38c8a9a5SSteve French scnprintf(name, MAX_NAME_LEN, "smbd_request_%p", info); 1427*38c8a9a5SSteve French info->request_cache = 1428*38c8a9a5SSteve French kmem_cache_create( 1429*38c8a9a5SSteve French name, 1430*38c8a9a5SSteve French sizeof(struct smbd_request) + 1431*38c8a9a5SSteve French sizeof(struct smbd_data_transfer), 1432*38c8a9a5SSteve French 0, SLAB_HWCACHE_ALIGN, NULL); 1433*38c8a9a5SSteve French if (!info->request_cache) 1434*38c8a9a5SSteve French return -ENOMEM; 1435*38c8a9a5SSteve French 1436*38c8a9a5SSteve French info->request_mempool = 1437*38c8a9a5SSteve French mempool_create(info->send_credit_target, mempool_alloc_slab, 1438*38c8a9a5SSteve French mempool_free_slab, info->request_cache); 1439*38c8a9a5SSteve French if (!info->request_mempool) 1440*38c8a9a5SSteve French goto out1; 1441*38c8a9a5SSteve French 1442*38c8a9a5SSteve French scnprintf(name, MAX_NAME_LEN, "smbd_response_%p", info); 1443*38c8a9a5SSteve French info->response_cache = 1444*38c8a9a5SSteve French kmem_cache_create( 1445*38c8a9a5SSteve French name, 1446*38c8a9a5SSteve French sizeof(struct smbd_response) + 1447*38c8a9a5SSteve French info->max_receive_size, 1448*38c8a9a5SSteve French 0, SLAB_HWCACHE_ALIGN, NULL); 1449*38c8a9a5SSteve French if (!info->response_cache) 1450*38c8a9a5SSteve French goto out2; 1451*38c8a9a5SSteve French 1452*38c8a9a5SSteve French info->response_mempool = 1453*38c8a9a5SSteve French mempool_create(info->receive_credit_max, mempool_alloc_slab, 1454*38c8a9a5SSteve French mempool_free_slab, info->response_cache); 1455*38c8a9a5SSteve French if (!info->response_mempool) 1456*38c8a9a5SSteve French goto out3; 1457*38c8a9a5SSteve French 1458*38c8a9a5SSteve French scnprintf(name, MAX_NAME_LEN, "smbd_%p", info); 1459*38c8a9a5SSteve French info->workqueue = create_workqueue(name); 1460*38c8a9a5SSteve French if (!info->workqueue) 1461*38c8a9a5SSteve French goto out4; 1462*38c8a9a5SSteve French 1463*38c8a9a5SSteve French rc = allocate_receive_buffers(info, info->receive_credit_max); 1464*38c8a9a5SSteve French if (rc) { 1465*38c8a9a5SSteve French log_rdma_event(ERR, "failed to allocate receive buffers\n"); 1466*38c8a9a5SSteve French goto out5; 1467*38c8a9a5SSteve French } 1468*38c8a9a5SSteve French 1469*38c8a9a5SSteve French return 0; 1470*38c8a9a5SSteve French 1471*38c8a9a5SSteve French out5: 1472*38c8a9a5SSteve French destroy_workqueue(info->workqueue); 1473*38c8a9a5SSteve French out4: 1474*38c8a9a5SSteve French mempool_destroy(info->response_mempool); 1475*38c8a9a5SSteve French out3: 1476*38c8a9a5SSteve French kmem_cache_destroy(info->response_cache); 1477*38c8a9a5SSteve French out2: 1478*38c8a9a5SSteve French mempool_destroy(info->request_mempool); 1479*38c8a9a5SSteve French out1: 1480*38c8a9a5SSteve French kmem_cache_destroy(info->request_cache); 1481*38c8a9a5SSteve French return -ENOMEM; 1482*38c8a9a5SSteve French } 1483*38c8a9a5SSteve French 1484*38c8a9a5SSteve French /* Create a SMBD connection, called by upper layer */ 1485*38c8a9a5SSteve French static struct smbd_connection *_smbd_get_connection( 1486*38c8a9a5SSteve French struct TCP_Server_Info *server, struct sockaddr *dstaddr, int port) 1487*38c8a9a5SSteve French { 1488*38c8a9a5SSteve French int rc; 1489*38c8a9a5SSteve French struct smbd_connection *info; 1490*38c8a9a5SSteve French struct rdma_conn_param conn_param; 1491*38c8a9a5SSteve French struct ib_qp_init_attr qp_attr; 1492*38c8a9a5SSteve French struct sockaddr_in *addr_in = (struct sockaddr_in *) dstaddr; 1493*38c8a9a5SSteve French struct ib_port_immutable port_immutable; 1494*38c8a9a5SSteve French u32 ird_ord_hdr[2]; 1495*38c8a9a5SSteve French 1496*38c8a9a5SSteve French info = kzalloc(sizeof(struct smbd_connection), GFP_KERNEL); 1497*38c8a9a5SSteve French if (!info) 1498*38c8a9a5SSteve French return NULL; 1499*38c8a9a5SSteve French 1500*38c8a9a5SSteve French info->transport_status = SMBD_CONNECTING; 1501*38c8a9a5SSteve French rc = smbd_ia_open(info, dstaddr, port); 1502*38c8a9a5SSteve French if (rc) { 1503*38c8a9a5SSteve French log_rdma_event(INFO, "smbd_ia_open rc=%d\n", rc); 1504*38c8a9a5SSteve French goto create_id_failed; 1505*38c8a9a5SSteve French } 1506*38c8a9a5SSteve French 1507*38c8a9a5SSteve French if (smbd_send_credit_target > info->id->device->attrs.max_cqe || 1508*38c8a9a5SSteve French smbd_send_credit_target > info->id->device->attrs.max_qp_wr) { 1509*38c8a9a5SSteve French log_rdma_event(ERR, "consider lowering send_credit_target = %d. Possible CQE overrun, device reporting max_cqe %d max_qp_wr %d\n", 1510*38c8a9a5SSteve French smbd_send_credit_target, 1511*38c8a9a5SSteve French info->id->device->attrs.max_cqe, 1512*38c8a9a5SSteve French info->id->device->attrs.max_qp_wr); 1513*38c8a9a5SSteve French goto config_failed; 1514*38c8a9a5SSteve French } 1515*38c8a9a5SSteve French 1516*38c8a9a5SSteve French if (smbd_receive_credit_max > info->id->device->attrs.max_cqe || 1517*38c8a9a5SSteve French smbd_receive_credit_max > info->id->device->attrs.max_qp_wr) { 1518*38c8a9a5SSteve French log_rdma_event(ERR, "consider lowering receive_credit_max = %d. Possible CQE overrun, device reporting max_cqe %d max_qp_wr %d\n", 1519*38c8a9a5SSteve French smbd_receive_credit_max, 1520*38c8a9a5SSteve French info->id->device->attrs.max_cqe, 1521*38c8a9a5SSteve French info->id->device->attrs.max_qp_wr); 1522*38c8a9a5SSteve French goto config_failed; 1523*38c8a9a5SSteve French } 1524*38c8a9a5SSteve French 1525*38c8a9a5SSteve French info->receive_credit_max = smbd_receive_credit_max; 1526*38c8a9a5SSteve French info->send_credit_target = smbd_send_credit_target; 1527*38c8a9a5SSteve French info->max_send_size = smbd_max_send_size; 1528*38c8a9a5SSteve French info->max_fragmented_recv_size = smbd_max_fragmented_recv_size; 1529*38c8a9a5SSteve French info->max_receive_size = smbd_max_receive_size; 1530*38c8a9a5SSteve French info->keep_alive_interval = smbd_keep_alive_interval; 1531*38c8a9a5SSteve French 1532*38c8a9a5SSteve French if (info->id->device->attrs.max_send_sge < SMBDIRECT_MAX_SEND_SGE || 1533*38c8a9a5SSteve French info->id->device->attrs.max_recv_sge < SMBDIRECT_MAX_RECV_SGE) { 1534*38c8a9a5SSteve French log_rdma_event(ERR, 1535*38c8a9a5SSteve French "device %.*s max_send_sge/max_recv_sge = %d/%d too small\n", 1536*38c8a9a5SSteve French IB_DEVICE_NAME_MAX, 1537*38c8a9a5SSteve French info->id->device->name, 1538*38c8a9a5SSteve French info->id->device->attrs.max_send_sge, 1539*38c8a9a5SSteve French info->id->device->attrs.max_recv_sge); 1540*38c8a9a5SSteve French goto config_failed; 1541*38c8a9a5SSteve French } 1542*38c8a9a5SSteve French 1543*38c8a9a5SSteve French info->send_cq = NULL; 1544*38c8a9a5SSteve French info->recv_cq = NULL; 1545*38c8a9a5SSteve French info->send_cq = 1546*38c8a9a5SSteve French ib_alloc_cq_any(info->id->device, info, 1547*38c8a9a5SSteve French info->send_credit_target, IB_POLL_SOFTIRQ); 1548*38c8a9a5SSteve French if (IS_ERR(info->send_cq)) { 1549*38c8a9a5SSteve French info->send_cq = NULL; 1550*38c8a9a5SSteve French goto alloc_cq_failed; 1551*38c8a9a5SSteve French } 1552*38c8a9a5SSteve French 1553*38c8a9a5SSteve French info->recv_cq = 1554*38c8a9a5SSteve French ib_alloc_cq_any(info->id->device, info, 1555*38c8a9a5SSteve French info->receive_credit_max, IB_POLL_SOFTIRQ); 1556*38c8a9a5SSteve French if (IS_ERR(info->recv_cq)) { 1557*38c8a9a5SSteve French info->recv_cq = NULL; 1558*38c8a9a5SSteve French goto alloc_cq_failed; 1559*38c8a9a5SSteve French } 1560*38c8a9a5SSteve French 1561*38c8a9a5SSteve French memset(&qp_attr, 0, sizeof(qp_attr)); 1562*38c8a9a5SSteve French qp_attr.event_handler = smbd_qp_async_error_upcall; 1563*38c8a9a5SSteve French qp_attr.qp_context = info; 1564*38c8a9a5SSteve French qp_attr.cap.max_send_wr = info->send_credit_target; 1565*38c8a9a5SSteve French qp_attr.cap.max_recv_wr = info->receive_credit_max; 1566*38c8a9a5SSteve French qp_attr.cap.max_send_sge = SMBDIRECT_MAX_SEND_SGE; 1567*38c8a9a5SSteve French qp_attr.cap.max_recv_sge = SMBDIRECT_MAX_RECV_SGE; 1568*38c8a9a5SSteve French qp_attr.cap.max_inline_data = 0; 1569*38c8a9a5SSteve French qp_attr.sq_sig_type = IB_SIGNAL_REQ_WR; 1570*38c8a9a5SSteve French qp_attr.qp_type = IB_QPT_RC; 1571*38c8a9a5SSteve French qp_attr.send_cq = info->send_cq; 1572*38c8a9a5SSteve French qp_attr.recv_cq = info->recv_cq; 1573*38c8a9a5SSteve French qp_attr.port_num = ~0; 1574*38c8a9a5SSteve French 1575*38c8a9a5SSteve French rc = rdma_create_qp(info->id, info->pd, &qp_attr); 1576*38c8a9a5SSteve French if (rc) { 1577*38c8a9a5SSteve French log_rdma_event(ERR, "rdma_create_qp failed %i\n", rc); 1578*38c8a9a5SSteve French goto create_qp_failed; 1579*38c8a9a5SSteve French } 1580*38c8a9a5SSteve French 1581*38c8a9a5SSteve French memset(&conn_param, 0, sizeof(conn_param)); 1582*38c8a9a5SSteve French conn_param.initiator_depth = 0; 1583*38c8a9a5SSteve French 1584*38c8a9a5SSteve French conn_param.responder_resources = 1585*38c8a9a5SSteve French info->id->device->attrs.max_qp_rd_atom 1586*38c8a9a5SSteve French < SMBD_CM_RESPONDER_RESOURCES ? 1587*38c8a9a5SSteve French info->id->device->attrs.max_qp_rd_atom : 1588*38c8a9a5SSteve French SMBD_CM_RESPONDER_RESOURCES; 1589*38c8a9a5SSteve French info->responder_resources = conn_param.responder_resources; 1590*38c8a9a5SSteve French log_rdma_mr(INFO, "responder_resources=%d\n", 1591*38c8a9a5SSteve French info->responder_resources); 1592*38c8a9a5SSteve French 1593*38c8a9a5SSteve French /* Need to send IRD/ORD in private data for iWARP */ 1594*38c8a9a5SSteve French info->id->device->ops.get_port_immutable( 1595*38c8a9a5SSteve French info->id->device, info->id->port_num, &port_immutable); 1596*38c8a9a5SSteve French if (port_immutable.core_cap_flags & RDMA_CORE_PORT_IWARP) { 1597*38c8a9a5SSteve French ird_ord_hdr[0] = info->responder_resources; 1598*38c8a9a5SSteve French ird_ord_hdr[1] = 1; 1599*38c8a9a5SSteve French conn_param.private_data = ird_ord_hdr; 1600*38c8a9a5SSteve French conn_param.private_data_len = sizeof(ird_ord_hdr); 1601*38c8a9a5SSteve French } else { 1602*38c8a9a5SSteve French conn_param.private_data = NULL; 1603*38c8a9a5SSteve French conn_param.private_data_len = 0; 1604*38c8a9a5SSteve French } 1605*38c8a9a5SSteve French 1606*38c8a9a5SSteve French conn_param.retry_count = SMBD_CM_RETRY; 1607*38c8a9a5SSteve French conn_param.rnr_retry_count = SMBD_CM_RNR_RETRY; 1608*38c8a9a5SSteve French conn_param.flow_control = 0; 1609*38c8a9a5SSteve French 1610*38c8a9a5SSteve French log_rdma_event(INFO, "connecting to IP %pI4 port %d\n", 1611*38c8a9a5SSteve French &addr_in->sin_addr, port); 1612*38c8a9a5SSteve French 1613*38c8a9a5SSteve French init_waitqueue_head(&info->conn_wait); 1614*38c8a9a5SSteve French init_waitqueue_head(&info->disconn_wait); 1615*38c8a9a5SSteve French init_waitqueue_head(&info->wait_reassembly_queue); 1616*38c8a9a5SSteve French rc = rdma_connect(info->id, &conn_param); 1617*38c8a9a5SSteve French if (rc) { 1618*38c8a9a5SSteve French log_rdma_event(ERR, "rdma_connect() failed with %i\n", rc); 1619*38c8a9a5SSteve French goto rdma_connect_failed; 1620*38c8a9a5SSteve French } 1621*38c8a9a5SSteve French 1622*38c8a9a5SSteve French wait_event_interruptible( 1623*38c8a9a5SSteve French info->conn_wait, info->transport_status != SMBD_CONNECTING); 1624*38c8a9a5SSteve French 1625*38c8a9a5SSteve French if (info->transport_status != SMBD_CONNECTED) { 1626*38c8a9a5SSteve French log_rdma_event(ERR, "rdma_connect failed port=%d\n", port); 1627*38c8a9a5SSteve French goto rdma_connect_failed; 1628*38c8a9a5SSteve French } 1629*38c8a9a5SSteve French 1630*38c8a9a5SSteve French log_rdma_event(INFO, "rdma_connect connected\n"); 1631*38c8a9a5SSteve French 1632*38c8a9a5SSteve French rc = allocate_caches_and_workqueue(info); 1633*38c8a9a5SSteve French if (rc) { 1634*38c8a9a5SSteve French log_rdma_event(ERR, "cache allocation failed\n"); 1635*38c8a9a5SSteve French goto allocate_cache_failed; 1636*38c8a9a5SSteve French } 1637*38c8a9a5SSteve French 1638*38c8a9a5SSteve French init_waitqueue_head(&info->wait_send_queue); 1639*38c8a9a5SSteve French INIT_DELAYED_WORK(&info->idle_timer_work, idle_connection_timer); 1640*38c8a9a5SSteve French queue_delayed_work(info->workqueue, &info->idle_timer_work, 1641*38c8a9a5SSteve French info->keep_alive_interval*HZ); 1642*38c8a9a5SSteve French 1643*38c8a9a5SSteve French init_waitqueue_head(&info->wait_send_pending); 1644*38c8a9a5SSteve French atomic_set(&info->send_pending, 0); 1645*38c8a9a5SSteve French 1646*38c8a9a5SSteve French init_waitqueue_head(&info->wait_post_send); 1647*38c8a9a5SSteve French 1648*38c8a9a5SSteve French INIT_WORK(&info->disconnect_work, smbd_disconnect_rdma_work); 1649*38c8a9a5SSteve French INIT_WORK(&info->post_send_credits_work, smbd_post_send_credits); 1650*38c8a9a5SSteve French info->new_credits_offered = 0; 1651*38c8a9a5SSteve French spin_lock_init(&info->lock_new_credits_offered); 1652*38c8a9a5SSteve French 1653*38c8a9a5SSteve French rc = smbd_negotiate(info); 1654*38c8a9a5SSteve French if (rc) { 1655*38c8a9a5SSteve French log_rdma_event(ERR, "smbd_negotiate rc=%d\n", rc); 1656*38c8a9a5SSteve French goto negotiation_failed; 1657*38c8a9a5SSteve French } 1658*38c8a9a5SSteve French 1659*38c8a9a5SSteve French rc = allocate_mr_list(info); 1660*38c8a9a5SSteve French if (rc) { 1661*38c8a9a5SSteve French log_rdma_mr(ERR, "memory registration allocation failed\n"); 1662*38c8a9a5SSteve French goto allocate_mr_failed; 1663*38c8a9a5SSteve French } 1664*38c8a9a5SSteve French 1665*38c8a9a5SSteve French return info; 1666*38c8a9a5SSteve French 1667*38c8a9a5SSteve French allocate_mr_failed: 1668*38c8a9a5SSteve French /* At this point, need to a full transport shutdown */ 1669*38c8a9a5SSteve French server->smbd_conn = info; 1670*38c8a9a5SSteve French smbd_destroy(server); 1671*38c8a9a5SSteve French return NULL; 1672*38c8a9a5SSteve French 1673*38c8a9a5SSteve French negotiation_failed: 1674*38c8a9a5SSteve French cancel_delayed_work_sync(&info->idle_timer_work); 1675*38c8a9a5SSteve French destroy_caches_and_workqueue(info); 1676*38c8a9a5SSteve French info->transport_status = SMBD_NEGOTIATE_FAILED; 1677*38c8a9a5SSteve French init_waitqueue_head(&info->conn_wait); 1678*38c8a9a5SSteve French rdma_disconnect(info->id); 1679*38c8a9a5SSteve French wait_event(info->conn_wait, 1680*38c8a9a5SSteve French info->transport_status == SMBD_DISCONNECTED); 1681*38c8a9a5SSteve French 1682*38c8a9a5SSteve French allocate_cache_failed: 1683*38c8a9a5SSteve French rdma_connect_failed: 1684*38c8a9a5SSteve French rdma_destroy_qp(info->id); 1685*38c8a9a5SSteve French 1686*38c8a9a5SSteve French create_qp_failed: 1687*38c8a9a5SSteve French alloc_cq_failed: 1688*38c8a9a5SSteve French if (info->send_cq) 1689*38c8a9a5SSteve French ib_free_cq(info->send_cq); 1690*38c8a9a5SSteve French if (info->recv_cq) 1691*38c8a9a5SSteve French ib_free_cq(info->recv_cq); 1692*38c8a9a5SSteve French 1693*38c8a9a5SSteve French config_failed: 1694*38c8a9a5SSteve French ib_dealloc_pd(info->pd); 1695*38c8a9a5SSteve French rdma_destroy_id(info->id); 1696*38c8a9a5SSteve French 1697*38c8a9a5SSteve French create_id_failed: 1698*38c8a9a5SSteve French kfree(info); 1699*38c8a9a5SSteve French return NULL; 1700*38c8a9a5SSteve French } 1701*38c8a9a5SSteve French 1702*38c8a9a5SSteve French struct smbd_connection *smbd_get_connection( 1703*38c8a9a5SSteve French struct TCP_Server_Info *server, struct sockaddr *dstaddr) 1704*38c8a9a5SSteve French { 1705*38c8a9a5SSteve French struct smbd_connection *ret; 1706*38c8a9a5SSteve French int port = SMBD_PORT; 1707*38c8a9a5SSteve French 1708*38c8a9a5SSteve French try_again: 1709*38c8a9a5SSteve French ret = _smbd_get_connection(server, dstaddr, port); 1710*38c8a9a5SSteve French 1711*38c8a9a5SSteve French /* Try SMB_PORT if SMBD_PORT doesn't work */ 1712*38c8a9a5SSteve French if (!ret && port == SMBD_PORT) { 1713*38c8a9a5SSteve French port = SMB_PORT; 1714*38c8a9a5SSteve French goto try_again; 1715*38c8a9a5SSteve French } 1716*38c8a9a5SSteve French return ret; 1717*38c8a9a5SSteve French } 1718*38c8a9a5SSteve French 1719*38c8a9a5SSteve French /* 1720*38c8a9a5SSteve French * Receive data from receive reassembly queue 1721*38c8a9a5SSteve French * All the incoming data packets are placed in reassembly queue 1722*38c8a9a5SSteve French * buf: the buffer to read data into 1723*38c8a9a5SSteve French * size: the length of data to read 1724*38c8a9a5SSteve French * return value: actual data read 1725*38c8a9a5SSteve French * Note: this implementation copies the data from reassebmly queue to receive 1726*38c8a9a5SSteve French * buffers used by upper layer. This is not the optimal code path. A better way 1727*38c8a9a5SSteve French * to do it is to not have upper layer allocate its receive buffers but rather 1728*38c8a9a5SSteve French * borrow the buffer from reassembly queue, and return it after data is 1729*38c8a9a5SSteve French * consumed. But this will require more changes to upper layer code, and also 1730*38c8a9a5SSteve French * need to consider packet boundaries while they still being reassembled. 1731*38c8a9a5SSteve French */ 1732*38c8a9a5SSteve French static int smbd_recv_buf(struct smbd_connection *info, char *buf, 1733*38c8a9a5SSteve French unsigned int size) 1734*38c8a9a5SSteve French { 1735*38c8a9a5SSteve French struct smbd_response *response; 1736*38c8a9a5SSteve French struct smbd_data_transfer *data_transfer; 1737*38c8a9a5SSteve French int to_copy, to_read, data_read, offset; 1738*38c8a9a5SSteve French u32 data_length, remaining_data_length, data_offset; 1739*38c8a9a5SSteve French int rc; 1740*38c8a9a5SSteve French 1741*38c8a9a5SSteve French again: 1742*38c8a9a5SSteve French /* 1743*38c8a9a5SSteve French * No need to hold the reassembly queue lock all the time as we are 1744*38c8a9a5SSteve French * the only one reading from the front of the queue. The transport 1745*38c8a9a5SSteve French * may add more entries to the back of the queue at the same time 1746*38c8a9a5SSteve French */ 1747*38c8a9a5SSteve French log_read(INFO, "size=%d info->reassembly_data_length=%d\n", size, 1748*38c8a9a5SSteve French info->reassembly_data_length); 1749*38c8a9a5SSteve French if (info->reassembly_data_length >= size) { 1750*38c8a9a5SSteve French int queue_length; 1751*38c8a9a5SSteve French int queue_removed = 0; 1752*38c8a9a5SSteve French 1753*38c8a9a5SSteve French /* 1754*38c8a9a5SSteve French * Need to make sure reassembly_data_length is read before 1755*38c8a9a5SSteve French * reading reassembly_queue_length and calling 1756*38c8a9a5SSteve French * _get_first_reassembly. This call is lock free 1757*38c8a9a5SSteve French * as we never read at the end of the queue which are being 1758*38c8a9a5SSteve French * updated in SOFTIRQ as more data is received 1759*38c8a9a5SSteve French */ 1760*38c8a9a5SSteve French virt_rmb(); 1761*38c8a9a5SSteve French queue_length = info->reassembly_queue_length; 1762*38c8a9a5SSteve French data_read = 0; 1763*38c8a9a5SSteve French to_read = size; 1764*38c8a9a5SSteve French offset = info->first_entry_offset; 1765*38c8a9a5SSteve French while (data_read < size) { 1766*38c8a9a5SSteve French response = _get_first_reassembly(info); 1767*38c8a9a5SSteve French data_transfer = smbd_response_payload(response); 1768*38c8a9a5SSteve French data_length = le32_to_cpu(data_transfer->data_length); 1769*38c8a9a5SSteve French remaining_data_length = 1770*38c8a9a5SSteve French le32_to_cpu( 1771*38c8a9a5SSteve French data_transfer->remaining_data_length); 1772*38c8a9a5SSteve French data_offset = le32_to_cpu(data_transfer->data_offset); 1773*38c8a9a5SSteve French 1774*38c8a9a5SSteve French /* 1775*38c8a9a5SSteve French * The upper layer expects RFC1002 length at the 1776*38c8a9a5SSteve French * beginning of the payload. Return it to indicate 1777*38c8a9a5SSteve French * the total length of the packet. This minimize the 1778*38c8a9a5SSteve French * change to upper layer packet processing logic. This 1779*38c8a9a5SSteve French * will be eventually remove when an intermediate 1780*38c8a9a5SSteve French * transport layer is added 1781*38c8a9a5SSteve French */ 1782*38c8a9a5SSteve French if (response->first_segment && size == 4) { 1783*38c8a9a5SSteve French unsigned int rfc1002_len = 1784*38c8a9a5SSteve French data_length + remaining_data_length; 1785*38c8a9a5SSteve French *((__be32 *)buf) = cpu_to_be32(rfc1002_len); 1786*38c8a9a5SSteve French data_read = 4; 1787*38c8a9a5SSteve French response->first_segment = false; 1788*38c8a9a5SSteve French log_read(INFO, "returning rfc1002 length %d\n", 1789*38c8a9a5SSteve French rfc1002_len); 1790*38c8a9a5SSteve French goto read_rfc1002_done; 1791*38c8a9a5SSteve French } 1792*38c8a9a5SSteve French 1793*38c8a9a5SSteve French to_copy = min_t(int, data_length - offset, to_read); 1794*38c8a9a5SSteve French memcpy( 1795*38c8a9a5SSteve French buf + data_read, 1796*38c8a9a5SSteve French (char *)data_transfer + data_offset + offset, 1797*38c8a9a5SSteve French to_copy); 1798*38c8a9a5SSteve French 1799*38c8a9a5SSteve French /* move on to the next buffer? */ 1800*38c8a9a5SSteve French if (to_copy == data_length - offset) { 1801*38c8a9a5SSteve French queue_length--; 1802*38c8a9a5SSteve French /* 1803*38c8a9a5SSteve French * No need to lock if we are not at the 1804*38c8a9a5SSteve French * end of the queue 1805*38c8a9a5SSteve French */ 1806*38c8a9a5SSteve French if (queue_length) 1807*38c8a9a5SSteve French list_del(&response->list); 1808*38c8a9a5SSteve French else { 1809*38c8a9a5SSteve French spin_lock_irq( 1810*38c8a9a5SSteve French &info->reassembly_queue_lock); 1811*38c8a9a5SSteve French list_del(&response->list); 1812*38c8a9a5SSteve French spin_unlock_irq( 1813*38c8a9a5SSteve French &info->reassembly_queue_lock); 1814*38c8a9a5SSteve French } 1815*38c8a9a5SSteve French queue_removed++; 1816*38c8a9a5SSteve French info->count_reassembly_queue--; 1817*38c8a9a5SSteve French info->count_dequeue_reassembly_queue++; 1818*38c8a9a5SSteve French put_receive_buffer(info, response); 1819*38c8a9a5SSteve French offset = 0; 1820*38c8a9a5SSteve French log_read(INFO, "put_receive_buffer offset=0\n"); 1821*38c8a9a5SSteve French } else 1822*38c8a9a5SSteve French offset += to_copy; 1823*38c8a9a5SSteve French 1824*38c8a9a5SSteve French to_read -= to_copy; 1825*38c8a9a5SSteve French data_read += to_copy; 1826*38c8a9a5SSteve French 1827*38c8a9a5SSteve French log_read(INFO, "_get_first_reassembly memcpy %d bytes data_transfer_length-offset=%d after that to_read=%d data_read=%d offset=%d\n", 1828*38c8a9a5SSteve French to_copy, data_length - offset, 1829*38c8a9a5SSteve French to_read, data_read, offset); 1830*38c8a9a5SSteve French } 1831*38c8a9a5SSteve French 1832*38c8a9a5SSteve French spin_lock_irq(&info->reassembly_queue_lock); 1833*38c8a9a5SSteve French info->reassembly_data_length -= data_read; 1834*38c8a9a5SSteve French info->reassembly_queue_length -= queue_removed; 1835*38c8a9a5SSteve French spin_unlock_irq(&info->reassembly_queue_lock); 1836*38c8a9a5SSteve French 1837*38c8a9a5SSteve French info->first_entry_offset = offset; 1838*38c8a9a5SSteve French log_read(INFO, "returning to thread data_read=%d reassembly_data_length=%d first_entry_offset=%d\n", 1839*38c8a9a5SSteve French data_read, info->reassembly_data_length, 1840*38c8a9a5SSteve French info->first_entry_offset); 1841*38c8a9a5SSteve French read_rfc1002_done: 1842*38c8a9a5SSteve French return data_read; 1843*38c8a9a5SSteve French } 1844*38c8a9a5SSteve French 1845*38c8a9a5SSteve French log_read(INFO, "wait_event on more data\n"); 1846*38c8a9a5SSteve French rc = wait_event_interruptible( 1847*38c8a9a5SSteve French info->wait_reassembly_queue, 1848*38c8a9a5SSteve French info->reassembly_data_length >= size || 1849*38c8a9a5SSteve French info->transport_status != SMBD_CONNECTED); 1850*38c8a9a5SSteve French /* Don't return any data if interrupted */ 1851*38c8a9a5SSteve French if (rc) 1852*38c8a9a5SSteve French return rc; 1853*38c8a9a5SSteve French 1854*38c8a9a5SSteve French if (info->transport_status != SMBD_CONNECTED) { 1855*38c8a9a5SSteve French log_read(ERR, "disconnected\n"); 1856*38c8a9a5SSteve French return -ECONNABORTED; 1857*38c8a9a5SSteve French } 1858*38c8a9a5SSteve French 1859*38c8a9a5SSteve French goto again; 1860*38c8a9a5SSteve French } 1861*38c8a9a5SSteve French 1862*38c8a9a5SSteve French /* 1863*38c8a9a5SSteve French * Receive a page from receive reassembly queue 1864*38c8a9a5SSteve French * page: the page to read data into 1865*38c8a9a5SSteve French * to_read: the length of data to read 1866*38c8a9a5SSteve French * return value: actual data read 1867*38c8a9a5SSteve French */ 1868*38c8a9a5SSteve French static int smbd_recv_page(struct smbd_connection *info, 1869*38c8a9a5SSteve French struct page *page, unsigned int page_offset, 1870*38c8a9a5SSteve French unsigned int to_read) 1871*38c8a9a5SSteve French { 1872*38c8a9a5SSteve French int ret; 1873*38c8a9a5SSteve French char *to_address; 1874*38c8a9a5SSteve French void *page_address; 1875*38c8a9a5SSteve French 1876*38c8a9a5SSteve French /* make sure we have the page ready for read */ 1877*38c8a9a5SSteve French ret = wait_event_interruptible( 1878*38c8a9a5SSteve French info->wait_reassembly_queue, 1879*38c8a9a5SSteve French info->reassembly_data_length >= to_read || 1880*38c8a9a5SSteve French info->transport_status != SMBD_CONNECTED); 1881*38c8a9a5SSteve French if (ret) 1882*38c8a9a5SSteve French return ret; 1883*38c8a9a5SSteve French 1884*38c8a9a5SSteve French /* now we can read from reassembly queue and not sleep */ 1885*38c8a9a5SSteve French page_address = kmap_atomic(page); 1886*38c8a9a5SSteve French to_address = (char *) page_address + page_offset; 1887*38c8a9a5SSteve French 1888*38c8a9a5SSteve French log_read(INFO, "reading from page=%p address=%p to_read=%d\n", 1889*38c8a9a5SSteve French page, to_address, to_read); 1890*38c8a9a5SSteve French 1891*38c8a9a5SSteve French ret = smbd_recv_buf(info, to_address, to_read); 1892*38c8a9a5SSteve French kunmap_atomic(page_address); 1893*38c8a9a5SSteve French 1894*38c8a9a5SSteve French return ret; 1895*38c8a9a5SSteve French } 1896*38c8a9a5SSteve French 1897*38c8a9a5SSteve French /* 1898*38c8a9a5SSteve French * Receive data from transport 1899*38c8a9a5SSteve French * msg: a msghdr point to the buffer, can be ITER_KVEC or ITER_BVEC 1900*38c8a9a5SSteve French * return: total bytes read, or 0. SMB Direct will not do partial read. 1901*38c8a9a5SSteve French */ 1902*38c8a9a5SSteve French int smbd_recv(struct smbd_connection *info, struct msghdr *msg) 1903*38c8a9a5SSteve French { 1904*38c8a9a5SSteve French char *buf; 1905*38c8a9a5SSteve French struct page *page; 1906*38c8a9a5SSteve French unsigned int to_read, page_offset; 1907*38c8a9a5SSteve French int rc; 1908*38c8a9a5SSteve French 1909*38c8a9a5SSteve French if (iov_iter_rw(&msg->msg_iter) == WRITE) { 1910*38c8a9a5SSteve French /* It's a bug in upper layer to get there */ 1911*38c8a9a5SSteve French cifs_dbg(VFS, "Invalid msg iter dir %u\n", 1912*38c8a9a5SSteve French iov_iter_rw(&msg->msg_iter)); 1913*38c8a9a5SSteve French rc = -EINVAL; 1914*38c8a9a5SSteve French goto out; 1915*38c8a9a5SSteve French } 1916*38c8a9a5SSteve French 1917*38c8a9a5SSteve French switch (iov_iter_type(&msg->msg_iter)) { 1918*38c8a9a5SSteve French case ITER_KVEC: 1919*38c8a9a5SSteve French buf = msg->msg_iter.kvec->iov_base; 1920*38c8a9a5SSteve French to_read = msg->msg_iter.kvec->iov_len; 1921*38c8a9a5SSteve French rc = smbd_recv_buf(info, buf, to_read); 1922*38c8a9a5SSteve French break; 1923*38c8a9a5SSteve French 1924*38c8a9a5SSteve French case ITER_BVEC: 1925*38c8a9a5SSteve French page = msg->msg_iter.bvec->bv_page; 1926*38c8a9a5SSteve French page_offset = msg->msg_iter.bvec->bv_offset; 1927*38c8a9a5SSteve French to_read = msg->msg_iter.bvec->bv_len; 1928*38c8a9a5SSteve French rc = smbd_recv_page(info, page, page_offset, to_read); 1929*38c8a9a5SSteve French break; 1930*38c8a9a5SSteve French 1931*38c8a9a5SSteve French default: 1932*38c8a9a5SSteve French /* It's a bug in upper layer to get there */ 1933*38c8a9a5SSteve French cifs_dbg(VFS, "Invalid msg type %d\n", 1934*38c8a9a5SSteve French iov_iter_type(&msg->msg_iter)); 1935*38c8a9a5SSteve French rc = -EINVAL; 1936*38c8a9a5SSteve French } 1937*38c8a9a5SSteve French 1938*38c8a9a5SSteve French out: 1939*38c8a9a5SSteve French /* SMBDirect will read it all or nothing */ 1940*38c8a9a5SSteve French if (rc > 0) 1941*38c8a9a5SSteve French msg->msg_iter.count = 0; 1942*38c8a9a5SSteve French return rc; 1943*38c8a9a5SSteve French } 1944*38c8a9a5SSteve French 1945*38c8a9a5SSteve French /* 1946*38c8a9a5SSteve French * Send data to transport 1947*38c8a9a5SSteve French * Each rqst is transported as a SMBDirect payload 1948*38c8a9a5SSteve French * rqst: the data to write 1949*38c8a9a5SSteve French * return value: 0 if successfully write, otherwise error code 1950*38c8a9a5SSteve French */ 1951*38c8a9a5SSteve French int smbd_send(struct TCP_Server_Info *server, 1952*38c8a9a5SSteve French int num_rqst, struct smb_rqst *rqst_array) 1953*38c8a9a5SSteve French { 1954*38c8a9a5SSteve French struct smbd_connection *info = server->smbd_conn; 1955*38c8a9a5SSteve French struct smb_rqst *rqst; 1956*38c8a9a5SSteve French struct iov_iter iter; 1957*38c8a9a5SSteve French unsigned int remaining_data_length, klen; 1958*38c8a9a5SSteve French int rc, i, rqst_idx; 1959*38c8a9a5SSteve French 1960*38c8a9a5SSteve French if (info->transport_status != SMBD_CONNECTED) 1961*38c8a9a5SSteve French return -EAGAIN; 1962*38c8a9a5SSteve French 1963*38c8a9a5SSteve French /* 1964*38c8a9a5SSteve French * Add in the page array if there is one. The caller needs to set 1965*38c8a9a5SSteve French * rq_tailsz to PAGE_SIZE when the buffer has multiple pages and 1966*38c8a9a5SSteve French * ends at page boundary 1967*38c8a9a5SSteve French */ 1968*38c8a9a5SSteve French remaining_data_length = 0; 1969*38c8a9a5SSteve French for (i = 0; i < num_rqst; i++) 1970*38c8a9a5SSteve French remaining_data_length += smb_rqst_len(server, &rqst_array[i]); 1971*38c8a9a5SSteve French 1972*38c8a9a5SSteve French if (unlikely(remaining_data_length > info->max_fragmented_send_size)) { 1973*38c8a9a5SSteve French /* assertion: payload never exceeds negotiated maximum */ 1974*38c8a9a5SSteve French log_write(ERR, "payload size %d > max size %d\n", 1975*38c8a9a5SSteve French remaining_data_length, info->max_fragmented_send_size); 1976*38c8a9a5SSteve French return -EINVAL; 1977*38c8a9a5SSteve French } 1978*38c8a9a5SSteve French 1979*38c8a9a5SSteve French log_write(INFO, "num_rqst=%d total length=%u\n", 1980*38c8a9a5SSteve French num_rqst, remaining_data_length); 1981*38c8a9a5SSteve French 1982*38c8a9a5SSteve French rqst_idx = 0; 1983*38c8a9a5SSteve French do { 1984*38c8a9a5SSteve French rqst = &rqst_array[rqst_idx]; 1985*38c8a9a5SSteve French 1986*38c8a9a5SSteve French cifs_dbg(FYI, "Sending smb (RDMA): idx=%d smb_len=%lu\n", 1987*38c8a9a5SSteve French rqst_idx, smb_rqst_len(server, rqst)); 1988*38c8a9a5SSteve French for (i = 0; i < rqst->rq_nvec; i++) 1989*38c8a9a5SSteve French dump_smb(rqst->rq_iov[i].iov_base, rqst->rq_iov[i].iov_len); 1990*38c8a9a5SSteve French 1991*38c8a9a5SSteve French log_write(INFO, "RDMA-WR[%u] nvec=%d len=%u iter=%zu rqlen=%lu\n", 1992*38c8a9a5SSteve French rqst_idx, rqst->rq_nvec, remaining_data_length, 1993*38c8a9a5SSteve French iov_iter_count(&rqst->rq_iter), smb_rqst_len(server, rqst)); 1994*38c8a9a5SSteve French 1995*38c8a9a5SSteve French /* Send the metadata pages. */ 1996*38c8a9a5SSteve French klen = 0; 1997*38c8a9a5SSteve French for (i = 0; i < rqst->rq_nvec; i++) 1998*38c8a9a5SSteve French klen += rqst->rq_iov[i].iov_len; 1999*38c8a9a5SSteve French iov_iter_kvec(&iter, ITER_SOURCE, rqst->rq_iov, rqst->rq_nvec, klen); 2000*38c8a9a5SSteve French 2001*38c8a9a5SSteve French rc = smbd_post_send_iter(info, &iter, &remaining_data_length); 2002*38c8a9a5SSteve French if (rc < 0) 2003*38c8a9a5SSteve French break; 2004*38c8a9a5SSteve French 2005*38c8a9a5SSteve French if (iov_iter_count(&rqst->rq_iter) > 0) { 2006*38c8a9a5SSteve French /* And then the data pages if there are any */ 2007*38c8a9a5SSteve French rc = smbd_post_send_iter(info, &rqst->rq_iter, 2008*38c8a9a5SSteve French &remaining_data_length); 2009*38c8a9a5SSteve French if (rc < 0) 2010*38c8a9a5SSteve French break; 2011*38c8a9a5SSteve French } 2012*38c8a9a5SSteve French 2013*38c8a9a5SSteve French } while (++rqst_idx < num_rqst); 2014*38c8a9a5SSteve French 2015*38c8a9a5SSteve French /* 2016*38c8a9a5SSteve French * As an optimization, we don't wait for individual I/O to finish 2017*38c8a9a5SSteve French * before sending the next one. 2018*38c8a9a5SSteve French * Send them all and wait for pending send count to get to 0 2019*38c8a9a5SSteve French * that means all the I/Os have been out and we are good to return 2020*38c8a9a5SSteve French */ 2021*38c8a9a5SSteve French 2022*38c8a9a5SSteve French wait_event(info->wait_send_pending, 2023*38c8a9a5SSteve French atomic_read(&info->send_pending) == 0); 2024*38c8a9a5SSteve French 2025*38c8a9a5SSteve French return rc; 2026*38c8a9a5SSteve French } 2027*38c8a9a5SSteve French 2028*38c8a9a5SSteve French static void register_mr_done(struct ib_cq *cq, struct ib_wc *wc) 2029*38c8a9a5SSteve French { 2030*38c8a9a5SSteve French struct smbd_mr *mr; 2031*38c8a9a5SSteve French struct ib_cqe *cqe; 2032*38c8a9a5SSteve French 2033*38c8a9a5SSteve French if (wc->status) { 2034*38c8a9a5SSteve French log_rdma_mr(ERR, "status=%d\n", wc->status); 2035*38c8a9a5SSteve French cqe = wc->wr_cqe; 2036*38c8a9a5SSteve French mr = container_of(cqe, struct smbd_mr, cqe); 2037*38c8a9a5SSteve French smbd_disconnect_rdma_connection(mr->conn); 2038*38c8a9a5SSteve French } 2039*38c8a9a5SSteve French } 2040*38c8a9a5SSteve French 2041*38c8a9a5SSteve French /* 2042*38c8a9a5SSteve French * The work queue function that recovers MRs 2043*38c8a9a5SSteve French * We need to call ib_dereg_mr() and ib_alloc_mr() before this MR can be used 2044*38c8a9a5SSteve French * again. Both calls are slow, so finish them in a workqueue. This will not 2045*38c8a9a5SSteve French * block I/O path. 2046*38c8a9a5SSteve French * There is one workqueue that recovers MRs, there is no need to lock as the 2047*38c8a9a5SSteve French * I/O requests calling smbd_register_mr will never update the links in the 2048*38c8a9a5SSteve French * mr_list. 2049*38c8a9a5SSteve French */ 2050*38c8a9a5SSteve French static void smbd_mr_recovery_work(struct work_struct *work) 2051*38c8a9a5SSteve French { 2052*38c8a9a5SSteve French struct smbd_connection *info = 2053*38c8a9a5SSteve French container_of(work, struct smbd_connection, mr_recovery_work); 2054*38c8a9a5SSteve French struct smbd_mr *smbdirect_mr; 2055*38c8a9a5SSteve French int rc; 2056*38c8a9a5SSteve French 2057*38c8a9a5SSteve French list_for_each_entry(smbdirect_mr, &info->mr_list, list) { 2058*38c8a9a5SSteve French if (smbdirect_mr->state == MR_ERROR) { 2059*38c8a9a5SSteve French 2060*38c8a9a5SSteve French /* recover this MR entry */ 2061*38c8a9a5SSteve French rc = ib_dereg_mr(smbdirect_mr->mr); 2062*38c8a9a5SSteve French if (rc) { 2063*38c8a9a5SSteve French log_rdma_mr(ERR, 2064*38c8a9a5SSteve French "ib_dereg_mr failed rc=%x\n", 2065*38c8a9a5SSteve French rc); 2066*38c8a9a5SSteve French smbd_disconnect_rdma_connection(info); 2067*38c8a9a5SSteve French continue; 2068*38c8a9a5SSteve French } 2069*38c8a9a5SSteve French 2070*38c8a9a5SSteve French smbdirect_mr->mr = ib_alloc_mr( 2071*38c8a9a5SSteve French info->pd, info->mr_type, 2072*38c8a9a5SSteve French info->max_frmr_depth); 2073*38c8a9a5SSteve French if (IS_ERR(smbdirect_mr->mr)) { 2074*38c8a9a5SSteve French log_rdma_mr(ERR, "ib_alloc_mr failed mr_type=%x max_frmr_depth=%x\n", 2075*38c8a9a5SSteve French info->mr_type, 2076*38c8a9a5SSteve French info->max_frmr_depth); 2077*38c8a9a5SSteve French smbd_disconnect_rdma_connection(info); 2078*38c8a9a5SSteve French continue; 2079*38c8a9a5SSteve French } 2080*38c8a9a5SSteve French } else 2081*38c8a9a5SSteve French /* This MR is being used, don't recover it */ 2082*38c8a9a5SSteve French continue; 2083*38c8a9a5SSteve French 2084*38c8a9a5SSteve French smbdirect_mr->state = MR_READY; 2085*38c8a9a5SSteve French 2086*38c8a9a5SSteve French /* smbdirect_mr->state is updated by this function 2087*38c8a9a5SSteve French * and is read and updated by I/O issuing CPUs trying 2088*38c8a9a5SSteve French * to get a MR, the call to atomic_inc_return 2089*38c8a9a5SSteve French * implicates a memory barrier and guarantees this 2090*38c8a9a5SSteve French * value is updated before waking up any calls to 2091*38c8a9a5SSteve French * get_mr() from the I/O issuing CPUs 2092*38c8a9a5SSteve French */ 2093*38c8a9a5SSteve French if (atomic_inc_return(&info->mr_ready_count) == 1) 2094*38c8a9a5SSteve French wake_up_interruptible(&info->wait_mr); 2095*38c8a9a5SSteve French } 2096*38c8a9a5SSteve French } 2097*38c8a9a5SSteve French 2098*38c8a9a5SSteve French static void destroy_mr_list(struct smbd_connection *info) 2099*38c8a9a5SSteve French { 2100*38c8a9a5SSteve French struct smbd_mr *mr, *tmp; 2101*38c8a9a5SSteve French 2102*38c8a9a5SSteve French cancel_work_sync(&info->mr_recovery_work); 2103*38c8a9a5SSteve French list_for_each_entry_safe(mr, tmp, &info->mr_list, list) { 2104*38c8a9a5SSteve French if (mr->state == MR_INVALIDATED) 2105*38c8a9a5SSteve French ib_dma_unmap_sg(info->id->device, mr->sgt.sgl, 2106*38c8a9a5SSteve French mr->sgt.nents, mr->dir); 2107*38c8a9a5SSteve French ib_dereg_mr(mr->mr); 2108*38c8a9a5SSteve French kfree(mr->sgt.sgl); 2109*38c8a9a5SSteve French kfree(mr); 2110*38c8a9a5SSteve French } 2111*38c8a9a5SSteve French } 2112*38c8a9a5SSteve French 2113*38c8a9a5SSteve French /* 2114*38c8a9a5SSteve French * Allocate MRs used for RDMA read/write 2115*38c8a9a5SSteve French * The number of MRs will not exceed hardware capability in responder_resources 2116*38c8a9a5SSteve French * All MRs are kept in mr_list. The MR can be recovered after it's used 2117*38c8a9a5SSteve French * Recovery is done in smbd_mr_recovery_work. The content of list entry changes 2118*38c8a9a5SSteve French * as MRs are used and recovered for I/O, but the list links will not change 2119*38c8a9a5SSteve French */ 2120*38c8a9a5SSteve French static int allocate_mr_list(struct smbd_connection *info) 2121*38c8a9a5SSteve French { 2122*38c8a9a5SSteve French int i; 2123*38c8a9a5SSteve French struct smbd_mr *smbdirect_mr, *tmp; 2124*38c8a9a5SSteve French 2125*38c8a9a5SSteve French INIT_LIST_HEAD(&info->mr_list); 2126*38c8a9a5SSteve French init_waitqueue_head(&info->wait_mr); 2127*38c8a9a5SSteve French spin_lock_init(&info->mr_list_lock); 2128*38c8a9a5SSteve French atomic_set(&info->mr_ready_count, 0); 2129*38c8a9a5SSteve French atomic_set(&info->mr_used_count, 0); 2130*38c8a9a5SSteve French init_waitqueue_head(&info->wait_for_mr_cleanup); 2131*38c8a9a5SSteve French INIT_WORK(&info->mr_recovery_work, smbd_mr_recovery_work); 2132*38c8a9a5SSteve French /* Allocate more MRs (2x) than hardware responder_resources */ 2133*38c8a9a5SSteve French for (i = 0; i < info->responder_resources * 2; i++) { 2134*38c8a9a5SSteve French smbdirect_mr = kzalloc(sizeof(*smbdirect_mr), GFP_KERNEL); 2135*38c8a9a5SSteve French if (!smbdirect_mr) 2136*38c8a9a5SSteve French goto out; 2137*38c8a9a5SSteve French smbdirect_mr->mr = ib_alloc_mr(info->pd, info->mr_type, 2138*38c8a9a5SSteve French info->max_frmr_depth); 2139*38c8a9a5SSteve French if (IS_ERR(smbdirect_mr->mr)) { 2140*38c8a9a5SSteve French log_rdma_mr(ERR, "ib_alloc_mr failed mr_type=%x max_frmr_depth=%x\n", 2141*38c8a9a5SSteve French info->mr_type, info->max_frmr_depth); 2142*38c8a9a5SSteve French goto out; 2143*38c8a9a5SSteve French } 2144*38c8a9a5SSteve French smbdirect_mr->sgt.sgl = kcalloc(info->max_frmr_depth, 2145*38c8a9a5SSteve French sizeof(struct scatterlist), 2146*38c8a9a5SSteve French GFP_KERNEL); 2147*38c8a9a5SSteve French if (!smbdirect_mr->sgt.sgl) { 2148*38c8a9a5SSteve French log_rdma_mr(ERR, "failed to allocate sgl\n"); 2149*38c8a9a5SSteve French ib_dereg_mr(smbdirect_mr->mr); 2150*38c8a9a5SSteve French goto out; 2151*38c8a9a5SSteve French } 2152*38c8a9a5SSteve French smbdirect_mr->state = MR_READY; 2153*38c8a9a5SSteve French smbdirect_mr->conn = info; 2154*38c8a9a5SSteve French 2155*38c8a9a5SSteve French list_add_tail(&smbdirect_mr->list, &info->mr_list); 2156*38c8a9a5SSteve French atomic_inc(&info->mr_ready_count); 2157*38c8a9a5SSteve French } 2158*38c8a9a5SSteve French return 0; 2159*38c8a9a5SSteve French 2160*38c8a9a5SSteve French out: 2161*38c8a9a5SSteve French kfree(smbdirect_mr); 2162*38c8a9a5SSteve French 2163*38c8a9a5SSteve French list_for_each_entry_safe(smbdirect_mr, tmp, &info->mr_list, list) { 2164*38c8a9a5SSteve French list_del(&smbdirect_mr->list); 2165*38c8a9a5SSteve French ib_dereg_mr(smbdirect_mr->mr); 2166*38c8a9a5SSteve French kfree(smbdirect_mr->sgt.sgl); 2167*38c8a9a5SSteve French kfree(smbdirect_mr); 2168*38c8a9a5SSteve French } 2169*38c8a9a5SSteve French return -ENOMEM; 2170*38c8a9a5SSteve French } 2171*38c8a9a5SSteve French 2172*38c8a9a5SSteve French /* 2173*38c8a9a5SSteve French * Get a MR from mr_list. This function waits until there is at least one 2174*38c8a9a5SSteve French * MR available in the list. It may access the list while the 2175*38c8a9a5SSteve French * smbd_mr_recovery_work is recovering the MR list. This doesn't need a lock 2176*38c8a9a5SSteve French * as they never modify the same places. However, there may be several CPUs 2177*38c8a9a5SSteve French * issueing I/O trying to get MR at the same time, mr_list_lock is used to 2178*38c8a9a5SSteve French * protect this situation. 2179*38c8a9a5SSteve French */ 2180*38c8a9a5SSteve French static struct smbd_mr *get_mr(struct smbd_connection *info) 2181*38c8a9a5SSteve French { 2182*38c8a9a5SSteve French struct smbd_mr *ret; 2183*38c8a9a5SSteve French int rc; 2184*38c8a9a5SSteve French again: 2185*38c8a9a5SSteve French rc = wait_event_interruptible(info->wait_mr, 2186*38c8a9a5SSteve French atomic_read(&info->mr_ready_count) || 2187*38c8a9a5SSteve French info->transport_status != SMBD_CONNECTED); 2188*38c8a9a5SSteve French if (rc) { 2189*38c8a9a5SSteve French log_rdma_mr(ERR, "wait_event_interruptible rc=%x\n", rc); 2190*38c8a9a5SSteve French return NULL; 2191*38c8a9a5SSteve French } 2192*38c8a9a5SSteve French 2193*38c8a9a5SSteve French if (info->transport_status != SMBD_CONNECTED) { 2194*38c8a9a5SSteve French log_rdma_mr(ERR, "info->transport_status=%x\n", 2195*38c8a9a5SSteve French info->transport_status); 2196*38c8a9a5SSteve French return NULL; 2197*38c8a9a5SSteve French } 2198*38c8a9a5SSteve French 2199*38c8a9a5SSteve French spin_lock(&info->mr_list_lock); 2200*38c8a9a5SSteve French list_for_each_entry(ret, &info->mr_list, list) { 2201*38c8a9a5SSteve French if (ret->state == MR_READY) { 2202*38c8a9a5SSteve French ret->state = MR_REGISTERED; 2203*38c8a9a5SSteve French spin_unlock(&info->mr_list_lock); 2204*38c8a9a5SSteve French atomic_dec(&info->mr_ready_count); 2205*38c8a9a5SSteve French atomic_inc(&info->mr_used_count); 2206*38c8a9a5SSteve French return ret; 2207*38c8a9a5SSteve French } 2208*38c8a9a5SSteve French } 2209*38c8a9a5SSteve French 2210*38c8a9a5SSteve French spin_unlock(&info->mr_list_lock); 2211*38c8a9a5SSteve French /* 2212*38c8a9a5SSteve French * It is possible that we could fail to get MR because other processes may 2213*38c8a9a5SSteve French * try to acquire a MR at the same time. If this is the case, retry it. 2214*38c8a9a5SSteve French */ 2215*38c8a9a5SSteve French goto again; 2216*38c8a9a5SSteve French } 2217*38c8a9a5SSteve French 2218*38c8a9a5SSteve French /* 2219*38c8a9a5SSteve French * Transcribe the pages from an iterator into an MR scatterlist. 2220*38c8a9a5SSteve French */ 2221*38c8a9a5SSteve French static int smbd_iter_to_mr(struct smbd_connection *info, 2222*38c8a9a5SSteve French struct iov_iter *iter, 2223*38c8a9a5SSteve French struct sg_table *sgt, 2224*38c8a9a5SSteve French unsigned int max_sg) 2225*38c8a9a5SSteve French { 2226*38c8a9a5SSteve French int ret; 2227*38c8a9a5SSteve French 2228*38c8a9a5SSteve French memset(sgt->sgl, 0, max_sg * sizeof(struct scatterlist)); 2229*38c8a9a5SSteve French 2230*38c8a9a5SSteve French ret = netfs_extract_iter_to_sg(iter, iov_iter_count(iter), sgt, max_sg, 0); 2231*38c8a9a5SSteve French WARN_ON(ret < 0); 2232*38c8a9a5SSteve French if (sgt->nents > 0) 2233*38c8a9a5SSteve French sg_mark_end(&sgt->sgl[sgt->nents - 1]); 2234*38c8a9a5SSteve French return ret; 2235*38c8a9a5SSteve French } 2236*38c8a9a5SSteve French 2237*38c8a9a5SSteve French /* 2238*38c8a9a5SSteve French * Register memory for RDMA read/write 2239*38c8a9a5SSteve French * iter: the buffer to register memory with 2240*38c8a9a5SSteve French * writing: true if this is a RDMA write (SMB read), false for RDMA read 2241*38c8a9a5SSteve French * need_invalidate: true if this MR needs to be locally invalidated after I/O 2242*38c8a9a5SSteve French * return value: the MR registered, NULL if failed. 2243*38c8a9a5SSteve French */ 2244*38c8a9a5SSteve French struct smbd_mr *smbd_register_mr(struct smbd_connection *info, 2245*38c8a9a5SSteve French struct iov_iter *iter, 2246*38c8a9a5SSteve French bool writing, bool need_invalidate) 2247*38c8a9a5SSteve French { 2248*38c8a9a5SSteve French struct smbd_mr *smbdirect_mr; 2249*38c8a9a5SSteve French int rc, num_pages; 2250*38c8a9a5SSteve French enum dma_data_direction dir; 2251*38c8a9a5SSteve French struct ib_reg_wr *reg_wr; 2252*38c8a9a5SSteve French 2253*38c8a9a5SSteve French num_pages = iov_iter_npages(iter, info->max_frmr_depth + 1); 2254*38c8a9a5SSteve French if (num_pages > info->max_frmr_depth) { 2255*38c8a9a5SSteve French log_rdma_mr(ERR, "num_pages=%d max_frmr_depth=%d\n", 2256*38c8a9a5SSteve French num_pages, info->max_frmr_depth); 2257*38c8a9a5SSteve French WARN_ON_ONCE(1); 2258*38c8a9a5SSteve French return NULL; 2259*38c8a9a5SSteve French } 2260*38c8a9a5SSteve French 2261*38c8a9a5SSteve French smbdirect_mr = get_mr(info); 2262*38c8a9a5SSteve French if (!smbdirect_mr) { 2263*38c8a9a5SSteve French log_rdma_mr(ERR, "get_mr returning NULL\n"); 2264*38c8a9a5SSteve French return NULL; 2265*38c8a9a5SSteve French } 2266*38c8a9a5SSteve French 2267*38c8a9a5SSteve French dir = writing ? DMA_FROM_DEVICE : DMA_TO_DEVICE; 2268*38c8a9a5SSteve French smbdirect_mr->dir = dir; 2269*38c8a9a5SSteve French smbdirect_mr->need_invalidate = need_invalidate; 2270*38c8a9a5SSteve French smbdirect_mr->sgt.nents = 0; 2271*38c8a9a5SSteve French smbdirect_mr->sgt.orig_nents = 0; 2272*38c8a9a5SSteve French 2273*38c8a9a5SSteve French log_rdma_mr(INFO, "num_pages=0x%x count=0x%zx depth=%u\n", 2274*38c8a9a5SSteve French num_pages, iov_iter_count(iter), info->max_frmr_depth); 2275*38c8a9a5SSteve French smbd_iter_to_mr(info, iter, &smbdirect_mr->sgt, info->max_frmr_depth); 2276*38c8a9a5SSteve French 2277*38c8a9a5SSteve French rc = ib_dma_map_sg(info->id->device, smbdirect_mr->sgt.sgl, 2278*38c8a9a5SSteve French smbdirect_mr->sgt.nents, dir); 2279*38c8a9a5SSteve French if (!rc) { 2280*38c8a9a5SSteve French log_rdma_mr(ERR, "ib_dma_map_sg num_pages=%x dir=%x rc=%x\n", 2281*38c8a9a5SSteve French num_pages, dir, rc); 2282*38c8a9a5SSteve French goto dma_map_error; 2283*38c8a9a5SSteve French } 2284*38c8a9a5SSteve French 2285*38c8a9a5SSteve French rc = ib_map_mr_sg(smbdirect_mr->mr, smbdirect_mr->sgt.sgl, 2286*38c8a9a5SSteve French smbdirect_mr->sgt.nents, NULL, PAGE_SIZE); 2287*38c8a9a5SSteve French if (rc != smbdirect_mr->sgt.nents) { 2288*38c8a9a5SSteve French log_rdma_mr(ERR, 2289*38c8a9a5SSteve French "ib_map_mr_sg failed rc = %d nents = %x\n", 2290*38c8a9a5SSteve French rc, smbdirect_mr->sgt.nents); 2291*38c8a9a5SSteve French goto map_mr_error; 2292*38c8a9a5SSteve French } 2293*38c8a9a5SSteve French 2294*38c8a9a5SSteve French ib_update_fast_reg_key(smbdirect_mr->mr, 2295*38c8a9a5SSteve French ib_inc_rkey(smbdirect_mr->mr->rkey)); 2296*38c8a9a5SSteve French reg_wr = &smbdirect_mr->wr; 2297*38c8a9a5SSteve French reg_wr->wr.opcode = IB_WR_REG_MR; 2298*38c8a9a5SSteve French smbdirect_mr->cqe.done = register_mr_done; 2299*38c8a9a5SSteve French reg_wr->wr.wr_cqe = &smbdirect_mr->cqe; 2300*38c8a9a5SSteve French reg_wr->wr.num_sge = 0; 2301*38c8a9a5SSteve French reg_wr->wr.send_flags = IB_SEND_SIGNALED; 2302*38c8a9a5SSteve French reg_wr->mr = smbdirect_mr->mr; 2303*38c8a9a5SSteve French reg_wr->key = smbdirect_mr->mr->rkey; 2304*38c8a9a5SSteve French reg_wr->access = writing ? 2305*38c8a9a5SSteve French IB_ACCESS_REMOTE_WRITE | IB_ACCESS_LOCAL_WRITE : 2306*38c8a9a5SSteve French IB_ACCESS_REMOTE_READ; 2307*38c8a9a5SSteve French 2308*38c8a9a5SSteve French /* 2309*38c8a9a5SSteve French * There is no need for waiting for complemtion on ib_post_send 2310*38c8a9a5SSteve French * on IB_WR_REG_MR. Hardware enforces a barrier and order of execution 2311*38c8a9a5SSteve French * on the next ib_post_send when we actaully send I/O to remote peer 2312*38c8a9a5SSteve French */ 2313*38c8a9a5SSteve French rc = ib_post_send(info->id->qp, ®_wr->wr, NULL); 2314*38c8a9a5SSteve French if (!rc) 2315*38c8a9a5SSteve French return smbdirect_mr; 2316*38c8a9a5SSteve French 2317*38c8a9a5SSteve French log_rdma_mr(ERR, "ib_post_send failed rc=%x reg_wr->key=%x\n", 2318*38c8a9a5SSteve French rc, reg_wr->key); 2319*38c8a9a5SSteve French 2320*38c8a9a5SSteve French /* If all failed, attempt to recover this MR by setting it MR_ERROR*/ 2321*38c8a9a5SSteve French map_mr_error: 2322*38c8a9a5SSteve French ib_dma_unmap_sg(info->id->device, smbdirect_mr->sgt.sgl, 2323*38c8a9a5SSteve French smbdirect_mr->sgt.nents, smbdirect_mr->dir); 2324*38c8a9a5SSteve French 2325*38c8a9a5SSteve French dma_map_error: 2326*38c8a9a5SSteve French smbdirect_mr->state = MR_ERROR; 2327*38c8a9a5SSteve French if (atomic_dec_and_test(&info->mr_used_count)) 2328*38c8a9a5SSteve French wake_up(&info->wait_for_mr_cleanup); 2329*38c8a9a5SSteve French 2330*38c8a9a5SSteve French smbd_disconnect_rdma_connection(info); 2331*38c8a9a5SSteve French 2332*38c8a9a5SSteve French return NULL; 2333*38c8a9a5SSteve French } 2334*38c8a9a5SSteve French 2335*38c8a9a5SSteve French static void local_inv_done(struct ib_cq *cq, struct ib_wc *wc) 2336*38c8a9a5SSteve French { 2337*38c8a9a5SSteve French struct smbd_mr *smbdirect_mr; 2338*38c8a9a5SSteve French struct ib_cqe *cqe; 2339*38c8a9a5SSteve French 2340*38c8a9a5SSteve French cqe = wc->wr_cqe; 2341*38c8a9a5SSteve French smbdirect_mr = container_of(cqe, struct smbd_mr, cqe); 2342*38c8a9a5SSteve French smbdirect_mr->state = MR_INVALIDATED; 2343*38c8a9a5SSteve French if (wc->status != IB_WC_SUCCESS) { 2344*38c8a9a5SSteve French log_rdma_mr(ERR, "invalidate failed status=%x\n", wc->status); 2345*38c8a9a5SSteve French smbdirect_mr->state = MR_ERROR; 2346*38c8a9a5SSteve French } 2347*38c8a9a5SSteve French complete(&smbdirect_mr->invalidate_done); 2348*38c8a9a5SSteve French } 2349*38c8a9a5SSteve French 2350*38c8a9a5SSteve French /* 2351*38c8a9a5SSteve French * Deregister a MR after I/O is done 2352*38c8a9a5SSteve French * This function may wait if remote invalidation is not used 2353*38c8a9a5SSteve French * and we have to locally invalidate the buffer to prevent data is being 2354*38c8a9a5SSteve French * modified by remote peer after upper layer consumes it 2355*38c8a9a5SSteve French */ 2356*38c8a9a5SSteve French int smbd_deregister_mr(struct smbd_mr *smbdirect_mr) 2357*38c8a9a5SSteve French { 2358*38c8a9a5SSteve French struct ib_send_wr *wr; 2359*38c8a9a5SSteve French struct smbd_connection *info = smbdirect_mr->conn; 2360*38c8a9a5SSteve French int rc = 0; 2361*38c8a9a5SSteve French 2362*38c8a9a5SSteve French if (smbdirect_mr->need_invalidate) { 2363*38c8a9a5SSteve French /* Need to finish local invalidation before returning */ 2364*38c8a9a5SSteve French wr = &smbdirect_mr->inv_wr; 2365*38c8a9a5SSteve French wr->opcode = IB_WR_LOCAL_INV; 2366*38c8a9a5SSteve French smbdirect_mr->cqe.done = local_inv_done; 2367*38c8a9a5SSteve French wr->wr_cqe = &smbdirect_mr->cqe; 2368*38c8a9a5SSteve French wr->num_sge = 0; 2369*38c8a9a5SSteve French wr->ex.invalidate_rkey = smbdirect_mr->mr->rkey; 2370*38c8a9a5SSteve French wr->send_flags = IB_SEND_SIGNALED; 2371*38c8a9a5SSteve French 2372*38c8a9a5SSteve French init_completion(&smbdirect_mr->invalidate_done); 2373*38c8a9a5SSteve French rc = ib_post_send(info->id->qp, wr, NULL); 2374*38c8a9a5SSteve French if (rc) { 2375*38c8a9a5SSteve French log_rdma_mr(ERR, "ib_post_send failed rc=%x\n", rc); 2376*38c8a9a5SSteve French smbd_disconnect_rdma_connection(info); 2377*38c8a9a5SSteve French goto done; 2378*38c8a9a5SSteve French } 2379*38c8a9a5SSteve French wait_for_completion(&smbdirect_mr->invalidate_done); 2380*38c8a9a5SSteve French smbdirect_mr->need_invalidate = false; 2381*38c8a9a5SSteve French } else 2382*38c8a9a5SSteve French /* 2383*38c8a9a5SSteve French * For remote invalidation, just set it to MR_INVALIDATED 2384*38c8a9a5SSteve French * and defer to mr_recovery_work to recover the MR for next use 2385*38c8a9a5SSteve French */ 2386*38c8a9a5SSteve French smbdirect_mr->state = MR_INVALIDATED; 2387*38c8a9a5SSteve French 2388*38c8a9a5SSteve French if (smbdirect_mr->state == MR_INVALIDATED) { 2389*38c8a9a5SSteve French ib_dma_unmap_sg( 2390*38c8a9a5SSteve French info->id->device, smbdirect_mr->sgt.sgl, 2391*38c8a9a5SSteve French smbdirect_mr->sgt.nents, 2392*38c8a9a5SSteve French smbdirect_mr->dir); 2393*38c8a9a5SSteve French smbdirect_mr->state = MR_READY; 2394*38c8a9a5SSteve French if (atomic_inc_return(&info->mr_ready_count) == 1) 2395*38c8a9a5SSteve French wake_up_interruptible(&info->wait_mr); 2396*38c8a9a5SSteve French } else 2397*38c8a9a5SSteve French /* 2398*38c8a9a5SSteve French * Schedule the work to do MR recovery for future I/Os MR 2399*38c8a9a5SSteve French * recovery is slow and don't want it to block current I/O 2400*38c8a9a5SSteve French */ 2401*38c8a9a5SSteve French queue_work(info->workqueue, &info->mr_recovery_work); 2402*38c8a9a5SSteve French 2403*38c8a9a5SSteve French done: 2404*38c8a9a5SSteve French if (atomic_dec_and_test(&info->mr_used_count)) 2405*38c8a9a5SSteve French wake_up(&info->wait_for_mr_cleanup); 2406*38c8a9a5SSteve French 2407*38c8a9a5SSteve French return rc; 2408*38c8a9a5SSteve French } 2409*38c8a9a5SSteve French 2410*38c8a9a5SSteve French static bool smb_set_sge(struct smb_extract_to_rdma *rdma, 2411*38c8a9a5SSteve French struct page *lowest_page, size_t off, size_t len) 2412*38c8a9a5SSteve French { 2413*38c8a9a5SSteve French struct ib_sge *sge = &rdma->sge[rdma->nr_sge]; 2414*38c8a9a5SSteve French u64 addr; 2415*38c8a9a5SSteve French 2416*38c8a9a5SSteve French addr = ib_dma_map_page(rdma->device, lowest_page, 2417*38c8a9a5SSteve French off, len, rdma->direction); 2418*38c8a9a5SSteve French if (ib_dma_mapping_error(rdma->device, addr)) 2419*38c8a9a5SSteve French return false; 2420*38c8a9a5SSteve French 2421*38c8a9a5SSteve French sge->addr = addr; 2422*38c8a9a5SSteve French sge->length = len; 2423*38c8a9a5SSteve French sge->lkey = rdma->local_dma_lkey; 2424*38c8a9a5SSteve French rdma->nr_sge++; 2425*38c8a9a5SSteve French return true; 2426*38c8a9a5SSteve French } 2427*38c8a9a5SSteve French 2428*38c8a9a5SSteve French /* 2429*38c8a9a5SSteve French * Extract page fragments from a BVEC-class iterator and add them to an RDMA 2430*38c8a9a5SSteve French * element list. The pages are not pinned. 2431*38c8a9a5SSteve French */ 2432*38c8a9a5SSteve French static ssize_t smb_extract_bvec_to_rdma(struct iov_iter *iter, 2433*38c8a9a5SSteve French struct smb_extract_to_rdma *rdma, 2434*38c8a9a5SSteve French ssize_t maxsize) 2435*38c8a9a5SSteve French { 2436*38c8a9a5SSteve French const struct bio_vec *bv = iter->bvec; 2437*38c8a9a5SSteve French unsigned long start = iter->iov_offset; 2438*38c8a9a5SSteve French unsigned int i; 2439*38c8a9a5SSteve French ssize_t ret = 0; 2440*38c8a9a5SSteve French 2441*38c8a9a5SSteve French for (i = 0; i < iter->nr_segs; i++) { 2442*38c8a9a5SSteve French size_t off, len; 2443*38c8a9a5SSteve French 2444*38c8a9a5SSteve French len = bv[i].bv_len; 2445*38c8a9a5SSteve French if (start >= len) { 2446*38c8a9a5SSteve French start -= len; 2447*38c8a9a5SSteve French continue; 2448*38c8a9a5SSteve French } 2449*38c8a9a5SSteve French 2450*38c8a9a5SSteve French len = min_t(size_t, maxsize, len - start); 2451*38c8a9a5SSteve French off = bv[i].bv_offset + start; 2452*38c8a9a5SSteve French 2453*38c8a9a5SSteve French if (!smb_set_sge(rdma, bv[i].bv_page, off, len)) 2454*38c8a9a5SSteve French return -EIO; 2455*38c8a9a5SSteve French 2456*38c8a9a5SSteve French ret += len; 2457*38c8a9a5SSteve French maxsize -= len; 2458*38c8a9a5SSteve French if (rdma->nr_sge >= rdma->max_sge || maxsize <= 0) 2459*38c8a9a5SSteve French break; 2460*38c8a9a5SSteve French start = 0; 2461*38c8a9a5SSteve French } 2462*38c8a9a5SSteve French 2463*38c8a9a5SSteve French return ret; 2464*38c8a9a5SSteve French } 2465*38c8a9a5SSteve French 2466*38c8a9a5SSteve French /* 2467*38c8a9a5SSteve French * Extract fragments from a KVEC-class iterator and add them to an RDMA list. 2468*38c8a9a5SSteve French * This can deal with vmalloc'd buffers as well as kmalloc'd or static buffers. 2469*38c8a9a5SSteve French * The pages are not pinned. 2470*38c8a9a5SSteve French */ 2471*38c8a9a5SSteve French static ssize_t smb_extract_kvec_to_rdma(struct iov_iter *iter, 2472*38c8a9a5SSteve French struct smb_extract_to_rdma *rdma, 2473*38c8a9a5SSteve French ssize_t maxsize) 2474*38c8a9a5SSteve French { 2475*38c8a9a5SSteve French const struct kvec *kv = iter->kvec; 2476*38c8a9a5SSteve French unsigned long start = iter->iov_offset; 2477*38c8a9a5SSteve French unsigned int i; 2478*38c8a9a5SSteve French ssize_t ret = 0; 2479*38c8a9a5SSteve French 2480*38c8a9a5SSteve French for (i = 0; i < iter->nr_segs; i++) { 2481*38c8a9a5SSteve French struct page *page; 2482*38c8a9a5SSteve French unsigned long kaddr; 2483*38c8a9a5SSteve French size_t off, len, seg; 2484*38c8a9a5SSteve French 2485*38c8a9a5SSteve French len = kv[i].iov_len; 2486*38c8a9a5SSteve French if (start >= len) { 2487*38c8a9a5SSteve French start -= len; 2488*38c8a9a5SSteve French continue; 2489*38c8a9a5SSteve French } 2490*38c8a9a5SSteve French 2491*38c8a9a5SSteve French kaddr = (unsigned long)kv[i].iov_base + start; 2492*38c8a9a5SSteve French off = kaddr & ~PAGE_MASK; 2493*38c8a9a5SSteve French len = min_t(size_t, maxsize, len - start); 2494*38c8a9a5SSteve French kaddr &= PAGE_MASK; 2495*38c8a9a5SSteve French 2496*38c8a9a5SSteve French maxsize -= len; 2497*38c8a9a5SSteve French do { 2498*38c8a9a5SSteve French seg = min_t(size_t, len, PAGE_SIZE - off); 2499*38c8a9a5SSteve French 2500*38c8a9a5SSteve French if (is_vmalloc_or_module_addr((void *)kaddr)) 2501*38c8a9a5SSteve French page = vmalloc_to_page((void *)kaddr); 2502*38c8a9a5SSteve French else 2503*38c8a9a5SSteve French page = virt_to_page(kaddr); 2504*38c8a9a5SSteve French 2505*38c8a9a5SSteve French if (!smb_set_sge(rdma, page, off, seg)) 2506*38c8a9a5SSteve French return -EIO; 2507*38c8a9a5SSteve French 2508*38c8a9a5SSteve French ret += seg; 2509*38c8a9a5SSteve French len -= seg; 2510*38c8a9a5SSteve French kaddr += PAGE_SIZE; 2511*38c8a9a5SSteve French off = 0; 2512*38c8a9a5SSteve French } while (len > 0 && rdma->nr_sge < rdma->max_sge); 2513*38c8a9a5SSteve French 2514*38c8a9a5SSteve French if (rdma->nr_sge >= rdma->max_sge || maxsize <= 0) 2515*38c8a9a5SSteve French break; 2516*38c8a9a5SSteve French start = 0; 2517*38c8a9a5SSteve French } 2518*38c8a9a5SSteve French 2519*38c8a9a5SSteve French return ret; 2520*38c8a9a5SSteve French } 2521*38c8a9a5SSteve French 2522*38c8a9a5SSteve French /* 2523*38c8a9a5SSteve French * Extract folio fragments from an XARRAY-class iterator and add them to an 2524*38c8a9a5SSteve French * RDMA list. The folios are not pinned. 2525*38c8a9a5SSteve French */ 2526*38c8a9a5SSteve French static ssize_t smb_extract_xarray_to_rdma(struct iov_iter *iter, 2527*38c8a9a5SSteve French struct smb_extract_to_rdma *rdma, 2528*38c8a9a5SSteve French ssize_t maxsize) 2529*38c8a9a5SSteve French { 2530*38c8a9a5SSteve French struct xarray *xa = iter->xarray; 2531*38c8a9a5SSteve French struct folio *folio; 2532*38c8a9a5SSteve French loff_t start = iter->xarray_start + iter->iov_offset; 2533*38c8a9a5SSteve French pgoff_t index = start / PAGE_SIZE; 2534*38c8a9a5SSteve French ssize_t ret = 0; 2535*38c8a9a5SSteve French size_t off, len; 2536*38c8a9a5SSteve French XA_STATE(xas, xa, index); 2537*38c8a9a5SSteve French 2538*38c8a9a5SSteve French rcu_read_lock(); 2539*38c8a9a5SSteve French 2540*38c8a9a5SSteve French xas_for_each(&xas, folio, ULONG_MAX) { 2541*38c8a9a5SSteve French if (xas_retry(&xas, folio)) 2542*38c8a9a5SSteve French continue; 2543*38c8a9a5SSteve French if (WARN_ON(xa_is_value(folio))) 2544*38c8a9a5SSteve French break; 2545*38c8a9a5SSteve French if (WARN_ON(folio_test_hugetlb(folio))) 2546*38c8a9a5SSteve French break; 2547*38c8a9a5SSteve French 2548*38c8a9a5SSteve French off = offset_in_folio(folio, start); 2549*38c8a9a5SSteve French len = min_t(size_t, maxsize, folio_size(folio) - off); 2550*38c8a9a5SSteve French 2551*38c8a9a5SSteve French if (!smb_set_sge(rdma, folio_page(folio, 0), off, len)) { 2552*38c8a9a5SSteve French rcu_read_unlock(); 2553*38c8a9a5SSteve French return -EIO; 2554*38c8a9a5SSteve French } 2555*38c8a9a5SSteve French 2556*38c8a9a5SSteve French maxsize -= len; 2557*38c8a9a5SSteve French ret += len; 2558*38c8a9a5SSteve French if (rdma->nr_sge >= rdma->max_sge || maxsize <= 0) 2559*38c8a9a5SSteve French break; 2560*38c8a9a5SSteve French } 2561*38c8a9a5SSteve French 2562*38c8a9a5SSteve French rcu_read_unlock(); 2563*38c8a9a5SSteve French return ret; 2564*38c8a9a5SSteve French } 2565*38c8a9a5SSteve French 2566*38c8a9a5SSteve French /* 2567*38c8a9a5SSteve French * Extract page fragments from up to the given amount of the source iterator 2568*38c8a9a5SSteve French * and build up an RDMA list that refers to all of those bits. The RDMA list 2569*38c8a9a5SSteve French * is appended to, up to the maximum number of elements set in the parameter 2570*38c8a9a5SSteve French * block. 2571*38c8a9a5SSteve French * 2572*38c8a9a5SSteve French * The extracted page fragments are not pinned or ref'd in any way; if an 2573*38c8a9a5SSteve French * IOVEC/UBUF-type iterator is to be used, it should be converted to a 2574*38c8a9a5SSteve French * BVEC-type iterator and the pages pinned, ref'd or otherwise held in some 2575*38c8a9a5SSteve French * way. 2576*38c8a9a5SSteve French */ 2577*38c8a9a5SSteve French static ssize_t smb_extract_iter_to_rdma(struct iov_iter *iter, size_t len, 2578*38c8a9a5SSteve French struct smb_extract_to_rdma *rdma) 2579*38c8a9a5SSteve French { 2580*38c8a9a5SSteve French ssize_t ret; 2581*38c8a9a5SSteve French int before = rdma->nr_sge; 2582*38c8a9a5SSteve French 2583*38c8a9a5SSteve French switch (iov_iter_type(iter)) { 2584*38c8a9a5SSteve French case ITER_BVEC: 2585*38c8a9a5SSteve French ret = smb_extract_bvec_to_rdma(iter, rdma, len); 2586*38c8a9a5SSteve French break; 2587*38c8a9a5SSteve French case ITER_KVEC: 2588*38c8a9a5SSteve French ret = smb_extract_kvec_to_rdma(iter, rdma, len); 2589*38c8a9a5SSteve French break; 2590*38c8a9a5SSteve French case ITER_XARRAY: 2591*38c8a9a5SSteve French ret = smb_extract_xarray_to_rdma(iter, rdma, len); 2592*38c8a9a5SSteve French break; 2593*38c8a9a5SSteve French default: 2594*38c8a9a5SSteve French WARN_ON_ONCE(1); 2595*38c8a9a5SSteve French return -EIO; 2596*38c8a9a5SSteve French } 2597*38c8a9a5SSteve French 2598*38c8a9a5SSteve French if (ret > 0) { 2599*38c8a9a5SSteve French iov_iter_advance(iter, ret); 2600*38c8a9a5SSteve French } else if (ret < 0) { 2601*38c8a9a5SSteve French while (rdma->nr_sge > before) { 2602*38c8a9a5SSteve French struct ib_sge *sge = &rdma->sge[rdma->nr_sge--]; 2603*38c8a9a5SSteve French 2604*38c8a9a5SSteve French ib_dma_unmap_single(rdma->device, sge->addr, sge->length, 2605*38c8a9a5SSteve French rdma->direction); 2606*38c8a9a5SSteve French sge->addr = 0; 2607*38c8a9a5SSteve French } 2608*38c8a9a5SSteve French } 2609*38c8a9a5SSteve French 2610*38c8a9a5SSteve French return ret; 2611*38c8a9a5SSteve French } 2612