1 /* 2 * Copyright (c) 2009, 2018 Oracle and/or its affiliates. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 * 32 */ 33 #include <linux/module.h> 34 #include <rdma/rdma_cm.h> 35 36 #include "rds_single_path.h" 37 #include "rdma_transport.h" 38 #include "ib.h" 39 40 /* Global IPv4 and IPv6 RDS RDMA listener cm_id */ 41 static struct rdma_cm_id *rds_rdma_listen_id; 42 #if IS_ENABLED(CONFIG_IPV6) 43 static struct rdma_cm_id *rds6_rdma_listen_id; 44 #endif 45 46 static int rds_rdma_cm_event_handler_cmn(struct rdma_cm_id *cm_id, 47 struct rdma_cm_event *event, 48 bool isv6) 49 { 50 /* this can be null in the listening path */ 51 struct rds_connection *conn = cm_id->context; 52 struct rds_transport *trans; 53 int ret = 0; 54 int *err; 55 u8 len; 56 57 rdsdebug("conn %p id %p handling event %u (%s)\n", conn, cm_id, 58 event->event, rdma_event_msg(event->event)); 59 60 if (cm_id->device->node_type == RDMA_NODE_IB_CA) 61 trans = &rds_ib_transport; 62 63 /* Prevent shutdown from tearing down the connection 64 * while we're executing. */ 65 if (conn) { 66 mutex_lock(&conn->c_cm_lock); 67 68 /* If the connection is being shut down, bail out 69 * right away. We return 0 so cm_id doesn't get 70 * destroyed prematurely */ 71 if (rds_conn_state(conn) == RDS_CONN_DISCONNECTING) { 72 /* Reject incoming connections while we're tearing 73 * down an existing one. */ 74 if (event->event == RDMA_CM_EVENT_CONNECT_REQUEST) 75 ret = 1; 76 goto out; 77 } 78 } 79 80 switch (event->event) { 81 case RDMA_CM_EVENT_CONNECT_REQUEST: 82 ret = trans->cm_handle_connect(cm_id, event, isv6); 83 break; 84 85 case RDMA_CM_EVENT_ADDR_RESOLVED: 86 rdma_set_service_type(cm_id, conn->c_tos); 87 /* XXX do we need to clean up if this fails? */ 88 ret = rdma_resolve_route(cm_id, 89 RDS_RDMA_RESOLVE_TIMEOUT_MS); 90 break; 91 92 case RDMA_CM_EVENT_ROUTE_RESOLVED: 93 /* Connection could have been dropped so make sure the 94 * cm_id is valid before proceeding 95 */ 96 if (conn) { 97 struct rds_ib_connection *ibic; 98 99 ibic = conn->c_transport_data; 100 if (ibic && ibic->i_cm_id == cm_id) 101 ret = trans->cm_initiate_connect(cm_id, isv6); 102 else 103 rds_conn_drop(conn); 104 } 105 break; 106 107 case RDMA_CM_EVENT_ESTABLISHED: 108 if (conn) 109 trans->cm_connect_complete(conn, event); 110 break; 111 112 case RDMA_CM_EVENT_REJECTED: 113 if (!conn) 114 break; 115 err = (int *)rdma_consumer_reject_data(cm_id, event, &len); 116 if (!err || 117 (err && len >= sizeof(*err) && 118 ((*err) <= RDS_RDMA_REJ_INCOMPAT))) { 119 pr_warn("RDS/RDMA: conn <%pI6c, %pI6c> rejected, dropping connection\n", 120 &conn->c_laddr, &conn->c_faddr); 121 122 if (!conn->c_tos) 123 conn->c_proposed_version = RDS_PROTOCOL_COMPAT_VERSION; 124 125 rds_conn_drop(conn); 126 } 127 rdsdebug("Connection rejected: %s\n", 128 rdma_reject_msg(cm_id, event->status)); 129 break; 130 case RDMA_CM_EVENT_ADDR_ERROR: 131 case RDMA_CM_EVENT_ROUTE_ERROR: 132 case RDMA_CM_EVENT_CONNECT_ERROR: 133 case RDMA_CM_EVENT_UNREACHABLE: 134 case RDMA_CM_EVENT_DEVICE_REMOVAL: 135 case RDMA_CM_EVENT_ADDR_CHANGE: 136 if (conn) 137 rds_conn_drop(conn); 138 break; 139 140 case RDMA_CM_EVENT_DISCONNECTED: 141 if (!conn) 142 break; 143 rdsdebug("DISCONNECT event - dropping connection " 144 "%pI6c->%pI6c\n", &conn->c_laddr, 145 &conn->c_faddr); 146 rds_conn_drop(conn); 147 break; 148 149 case RDMA_CM_EVENT_TIMEWAIT_EXIT: 150 if (conn) { 151 pr_info("RDS: RDMA_CM_EVENT_TIMEWAIT_EXIT event: dropping connection %pI6c->%pI6c\n", 152 &conn->c_laddr, &conn->c_faddr); 153 rds_conn_drop(conn); 154 } 155 break; 156 157 default: 158 /* things like device disconnect? */ 159 printk(KERN_ERR "RDS: unknown event %u (%s)!\n", 160 event->event, rdma_event_msg(event->event)); 161 break; 162 } 163 164 out: 165 if (conn) 166 mutex_unlock(&conn->c_cm_lock); 167 168 rdsdebug("id %p event %u (%s) handling ret %d\n", cm_id, event->event, 169 rdma_event_msg(event->event), ret); 170 171 return ret; 172 } 173 174 int rds_rdma_cm_event_handler(struct rdma_cm_id *cm_id, 175 struct rdma_cm_event *event) 176 { 177 return rds_rdma_cm_event_handler_cmn(cm_id, event, false); 178 } 179 180 #if IS_ENABLED(CONFIG_IPV6) 181 int rds6_rdma_cm_event_handler(struct rdma_cm_id *cm_id, 182 struct rdma_cm_event *event) 183 { 184 return rds_rdma_cm_event_handler_cmn(cm_id, event, true); 185 } 186 #endif 187 188 static int rds_rdma_listen_init_common(rdma_cm_event_handler handler, 189 struct sockaddr *sa, 190 struct rdma_cm_id **ret_cm_id) 191 { 192 struct rdma_cm_id *cm_id; 193 int ret; 194 195 cm_id = rdma_create_id(&init_net, handler, NULL, 196 RDMA_PS_TCP, IB_QPT_RC); 197 if (IS_ERR(cm_id)) { 198 ret = PTR_ERR(cm_id); 199 printk(KERN_ERR "RDS/RDMA: failed to setup listener, " 200 "rdma_create_id() returned %d\n", ret); 201 return ret; 202 } 203 204 /* 205 * XXX I bet this binds the cm_id to a device. If we want to support 206 * fail-over we'll have to take this into consideration. 207 */ 208 ret = rdma_bind_addr(cm_id, sa); 209 if (ret) { 210 printk(KERN_ERR "RDS/RDMA: failed to setup listener, " 211 "rdma_bind_addr() returned %d\n", ret); 212 goto out; 213 } 214 215 ret = rdma_listen(cm_id, 128); 216 if (ret) { 217 printk(KERN_ERR "RDS/RDMA: failed to setup listener, " 218 "rdma_listen() returned %d\n", ret); 219 goto out; 220 } 221 222 rdsdebug("cm %p listening on port %u\n", cm_id, RDS_PORT); 223 224 *ret_cm_id = cm_id; 225 cm_id = NULL; 226 out: 227 if (cm_id) 228 rdma_destroy_id(cm_id); 229 return ret; 230 } 231 232 /* Initialize the RDS RDMA listeners. We create two listeners for 233 * compatibility reason. The one on RDS_PORT is used for IPv4 234 * requests only. The one on RDS_CM_PORT is used for IPv6 requests 235 * only. So only IPv6 enabled RDS module will communicate using this 236 * port. 237 */ 238 static int rds_rdma_listen_init(void) 239 { 240 int ret; 241 #if IS_ENABLED(CONFIG_IPV6) 242 struct sockaddr_in6 sin6; 243 #endif 244 struct sockaddr_in sin; 245 246 sin.sin_family = PF_INET; 247 sin.sin_addr.s_addr = htonl(INADDR_ANY); 248 sin.sin_port = htons(RDS_PORT); 249 ret = rds_rdma_listen_init_common(rds_rdma_cm_event_handler, 250 (struct sockaddr *)&sin, 251 &rds_rdma_listen_id); 252 if (ret != 0) 253 return ret; 254 255 #if IS_ENABLED(CONFIG_IPV6) 256 sin6.sin6_family = PF_INET6; 257 sin6.sin6_addr = in6addr_any; 258 sin6.sin6_port = htons(RDS_CM_PORT); 259 sin6.sin6_scope_id = 0; 260 sin6.sin6_flowinfo = 0; 261 ret = rds_rdma_listen_init_common(rds6_rdma_cm_event_handler, 262 (struct sockaddr *)&sin6, 263 &rds6_rdma_listen_id); 264 /* Keep going even when IPv6 is not enabled in the system. */ 265 if (ret != 0) 266 rdsdebug("Cannot set up IPv6 RDMA listener\n"); 267 #endif 268 return 0; 269 } 270 271 static void rds_rdma_listen_stop(void) 272 { 273 if (rds_rdma_listen_id) { 274 rdsdebug("cm %p\n", rds_rdma_listen_id); 275 rdma_destroy_id(rds_rdma_listen_id); 276 rds_rdma_listen_id = NULL; 277 } 278 #if IS_ENABLED(CONFIG_IPV6) 279 if (rds6_rdma_listen_id) { 280 rdsdebug("cm %p\n", rds6_rdma_listen_id); 281 rdma_destroy_id(rds6_rdma_listen_id); 282 rds6_rdma_listen_id = NULL; 283 } 284 #endif 285 } 286 287 static int rds_rdma_init(void) 288 { 289 int ret; 290 291 ret = rds_ib_init(); 292 if (ret) 293 goto out; 294 295 ret = rds_rdma_listen_init(); 296 if (ret) 297 rds_ib_exit(); 298 out: 299 return ret; 300 } 301 module_init(rds_rdma_init); 302 303 static void rds_rdma_exit(void) 304 { 305 /* stop listening first to ensure no new connections are attempted */ 306 rds_rdma_listen_stop(); 307 rds_ib_exit(); 308 } 309 module_exit(rds_rdma_exit); 310 311 MODULE_AUTHOR("Oracle Corporation <rds-devel@oss.oracle.com>"); 312 MODULE_DESCRIPTION("RDS: IB transport"); 313 MODULE_LICENSE("Dual BSD/GPL"); 314