1 /* 2 * Copyright (c) 2009, 2018 Oracle and/or its affiliates. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 * 32 */ 33 #include <linux/module.h> 34 #include <rdma/rdma_cm.h> 35 36 #include "rds_single_path.h" 37 #include "rdma_transport.h" 38 #include "ib.h" 39 40 /* Global IPv4 and IPv6 RDS RDMA listener cm_id */ 41 static struct rdma_cm_id *rds_rdma_listen_id; 42 #if IS_ENABLED(CONFIG_IPV6) 43 static struct rdma_cm_id *rds6_rdma_listen_id; 44 #endif 45 46 static int rds_rdma_cm_event_handler_cmn(struct rdma_cm_id *cm_id, 47 struct rdma_cm_event *event, 48 bool isv6) 49 { 50 /* this can be null in the listening path */ 51 struct rds_connection *conn = cm_id->context; 52 struct rds_transport *trans; 53 int ret = 0; 54 int *err; 55 u8 len; 56 57 rdsdebug("conn %p id %p handling event %u (%s)\n", conn, cm_id, 58 event->event, rdma_event_msg(event->event)); 59 60 if (cm_id->device->node_type == RDMA_NODE_IB_CA) 61 trans = &rds_ib_transport; 62 63 /* Prevent shutdown from tearing down the connection 64 * while we're executing. */ 65 if (conn) { 66 mutex_lock(&conn->c_cm_lock); 67 68 /* If the connection is being shut down, bail out 69 * right away. We return 0 so cm_id doesn't get 70 * destroyed prematurely */ 71 if (rds_conn_state(conn) == RDS_CONN_DISCONNECTING) { 72 /* Reject incoming connections while we're tearing 73 * down an existing one. */ 74 if (event->event == RDMA_CM_EVENT_CONNECT_REQUEST) 75 ret = 1; 76 goto out; 77 } 78 } 79 80 switch (event->event) { 81 case RDMA_CM_EVENT_CONNECT_REQUEST: 82 ret = trans->cm_handle_connect(cm_id, event, isv6); 83 break; 84 85 case RDMA_CM_EVENT_ADDR_RESOLVED: 86 rdma_set_service_type(cm_id, conn->c_tos); 87 /* XXX do we need to clean up if this fails? */ 88 ret = rdma_resolve_route(cm_id, 89 RDS_RDMA_RESOLVE_TIMEOUT_MS); 90 break; 91 92 case RDMA_CM_EVENT_ROUTE_RESOLVED: 93 /* Connection could have been dropped so make sure the 94 * cm_id is valid before proceeding 95 */ 96 if (conn) { 97 struct rds_ib_connection *ibic; 98 99 ibic = conn->c_transport_data; 100 if (ibic && ibic->i_cm_id == cm_id) 101 ret = trans->cm_initiate_connect(cm_id, isv6); 102 else 103 rds_conn_drop(conn); 104 } 105 break; 106 107 case RDMA_CM_EVENT_ESTABLISHED: 108 trans->cm_connect_complete(conn, event); 109 break; 110 111 case RDMA_CM_EVENT_REJECTED: 112 if (!conn) 113 break; 114 err = (int *)rdma_consumer_reject_data(cm_id, event, &len); 115 if (!err || (err && ((*err) == RDS_RDMA_REJ_INCOMPAT))) { 116 pr_warn("RDS/RDMA: conn <%pI6c, %pI6c> rejected, dropping connection\n", 117 &conn->c_laddr, &conn->c_faddr); 118 conn->c_proposed_version = RDS_PROTOCOL_COMPAT_VERSION; 119 conn->c_tos = 0; 120 rds_conn_drop(conn); 121 } 122 rdsdebug("Connection rejected: %s\n", 123 rdma_reject_msg(cm_id, event->status)); 124 break; 125 /* FALLTHROUGH */ 126 case RDMA_CM_EVENT_ADDR_ERROR: 127 case RDMA_CM_EVENT_ROUTE_ERROR: 128 case RDMA_CM_EVENT_CONNECT_ERROR: 129 case RDMA_CM_EVENT_UNREACHABLE: 130 case RDMA_CM_EVENT_DEVICE_REMOVAL: 131 case RDMA_CM_EVENT_ADDR_CHANGE: 132 if (conn) 133 rds_conn_drop(conn); 134 break; 135 136 case RDMA_CM_EVENT_DISCONNECTED: 137 rdsdebug("DISCONNECT event - dropping connection " 138 "%pI6c->%pI6c\n", &conn->c_laddr, 139 &conn->c_faddr); 140 rds_conn_drop(conn); 141 break; 142 143 case RDMA_CM_EVENT_TIMEWAIT_EXIT: 144 if (conn) { 145 pr_info("RDS: RDMA_CM_EVENT_TIMEWAIT_EXIT event: dropping connection %pI6c->%pI6c\n", 146 &conn->c_laddr, &conn->c_faddr); 147 rds_conn_drop(conn); 148 } 149 break; 150 151 default: 152 /* things like device disconnect? */ 153 printk(KERN_ERR "RDS: unknown event %u (%s)!\n", 154 event->event, rdma_event_msg(event->event)); 155 break; 156 } 157 158 out: 159 if (conn) 160 mutex_unlock(&conn->c_cm_lock); 161 162 rdsdebug("id %p event %u (%s) handling ret %d\n", cm_id, event->event, 163 rdma_event_msg(event->event), ret); 164 165 return ret; 166 } 167 168 int rds_rdma_cm_event_handler(struct rdma_cm_id *cm_id, 169 struct rdma_cm_event *event) 170 { 171 return rds_rdma_cm_event_handler_cmn(cm_id, event, false); 172 } 173 174 #if IS_ENABLED(CONFIG_IPV6) 175 int rds6_rdma_cm_event_handler(struct rdma_cm_id *cm_id, 176 struct rdma_cm_event *event) 177 { 178 return rds_rdma_cm_event_handler_cmn(cm_id, event, true); 179 } 180 #endif 181 182 static int rds_rdma_listen_init_common(rdma_cm_event_handler handler, 183 struct sockaddr *sa, 184 struct rdma_cm_id **ret_cm_id) 185 { 186 struct rdma_cm_id *cm_id; 187 int ret; 188 189 cm_id = rdma_create_id(&init_net, handler, NULL, 190 RDMA_PS_TCP, IB_QPT_RC); 191 if (IS_ERR(cm_id)) { 192 ret = PTR_ERR(cm_id); 193 printk(KERN_ERR "RDS/RDMA: failed to setup listener, " 194 "rdma_create_id() returned %d\n", ret); 195 return ret; 196 } 197 198 /* 199 * XXX I bet this binds the cm_id to a device. If we want to support 200 * fail-over we'll have to take this into consideration. 201 */ 202 ret = rdma_bind_addr(cm_id, sa); 203 if (ret) { 204 printk(KERN_ERR "RDS/RDMA: failed to setup listener, " 205 "rdma_bind_addr() returned %d\n", ret); 206 goto out; 207 } 208 209 ret = rdma_listen(cm_id, 128); 210 if (ret) { 211 printk(KERN_ERR "RDS/RDMA: failed to setup listener, " 212 "rdma_listen() returned %d\n", ret); 213 goto out; 214 } 215 216 rdsdebug("cm %p listening on port %u\n", cm_id, RDS_PORT); 217 218 *ret_cm_id = cm_id; 219 cm_id = NULL; 220 out: 221 if (cm_id) 222 rdma_destroy_id(cm_id); 223 return ret; 224 } 225 226 /* Initialize the RDS RDMA listeners. We create two listeners for 227 * compatibility reason. The one on RDS_PORT is used for IPv4 228 * requests only. The one on RDS_CM_PORT is used for IPv6 requests 229 * only. So only IPv6 enabled RDS module will communicate using this 230 * port. 231 */ 232 static int rds_rdma_listen_init(void) 233 { 234 int ret; 235 #if IS_ENABLED(CONFIG_IPV6) 236 struct sockaddr_in6 sin6; 237 #endif 238 struct sockaddr_in sin; 239 240 sin.sin_family = PF_INET; 241 sin.sin_addr.s_addr = htonl(INADDR_ANY); 242 sin.sin_port = htons(RDS_PORT); 243 ret = rds_rdma_listen_init_common(rds_rdma_cm_event_handler, 244 (struct sockaddr *)&sin, 245 &rds_rdma_listen_id); 246 if (ret != 0) 247 return ret; 248 249 #if IS_ENABLED(CONFIG_IPV6) 250 sin6.sin6_family = PF_INET6; 251 sin6.sin6_addr = in6addr_any; 252 sin6.sin6_port = htons(RDS_CM_PORT); 253 sin6.sin6_scope_id = 0; 254 sin6.sin6_flowinfo = 0; 255 ret = rds_rdma_listen_init_common(rds6_rdma_cm_event_handler, 256 (struct sockaddr *)&sin6, 257 &rds6_rdma_listen_id); 258 /* Keep going even when IPv6 is not enabled in the system. */ 259 if (ret != 0) 260 rdsdebug("Cannot set up IPv6 RDMA listener\n"); 261 #endif 262 return 0; 263 } 264 265 static void rds_rdma_listen_stop(void) 266 { 267 if (rds_rdma_listen_id) { 268 rdsdebug("cm %p\n", rds_rdma_listen_id); 269 rdma_destroy_id(rds_rdma_listen_id); 270 rds_rdma_listen_id = NULL; 271 } 272 #if IS_ENABLED(CONFIG_IPV6) 273 if (rds6_rdma_listen_id) { 274 rdsdebug("cm %p\n", rds6_rdma_listen_id); 275 rdma_destroy_id(rds6_rdma_listen_id); 276 rds6_rdma_listen_id = NULL; 277 } 278 #endif 279 } 280 281 static int rds_rdma_init(void) 282 { 283 int ret; 284 285 ret = rds_ib_init(); 286 if (ret) 287 goto out; 288 289 ret = rds_rdma_listen_init(); 290 if (ret) 291 rds_ib_exit(); 292 out: 293 return ret; 294 } 295 module_init(rds_rdma_init); 296 297 static void rds_rdma_exit(void) 298 { 299 /* stop listening first to ensure no new connections are attempted */ 300 rds_rdma_listen_stop(); 301 rds_ib_exit(); 302 } 303 module_exit(rds_rdma_exit); 304 305 MODULE_AUTHOR("Oracle Corporation <rds-devel@oss.oracle.com>"); 306 MODULE_DESCRIPTION("RDS: IB transport"); 307 MODULE_LICENSE("Dual BSD/GPL"); 308