1 /* 2 * Copyright (c) 2009, 2018 Oracle and/or its affiliates. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 * 32 */ 33 #include <linux/module.h> 34 #include <rdma/rdma_cm.h> 35 36 #include "rds_single_path.h" 37 #include "rdma_transport.h" 38 #include "ib.h" 39 40 /* Global IPv4 and IPv6 RDS RDMA listener cm_id */ 41 static struct rdma_cm_id *rds_rdma_listen_id; 42 #if IS_ENABLED(CONFIG_IPV6) 43 static struct rdma_cm_id *rds6_rdma_listen_id; 44 #endif 45 46 static int rds_rdma_cm_event_handler_cmn(struct rdma_cm_id *cm_id, 47 struct rdma_cm_event *event, 48 bool isv6) 49 { 50 /* this can be null in the listening path */ 51 struct rds_connection *conn = cm_id->context; 52 struct rds_transport *trans; 53 int ret = 0; 54 int *err; 55 u8 len; 56 57 rdsdebug("conn %p id %p handling event %u (%s)\n", conn, cm_id, 58 event->event, rdma_event_msg(event->event)); 59 60 if (cm_id->device->node_type == RDMA_NODE_IB_CA) 61 trans = &rds_ib_transport; 62 63 /* Prevent shutdown from tearing down the connection 64 * while we're executing. */ 65 if (conn) { 66 mutex_lock(&conn->c_cm_lock); 67 68 /* If the connection is being shut down, bail out 69 * right away. We return 0 so cm_id doesn't get 70 * destroyed prematurely */ 71 if (rds_conn_state(conn) == RDS_CONN_DISCONNECTING) { 72 /* Reject incoming connections while we're tearing 73 * down an existing one. */ 74 if (event->event == RDMA_CM_EVENT_CONNECT_REQUEST) 75 ret = 1; 76 goto out; 77 } 78 } 79 80 switch (event->event) { 81 case RDMA_CM_EVENT_CONNECT_REQUEST: 82 ret = trans->cm_handle_connect(cm_id, event, isv6); 83 break; 84 85 case RDMA_CM_EVENT_ADDR_RESOLVED: 86 rdma_set_service_type(cm_id, conn->c_tos); 87 /* XXX do we need to clean up if this fails? */ 88 ret = rdma_resolve_route(cm_id, 89 RDS_RDMA_RESOLVE_TIMEOUT_MS); 90 break; 91 92 case RDMA_CM_EVENT_ROUTE_RESOLVED: 93 /* Connection could have been dropped so make sure the 94 * cm_id is valid before proceeding 95 */ 96 if (conn) { 97 struct rds_ib_connection *ibic; 98 99 ibic = conn->c_transport_data; 100 if (ibic && ibic->i_cm_id == cm_id) 101 ret = trans->cm_initiate_connect(cm_id, isv6); 102 else 103 rds_conn_drop(conn); 104 } 105 break; 106 107 case RDMA_CM_EVENT_ESTABLISHED: 108 trans->cm_connect_complete(conn, event); 109 break; 110 111 case RDMA_CM_EVENT_REJECTED: 112 if (!conn) 113 break; 114 err = (int *)rdma_consumer_reject_data(cm_id, event, &len); 115 if (!err || 116 (err && len >= sizeof(*err) && 117 ((*err) <= RDS_RDMA_REJ_INCOMPAT))) { 118 pr_warn("RDS/RDMA: conn <%pI6c, %pI6c> rejected, dropping connection\n", 119 &conn->c_laddr, &conn->c_faddr); 120 121 if (!conn->c_tos) 122 conn->c_proposed_version = RDS_PROTOCOL_COMPAT_VERSION; 123 124 rds_conn_drop(conn); 125 } 126 rdsdebug("Connection rejected: %s\n", 127 rdma_reject_msg(cm_id, event->status)); 128 break; 129 case RDMA_CM_EVENT_ADDR_ERROR: 130 case RDMA_CM_EVENT_ROUTE_ERROR: 131 case RDMA_CM_EVENT_CONNECT_ERROR: 132 case RDMA_CM_EVENT_UNREACHABLE: 133 case RDMA_CM_EVENT_DEVICE_REMOVAL: 134 case RDMA_CM_EVENT_ADDR_CHANGE: 135 if (conn) 136 rds_conn_drop(conn); 137 break; 138 139 case RDMA_CM_EVENT_DISCONNECTED: 140 rdsdebug("DISCONNECT event - dropping connection " 141 "%pI6c->%pI6c\n", &conn->c_laddr, 142 &conn->c_faddr); 143 rds_conn_drop(conn); 144 break; 145 146 case RDMA_CM_EVENT_TIMEWAIT_EXIT: 147 if (conn) { 148 pr_info("RDS: RDMA_CM_EVENT_TIMEWAIT_EXIT event: dropping connection %pI6c->%pI6c\n", 149 &conn->c_laddr, &conn->c_faddr); 150 rds_conn_drop(conn); 151 } 152 break; 153 154 default: 155 /* things like device disconnect? */ 156 printk(KERN_ERR "RDS: unknown event %u (%s)!\n", 157 event->event, rdma_event_msg(event->event)); 158 break; 159 } 160 161 out: 162 if (conn) 163 mutex_unlock(&conn->c_cm_lock); 164 165 rdsdebug("id %p event %u (%s) handling ret %d\n", cm_id, event->event, 166 rdma_event_msg(event->event), ret); 167 168 return ret; 169 } 170 171 int rds_rdma_cm_event_handler(struct rdma_cm_id *cm_id, 172 struct rdma_cm_event *event) 173 { 174 return rds_rdma_cm_event_handler_cmn(cm_id, event, false); 175 } 176 177 #if IS_ENABLED(CONFIG_IPV6) 178 int rds6_rdma_cm_event_handler(struct rdma_cm_id *cm_id, 179 struct rdma_cm_event *event) 180 { 181 return rds_rdma_cm_event_handler_cmn(cm_id, event, true); 182 } 183 #endif 184 185 static int rds_rdma_listen_init_common(rdma_cm_event_handler handler, 186 struct sockaddr *sa, 187 struct rdma_cm_id **ret_cm_id) 188 { 189 struct rdma_cm_id *cm_id; 190 int ret; 191 192 cm_id = rdma_create_id(&init_net, handler, NULL, 193 RDMA_PS_TCP, IB_QPT_RC); 194 if (IS_ERR(cm_id)) { 195 ret = PTR_ERR(cm_id); 196 printk(KERN_ERR "RDS/RDMA: failed to setup listener, " 197 "rdma_create_id() returned %d\n", ret); 198 return ret; 199 } 200 201 /* 202 * XXX I bet this binds the cm_id to a device. If we want to support 203 * fail-over we'll have to take this into consideration. 204 */ 205 ret = rdma_bind_addr(cm_id, sa); 206 if (ret) { 207 printk(KERN_ERR "RDS/RDMA: failed to setup listener, " 208 "rdma_bind_addr() returned %d\n", ret); 209 goto out; 210 } 211 212 ret = rdma_listen(cm_id, 128); 213 if (ret) { 214 printk(KERN_ERR "RDS/RDMA: failed to setup listener, " 215 "rdma_listen() returned %d\n", ret); 216 goto out; 217 } 218 219 rdsdebug("cm %p listening on port %u\n", cm_id, RDS_PORT); 220 221 *ret_cm_id = cm_id; 222 cm_id = NULL; 223 out: 224 if (cm_id) 225 rdma_destroy_id(cm_id); 226 return ret; 227 } 228 229 /* Initialize the RDS RDMA listeners. We create two listeners for 230 * compatibility reason. The one on RDS_PORT is used for IPv4 231 * requests only. The one on RDS_CM_PORT is used for IPv6 requests 232 * only. So only IPv6 enabled RDS module will communicate using this 233 * port. 234 */ 235 static int rds_rdma_listen_init(void) 236 { 237 int ret; 238 #if IS_ENABLED(CONFIG_IPV6) 239 struct sockaddr_in6 sin6; 240 #endif 241 struct sockaddr_in sin; 242 243 sin.sin_family = PF_INET; 244 sin.sin_addr.s_addr = htonl(INADDR_ANY); 245 sin.sin_port = htons(RDS_PORT); 246 ret = rds_rdma_listen_init_common(rds_rdma_cm_event_handler, 247 (struct sockaddr *)&sin, 248 &rds_rdma_listen_id); 249 if (ret != 0) 250 return ret; 251 252 #if IS_ENABLED(CONFIG_IPV6) 253 sin6.sin6_family = PF_INET6; 254 sin6.sin6_addr = in6addr_any; 255 sin6.sin6_port = htons(RDS_CM_PORT); 256 sin6.sin6_scope_id = 0; 257 sin6.sin6_flowinfo = 0; 258 ret = rds_rdma_listen_init_common(rds6_rdma_cm_event_handler, 259 (struct sockaddr *)&sin6, 260 &rds6_rdma_listen_id); 261 /* Keep going even when IPv6 is not enabled in the system. */ 262 if (ret != 0) 263 rdsdebug("Cannot set up IPv6 RDMA listener\n"); 264 #endif 265 return 0; 266 } 267 268 static void rds_rdma_listen_stop(void) 269 { 270 if (rds_rdma_listen_id) { 271 rdsdebug("cm %p\n", rds_rdma_listen_id); 272 rdma_destroy_id(rds_rdma_listen_id); 273 rds_rdma_listen_id = NULL; 274 } 275 #if IS_ENABLED(CONFIG_IPV6) 276 if (rds6_rdma_listen_id) { 277 rdsdebug("cm %p\n", rds6_rdma_listen_id); 278 rdma_destroy_id(rds6_rdma_listen_id); 279 rds6_rdma_listen_id = NULL; 280 } 281 #endif 282 } 283 284 static int rds_rdma_init(void) 285 { 286 int ret; 287 288 ret = rds_ib_init(); 289 if (ret) 290 goto out; 291 292 ret = rds_rdma_listen_init(); 293 if (ret) 294 rds_ib_exit(); 295 out: 296 return ret; 297 } 298 module_init(rds_rdma_init); 299 300 static void rds_rdma_exit(void) 301 { 302 /* stop listening first to ensure no new connections are attempted */ 303 rds_rdma_listen_stop(); 304 rds_ib_exit(); 305 } 306 module_exit(rds_rdma_exit); 307 308 MODULE_AUTHOR("Oracle Corporation <rds-devel@oss.oracle.com>"); 309 MODULE_DESCRIPTION("RDS: IB transport"); 310 MODULE_LICENSE("Dual BSD/GPL"); 311