1 /*
2  * Copyright(c) 2017 Intel Corporation.
3  *
4  * This file is provided under a dual BSD/GPLv2 license.  When using or
5  * redistributing this file, you may do so under either license.
6  *
7  * GPL LICENSE SUMMARY
8  *
9  * This program is free software; you can redistribute it and/or modify
10  * it under the terms of version 2 of the GNU General Public License as
11  * published by the Free Software Foundation.
12  *
13  * This program is distributed in the hope that it will be useful, but
14  * WITHOUT ANY WARRANTY; without even the implied warranty of
15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
16  * General Public License for more details.
17  *
18  * BSD LICENSE
19  *
20  * Redistribution and use in source and binary forms, with or without
21  * modification, are permitted provided that the following conditions
22  * are met:
23  *
24  *  - Redistributions of source code must retain the above copyright
25  *    notice, this list of conditions and the following disclaimer.
26  *  - Redistributions in binary form must reproduce the above copyright
27  *    notice, this list of conditions and the following disclaimer in
28  *    the documentation and/or other materials provided with the
29  *    distribution.
30  *  - Neither the name of Intel Corporation nor the names of its
31  *    contributors may be used to endorse or promote products derived
32  *    from this software without specific prior written permission.
33  *
34  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
35  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
36  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
37  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
38  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
39  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
40  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
41  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
42  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
43  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
44  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
45  *
46  */
47 
48 /*
49  * This file contains OPA Virtual Network Interface Controller (VNIC)
50  * Ethernet Management Agent (EMA) driver
51  */
52 
53 #include <linux/module.h>
54 #include <rdma/ib_addr.h>
55 #include <rdma/ib_smi.h>
56 
57 #include "opa_vnic_internal.h"
58 
59 #define DRV_VERSION "1.0"
60 char opa_vnic_driver_name[] = "opa_vnic";
61 const char opa_vnic_driver_version[] = DRV_VERSION;
62 
63 /*
64  * The trap service level is kept in bits 3 to 7 in the trap_sl_rsvd
65  * field in the class port info MAD.
66  */
67 #define GET_TRAP_SL_FROM_CLASS_PORT_INFO(x)  (((x) >> 3) & 0x1f)
68 
69 /* Cap trap bursts to a reasonable limit good for normal cases */
70 #define OPA_VNIC_TRAP_BURST_LIMIT 4
71 
72 /*
73  * VNIC trap limit timeout.
74  * Inverse of cap2_mask response time out (1.0737 secs) = 0.9
75  * secs approx IB spec 13.4.6.2.1 PortInfoSubnetTimeout and
76  * 13.4.9 Traps.
77  */
78 #define OPA_VNIC_TRAP_TIMEOUT  ((4096 * (1UL << 18)) / 1000)
79 
80 #define OPA_VNIC_UNSUP_ATTR  \
81 		cpu_to_be16(IB_MGMT_MAD_STATUS_UNSUPPORTED_METHOD_ATTRIB)
82 
83 #define OPA_VNIC_INVAL_ATTR  \
84 		cpu_to_be16(IB_MGMT_MAD_STATUS_INVALID_ATTRIB_VALUE)
85 
86 #define OPA_VNIC_CLASS_CAP_TRAP   0x1
87 
88 /* Maximum number of VNIC ports supported */
89 #define OPA_VNIC_MAX_NUM_VPORT    255
90 
91 /**
92  * struct opa_vnic_vema_port -- VNIC VEMA port details
93  * @cport: pointer to port
94  * @mad_agent: pointer to mad agent for port
95  * @class_port_info: Class port info information.
96  * @tid: Transaction id
97  * @port_num: OPA port number
98  * @vport_idr: vnic ports idr
99  * @event_handler: ib event handler
100  * @lock: adapter interface lock
101  */
102 struct opa_vnic_vema_port {
103 	struct opa_vnic_ctrl_port      *cport;
104 	struct ib_mad_agent            *mad_agent;
105 	struct opa_class_port_info      class_port_info;
106 	u64                             tid;
107 	u8                              port_num;
108 	struct idr                      vport_idr;
109 	struct ib_event_handler         event_handler;
110 
111 	/* Lock to query/update network adapter */
112 	struct mutex                    lock;
113 };
114 
115 static void opa_vnic_vema_add_one(struct ib_device *device);
116 static void opa_vnic_vema_rem_one(struct ib_device *device,
117 				  void *client_data);
118 
119 static struct ib_client opa_vnic_client = {
120 	.name   = opa_vnic_driver_name,
121 	.add    = opa_vnic_vema_add_one,
122 	.remove = opa_vnic_vema_rem_one,
123 };
124 
125 /**
126  * vema_get_vport_num -- Get the vnic from the mad
127  * @recvd_mad:  Received mad
128  *
129  * Return: returns value of the vnic port number
130  */
131 static inline u8 vema_get_vport_num(struct opa_vnic_vema_mad *recvd_mad)
132 {
133 	return be32_to_cpu(recvd_mad->mad_hdr.attr_mod) & 0xff;
134 }
135 
136 /**
137  * vema_get_vport_adapter -- Get vnic port adapter from recvd mad
138  * @recvd_mad: received mad
139  * @port: ptr to port struct on which MAD was recvd
140  *
141  * Return: vnic adapter
142  */
143 static inline struct opa_vnic_adapter *
144 vema_get_vport_adapter(struct opa_vnic_vema_mad *recvd_mad,
145 		       struct opa_vnic_vema_port *port)
146 {
147 	u8 vport_num = vema_get_vport_num(recvd_mad);
148 
149 	return idr_find(&port->vport_idr, vport_num);
150 }
151 
152 /**
153  * vema_mac_tbl_req_ok -- Check if mac request has correct values
154  * @mac_tbl: mac table
155  *
156  * This function checks for the validity of the offset and number of
157  * entries required.
158  *
159  * Return: true if offset and num_entries are valid
160  */
161 static inline bool vema_mac_tbl_req_ok(struct opa_veswport_mactable *mac_tbl)
162 {
163 	u16 offset, num_entries;
164 	u16 req_entries = ((OPA_VNIC_EMA_DATA - sizeof(*mac_tbl)) /
165 			   sizeof(mac_tbl->tbl_entries[0]));
166 
167 	offset = be16_to_cpu(mac_tbl->offset);
168 	num_entries = be16_to_cpu(mac_tbl->num_entries);
169 
170 	return ((num_entries <= req_entries) &&
171 		(offset + num_entries <= OPA_VNIC_MAC_TBL_MAX_ENTRIES));
172 }
173 
174 /*
175  * Return the power on default values in the port info structure
176  * in big endian format as required by MAD.
177  */
178 static inline void vema_get_pod_values(struct opa_veswport_info *port_info)
179 {
180 	memset(port_info, 0, sizeof(*port_info));
181 	port_info->vport.max_mac_tbl_ent =
182 		cpu_to_be16(OPA_VNIC_MAC_TBL_MAX_ENTRIES);
183 	port_info->vport.max_smac_ent =
184 		cpu_to_be16(OPA_VNIC_MAX_SMAC_LIMIT);
185 	port_info->vport.oper_state = OPA_VNIC_STATE_DROP_ALL;
186 	port_info->vport.config_state = OPA_VNIC_STATE_DROP_ALL;
187 }
188 
189 /**
190  * vema_add_vport -- Add a new vnic port
191  * @port: ptr to opa_vnic_vema_port struct
192  * @vport_num: vnic port number (to be added)
193  *
194  * Return a pointer to the vnic adapter structure
195  */
196 static struct opa_vnic_adapter *vema_add_vport(struct opa_vnic_vema_port *port,
197 					       u8 vport_num)
198 {
199 	struct opa_vnic_ctrl_port *cport = port->cport;
200 	struct opa_vnic_adapter *adapter;
201 
202 	adapter = opa_vnic_add_netdev(cport->ibdev, port->port_num, vport_num);
203 	if (!IS_ERR(adapter)) {
204 		int rc;
205 
206 		adapter->cport = cport;
207 		rc = idr_alloc(&port->vport_idr, adapter, vport_num,
208 			       vport_num + 1, GFP_NOWAIT);
209 		if (rc < 0) {
210 			opa_vnic_rem_netdev(adapter);
211 			adapter = ERR_PTR(rc);
212 		}
213 	}
214 
215 	return adapter;
216 }
217 
218 /**
219  * vema_get_class_port_info -- Get class info for port
220  * @port:  Port on whic MAD was received
221  * @recvd_mad: pointer to the received mad
222  * @rsp_mad:   pointer to respose mad
223  *
224  * This function copies the latest class port info value set for the
225  * port and stores it for generating traps
226  */
227 static void vema_get_class_port_info(struct opa_vnic_vema_port *port,
228 				     struct opa_vnic_vema_mad *recvd_mad,
229 				     struct opa_vnic_vema_mad *rsp_mad)
230 {
231 	struct opa_class_port_info *port_info;
232 
233 	port_info = (struct opa_class_port_info *)rsp_mad->data;
234 	memcpy(port_info, &port->class_port_info, sizeof(*port_info));
235 	port_info->base_version = OPA_MGMT_BASE_VERSION,
236 	port_info->class_version = OPA_EMA_CLASS_VERSION;
237 
238 	/*
239 	 * Set capability mask bit indicating agent generates traps,
240 	 * and set the maximum number of VNIC ports supported.
241 	 */
242 	port_info->cap_mask = cpu_to_be16((OPA_VNIC_CLASS_CAP_TRAP |
243 					   (OPA_VNIC_MAX_NUM_VPORT << 8)));
244 
245 	/*
246 	 * Since a get routine is always sent by the EM first we
247 	 * set the expected response time to
248 	 * 4.096 usec * 2^18 == 1.0737 sec here.
249 	 */
250 	port_info->cap_mask2_resp_time = cpu_to_be32(18);
251 }
252 
253 /**
254  * vema_set_class_port_info -- Get class info for port
255  * @port:  Port on whic MAD was received
256  * @recvd_mad: pointer to the received mad
257  * @rsp_mad:   pointer to respose mad
258  *
259  * This function updates the port class info for the specific vnic
260  * and sets up the response mad data
261  */
262 static void vema_set_class_port_info(struct opa_vnic_vema_port *port,
263 				     struct opa_vnic_vema_mad *recvd_mad,
264 				     struct opa_vnic_vema_mad *rsp_mad)
265 {
266 	memcpy(&port->class_port_info, recvd_mad->data,
267 	       sizeof(port->class_port_info));
268 
269 	vema_get_class_port_info(port, recvd_mad, rsp_mad);
270 }
271 
272 /**
273  * vema_get_veswport_info -- Get veswport info
274  * @port:      source port on which MAD was received
275  * @recvd_mad: pointer to the received mad
276  * @rsp_mad:   pointer to respose mad
277  */
278 static void vema_get_veswport_info(struct opa_vnic_vema_port *port,
279 				   struct opa_vnic_vema_mad *recvd_mad,
280 				   struct opa_vnic_vema_mad *rsp_mad)
281 {
282 	struct opa_veswport_info *port_info =
283 				  (struct opa_veswport_info *)rsp_mad->data;
284 	struct opa_vnic_adapter *adapter;
285 
286 	adapter = vema_get_vport_adapter(recvd_mad, port);
287 	if (adapter) {
288 		memset(port_info, 0, sizeof(*port_info));
289 		opa_vnic_get_vesw_info(adapter, &port_info->vesw);
290 		opa_vnic_get_per_veswport_info(adapter,
291 					       &port_info->vport);
292 	} else {
293 		vema_get_pod_values(port_info);
294 	}
295 }
296 
297 /**
298  * vema_set_veswport_info -- Set veswport info
299  * @port:      source port on which MAD was received
300  * @recvd_mad: pointer to the received mad
301  * @rsp_mad:   pointer to respose mad
302  *
303  * This function gets the port class infor for vnic
304  */
305 static void vema_set_veswport_info(struct opa_vnic_vema_port *port,
306 				   struct opa_vnic_vema_mad *recvd_mad,
307 				   struct opa_vnic_vema_mad *rsp_mad)
308 {
309 	struct opa_vnic_ctrl_port *cport = port->cport;
310 	struct opa_veswport_info *port_info;
311 	struct opa_vnic_adapter *adapter;
312 	u8 vport_num;
313 
314 	vport_num = vema_get_vport_num(recvd_mad);
315 
316 	adapter = vema_get_vport_adapter(recvd_mad, port);
317 	if (!adapter) {
318 		adapter = vema_add_vport(port, vport_num);
319 		if (IS_ERR(adapter)) {
320 			c_err("failed to add vport %d: %ld\n",
321 			      vport_num, PTR_ERR(adapter));
322 			goto err_exit;
323 		}
324 	}
325 
326 	port_info = (struct opa_veswport_info *)recvd_mad->data;
327 	opa_vnic_set_vesw_info(adapter, &port_info->vesw);
328 	opa_vnic_set_per_veswport_info(adapter, &port_info->vport);
329 
330 	/* Process the new config settings */
331 	opa_vnic_process_vema_config(adapter);
332 
333 	vema_get_veswport_info(port, recvd_mad, rsp_mad);
334 	return;
335 
336 err_exit:
337 	rsp_mad->mad_hdr.status = OPA_VNIC_INVAL_ATTR;
338 }
339 
340 /**
341  * vema_get_mac_entries -- Get MAC entries in VNIC MAC table
342  * @port:      source port on which MAD was received
343  * @recvd_mad: pointer to the received mad
344  * @rsp_mad:   pointer to respose mad
345  *
346  * This function gets the MAC entries that are programmed into
347  * the VNIC MAC forwarding table. It checks for the validity of
348  * the index into the MAC table and the number of entries that
349  * are to be retrieved.
350  */
351 static void vema_get_mac_entries(struct opa_vnic_vema_port *port,
352 				 struct opa_vnic_vema_mad *recvd_mad,
353 				 struct opa_vnic_vema_mad *rsp_mad)
354 {
355 	struct opa_veswport_mactable *mac_tbl_in, *mac_tbl_out;
356 	struct opa_vnic_adapter *adapter;
357 
358 	adapter = vema_get_vport_adapter(recvd_mad, port);
359 	if (!adapter) {
360 		rsp_mad->mad_hdr.status = OPA_VNIC_INVAL_ATTR;
361 		return;
362 	}
363 
364 	mac_tbl_in = (struct opa_veswport_mactable *)recvd_mad->data;
365 	mac_tbl_out = (struct opa_veswport_mactable *)rsp_mad->data;
366 
367 	if (vema_mac_tbl_req_ok(mac_tbl_in)) {
368 		mac_tbl_out->offset = mac_tbl_in->offset;
369 		mac_tbl_out->num_entries = mac_tbl_in->num_entries;
370 		opa_vnic_query_mac_tbl(adapter, mac_tbl_out);
371 	} else {
372 		rsp_mad->mad_hdr.status = OPA_VNIC_INVAL_ATTR;
373 	}
374 }
375 
376 /**
377  * vema_set_mac_entries -- Set MAC entries in VNIC MAC table
378  * @port:      source port on which MAD was received
379  * @recvd_mad: pointer to the received mad
380  * @rsp_mad:   pointer to respose mad
381  *
382  * This function sets the MAC entries in the VNIC forwarding table
383  * It checks for the validity of the index and the number of forwarding
384  * table entries to be programmed.
385  */
386 static void vema_set_mac_entries(struct opa_vnic_vema_port *port,
387 				 struct opa_vnic_vema_mad *recvd_mad,
388 				 struct opa_vnic_vema_mad *rsp_mad)
389 {
390 	struct opa_veswport_mactable *mac_tbl;
391 	struct opa_vnic_adapter *adapter;
392 
393 	adapter = vema_get_vport_adapter(recvd_mad, port);
394 	if (!adapter) {
395 		rsp_mad->mad_hdr.status = OPA_VNIC_INVAL_ATTR;
396 		return;
397 	}
398 
399 	mac_tbl = (struct opa_veswport_mactable *)recvd_mad->data;
400 	if (vema_mac_tbl_req_ok(mac_tbl)) {
401 		if (opa_vnic_update_mac_tbl(adapter, mac_tbl))
402 			rsp_mad->mad_hdr.status = OPA_VNIC_UNSUP_ATTR;
403 	} else {
404 		rsp_mad->mad_hdr.status = OPA_VNIC_UNSUP_ATTR;
405 	}
406 	vema_get_mac_entries(port, recvd_mad, rsp_mad);
407 }
408 
409 /**
410  * vema_set_delete_vesw -- Reset VESW info to POD values
411  * @port:      source port on which MAD was received
412  * @recvd_mad: pointer to the received mad
413  * @rsp_mad:   pointer to respose mad
414  *
415  * This function clears all the fields of veswport info for the requested vesw
416  * and sets them back to the power-on default values. It does not delete the
417  * vesw.
418  */
419 static void vema_set_delete_vesw(struct opa_vnic_vema_port *port,
420 				 struct opa_vnic_vema_mad *recvd_mad,
421 				 struct opa_vnic_vema_mad *rsp_mad)
422 {
423 	struct opa_veswport_info *port_info =
424 				  (struct opa_veswport_info *)rsp_mad->data;
425 	struct opa_vnic_adapter *adapter;
426 
427 	adapter = vema_get_vport_adapter(recvd_mad, port);
428 	if (!adapter) {
429 		rsp_mad->mad_hdr.status = OPA_VNIC_INVAL_ATTR;
430 		return;
431 	}
432 
433 	vema_get_pod_values(port_info);
434 	opa_vnic_set_vesw_info(adapter, &port_info->vesw);
435 	opa_vnic_set_per_veswport_info(adapter, &port_info->vport);
436 
437 	/* Process the new config settings */
438 	opa_vnic_process_vema_config(adapter);
439 
440 	opa_vnic_release_mac_tbl(adapter);
441 
442 	vema_get_veswport_info(port, recvd_mad, rsp_mad);
443 }
444 
445 /**
446  * vema_get_mac_list -- Get the unicast/multicast macs.
447  * @port:      source port on which MAD was received
448  * @recvd_mad: Received mad contains fields to set vnic parameters
449  * @rsp_mad:   Response mad to be built
450  * @attr_id:   Attribute ID indicating multicast or unicast mac list
451  */
452 static void vema_get_mac_list(struct opa_vnic_vema_port *port,
453 			      struct opa_vnic_vema_mad *recvd_mad,
454 			      struct opa_vnic_vema_mad *rsp_mad,
455 			      u16 attr_id)
456 {
457 	struct opa_veswport_iface_macs *macs_in, *macs_out;
458 	int max_entries = (OPA_VNIC_EMA_DATA - sizeof(*macs_out)) / ETH_ALEN;
459 	struct opa_vnic_adapter *adapter;
460 
461 	adapter = vema_get_vport_adapter(recvd_mad, port);
462 	if (!adapter) {
463 		rsp_mad->mad_hdr.status = OPA_VNIC_INVAL_ATTR;
464 		return;
465 	}
466 
467 	macs_in = (struct opa_veswport_iface_macs *)recvd_mad->data;
468 	macs_out = (struct opa_veswport_iface_macs *)rsp_mad->data;
469 
470 	macs_out->start_idx = macs_in->start_idx;
471 	if (macs_in->num_macs_in_msg)
472 		macs_out->num_macs_in_msg = macs_in->num_macs_in_msg;
473 	else
474 		macs_out->num_macs_in_msg = cpu_to_be16(max_entries);
475 
476 	if (attr_id == OPA_EM_ATTR_IFACE_MCAST_MACS)
477 		opa_vnic_query_mcast_macs(adapter, macs_out);
478 	else
479 		opa_vnic_query_ucast_macs(adapter, macs_out);
480 }
481 
482 /**
483  * vema_get_summary_counters -- Gets summary counters.
484  * @port:      source port on which MAD was received
485  * @recvd_mad: Received mad contains fields to set vnic parameters
486  * @rsp_mad:   Response mad to be built
487  */
488 static void vema_get_summary_counters(struct opa_vnic_vema_port *port,
489 				      struct opa_vnic_vema_mad *recvd_mad,
490 				      struct opa_vnic_vema_mad *rsp_mad)
491 {
492 	struct opa_veswport_summary_counters *cntrs;
493 	struct opa_vnic_adapter *adapter;
494 
495 	adapter = vema_get_vport_adapter(recvd_mad, port);
496 	if (adapter) {
497 		cntrs = (struct opa_veswport_summary_counters *)rsp_mad->data;
498 		opa_vnic_get_summary_counters(adapter, cntrs);
499 	} else {
500 		rsp_mad->mad_hdr.status = OPA_VNIC_INVAL_ATTR;
501 	}
502 }
503 
504 /**
505  * vema_get_error_counters -- Gets summary counters.
506  * @port:      source port on which MAD was received
507  * @recvd_mad: Received mad contains fields to set vnic parameters
508  * @rsp_mad:   Response mad to be built
509  */
510 static void vema_get_error_counters(struct opa_vnic_vema_port *port,
511 				    struct opa_vnic_vema_mad *recvd_mad,
512 				    struct opa_vnic_vema_mad *rsp_mad)
513 {
514 	struct opa_veswport_error_counters *cntrs;
515 	struct opa_vnic_adapter *adapter;
516 
517 	adapter = vema_get_vport_adapter(recvd_mad, port);
518 	if (adapter) {
519 		cntrs = (struct opa_veswport_error_counters *)rsp_mad->data;
520 		opa_vnic_get_error_counters(adapter, cntrs);
521 	} else {
522 		rsp_mad->mad_hdr.status = OPA_VNIC_INVAL_ATTR;
523 	}
524 }
525 
526 /**
527  * vema_get -- Process received get MAD
528  * @port:      source port on which MAD was received
529  * @recvd_mad: Received mad
530  * @rsp_mad:   Response mad to be built
531  */
532 static void vema_get(struct opa_vnic_vema_port *port,
533 		     struct opa_vnic_vema_mad *recvd_mad,
534 		     struct opa_vnic_vema_mad *rsp_mad)
535 {
536 	u16 attr_id = be16_to_cpu(recvd_mad->mad_hdr.attr_id);
537 
538 	switch (attr_id) {
539 	case OPA_EM_ATTR_CLASS_PORT_INFO:
540 		vema_get_class_port_info(port, recvd_mad, rsp_mad);
541 		break;
542 	case OPA_EM_ATTR_VESWPORT_INFO:
543 		vema_get_veswport_info(port, recvd_mad, rsp_mad);
544 		break;
545 	case OPA_EM_ATTR_VESWPORT_MAC_ENTRIES:
546 		vema_get_mac_entries(port, recvd_mad, rsp_mad);
547 		break;
548 	case OPA_EM_ATTR_IFACE_UCAST_MACS:
549 		/* fall through */
550 	case OPA_EM_ATTR_IFACE_MCAST_MACS:
551 		vema_get_mac_list(port, recvd_mad, rsp_mad, attr_id);
552 		break;
553 	case OPA_EM_ATTR_VESWPORT_SUMMARY_COUNTERS:
554 		vema_get_summary_counters(port, recvd_mad, rsp_mad);
555 		break;
556 	case OPA_EM_ATTR_VESWPORT_ERROR_COUNTERS:
557 		vema_get_error_counters(port, recvd_mad, rsp_mad);
558 		break;
559 	default:
560 		rsp_mad->mad_hdr.status = OPA_VNIC_UNSUP_ATTR;
561 		break;
562 	}
563 }
564 
565 /**
566  * vema_set -- Process received set MAD
567  * @port:      source port on which MAD was received
568  * @recvd_mad: Received mad contains fields to set vnic parameters
569  * @rsp_mad:   Response mad to be built
570  */
571 static void vema_set(struct opa_vnic_vema_port *port,
572 		     struct opa_vnic_vema_mad *recvd_mad,
573 		     struct opa_vnic_vema_mad *rsp_mad)
574 {
575 	u16 attr_id = be16_to_cpu(recvd_mad->mad_hdr.attr_id);
576 
577 	switch (attr_id) {
578 	case OPA_EM_ATTR_CLASS_PORT_INFO:
579 		vema_set_class_port_info(port, recvd_mad, rsp_mad);
580 		break;
581 	case OPA_EM_ATTR_VESWPORT_INFO:
582 		vema_set_veswport_info(port, recvd_mad, rsp_mad);
583 		break;
584 	case OPA_EM_ATTR_VESWPORT_MAC_ENTRIES:
585 		vema_set_mac_entries(port, recvd_mad, rsp_mad);
586 		break;
587 	case OPA_EM_ATTR_DELETE_VESW:
588 		vema_set_delete_vesw(port, recvd_mad, rsp_mad);
589 		break;
590 	default:
591 		rsp_mad->mad_hdr.status = OPA_VNIC_UNSUP_ATTR;
592 		break;
593 	}
594 }
595 
596 /**
597  * vema_send -- Send handler for VEMA MAD agent
598  * @mad_agent: pointer to the mad agent
599  * @mad_wc:    pointer to mad send work completion information
600  *
601  * Free all the data structures associated with the sent MAD
602  */
603 static void vema_send(struct ib_mad_agent *mad_agent,
604 		      struct ib_mad_send_wc *mad_wc)
605 {
606 	rdma_destroy_ah(mad_wc->send_buf->ah);
607 	ib_free_send_mad(mad_wc->send_buf);
608 }
609 
610 /**
611  * vema_recv -- Recv handler for VEMA MAD agent
612  * @mad_agent: pointer to the mad agent
613  * @send_buf: Send buffer if found, else NULL
614  * @mad_wc:    pointer to mad send work completion information
615  *
616  * Handle only set and get methods and respond to other methods
617  * as unsupported. Allocate response buffer and address handle
618  * for the response MAD.
619  */
620 static void vema_recv(struct ib_mad_agent *mad_agent,
621 		      struct ib_mad_send_buf *send_buf,
622 		      struct ib_mad_recv_wc *mad_wc)
623 {
624 	struct opa_vnic_vema_port *port;
625 	struct ib_ah              *ah;
626 	struct ib_mad_send_buf    *rsp;
627 	struct opa_vnic_vema_mad  *vema_mad;
628 
629 	if (!mad_wc || !mad_wc->recv_buf.mad)
630 		return;
631 
632 	port = mad_agent->context;
633 	ah = ib_create_ah_from_wc(mad_agent->qp->pd, mad_wc->wc,
634 				  mad_wc->recv_buf.grh, mad_agent->port_num);
635 	if (IS_ERR(ah))
636 		goto free_recv_mad;
637 
638 	rsp = ib_create_send_mad(mad_agent, mad_wc->wc->src_qp,
639 				 mad_wc->wc->pkey_index, 0,
640 				 IB_MGMT_VENDOR_HDR, OPA_VNIC_EMA_DATA,
641 				 GFP_KERNEL, OPA_MGMT_BASE_VERSION);
642 	if (IS_ERR(rsp))
643 		goto err_rsp;
644 
645 	rsp->ah = ah;
646 	vema_mad = rsp->mad;
647 	memcpy(vema_mad, mad_wc->recv_buf.mad, IB_MGMT_VENDOR_HDR);
648 	vema_mad->mad_hdr.method = IB_MGMT_METHOD_GET_RESP;
649 	vema_mad->mad_hdr.status = 0;
650 
651 	/* Lock ensures network adapter is not removed */
652 	mutex_lock(&port->lock);
653 
654 	switch (mad_wc->recv_buf.mad->mad_hdr.method) {
655 	case IB_MGMT_METHOD_GET:
656 		vema_get(port, (struct opa_vnic_vema_mad *)mad_wc->recv_buf.mad,
657 			 vema_mad);
658 		break;
659 	case IB_MGMT_METHOD_SET:
660 		vema_set(port, (struct opa_vnic_vema_mad *)mad_wc->recv_buf.mad,
661 			 vema_mad);
662 		break;
663 	default:
664 		vema_mad->mad_hdr.status = OPA_VNIC_UNSUP_ATTR;
665 		break;
666 	}
667 	mutex_unlock(&port->lock);
668 
669 	if (!ib_post_send_mad(rsp, NULL)) {
670 		/*
671 		 * with post send successful ah and send mad
672 		 * will be destroyed in send handler
673 		 */
674 		goto free_recv_mad;
675 	}
676 
677 	ib_free_send_mad(rsp);
678 
679 err_rsp:
680 	rdma_destroy_ah(ah);
681 free_recv_mad:
682 	ib_free_recv_mad(mad_wc);
683 }
684 
685 /**
686  * vema_get_port -- Gets the opa_vnic_vema_port
687  * @cport: pointer to control dev
688  * @port_num: Port number
689  *
690  * This function loops through the ports and returns
691  * the opa_vnic_vema port structure that is associated
692  * with the OPA port number
693  *
694  * Return: ptr to requested opa_vnic_vema_port strucure
695  *         if success, NULL if not
696  */
697 static struct opa_vnic_vema_port *
698 vema_get_port(struct opa_vnic_ctrl_port *cport, u8 port_num)
699 {
700 	struct opa_vnic_vema_port *port = (void *)cport + sizeof(*cport);
701 
702 	if (port_num > cport->num_ports)
703 		return NULL;
704 
705 	return port + (port_num - 1);
706 }
707 
708 /**
709  * opa_vnic_vema_send_trap -- This function sends a trap to the EM
710  * @cport: pointer to vnic control port
711  * @data: pointer to trap data filled by calling function
712  * @lid:  issuers lid (encap_slid from vesw_port_info)
713  *
714  * This function is called from the VNIC driver to send a trap if there
715  * is somethng the EM should be notified about. These events currently
716  * are
717  * 1) UNICAST INTERFACE MACADDRESS changes
718  * 2) MULTICAST INTERFACE MACADDRESS changes
719  * 3) ETHERNET LINK STATUS changes
720  * While allocating the send mad the remote site qpn used is 1
721  * as this is the well known QP.
722  *
723  */
724 void opa_vnic_vema_send_trap(struct opa_vnic_adapter *adapter,
725 			     struct __opa_veswport_trap *data, u32 lid)
726 {
727 	struct opa_vnic_ctrl_port *cport = adapter->cport;
728 	struct ib_mad_send_buf *send_buf;
729 	struct opa_vnic_vema_port *port;
730 	struct ib_device *ibp;
731 	struct opa_vnic_vema_mad_trap *trap_mad;
732 	struct opa_class_port_info *class;
733 	struct rdma_ah_attr ah_attr;
734 	struct ib_ah *ah;
735 	struct opa_veswport_trap *trap;
736 	u32 trap_lid;
737 	u16 pkey_idx;
738 
739 	if (!cport)
740 		goto err_exit;
741 	ibp = cport->ibdev;
742 	port = vema_get_port(cport, data->opaportnum);
743 	if (!port || !port->mad_agent)
744 		goto err_exit;
745 
746 	if (time_before(jiffies, adapter->trap_timeout)) {
747 		if (adapter->trap_count == OPA_VNIC_TRAP_BURST_LIMIT) {
748 			v_warn("Trap rate exceeded\n");
749 			goto err_exit;
750 		} else {
751 			adapter->trap_count++;
752 		}
753 	} else {
754 		adapter->trap_count = 0;
755 	}
756 
757 	class = &port->class_port_info;
758 	/* Set up address handle */
759 	memset(&ah_attr, 0, sizeof(ah_attr));
760 	ah_attr.type = rdma_ah_find_type(ibp, port->port_num);
761 	rdma_ah_set_sl(&ah_attr,
762 		       GET_TRAP_SL_FROM_CLASS_PORT_INFO(class->trap_sl_rsvd));
763 	rdma_ah_set_port_num(&ah_attr, port->port_num);
764 	trap_lid = be32_to_cpu(class->trap_lid);
765 	/*
766 	 * check for trap lid validity, must not be zero
767 	 * The trap sink could change after we fashion the MAD but since traps
768 	 * are not guaranteed we won't use a lock as anyway the change will take
769 	 * place even with locking.
770 	 */
771 	if (!trap_lid) {
772 		c_err("%s: Invalid dlid\n", __func__);
773 		goto err_exit;
774 	}
775 
776 	rdma_ah_set_dlid(&ah_attr, trap_lid);
777 	ah = rdma_create_ah(port->mad_agent->qp->pd, &ah_attr);
778 	if (IS_ERR(ah)) {
779 		c_err("%s:Couldn't create new AH = %p\n", __func__, ah);
780 		c_err("%s:dlid = %d, sl = %d, port = %d\n", __func__,
781 		      rdma_ah_get_dlid(&ah_attr), rdma_ah_get_sl(&ah_attr),
782 		      rdma_ah_get_port_num(&ah_attr));
783 		goto err_exit;
784 	}
785 
786 	if (ib_find_pkey(ibp, data->opaportnum, IB_DEFAULT_PKEY_FULL,
787 			 &pkey_idx) < 0) {
788 		c_err("%s:full key not found, defaulting to partial\n",
789 		      __func__);
790 		if (ib_find_pkey(ibp, data->opaportnum, IB_DEFAULT_PKEY_PARTIAL,
791 				 &pkey_idx) < 0)
792 			pkey_idx = 1;
793 	}
794 
795 	send_buf = ib_create_send_mad(port->mad_agent, 1, pkey_idx, 0,
796 				      IB_MGMT_VENDOR_HDR, IB_MGMT_MAD_DATA,
797 				      GFP_ATOMIC, OPA_MGMT_BASE_VERSION);
798 	if (IS_ERR(send_buf)) {
799 		c_err("%s:Couldn't allocate send buf\n", __func__);
800 		goto err_sndbuf;
801 	}
802 
803 	send_buf->ah = ah;
804 
805 	/* Set up common MAD hdr */
806 	trap_mad = send_buf->mad;
807 	trap_mad->mad_hdr.base_version = OPA_MGMT_BASE_VERSION;
808 	trap_mad->mad_hdr.mgmt_class = OPA_MGMT_CLASS_INTEL_EMA;
809 	trap_mad->mad_hdr.class_version = OPA_EMA_CLASS_VERSION;
810 	trap_mad->mad_hdr.method = IB_MGMT_METHOD_TRAP;
811 	port->tid++;
812 	trap_mad->mad_hdr.tid = cpu_to_be64(port->tid);
813 	trap_mad->mad_hdr.attr_id = IB_SMP_ATTR_NOTICE;
814 
815 	/* Set up vendor OUI */
816 	trap_mad->oui[0] = INTEL_OUI_1;
817 	trap_mad->oui[1] = INTEL_OUI_2;
818 	trap_mad->oui[2] = INTEL_OUI_3;
819 
820 	/* Setup notice attribute portion */
821 	trap_mad->notice.gen_type = OPA_INTEL_EMA_NOTICE_TYPE_INFO << 1;
822 	trap_mad->notice.oui_1 = INTEL_OUI_1;
823 	trap_mad->notice.oui_2 = INTEL_OUI_2;
824 	trap_mad->notice.oui_3 = INTEL_OUI_3;
825 	trap_mad->notice.issuer_lid = cpu_to_be32(lid);
826 
827 	/* copy the actual trap data */
828 	trap = (struct opa_veswport_trap *)trap_mad->notice.raw_data;
829 	trap->fabric_id = cpu_to_be16(data->fabric_id);
830 	trap->veswid = cpu_to_be16(data->veswid);
831 	trap->veswportnum = cpu_to_be32(data->veswportnum);
832 	trap->opaportnum = cpu_to_be16(data->opaportnum);
833 	trap->veswportindex = data->veswportindex;
834 	trap->opcode = data->opcode;
835 
836 	/* If successful send set up rate limit timeout else bail */
837 	if (ib_post_send_mad(send_buf, NULL)) {
838 		ib_free_send_mad(send_buf);
839 	} else {
840 		if (adapter->trap_count)
841 			return;
842 		adapter->trap_timeout = jiffies +
843 					usecs_to_jiffies(OPA_VNIC_TRAP_TIMEOUT);
844 		return;
845 	}
846 
847 err_sndbuf:
848 	rdma_destroy_ah(ah);
849 err_exit:
850 	v_err("Aborting trap\n");
851 }
852 
853 static int vema_rem_vport(int id, void *p, void *data)
854 {
855 	struct opa_vnic_adapter *adapter = p;
856 
857 	opa_vnic_rem_netdev(adapter);
858 	return 0;
859 }
860 
861 static int vema_enable_vport(int id, void *p, void *data)
862 {
863 	struct opa_vnic_adapter *adapter = p;
864 
865 	netif_carrier_on(adapter->netdev);
866 	return 0;
867 }
868 
869 static int vema_disable_vport(int id, void *p, void *data)
870 {
871 	struct opa_vnic_adapter *adapter = p;
872 
873 	netif_carrier_off(adapter->netdev);
874 	return 0;
875 }
876 
877 static void opa_vnic_event(struct ib_event_handler *handler,
878 			   struct ib_event *record)
879 {
880 	struct opa_vnic_vema_port *port =
881 		container_of(handler, struct opa_vnic_vema_port, event_handler);
882 	struct opa_vnic_ctrl_port *cport = port->cport;
883 
884 	if (record->element.port_num != port->port_num)
885 		return;
886 
887 	c_dbg("OPA_VNIC received event %d on device %s port %d\n",
888 	      record->event, record->device->name, record->element.port_num);
889 
890 	if (record->event == IB_EVENT_PORT_ERR)
891 		idr_for_each(&port->vport_idr, vema_disable_vport, NULL);
892 	if (record->event == IB_EVENT_PORT_ACTIVE)
893 		idr_for_each(&port->vport_idr, vema_enable_vport, NULL);
894 }
895 
896 /**
897  * vema_unregister -- Unregisters agent
898  * @cport: pointer to control port
899  *
900  * This deletes the registration by VEMA for MADs
901  */
902 static void vema_unregister(struct opa_vnic_ctrl_port *cport)
903 {
904 	int i;
905 
906 	for (i = 1; i <= cport->num_ports; i++) {
907 		struct opa_vnic_vema_port *port = vema_get_port(cport, i);
908 
909 		if (!port->mad_agent)
910 			continue;
911 
912 		/* Lock ensures no MAD is being processed */
913 		mutex_lock(&port->lock);
914 		idr_for_each(&port->vport_idr, vema_rem_vport, NULL);
915 		mutex_unlock(&port->lock);
916 
917 		ib_unregister_mad_agent(port->mad_agent);
918 		port->mad_agent = NULL;
919 		mutex_destroy(&port->lock);
920 		idr_destroy(&port->vport_idr);
921 		ib_unregister_event_handler(&port->event_handler);
922 	}
923 }
924 
925 /**
926  * vema_register -- Registers agent
927  * @cport: pointer to control port
928  *
929  * This function registers the handlers for the VEMA MADs
930  *
931  * Return: returns 0 on success. non zero otherwise
932  */
933 static int vema_register(struct opa_vnic_ctrl_port *cport)
934 {
935 	struct ib_mad_reg_req reg_req = {
936 		.mgmt_class = OPA_MGMT_CLASS_INTEL_EMA,
937 		.mgmt_class_version = OPA_MGMT_BASE_VERSION,
938 		.oui = { INTEL_OUI_1, INTEL_OUI_2, INTEL_OUI_3 }
939 	};
940 	int i;
941 
942 	set_bit(IB_MGMT_METHOD_GET, reg_req.method_mask);
943 	set_bit(IB_MGMT_METHOD_SET, reg_req.method_mask);
944 
945 	/* register ib event handler and mad agent for each port on dev */
946 	for (i = 1; i <= cport->num_ports; i++) {
947 		struct opa_vnic_vema_port *port = vema_get_port(cport, i);
948 		int ret;
949 
950 		port->cport = cport;
951 		port->port_num = i;
952 
953 		INIT_IB_EVENT_HANDLER(&port->event_handler,
954 				      cport->ibdev, opa_vnic_event);
955 		ret = ib_register_event_handler(&port->event_handler);
956 		if (ret) {
957 			c_err("port %d: event handler register failed\n", i);
958 			vema_unregister(cport);
959 			return ret;
960 		}
961 
962 		idr_init(&port->vport_idr);
963 		mutex_init(&port->lock);
964 		port->mad_agent = ib_register_mad_agent(cport->ibdev, i,
965 							IB_QPT_GSI, &reg_req,
966 							IB_MGMT_RMPP_VERSION,
967 							vema_send, vema_recv,
968 							port, 0);
969 		if (IS_ERR(port->mad_agent)) {
970 			ret = PTR_ERR(port->mad_agent);
971 			port->mad_agent = NULL;
972 			mutex_destroy(&port->lock);
973 			idr_destroy(&port->vport_idr);
974 			vema_unregister(cport);
975 			return ret;
976 		}
977 	}
978 
979 	return 0;
980 }
981 
982 /**
983  * opa_vnic_vema_add_one -- Handle new ib device
984  * @device: ib device pointer
985  *
986  * Allocate the vnic control port and initialize it.
987  */
988 static void opa_vnic_vema_add_one(struct ib_device *device)
989 {
990 	struct opa_vnic_ctrl_port *cport;
991 	int rc, size = sizeof(*cport);
992 
993 	if (!rdma_cap_opa_vnic(device))
994 		return;
995 
996 	size += device->phys_port_cnt * sizeof(struct opa_vnic_vema_port);
997 	cport = kzalloc(size, GFP_KERNEL);
998 	if (!cport)
999 		return;
1000 
1001 	cport->num_ports = device->phys_port_cnt;
1002 	cport->ibdev = device;
1003 
1004 	/* Initialize opa vnic management agent (vema) */
1005 	rc = vema_register(cport);
1006 	if (!rc)
1007 		c_info("VNIC client initialized\n");
1008 
1009 	ib_set_client_data(device, &opa_vnic_client, cport);
1010 }
1011 
1012 /**
1013  * opa_vnic_vema_rem_one -- Handle ib device removal
1014  * @device: ib device pointer
1015  * @client_data: ib client data
1016  *
1017  * Uninitialize and free the vnic control port.
1018  */
1019 static void opa_vnic_vema_rem_one(struct ib_device *device,
1020 				  void *client_data)
1021 {
1022 	struct opa_vnic_ctrl_port *cport = client_data;
1023 
1024 	if (!cport)
1025 		return;
1026 
1027 	c_info("removing VNIC client\n");
1028 	vema_unregister(cport);
1029 	kfree(cport);
1030 }
1031 
1032 static int __init opa_vnic_init(void)
1033 {
1034 	int rc;
1035 
1036 	pr_info("OPA Virtual Network Driver - v%s\n",
1037 		opa_vnic_driver_version);
1038 
1039 	rc = ib_register_client(&opa_vnic_client);
1040 	if (rc)
1041 		pr_err("VNIC driver register failed %d\n", rc);
1042 
1043 	return rc;
1044 }
1045 module_init(opa_vnic_init);
1046 
1047 static void opa_vnic_deinit(void)
1048 {
1049 	ib_unregister_client(&opa_vnic_client);
1050 }
1051 module_exit(opa_vnic_deinit);
1052 
1053 MODULE_LICENSE("Dual BSD/GPL");
1054 MODULE_AUTHOR("Intel Corporation");
1055 MODULE_DESCRIPTION("Intel OPA Virtual Network driver");
1056 MODULE_VERSION(DRV_VERSION);
1057