xref: /openbmc/linux/drivers/hv/channel.c (revision f5ad1c74)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (c) 2009, Microsoft Corporation.
4  *
5  * Authors:
6  *   Haiyang Zhang <haiyangz@microsoft.com>
7  *   Hank Janssen  <hjanssen@microsoft.com>
8  */
9 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
10 
11 #include <linux/kernel.h>
12 #include <linux/sched.h>
13 #include <linux/wait.h>
14 #include <linux/mm.h>
15 #include <linux/slab.h>
16 #include <linux/module.h>
17 #include <linux/hyperv.h>
18 #include <linux/uio.h>
19 #include <linux/interrupt.h>
20 #include <asm/page.h>
21 #include <asm/mshyperv.h>
22 
23 #include "hyperv_vmbus.h"
24 
25 /*
26  * hv_gpadl_size - Return the real size of a gpadl, the size that Hyper-V uses
27  *
28  * For BUFFER gpadl, Hyper-V uses the exact same size as the guest does.
29  *
30  * For RING gpadl, in each ring, the guest uses one PAGE_SIZE as the header
31  * (because of the alignment requirement), however, the hypervisor only
32  * uses the first HV_HYP_PAGE_SIZE as the header, therefore leaving a
33  * (PAGE_SIZE - HV_HYP_PAGE_SIZE) gap. And since there are two rings in a
34  * ringbuffer, the total size for a RING gpadl that Hyper-V uses is the
35  * total size that the guest uses minus twice of the gap size.
36  */
37 static inline u32 hv_gpadl_size(enum hv_gpadl_type type, u32 size)
38 {
39 	switch (type) {
40 	case HV_GPADL_BUFFER:
41 		return size;
42 	case HV_GPADL_RING:
43 		/* The size of a ringbuffer must be page-aligned */
44 		BUG_ON(size % PAGE_SIZE);
45 		/*
46 		 * Two things to notice here:
47 		 * 1) We're processing two ring buffers as a unit
48 		 * 2) We're skipping any space larger than HV_HYP_PAGE_SIZE in
49 		 * the first guest-size page of each of the two ring buffers.
50 		 * So we effectively subtract out two guest-size pages, and add
51 		 * back two Hyper-V size pages.
52 		 */
53 		return size - 2 * (PAGE_SIZE - HV_HYP_PAGE_SIZE);
54 	}
55 	BUG();
56 	return 0;
57 }
58 
59 /*
60  * hv_ring_gpadl_send_hvpgoffset - Calculate the send offset (in unit of
61  *                                 HV_HYP_PAGE) in a ring gpadl based on the
62  *                                 offset in the guest
63  *
64  * @offset: the offset (in bytes) where the send ringbuffer starts in the
65  *               virtual address space of the guest
66  */
67 static inline u32 hv_ring_gpadl_send_hvpgoffset(u32 offset)
68 {
69 
70 	/*
71 	 * For RING gpadl, in each ring, the guest uses one PAGE_SIZE as the
72 	 * header (because of the alignment requirement), however, the
73 	 * hypervisor only uses the first HV_HYP_PAGE_SIZE as the header,
74 	 * therefore leaving a (PAGE_SIZE - HV_HYP_PAGE_SIZE) gap.
75 	 *
76 	 * And to calculate the effective send offset in gpadl, we need to
77 	 * substract this gap.
78 	 */
79 	return (offset - (PAGE_SIZE - HV_HYP_PAGE_SIZE)) >> HV_HYP_PAGE_SHIFT;
80 }
81 
82 /*
83  * hv_gpadl_hvpfn - Return the Hyper-V page PFN of the @i th Hyper-V page in
84  *                  the gpadl
85  *
86  * @type: the type of the gpadl
87  * @kbuffer: the pointer to the gpadl in the guest
88  * @size: the total size (in bytes) of the gpadl
89  * @send_offset: the offset (in bytes) where the send ringbuffer starts in the
90  *               virtual address space of the guest
91  * @i: the index
92  */
93 static inline u64 hv_gpadl_hvpfn(enum hv_gpadl_type type, void *kbuffer,
94 				 u32 size, u32 send_offset, int i)
95 {
96 	int send_idx = hv_ring_gpadl_send_hvpgoffset(send_offset);
97 	unsigned long delta = 0UL;
98 
99 	switch (type) {
100 	case HV_GPADL_BUFFER:
101 		break;
102 	case HV_GPADL_RING:
103 		if (i == 0)
104 			delta = 0;
105 		else if (i <= send_idx)
106 			delta = PAGE_SIZE - HV_HYP_PAGE_SIZE;
107 		else
108 			delta = 2 * (PAGE_SIZE - HV_HYP_PAGE_SIZE);
109 		break;
110 	default:
111 		BUG();
112 		break;
113 	}
114 
115 	return virt_to_hvpfn(kbuffer + delta + (HV_HYP_PAGE_SIZE * i));
116 }
117 
118 /*
119  * vmbus_setevent- Trigger an event notification on the specified
120  * channel.
121  */
122 void vmbus_setevent(struct vmbus_channel *channel)
123 {
124 	struct hv_monitor_page *monitorpage;
125 
126 	trace_vmbus_setevent(channel);
127 
128 	/*
129 	 * For channels marked as in "low latency" mode
130 	 * bypass the monitor page mechanism.
131 	 */
132 	if (channel->offermsg.monitor_allocated && !channel->low_latency) {
133 		vmbus_send_interrupt(channel->offermsg.child_relid);
134 
135 		/* Get the child to parent monitor page */
136 		monitorpage = vmbus_connection.monitor_pages[1];
137 
138 		sync_set_bit(channel->monitor_bit,
139 			(unsigned long *)&monitorpage->trigger_group
140 					[channel->monitor_grp].pending);
141 
142 	} else {
143 		vmbus_set_event(channel);
144 	}
145 }
146 EXPORT_SYMBOL_GPL(vmbus_setevent);
147 
148 /* vmbus_free_ring - drop mapping of ring buffer */
149 void vmbus_free_ring(struct vmbus_channel *channel)
150 {
151 	hv_ringbuffer_cleanup(&channel->outbound);
152 	hv_ringbuffer_cleanup(&channel->inbound);
153 
154 	if (channel->ringbuffer_page) {
155 		__free_pages(channel->ringbuffer_page,
156 			     get_order(channel->ringbuffer_pagecount
157 				       << PAGE_SHIFT));
158 		channel->ringbuffer_page = NULL;
159 	}
160 }
161 EXPORT_SYMBOL_GPL(vmbus_free_ring);
162 
163 /* vmbus_alloc_ring - allocate and map pages for ring buffer */
164 int vmbus_alloc_ring(struct vmbus_channel *newchannel,
165 		     u32 send_size, u32 recv_size)
166 {
167 	struct page *page;
168 	int order;
169 
170 	if (send_size % PAGE_SIZE || recv_size % PAGE_SIZE)
171 		return -EINVAL;
172 
173 	/* Allocate the ring buffer */
174 	order = get_order(send_size + recv_size);
175 	page = alloc_pages_node(cpu_to_node(newchannel->target_cpu),
176 				GFP_KERNEL|__GFP_ZERO, order);
177 
178 	if (!page)
179 		page = alloc_pages(GFP_KERNEL|__GFP_ZERO, order);
180 
181 	if (!page)
182 		return -ENOMEM;
183 
184 	newchannel->ringbuffer_page = page;
185 	newchannel->ringbuffer_pagecount = (send_size + recv_size) >> PAGE_SHIFT;
186 	newchannel->ringbuffer_send_offset = send_size >> PAGE_SHIFT;
187 
188 	return 0;
189 }
190 EXPORT_SYMBOL_GPL(vmbus_alloc_ring);
191 
192 /* Used for Hyper-V Socket: a guest client's connect() to the host */
193 int vmbus_send_tl_connect_request(const guid_t *shv_guest_servie_id,
194 				  const guid_t *shv_host_servie_id)
195 {
196 	struct vmbus_channel_tl_connect_request conn_msg;
197 	int ret;
198 
199 	memset(&conn_msg, 0, sizeof(conn_msg));
200 	conn_msg.header.msgtype = CHANNELMSG_TL_CONNECT_REQUEST;
201 	conn_msg.guest_endpoint_id = *shv_guest_servie_id;
202 	conn_msg.host_service_id = *shv_host_servie_id;
203 
204 	ret = vmbus_post_msg(&conn_msg, sizeof(conn_msg), true);
205 
206 	trace_vmbus_send_tl_connect_request(&conn_msg, ret);
207 
208 	return ret;
209 }
210 EXPORT_SYMBOL_GPL(vmbus_send_tl_connect_request);
211 
212 /*
213  * Set/change the vCPU (@target_vp) the channel (@child_relid) will interrupt.
214  *
215  * CHANNELMSG_MODIFYCHANNEL messages are aynchronous.  Also, Hyper-V does not
216  * ACK such messages.  IOW we can't know when the host will stop interrupting
217  * the "old" vCPU and start interrupting the "new" vCPU for the given channel.
218  *
219  * The CHANNELMSG_MODIFYCHANNEL message type is supported since VMBus version
220  * VERSION_WIN10_V4_1.
221  */
222 int vmbus_send_modifychannel(u32 child_relid, u32 target_vp)
223 {
224 	struct vmbus_channel_modifychannel conn_msg;
225 	int ret;
226 
227 	memset(&conn_msg, 0, sizeof(conn_msg));
228 	conn_msg.header.msgtype = CHANNELMSG_MODIFYCHANNEL;
229 	conn_msg.child_relid = child_relid;
230 	conn_msg.target_vp = target_vp;
231 
232 	ret = vmbus_post_msg(&conn_msg, sizeof(conn_msg), true);
233 
234 	trace_vmbus_send_modifychannel(&conn_msg, ret);
235 
236 	return ret;
237 }
238 EXPORT_SYMBOL_GPL(vmbus_send_modifychannel);
239 
240 /*
241  * create_gpadl_header - Creates a gpadl for the specified buffer
242  */
243 static int create_gpadl_header(enum hv_gpadl_type type, void *kbuffer,
244 			       u32 size, u32 send_offset,
245 			       struct vmbus_channel_msginfo **msginfo)
246 {
247 	int i;
248 	int pagecount;
249 	struct vmbus_channel_gpadl_header *gpadl_header;
250 	struct vmbus_channel_gpadl_body *gpadl_body;
251 	struct vmbus_channel_msginfo *msgheader;
252 	struct vmbus_channel_msginfo *msgbody = NULL;
253 	u32 msgsize;
254 
255 	int pfnsum, pfncount, pfnleft, pfncurr, pfnsize;
256 
257 	pagecount = hv_gpadl_size(type, size) >> HV_HYP_PAGE_SHIFT;
258 
259 	/* do we need a gpadl body msg */
260 	pfnsize = MAX_SIZE_CHANNEL_MESSAGE -
261 		  sizeof(struct vmbus_channel_gpadl_header) -
262 		  sizeof(struct gpa_range);
263 	pfncount = pfnsize / sizeof(u64);
264 
265 	if (pagecount > pfncount) {
266 		/* we need a gpadl body */
267 		/* fill in the header */
268 		msgsize = sizeof(struct vmbus_channel_msginfo) +
269 			  sizeof(struct vmbus_channel_gpadl_header) +
270 			  sizeof(struct gpa_range) + pfncount * sizeof(u64);
271 		msgheader =  kzalloc(msgsize, GFP_KERNEL);
272 		if (!msgheader)
273 			goto nomem;
274 
275 		INIT_LIST_HEAD(&msgheader->submsglist);
276 		msgheader->msgsize = msgsize;
277 
278 		gpadl_header = (struct vmbus_channel_gpadl_header *)
279 			msgheader->msg;
280 		gpadl_header->rangecount = 1;
281 		gpadl_header->range_buflen = sizeof(struct gpa_range) +
282 					 pagecount * sizeof(u64);
283 		gpadl_header->range[0].byte_offset = 0;
284 		gpadl_header->range[0].byte_count = hv_gpadl_size(type, size);
285 		for (i = 0; i < pfncount; i++)
286 			gpadl_header->range[0].pfn_array[i] = hv_gpadl_hvpfn(
287 				type, kbuffer, size, send_offset, i);
288 		*msginfo = msgheader;
289 
290 		pfnsum = pfncount;
291 		pfnleft = pagecount - pfncount;
292 
293 		/* how many pfns can we fit */
294 		pfnsize = MAX_SIZE_CHANNEL_MESSAGE -
295 			  sizeof(struct vmbus_channel_gpadl_body);
296 		pfncount = pfnsize / sizeof(u64);
297 
298 		/* fill in the body */
299 		while (pfnleft) {
300 			if (pfnleft > pfncount)
301 				pfncurr = pfncount;
302 			else
303 				pfncurr = pfnleft;
304 
305 			msgsize = sizeof(struct vmbus_channel_msginfo) +
306 				  sizeof(struct vmbus_channel_gpadl_body) +
307 				  pfncurr * sizeof(u64);
308 			msgbody = kzalloc(msgsize, GFP_KERNEL);
309 
310 			if (!msgbody) {
311 				struct vmbus_channel_msginfo *pos = NULL;
312 				struct vmbus_channel_msginfo *tmp = NULL;
313 				/*
314 				 * Free up all the allocated messages.
315 				 */
316 				list_for_each_entry_safe(pos, tmp,
317 					&msgheader->submsglist,
318 					msglistentry) {
319 
320 					list_del(&pos->msglistentry);
321 					kfree(pos);
322 				}
323 
324 				goto nomem;
325 			}
326 
327 			msgbody->msgsize = msgsize;
328 			gpadl_body =
329 				(struct vmbus_channel_gpadl_body *)msgbody->msg;
330 
331 			/*
332 			 * Gpadl is u32 and we are using a pointer which could
333 			 * be 64-bit
334 			 * This is governed by the guest/host protocol and
335 			 * so the hypervisor guarantees that this is ok.
336 			 */
337 			for (i = 0; i < pfncurr; i++)
338 				gpadl_body->pfn[i] = hv_gpadl_hvpfn(type,
339 					kbuffer, size, send_offset, pfnsum + i);
340 
341 			/* add to msg header */
342 			list_add_tail(&msgbody->msglistentry,
343 				      &msgheader->submsglist);
344 			pfnsum += pfncurr;
345 			pfnleft -= pfncurr;
346 		}
347 	} else {
348 		/* everything fits in a header */
349 		msgsize = sizeof(struct vmbus_channel_msginfo) +
350 			  sizeof(struct vmbus_channel_gpadl_header) +
351 			  sizeof(struct gpa_range) + pagecount * sizeof(u64);
352 		msgheader = kzalloc(msgsize, GFP_KERNEL);
353 		if (msgheader == NULL)
354 			goto nomem;
355 
356 		INIT_LIST_HEAD(&msgheader->submsglist);
357 		msgheader->msgsize = msgsize;
358 
359 		gpadl_header = (struct vmbus_channel_gpadl_header *)
360 			msgheader->msg;
361 		gpadl_header->rangecount = 1;
362 		gpadl_header->range_buflen = sizeof(struct gpa_range) +
363 					 pagecount * sizeof(u64);
364 		gpadl_header->range[0].byte_offset = 0;
365 		gpadl_header->range[0].byte_count = hv_gpadl_size(type, size);
366 		for (i = 0; i < pagecount; i++)
367 			gpadl_header->range[0].pfn_array[i] = hv_gpadl_hvpfn(
368 				type, kbuffer, size, send_offset, i);
369 
370 		*msginfo = msgheader;
371 	}
372 
373 	return 0;
374 nomem:
375 	kfree(msgheader);
376 	kfree(msgbody);
377 	return -ENOMEM;
378 }
379 
380 /*
381  * __vmbus_establish_gpadl - Establish a GPADL for a buffer or ringbuffer
382  *
383  * @channel: a channel
384  * @type: the type of the corresponding GPADL, only meaningful for the guest.
385  * @kbuffer: from kmalloc or vmalloc
386  * @size: page-size multiple
387  * @send_offset: the offset (in bytes) where the send ring buffer starts,
388  * 		 should be 0 for BUFFER type gpadl
389  * @gpadl_handle: some funky thing
390  */
391 static int __vmbus_establish_gpadl(struct vmbus_channel *channel,
392 				   enum hv_gpadl_type type, void *kbuffer,
393 				   u32 size, u32 send_offset,
394 				   u32 *gpadl_handle)
395 {
396 	struct vmbus_channel_gpadl_header *gpadlmsg;
397 	struct vmbus_channel_gpadl_body *gpadl_body;
398 	struct vmbus_channel_msginfo *msginfo = NULL;
399 	struct vmbus_channel_msginfo *submsginfo, *tmp;
400 	struct list_head *curr;
401 	u32 next_gpadl_handle;
402 	unsigned long flags;
403 	int ret = 0;
404 
405 	next_gpadl_handle =
406 		(atomic_inc_return(&vmbus_connection.next_gpadl_handle) - 1);
407 
408 	ret = create_gpadl_header(type, kbuffer, size, send_offset, &msginfo);
409 	if (ret)
410 		return ret;
411 
412 	init_completion(&msginfo->waitevent);
413 	msginfo->waiting_channel = channel;
414 
415 	gpadlmsg = (struct vmbus_channel_gpadl_header *)msginfo->msg;
416 	gpadlmsg->header.msgtype = CHANNELMSG_GPADL_HEADER;
417 	gpadlmsg->child_relid = channel->offermsg.child_relid;
418 	gpadlmsg->gpadl = next_gpadl_handle;
419 
420 
421 	spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
422 	list_add_tail(&msginfo->msglistentry,
423 		      &vmbus_connection.chn_msg_list);
424 
425 	spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
426 
427 	if (channel->rescind) {
428 		ret = -ENODEV;
429 		goto cleanup;
430 	}
431 
432 	ret = vmbus_post_msg(gpadlmsg, msginfo->msgsize -
433 			     sizeof(*msginfo), true);
434 
435 	trace_vmbus_establish_gpadl_header(gpadlmsg, ret);
436 
437 	if (ret != 0)
438 		goto cleanup;
439 
440 	list_for_each(curr, &msginfo->submsglist) {
441 		submsginfo = (struct vmbus_channel_msginfo *)curr;
442 		gpadl_body =
443 			(struct vmbus_channel_gpadl_body *)submsginfo->msg;
444 
445 		gpadl_body->header.msgtype =
446 			CHANNELMSG_GPADL_BODY;
447 		gpadl_body->gpadl = next_gpadl_handle;
448 
449 		ret = vmbus_post_msg(gpadl_body,
450 				     submsginfo->msgsize - sizeof(*submsginfo),
451 				     true);
452 
453 		trace_vmbus_establish_gpadl_body(gpadl_body, ret);
454 
455 		if (ret != 0)
456 			goto cleanup;
457 
458 	}
459 	wait_for_completion(&msginfo->waitevent);
460 
461 	if (msginfo->response.gpadl_created.creation_status != 0) {
462 		pr_err("Failed to establish GPADL: err = 0x%x\n",
463 		       msginfo->response.gpadl_created.creation_status);
464 
465 		ret = -EDQUOT;
466 		goto cleanup;
467 	}
468 
469 	if (channel->rescind) {
470 		ret = -ENODEV;
471 		goto cleanup;
472 	}
473 
474 	/* At this point, we received the gpadl created msg */
475 	*gpadl_handle = gpadlmsg->gpadl;
476 
477 cleanup:
478 	spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
479 	list_del(&msginfo->msglistentry);
480 	spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
481 	list_for_each_entry_safe(submsginfo, tmp, &msginfo->submsglist,
482 				 msglistentry) {
483 		kfree(submsginfo);
484 	}
485 
486 	kfree(msginfo);
487 	return ret;
488 }
489 
490 /*
491  * vmbus_establish_gpadl - Establish a GPADL for the specified buffer
492  *
493  * @channel: a channel
494  * @kbuffer: from kmalloc or vmalloc
495  * @size: page-size multiple
496  * @gpadl_handle: some funky thing
497  */
498 int vmbus_establish_gpadl(struct vmbus_channel *channel, void *kbuffer,
499 			  u32 size, u32 *gpadl_handle)
500 {
501 	return __vmbus_establish_gpadl(channel, HV_GPADL_BUFFER, kbuffer, size,
502 				       0U, gpadl_handle);
503 }
504 EXPORT_SYMBOL_GPL(vmbus_establish_gpadl);
505 
506 static int __vmbus_open(struct vmbus_channel *newchannel,
507 		       void *userdata, u32 userdatalen,
508 		       void (*onchannelcallback)(void *context), void *context)
509 {
510 	struct vmbus_channel_open_channel *open_msg;
511 	struct vmbus_channel_msginfo *open_info = NULL;
512 	struct page *page = newchannel->ringbuffer_page;
513 	u32 send_pages, recv_pages;
514 	unsigned long flags;
515 	int err;
516 
517 	if (userdatalen > MAX_USER_DEFINED_BYTES)
518 		return -EINVAL;
519 
520 	send_pages = newchannel->ringbuffer_send_offset;
521 	recv_pages = newchannel->ringbuffer_pagecount - send_pages;
522 
523 	if (newchannel->state != CHANNEL_OPEN_STATE)
524 		return -EINVAL;
525 
526 	newchannel->state = CHANNEL_OPENING_STATE;
527 	newchannel->onchannel_callback = onchannelcallback;
528 	newchannel->channel_callback_context = context;
529 
530 	err = hv_ringbuffer_init(&newchannel->outbound, page, send_pages);
531 	if (err)
532 		goto error_clean_ring;
533 
534 	err = hv_ringbuffer_init(&newchannel->inbound,
535 				 &page[send_pages], recv_pages);
536 	if (err)
537 		goto error_clean_ring;
538 
539 	/* Establish the gpadl for the ring buffer */
540 	newchannel->ringbuffer_gpadlhandle = 0;
541 
542 	err = __vmbus_establish_gpadl(newchannel, HV_GPADL_RING,
543 				      page_address(newchannel->ringbuffer_page),
544 				      (send_pages + recv_pages) << PAGE_SHIFT,
545 				      newchannel->ringbuffer_send_offset << PAGE_SHIFT,
546 				      &newchannel->ringbuffer_gpadlhandle);
547 	if (err)
548 		goto error_clean_ring;
549 
550 	/* Create and init the channel open message */
551 	open_info = kmalloc(sizeof(*open_info) +
552 			   sizeof(struct vmbus_channel_open_channel),
553 			   GFP_KERNEL);
554 	if (!open_info) {
555 		err = -ENOMEM;
556 		goto error_free_gpadl;
557 	}
558 
559 	init_completion(&open_info->waitevent);
560 	open_info->waiting_channel = newchannel;
561 
562 	open_msg = (struct vmbus_channel_open_channel *)open_info->msg;
563 	open_msg->header.msgtype = CHANNELMSG_OPENCHANNEL;
564 	open_msg->openid = newchannel->offermsg.child_relid;
565 	open_msg->child_relid = newchannel->offermsg.child_relid;
566 	open_msg->ringbuffer_gpadlhandle = newchannel->ringbuffer_gpadlhandle;
567 	/*
568 	 * The unit of ->downstream_ringbuffer_pageoffset is HV_HYP_PAGE and
569 	 * the unit of ->ringbuffer_send_offset (i.e. send_pages) is PAGE, so
570 	 * here we calculate it into HV_HYP_PAGE.
571 	 */
572 	open_msg->downstream_ringbuffer_pageoffset =
573 		hv_ring_gpadl_send_hvpgoffset(send_pages << PAGE_SHIFT);
574 	open_msg->target_vp = hv_cpu_number_to_vp_number(newchannel->target_cpu);
575 
576 	if (userdatalen)
577 		memcpy(open_msg->userdata, userdata, userdatalen);
578 
579 	spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
580 	list_add_tail(&open_info->msglistentry,
581 		      &vmbus_connection.chn_msg_list);
582 	spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
583 
584 	if (newchannel->rescind) {
585 		err = -ENODEV;
586 		goto error_free_info;
587 	}
588 
589 	err = vmbus_post_msg(open_msg,
590 			     sizeof(struct vmbus_channel_open_channel), true);
591 
592 	trace_vmbus_open(open_msg, err);
593 
594 	if (err != 0)
595 		goto error_clean_msglist;
596 
597 	wait_for_completion(&open_info->waitevent);
598 
599 	spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
600 	list_del(&open_info->msglistentry);
601 	spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
602 
603 	if (newchannel->rescind) {
604 		err = -ENODEV;
605 		goto error_free_info;
606 	}
607 
608 	if (open_info->response.open_result.status) {
609 		err = -EAGAIN;
610 		goto error_free_info;
611 	}
612 
613 	newchannel->state = CHANNEL_OPENED_STATE;
614 	kfree(open_info);
615 	return 0;
616 
617 error_clean_msglist:
618 	spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
619 	list_del(&open_info->msglistentry);
620 	spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
621 error_free_info:
622 	kfree(open_info);
623 error_free_gpadl:
624 	vmbus_teardown_gpadl(newchannel, newchannel->ringbuffer_gpadlhandle);
625 	newchannel->ringbuffer_gpadlhandle = 0;
626 error_clean_ring:
627 	hv_ringbuffer_cleanup(&newchannel->outbound);
628 	hv_ringbuffer_cleanup(&newchannel->inbound);
629 	newchannel->state = CHANNEL_OPEN_STATE;
630 	return err;
631 }
632 
633 /*
634  * vmbus_connect_ring - Open the channel but reuse ring buffer
635  */
636 int vmbus_connect_ring(struct vmbus_channel *newchannel,
637 		       void (*onchannelcallback)(void *context), void *context)
638 {
639 	return  __vmbus_open(newchannel, NULL, 0, onchannelcallback, context);
640 }
641 EXPORT_SYMBOL_GPL(vmbus_connect_ring);
642 
643 /*
644  * vmbus_open - Open the specified channel.
645  */
646 int vmbus_open(struct vmbus_channel *newchannel,
647 	       u32 send_ringbuffer_size, u32 recv_ringbuffer_size,
648 	       void *userdata, u32 userdatalen,
649 	       void (*onchannelcallback)(void *context), void *context)
650 {
651 	int err;
652 
653 	err = vmbus_alloc_ring(newchannel, send_ringbuffer_size,
654 			       recv_ringbuffer_size);
655 	if (err)
656 		return err;
657 
658 	err = __vmbus_open(newchannel, userdata, userdatalen,
659 			   onchannelcallback, context);
660 	if (err)
661 		vmbus_free_ring(newchannel);
662 
663 	return err;
664 }
665 EXPORT_SYMBOL_GPL(vmbus_open);
666 
667 /*
668  * vmbus_teardown_gpadl -Teardown the specified GPADL handle
669  */
670 int vmbus_teardown_gpadl(struct vmbus_channel *channel, u32 gpadl_handle)
671 {
672 	struct vmbus_channel_gpadl_teardown *msg;
673 	struct vmbus_channel_msginfo *info;
674 	unsigned long flags;
675 	int ret;
676 
677 	info = kmalloc(sizeof(*info) +
678 		       sizeof(struct vmbus_channel_gpadl_teardown), GFP_KERNEL);
679 	if (!info)
680 		return -ENOMEM;
681 
682 	init_completion(&info->waitevent);
683 	info->waiting_channel = channel;
684 
685 	msg = (struct vmbus_channel_gpadl_teardown *)info->msg;
686 
687 	msg->header.msgtype = CHANNELMSG_GPADL_TEARDOWN;
688 	msg->child_relid = channel->offermsg.child_relid;
689 	msg->gpadl = gpadl_handle;
690 
691 	spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
692 	list_add_tail(&info->msglistentry,
693 		      &vmbus_connection.chn_msg_list);
694 	spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
695 
696 	if (channel->rescind)
697 		goto post_msg_err;
698 
699 	ret = vmbus_post_msg(msg, sizeof(struct vmbus_channel_gpadl_teardown),
700 			     true);
701 
702 	trace_vmbus_teardown_gpadl(msg, ret);
703 
704 	if (ret)
705 		goto post_msg_err;
706 
707 	wait_for_completion(&info->waitevent);
708 
709 post_msg_err:
710 	/*
711 	 * If the channel has been rescinded;
712 	 * we will be awakened by the rescind
713 	 * handler; set the error code to zero so we don't leak memory.
714 	 */
715 	if (channel->rescind)
716 		ret = 0;
717 
718 	spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
719 	list_del(&info->msglistentry);
720 	spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
721 
722 	kfree(info);
723 	return ret;
724 }
725 EXPORT_SYMBOL_GPL(vmbus_teardown_gpadl);
726 
727 void vmbus_reset_channel_cb(struct vmbus_channel *channel)
728 {
729 	unsigned long flags;
730 
731 	/*
732 	 * vmbus_on_event(), running in the per-channel tasklet, can race
733 	 * with vmbus_close_internal() in the case of SMP guest, e.g., when
734 	 * the former is accessing channel->inbound.ring_buffer, the latter
735 	 * could be freeing the ring_buffer pages, so here we must stop it
736 	 * first.
737 	 *
738 	 * vmbus_chan_sched() might call the netvsc driver callback function
739 	 * that ends up scheduling NAPI work that accesses the ring buffer.
740 	 * At this point, we have to ensure that any such work is completed
741 	 * and that the channel ring buffer is no longer being accessed, cf.
742 	 * the calls to napi_disable() in netvsc_device_remove().
743 	 */
744 	tasklet_disable(&channel->callback_event);
745 
746 	/* See the inline comments in vmbus_chan_sched(). */
747 	spin_lock_irqsave(&channel->sched_lock, flags);
748 	channel->onchannel_callback = NULL;
749 	spin_unlock_irqrestore(&channel->sched_lock, flags);
750 
751 	channel->sc_creation_callback = NULL;
752 
753 	/* Re-enable tasklet for use on re-open */
754 	tasklet_enable(&channel->callback_event);
755 }
756 
757 static int vmbus_close_internal(struct vmbus_channel *channel)
758 {
759 	struct vmbus_channel_close_channel *msg;
760 	int ret;
761 
762 	vmbus_reset_channel_cb(channel);
763 
764 	/*
765 	 * In case a device driver's probe() fails (e.g.,
766 	 * util_probe() -> vmbus_open() returns -ENOMEM) and the device is
767 	 * rescinded later (e.g., we dynamically disable an Integrated Service
768 	 * in Hyper-V Manager), the driver's remove() invokes vmbus_close():
769 	 * here we should skip most of the below cleanup work.
770 	 */
771 	if (channel->state != CHANNEL_OPENED_STATE)
772 		return -EINVAL;
773 
774 	channel->state = CHANNEL_OPEN_STATE;
775 
776 	/* Send a closing message */
777 
778 	msg = &channel->close_msg.msg;
779 
780 	msg->header.msgtype = CHANNELMSG_CLOSECHANNEL;
781 	msg->child_relid = channel->offermsg.child_relid;
782 
783 	ret = vmbus_post_msg(msg, sizeof(struct vmbus_channel_close_channel),
784 			     true);
785 
786 	trace_vmbus_close_internal(msg, ret);
787 
788 	if (ret) {
789 		pr_err("Close failed: close post msg return is %d\n", ret);
790 		/*
791 		 * If we failed to post the close msg,
792 		 * it is perhaps better to leak memory.
793 		 */
794 	}
795 
796 	/* Tear down the gpadl for the channel's ring buffer */
797 	else if (channel->ringbuffer_gpadlhandle) {
798 		ret = vmbus_teardown_gpadl(channel,
799 					   channel->ringbuffer_gpadlhandle);
800 		if (ret) {
801 			pr_err("Close failed: teardown gpadl return %d\n", ret);
802 			/*
803 			 * If we failed to teardown gpadl,
804 			 * it is perhaps better to leak memory.
805 			 */
806 		}
807 
808 		channel->ringbuffer_gpadlhandle = 0;
809 	}
810 
811 	return ret;
812 }
813 
814 /* disconnect ring - close all channels */
815 int vmbus_disconnect_ring(struct vmbus_channel *channel)
816 {
817 	struct vmbus_channel *cur_channel, *tmp;
818 	int ret;
819 
820 	if (channel->primary_channel != NULL)
821 		return -EINVAL;
822 
823 	list_for_each_entry_safe(cur_channel, tmp, &channel->sc_list, sc_list) {
824 		if (cur_channel->rescind)
825 			wait_for_completion(&cur_channel->rescind_event);
826 
827 		mutex_lock(&vmbus_connection.channel_mutex);
828 		if (vmbus_close_internal(cur_channel) == 0) {
829 			vmbus_free_ring(cur_channel);
830 
831 			if (cur_channel->rescind)
832 				hv_process_channel_removal(cur_channel);
833 		}
834 		mutex_unlock(&vmbus_connection.channel_mutex);
835 	}
836 
837 	/*
838 	 * Now close the primary.
839 	 */
840 	mutex_lock(&vmbus_connection.channel_mutex);
841 	ret = vmbus_close_internal(channel);
842 	mutex_unlock(&vmbus_connection.channel_mutex);
843 
844 	return ret;
845 }
846 EXPORT_SYMBOL_GPL(vmbus_disconnect_ring);
847 
848 /*
849  * vmbus_close - Close the specified channel
850  */
851 void vmbus_close(struct vmbus_channel *channel)
852 {
853 	if (vmbus_disconnect_ring(channel) == 0)
854 		vmbus_free_ring(channel);
855 }
856 EXPORT_SYMBOL_GPL(vmbus_close);
857 
858 /**
859  * vmbus_sendpacket() - Send the specified buffer on the given channel
860  * @channel: Pointer to vmbus_channel structure
861  * @buffer: Pointer to the buffer you want to send the data from.
862  * @bufferlen: Maximum size of what the buffer holds.
863  * @requestid: Identifier of the request
864  * @type: Type of packet that is being sent e.g. negotiate, time
865  *	  packet etc.
866  * @flags: 0 or VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED
867  *
868  * Sends data in @buffer directly to Hyper-V via the vmbus.
869  * This will send the data unparsed to Hyper-V.
870  *
871  * Mainly used by Hyper-V drivers.
872  */
873 int vmbus_sendpacket(struct vmbus_channel *channel, void *buffer,
874 			   u32 bufferlen, u64 requestid,
875 			   enum vmbus_packet_type type, u32 flags)
876 {
877 	struct vmpacket_descriptor desc;
878 	u32 packetlen = sizeof(struct vmpacket_descriptor) + bufferlen;
879 	u32 packetlen_aligned = ALIGN(packetlen, sizeof(u64));
880 	struct kvec bufferlist[3];
881 	u64 aligned_data = 0;
882 	int num_vecs = ((bufferlen != 0) ? 3 : 1);
883 
884 
885 	/* Setup the descriptor */
886 	desc.type = type; /* VmbusPacketTypeDataInBand; */
887 	desc.flags = flags; /* VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED; */
888 	/* in 8-bytes granularity */
889 	desc.offset8 = sizeof(struct vmpacket_descriptor) >> 3;
890 	desc.len8 = (u16)(packetlen_aligned >> 3);
891 	desc.trans_id = requestid;
892 
893 	bufferlist[0].iov_base = &desc;
894 	bufferlist[0].iov_len = sizeof(struct vmpacket_descriptor);
895 	bufferlist[1].iov_base = buffer;
896 	bufferlist[1].iov_len = bufferlen;
897 	bufferlist[2].iov_base = &aligned_data;
898 	bufferlist[2].iov_len = (packetlen_aligned - packetlen);
899 
900 	return hv_ringbuffer_write(channel, bufferlist, num_vecs);
901 }
902 EXPORT_SYMBOL(vmbus_sendpacket);
903 
904 /*
905  * vmbus_sendpacket_pagebuffer - Send a range of single-page buffer
906  * packets using a GPADL Direct packet type. This interface allows you
907  * to control notifying the host. This will be useful for sending
908  * batched data. Also the sender can control the send flags
909  * explicitly.
910  */
911 int vmbus_sendpacket_pagebuffer(struct vmbus_channel *channel,
912 				struct hv_page_buffer pagebuffers[],
913 				u32 pagecount, void *buffer, u32 bufferlen,
914 				u64 requestid)
915 {
916 	int i;
917 	struct vmbus_channel_packet_page_buffer desc;
918 	u32 descsize;
919 	u32 packetlen;
920 	u32 packetlen_aligned;
921 	struct kvec bufferlist[3];
922 	u64 aligned_data = 0;
923 
924 	if (pagecount > MAX_PAGE_BUFFER_COUNT)
925 		return -EINVAL;
926 
927 	/*
928 	 * Adjust the size down since vmbus_channel_packet_page_buffer is the
929 	 * largest size we support
930 	 */
931 	descsize = sizeof(struct vmbus_channel_packet_page_buffer) -
932 			  ((MAX_PAGE_BUFFER_COUNT - pagecount) *
933 			  sizeof(struct hv_page_buffer));
934 	packetlen = descsize + bufferlen;
935 	packetlen_aligned = ALIGN(packetlen, sizeof(u64));
936 
937 	/* Setup the descriptor */
938 	desc.type = VM_PKT_DATA_USING_GPA_DIRECT;
939 	desc.flags = VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED;
940 	desc.dataoffset8 = descsize >> 3; /* in 8-bytes granularity */
941 	desc.length8 = (u16)(packetlen_aligned >> 3);
942 	desc.transactionid = requestid;
943 	desc.reserved = 0;
944 	desc.rangecount = pagecount;
945 
946 	for (i = 0; i < pagecount; i++) {
947 		desc.range[i].len = pagebuffers[i].len;
948 		desc.range[i].offset = pagebuffers[i].offset;
949 		desc.range[i].pfn	 = pagebuffers[i].pfn;
950 	}
951 
952 	bufferlist[0].iov_base = &desc;
953 	bufferlist[0].iov_len = descsize;
954 	bufferlist[1].iov_base = buffer;
955 	bufferlist[1].iov_len = bufferlen;
956 	bufferlist[2].iov_base = &aligned_data;
957 	bufferlist[2].iov_len = (packetlen_aligned - packetlen);
958 
959 	return hv_ringbuffer_write(channel, bufferlist, 3);
960 }
961 EXPORT_SYMBOL_GPL(vmbus_sendpacket_pagebuffer);
962 
963 /*
964  * vmbus_sendpacket_multipagebuffer - Send a multi-page buffer packet
965  * using a GPADL Direct packet type.
966  * The buffer includes the vmbus descriptor.
967  */
968 int vmbus_sendpacket_mpb_desc(struct vmbus_channel *channel,
969 			      struct vmbus_packet_mpb_array *desc,
970 			      u32 desc_size,
971 			      void *buffer, u32 bufferlen, u64 requestid)
972 {
973 	u32 packetlen;
974 	u32 packetlen_aligned;
975 	struct kvec bufferlist[3];
976 	u64 aligned_data = 0;
977 
978 	packetlen = desc_size + bufferlen;
979 	packetlen_aligned = ALIGN(packetlen, sizeof(u64));
980 
981 	/* Setup the descriptor */
982 	desc->type = VM_PKT_DATA_USING_GPA_DIRECT;
983 	desc->flags = VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED;
984 	desc->dataoffset8 = desc_size >> 3; /* in 8-bytes granularity */
985 	desc->length8 = (u16)(packetlen_aligned >> 3);
986 	desc->transactionid = requestid;
987 	desc->reserved = 0;
988 	desc->rangecount = 1;
989 
990 	bufferlist[0].iov_base = desc;
991 	bufferlist[0].iov_len = desc_size;
992 	bufferlist[1].iov_base = buffer;
993 	bufferlist[1].iov_len = bufferlen;
994 	bufferlist[2].iov_base = &aligned_data;
995 	bufferlist[2].iov_len = (packetlen_aligned - packetlen);
996 
997 	return hv_ringbuffer_write(channel, bufferlist, 3);
998 }
999 EXPORT_SYMBOL_GPL(vmbus_sendpacket_mpb_desc);
1000 
1001 /**
1002  * __vmbus_recvpacket() - Retrieve the user packet on the specified channel
1003  * @channel: Pointer to vmbus_channel structure
1004  * @buffer: Pointer to the buffer you want to receive the data into.
1005  * @bufferlen: Maximum size of what the buffer can hold.
1006  * @buffer_actual_len: The actual size of the data after it was received.
1007  * @requestid: Identifier of the request
1008  * @raw: true means keep the vmpacket_descriptor header in the received data.
1009  *
1010  * Receives directly from the hyper-v vmbus and puts the data it received
1011  * into Buffer. This will receive the data unparsed from hyper-v.
1012  *
1013  * Mainly used by Hyper-V drivers.
1014  */
1015 static inline int
1016 __vmbus_recvpacket(struct vmbus_channel *channel, void *buffer,
1017 		   u32 bufferlen, u32 *buffer_actual_len, u64 *requestid,
1018 		   bool raw)
1019 {
1020 	return hv_ringbuffer_read(channel, buffer, bufferlen,
1021 				  buffer_actual_len, requestid, raw);
1022 
1023 }
1024 
1025 int vmbus_recvpacket(struct vmbus_channel *channel, void *buffer,
1026 		     u32 bufferlen, u32 *buffer_actual_len,
1027 		     u64 *requestid)
1028 {
1029 	return __vmbus_recvpacket(channel, buffer, bufferlen,
1030 				  buffer_actual_len, requestid, false);
1031 }
1032 EXPORT_SYMBOL(vmbus_recvpacket);
1033 
1034 /*
1035  * vmbus_recvpacket_raw - Retrieve the raw packet on the specified channel
1036  */
1037 int vmbus_recvpacket_raw(struct vmbus_channel *channel, void *buffer,
1038 			      u32 bufferlen, u32 *buffer_actual_len,
1039 			      u64 *requestid)
1040 {
1041 	return __vmbus_recvpacket(channel, buffer, bufferlen,
1042 				  buffer_actual_len, requestid, true);
1043 }
1044 EXPORT_SYMBOL_GPL(vmbus_recvpacket_raw);
1045