1 /*
2  * VMware VMCI Driver
3  *
4  * Copyright (C) 2012 VMware, Inc. All rights reserved.
5  *
6  * This program is free software; you can redistribute it and/or modify it
7  * under the terms of the GNU General Public License as published by the
8  * Free Software Foundation version 2 and no later version.
9  *
10  * This program is distributed in the hope that it will be useful, but
11  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
12  * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
13  * for more details.
14  */
15 
16 #include <linux/vmw_vmci_defs.h>
17 #include <linux/vmw_vmci_api.h>
18 #include <linux/completion.h>
19 #include <linux/hash.h>
20 #include <linux/kernel.h>
21 #include <linux/list.h>
22 #include <linux/module.h>
23 #include <linux/sched.h>
24 #include <linux/slab.h>
25 
26 #include "vmci_datagram.h"
27 #include "vmci_doorbell.h"
28 #include "vmci_resource.h"
29 #include "vmci_driver.h"
30 #include "vmci_route.h"
31 
32 
33 #define VMCI_DOORBELL_INDEX_BITS	6
34 #define VMCI_DOORBELL_INDEX_TABLE_SIZE	(1 << VMCI_DOORBELL_INDEX_BITS)
35 #define VMCI_DOORBELL_HASH(_idx)	hash_32(_idx, VMCI_DOORBELL_INDEX_BITS)
36 
37 /*
38  * DoorbellEntry describes the a doorbell notification handle allocated by the
39  * host.
40  */
41 struct dbell_entry {
42 	struct vmci_resource resource;
43 	struct hlist_node node;
44 	struct work_struct work;
45 	vmci_callback notify_cb;
46 	void *client_data;
47 	u32 idx;
48 	u32 priv_flags;
49 	bool run_delayed;
50 	atomic_t active;	/* Only used by guest personality */
51 };
52 
53 /* The VMCI index table keeps track of currently registered doorbells. */
54 struct dbell_index_table {
55 	spinlock_t lock;	/* Index table lock */
56 	struct hlist_head entries[VMCI_DOORBELL_INDEX_TABLE_SIZE];
57 };
58 
59 static struct dbell_index_table vmci_doorbell_it = {
60 	.lock = __SPIN_LOCK_UNLOCKED(vmci_doorbell_it.lock),
61 };
62 
63 /*
64  * The max_notify_idx is one larger than the currently known bitmap index in
65  * use, and is used to determine how much of the bitmap needs to be scanned.
66  */
67 static u32 max_notify_idx;
68 
69 /*
70  * The notify_idx_count is used for determining whether there are free entries
71  * within the bitmap (if notify_idx_count + 1 < max_notify_idx).
72  */
73 static u32 notify_idx_count;
74 
75 /*
76  * The last_notify_idx_reserved is used to track the last index handed out - in
77  * the case where multiple handles share a notification index, we hand out
78  * indexes round robin based on last_notify_idx_reserved.
79  */
80 static u32 last_notify_idx_reserved;
81 
82 /* This is a one entry cache used to by the index allocation. */
83 static u32 last_notify_idx_released = PAGE_SIZE;
84 
85 
86 /*
87  * Utility function that retrieves the privilege flags associated
88  * with a given doorbell handle. For guest endpoints, the
89  * privileges are determined by the context ID, but for host
90  * endpoints privileges are associated with the complete
91  * handle. Hypervisor endpoints are not yet supported.
92  */
93 int vmci_dbell_get_priv_flags(struct vmci_handle handle, u32 *priv_flags)
94 {
95 	if (priv_flags == NULL || handle.context == VMCI_INVALID_ID)
96 		return VMCI_ERROR_INVALID_ARGS;
97 
98 	if (handle.context == VMCI_HOST_CONTEXT_ID) {
99 		struct dbell_entry *entry;
100 		struct vmci_resource *resource;
101 
102 		resource = vmci_resource_by_handle(handle,
103 						   VMCI_RESOURCE_TYPE_DOORBELL);
104 		if (!resource)
105 			return VMCI_ERROR_NOT_FOUND;
106 
107 		entry = container_of(resource, struct dbell_entry, resource);
108 		*priv_flags = entry->priv_flags;
109 		vmci_resource_put(resource);
110 	} else if (handle.context == VMCI_HYPERVISOR_CONTEXT_ID) {
111 		/*
112 		 * Hypervisor endpoints for notifications are not
113 		 * supported (yet).
114 		 */
115 		return VMCI_ERROR_INVALID_ARGS;
116 	} else {
117 		*priv_flags = vmci_context_get_priv_flags(handle.context);
118 	}
119 
120 	return VMCI_SUCCESS;
121 }
122 
123 /*
124  * Find doorbell entry by bitmap index.
125  */
126 static struct dbell_entry *dbell_index_table_find(u32 idx)
127 {
128 	u32 bucket = VMCI_DOORBELL_HASH(idx);
129 	struct dbell_entry *dbell;
130 
131 	hlist_for_each_entry(dbell, &vmci_doorbell_it.entries[bucket],
132 			     node) {
133 		if (idx == dbell->idx)
134 			return dbell;
135 	}
136 
137 	return NULL;
138 }
139 
140 /*
141  * Add the given entry to the index table.  This willi take a reference to the
142  * entry's resource so that the entry is not deleted before it is removed from
143  * the * table.
144  */
145 static void dbell_index_table_add(struct dbell_entry *entry)
146 {
147 	u32 bucket;
148 	u32 new_notify_idx;
149 
150 	vmci_resource_get(&entry->resource);
151 
152 	spin_lock_bh(&vmci_doorbell_it.lock);
153 
154 	/*
155 	 * Below we try to allocate an index in the notification
156 	 * bitmap with "not too much" sharing between resources. If we
157 	 * use less that the full bitmap, we either add to the end if
158 	 * there are no unused flags within the currently used area,
159 	 * or we search for unused ones. If we use the full bitmap, we
160 	 * allocate the index round robin.
161 	 */
162 	if (max_notify_idx < PAGE_SIZE || notify_idx_count < PAGE_SIZE) {
163 		if (last_notify_idx_released < max_notify_idx &&
164 		    !dbell_index_table_find(last_notify_idx_released)) {
165 			new_notify_idx = last_notify_idx_released;
166 			last_notify_idx_released = PAGE_SIZE;
167 		} else {
168 			bool reused = false;
169 			new_notify_idx = last_notify_idx_reserved;
170 			if (notify_idx_count + 1 < max_notify_idx) {
171 				do {
172 					if (!dbell_index_table_find
173 					    (new_notify_idx)) {
174 						reused = true;
175 						break;
176 					}
177 					new_notify_idx = (new_notify_idx + 1) %
178 					    max_notify_idx;
179 				} while (new_notify_idx !=
180 					 last_notify_idx_released);
181 			}
182 			if (!reused) {
183 				new_notify_idx = max_notify_idx;
184 				max_notify_idx++;
185 			}
186 		}
187 	} else {
188 		new_notify_idx = (last_notify_idx_reserved + 1) % PAGE_SIZE;
189 	}
190 
191 	last_notify_idx_reserved = new_notify_idx;
192 	notify_idx_count++;
193 
194 	entry->idx = new_notify_idx;
195 	bucket = VMCI_DOORBELL_HASH(entry->idx);
196 	hlist_add_head(&entry->node, &vmci_doorbell_it.entries[bucket]);
197 
198 	spin_unlock_bh(&vmci_doorbell_it.lock);
199 }
200 
201 /*
202  * Remove the given entry from the index table.  This will release() the
203  * entry's resource.
204  */
205 static void dbell_index_table_remove(struct dbell_entry *entry)
206 {
207 	spin_lock_bh(&vmci_doorbell_it.lock);
208 
209 	hlist_del_init(&entry->node);
210 
211 	notify_idx_count--;
212 	if (entry->idx == max_notify_idx - 1) {
213 		/*
214 		 * If we delete an entry with the maximum known
215 		 * notification index, we take the opportunity to
216 		 * prune the current max. As there might be other
217 		 * unused indices immediately below, we lower the
218 		 * maximum until we hit an index in use.
219 		 */
220 		while (max_notify_idx > 0 &&
221 		       !dbell_index_table_find(max_notify_idx - 1))
222 			max_notify_idx--;
223 	}
224 
225 	last_notify_idx_released = entry->idx;
226 
227 	spin_unlock_bh(&vmci_doorbell_it.lock);
228 
229 	vmci_resource_put(&entry->resource);
230 }
231 
232 /*
233  * Creates a link between the given doorbell handle and the given
234  * index in the bitmap in the device backend. A notification state
235  * is created in hypervisor.
236  */
237 static int dbell_link(struct vmci_handle handle, u32 notify_idx)
238 {
239 	struct vmci_doorbell_link_msg link_msg;
240 
241 	link_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
242 					    VMCI_DOORBELL_LINK);
243 	link_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
244 	link_msg.hdr.payload_size = sizeof(link_msg) - VMCI_DG_HEADERSIZE;
245 	link_msg.handle = handle;
246 	link_msg.notify_idx = notify_idx;
247 
248 	return vmci_send_datagram(&link_msg.hdr);
249 }
250 
251 /*
252  * Unlinks the given doorbell handle from an index in the bitmap in
253  * the device backend. The notification state is destroyed in hypervisor.
254  */
255 static int dbell_unlink(struct vmci_handle handle)
256 {
257 	struct vmci_doorbell_unlink_msg unlink_msg;
258 
259 	unlink_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
260 					      VMCI_DOORBELL_UNLINK);
261 	unlink_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
262 	unlink_msg.hdr.payload_size = sizeof(unlink_msg) - VMCI_DG_HEADERSIZE;
263 	unlink_msg.handle = handle;
264 
265 	return vmci_send_datagram(&unlink_msg.hdr);
266 }
267 
268 /*
269  * Notify another guest or the host.  We send a datagram down to the
270  * host via the hypervisor with the notification info.
271  */
272 static int dbell_notify_as_guest(struct vmci_handle handle, u32 priv_flags)
273 {
274 	struct vmci_doorbell_notify_msg notify_msg;
275 
276 	notify_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
277 					      VMCI_DOORBELL_NOTIFY);
278 	notify_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
279 	notify_msg.hdr.payload_size = sizeof(notify_msg) - VMCI_DG_HEADERSIZE;
280 	notify_msg.handle = handle;
281 
282 	return vmci_send_datagram(&notify_msg.hdr);
283 }
284 
285 /*
286  * Calls the specified callback in a delayed context.
287  */
288 static void dbell_delayed_dispatch(struct work_struct *work)
289 {
290 	struct dbell_entry *entry = container_of(work,
291 						 struct dbell_entry, work);
292 
293 	entry->notify_cb(entry->client_data);
294 	vmci_resource_put(&entry->resource);
295 }
296 
297 /*
298  * Dispatches a doorbell notification to the host context.
299  */
300 int vmci_dbell_host_context_notify(u32 src_cid, struct vmci_handle handle)
301 {
302 	struct dbell_entry *entry;
303 	struct vmci_resource *resource;
304 
305 	if (vmci_handle_is_invalid(handle)) {
306 		pr_devel("Notifying an invalid doorbell (handle=0x%x:0x%x)\n",
307 			 handle.context, handle.resource);
308 		return VMCI_ERROR_INVALID_ARGS;
309 	}
310 
311 	resource = vmci_resource_by_handle(handle,
312 					   VMCI_RESOURCE_TYPE_DOORBELL);
313 	if (!resource) {
314 		pr_devel("Notifying an unknown doorbell (handle=0x%x:0x%x)\n",
315 			 handle.context, handle.resource);
316 		return VMCI_ERROR_NOT_FOUND;
317 	}
318 
319 	entry = container_of(resource, struct dbell_entry, resource);
320 	if (entry->run_delayed) {
321 		schedule_work(&entry->work);
322 	} else {
323 		entry->notify_cb(entry->client_data);
324 		vmci_resource_put(resource);
325 	}
326 
327 	return VMCI_SUCCESS;
328 }
329 
330 /*
331  * Register the notification bitmap with the host.
332  */
333 bool vmci_dbell_register_notification_bitmap(u32 bitmap_ppn)
334 {
335 	int result;
336 	struct vmci_notify_bm_set_msg bitmap_set_msg;
337 
338 	bitmap_set_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
339 						  VMCI_SET_NOTIFY_BITMAP);
340 	bitmap_set_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
341 	bitmap_set_msg.hdr.payload_size = sizeof(bitmap_set_msg) -
342 	    VMCI_DG_HEADERSIZE;
343 	bitmap_set_msg.bitmap_ppn = bitmap_ppn;
344 
345 	result = vmci_send_datagram(&bitmap_set_msg.hdr);
346 	if (result != VMCI_SUCCESS) {
347 		pr_devel("Failed to register (PPN=%u) as notification bitmap (error=%d)\n",
348 			 bitmap_ppn, result);
349 		return false;
350 	}
351 	return true;
352 }
353 
354 /*
355  * Executes or schedules the handlers for a given notify index.
356  */
357 static void dbell_fire_entries(u32 notify_idx)
358 {
359 	u32 bucket = VMCI_DOORBELL_HASH(notify_idx);
360 	struct dbell_entry *dbell;
361 
362 	spin_lock_bh(&vmci_doorbell_it.lock);
363 
364 	hlist_for_each_entry(dbell, &vmci_doorbell_it.entries[bucket], node) {
365 		if (dbell->idx == notify_idx &&
366 		    atomic_read(&dbell->active) == 1) {
367 			if (dbell->run_delayed) {
368 				vmci_resource_get(&dbell->resource);
369 				schedule_work(&dbell->work);
370 			} else {
371 				dbell->notify_cb(dbell->client_data);
372 			}
373 		}
374 	}
375 
376 	spin_unlock_bh(&vmci_doorbell_it.lock);
377 }
378 
379 /*
380  * Scans the notification bitmap, collects pending notifications,
381  * resets the bitmap and invokes appropriate callbacks.
382  */
383 void vmci_dbell_scan_notification_entries(u8 *bitmap)
384 {
385 	u32 idx;
386 
387 	for (idx = 0; idx < max_notify_idx; idx++) {
388 		if (bitmap[idx] & 0x1) {
389 			bitmap[idx] &= ~1;
390 			dbell_fire_entries(idx);
391 		}
392 	}
393 }
394 
395 /*
396  * vmci_doorbell_create() - Creates a doorbell
397  * @handle:     A handle used to track the resource.  Can be invalid.
398  * @flags:      Flag that determines context of callback.
399  * @priv_flags: Privileges flags.
400  * @notify_cb:  The callback to be ivoked when the doorbell fires.
401  * @client_data:        A parameter to be passed to the callback.
402  *
403  * Creates a doorbell with the given callback. If the handle is
404  * VMCI_INVALID_HANDLE, a free handle will be assigned, if
405  * possible. The callback can be run immediately (potentially with
406  * locks held - the default) or delayed (in a kernel thread) by
407  * specifying the flag VMCI_FLAG_DELAYED_CB. If delayed execution
408  * is selected, a given callback may not be run if the kernel is
409  * unable to allocate memory for the delayed execution (highly
410  * unlikely).
411  */
412 int vmci_doorbell_create(struct vmci_handle *handle,
413 			 u32 flags,
414 			 u32 priv_flags,
415 			 vmci_callback notify_cb, void *client_data)
416 {
417 	struct dbell_entry *entry;
418 	struct vmci_handle new_handle;
419 	int result;
420 
421 	if (!handle || !notify_cb || flags & ~VMCI_FLAG_DELAYED_CB ||
422 	    priv_flags & ~VMCI_PRIVILEGE_ALL_FLAGS)
423 		return VMCI_ERROR_INVALID_ARGS;
424 
425 	entry = kmalloc(sizeof(*entry), GFP_KERNEL);
426 	if (entry == NULL) {
427 		pr_warn("Failed allocating memory for datagram entry\n");
428 		return VMCI_ERROR_NO_MEM;
429 	}
430 
431 	if (vmci_handle_is_invalid(*handle)) {
432 		u32 context_id = vmci_get_context_id();
433 
434 		/* Let resource code allocate a free ID for us */
435 		new_handle = vmci_make_handle(context_id, VMCI_INVALID_ID);
436 	} else {
437 		bool valid_context = false;
438 
439 		/*
440 		 * Validate the handle.  We must do both of the checks below
441 		 * because we can be acting as both a host and a guest at the
442 		 * same time. We always allow the host context ID, since the
443 		 * host functionality is in practice always there with the
444 		 * unified driver.
445 		 */
446 		if (handle->context == VMCI_HOST_CONTEXT_ID ||
447 		    (vmci_guest_code_active() &&
448 		     vmci_get_context_id() == handle->context)) {
449 			valid_context = true;
450 		}
451 
452 		if (!valid_context || handle->resource == VMCI_INVALID_ID) {
453 			pr_devel("Invalid argument (handle=0x%x:0x%x)\n",
454 				 handle->context, handle->resource);
455 			result = VMCI_ERROR_INVALID_ARGS;
456 			goto free_mem;
457 		}
458 
459 		new_handle = *handle;
460 	}
461 
462 	entry->idx = 0;
463 	INIT_HLIST_NODE(&entry->node);
464 	entry->priv_flags = priv_flags;
465 	INIT_WORK(&entry->work, dbell_delayed_dispatch);
466 	entry->run_delayed = flags & VMCI_FLAG_DELAYED_CB;
467 	entry->notify_cb = notify_cb;
468 	entry->client_data = client_data;
469 	atomic_set(&entry->active, 0);
470 
471 	result = vmci_resource_add(&entry->resource,
472 				   VMCI_RESOURCE_TYPE_DOORBELL,
473 				   new_handle);
474 	if (result != VMCI_SUCCESS) {
475 		pr_warn("Failed to add new resource (handle=0x%x:0x%x), error: %d\n",
476 			new_handle.context, new_handle.resource, result);
477 		goto free_mem;
478 	}
479 
480 	new_handle = vmci_resource_handle(&entry->resource);
481 	if (vmci_guest_code_active()) {
482 		dbell_index_table_add(entry);
483 		result = dbell_link(new_handle, entry->idx);
484 		if (VMCI_SUCCESS != result)
485 			goto destroy_resource;
486 
487 		atomic_set(&entry->active, 1);
488 	}
489 
490 	*handle = new_handle;
491 
492 	return result;
493 
494  destroy_resource:
495 	dbell_index_table_remove(entry);
496 	vmci_resource_remove(&entry->resource);
497  free_mem:
498 	kfree(entry);
499 	return result;
500 }
501 EXPORT_SYMBOL_GPL(vmci_doorbell_create);
502 
503 /*
504  * vmci_doorbell_destroy() - Destroy a doorbell.
505  * @handle:     The handle tracking the resource.
506  *
507  * Destroys a doorbell previously created with vmcii_doorbell_create. This
508  * operation may block waiting for a callback to finish.
509  */
510 int vmci_doorbell_destroy(struct vmci_handle handle)
511 {
512 	struct dbell_entry *entry;
513 	struct vmci_resource *resource;
514 
515 	if (vmci_handle_is_invalid(handle))
516 		return VMCI_ERROR_INVALID_ARGS;
517 
518 	resource = vmci_resource_by_handle(handle,
519 					   VMCI_RESOURCE_TYPE_DOORBELL);
520 	if (!resource) {
521 		pr_devel("Failed to destroy doorbell (handle=0x%x:0x%x)\n",
522 			 handle.context, handle.resource);
523 		return VMCI_ERROR_NOT_FOUND;
524 	}
525 
526 	entry = container_of(resource, struct dbell_entry, resource);
527 
528 	if (vmci_guest_code_active()) {
529 		int result;
530 
531 		dbell_index_table_remove(entry);
532 
533 		result = dbell_unlink(handle);
534 		if (VMCI_SUCCESS != result) {
535 
536 			/*
537 			 * The only reason this should fail would be
538 			 * an inconsistency between guest and
539 			 * hypervisor state, where the guest believes
540 			 * it has an active registration whereas the
541 			 * hypervisor doesn't. One case where this may
542 			 * happen is if a doorbell is unregistered
543 			 * following a hibernation at a time where the
544 			 * doorbell state hasn't been restored on the
545 			 * hypervisor side yet. Since the handle has
546 			 * now been removed in the guest, we just
547 			 * print a warning and return success.
548 			 */
549 			pr_devel("Unlink of doorbell (handle=0x%x:0x%x) unknown by hypervisor (error=%d)\n",
550 				 handle.context, handle.resource, result);
551 		}
552 	}
553 
554 	/*
555 	 * Now remove the resource from the table.  It might still be in use
556 	 * after this, in a callback or still on the delayed work queue.
557 	 */
558 	vmci_resource_put(&entry->resource);
559 	vmci_resource_remove(&entry->resource);
560 
561 	kfree(entry);
562 
563 	return VMCI_SUCCESS;
564 }
565 EXPORT_SYMBOL_GPL(vmci_doorbell_destroy);
566 
567 /*
568  * vmci_doorbell_notify() - Ring the doorbell (and hide in the bushes).
569  * @dst:        The handlle identifying the doorbell resource
570  * @priv_flags: Priviledge flags.
571  *
572  * Generates a notification on the doorbell identified by the
573  * handle. For host side generation of notifications, the caller
574  * can specify what the privilege of the calling side is.
575  */
576 int vmci_doorbell_notify(struct vmci_handle dst, u32 priv_flags)
577 {
578 	int retval;
579 	enum vmci_route route;
580 	struct vmci_handle src;
581 
582 	if (vmci_handle_is_invalid(dst) ||
583 	    (priv_flags & ~VMCI_PRIVILEGE_ALL_FLAGS))
584 		return VMCI_ERROR_INVALID_ARGS;
585 
586 	src = VMCI_INVALID_HANDLE;
587 	retval = vmci_route(&src, &dst, false, &route);
588 	if (retval < VMCI_SUCCESS)
589 		return retval;
590 
591 	if (VMCI_ROUTE_AS_HOST == route)
592 		return vmci_ctx_notify_dbell(VMCI_HOST_CONTEXT_ID,
593 					     dst, priv_flags);
594 
595 	if (VMCI_ROUTE_AS_GUEST == route)
596 		return dbell_notify_as_guest(dst, priv_flags);
597 
598 	pr_warn("Unknown route (%d) for doorbell\n", route);
599 	return VMCI_ERROR_DST_UNREACHABLE;
600 }
601 EXPORT_SYMBOL_GPL(vmci_doorbell_notify);
602