1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * VMware VMCI Driver
4  *
5  * Copyright (C) 2012 VMware, Inc. All rights reserved.
6  */
7 
8 #include <linux/vmw_vmci_defs.h>
9 #include <linux/vmw_vmci_api.h>
10 #include <linux/completion.h>
11 #include <linux/hash.h>
12 #include <linux/kernel.h>
13 #include <linux/list.h>
14 #include <linux/module.h>
15 #include <linux/sched.h>
16 #include <linux/slab.h>
17 
18 #include "vmci_datagram.h"
19 #include "vmci_doorbell.h"
20 #include "vmci_resource.h"
21 #include "vmci_driver.h"
22 #include "vmci_route.h"
23 
24 
25 #define VMCI_DOORBELL_INDEX_BITS	6
26 #define VMCI_DOORBELL_INDEX_TABLE_SIZE	(1 << VMCI_DOORBELL_INDEX_BITS)
27 #define VMCI_DOORBELL_HASH(_idx)	hash_32(_idx, VMCI_DOORBELL_INDEX_BITS)
28 
29 /*
30  * DoorbellEntry describes the a doorbell notification handle allocated by the
31  * host.
32  */
33 struct dbell_entry {
34 	struct vmci_resource resource;
35 	struct hlist_node node;
36 	struct work_struct work;
37 	vmci_callback notify_cb;
38 	void *client_data;
39 	u32 idx;
40 	u32 priv_flags;
41 	bool run_delayed;
42 	atomic_t active;	/* Only used by guest personality */
43 };
44 
45 /* The VMCI index table keeps track of currently registered doorbells. */
46 struct dbell_index_table {
47 	spinlock_t lock;	/* Index table lock */
48 	struct hlist_head entries[VMCI_DOORBELL_INDEX_TABLE_SIZE];
49 };
50 
51 static struct dbell_index_table vmci_doorbell_it = {
52 	.lock = __SPIN_LOCK_UNLOCKED(vmci_doorbell_it.lock),
53 };
54 
55 /*
56  * The max_notify_idx is one larger than the currently known bitmap index in
57  * use, and is used to determine how much of the bitmap needs to be scanned.
58  */
59 static u32 max_notify_idx;
60 
61 /*
62  * The notify_idx_count is used for determining whether there are free entries
63  * within the bitmap (if notify_idx_count + 1 < max_notify_idx).
64  */
65 static u32 notify_idx_count;
66 
67 /*
68  * The last_notify_idx_reserved is used to track the last index handed out - in
69  * the case where multiple handles share a notification index, we hand out
70  * indexes round robin based on last_notify_idx_reserved.
71  */
72 static u32 last_notify_idx_reserved;
73 
74 /* This is a one entry cache used to by the index allocation. */
75 static u32 last_notify_idx_released = PAGE_SIZE;
76 
77 
78 /*
79  * Utility function that retrieves the privilege flags associated
80  * with a given doorbell handle. For guest endpoints, the
81  * privileges are determined by the context ID, but for host
82  * endpoints privileges are associated with the complete
83  * handle. Hypervisor endpoints are not yet supported.
84  */
85 int vmci_dbell_get_priv_flags(struct vmci_handle handle, u32 *priv_flags)
86 {
87 	if (priv_flags == NULL || handle.context == VMCI_INVALID_ID)
88 		return VMCI_ERROR_INVALID_ARGS;
89 
90 	if (handle.context == VMCI_HOST_CONTEXT_ID) {
91 		struct dbell_entry *entry;
92 		struct vmci_resource *resource;
93 
94 		resource = vmci_resource_by_handle(handle,
95 						   VMCI_RESOURCE_TYPE_DOORBELL);
96 		if (!resource)
97 			return VMCI_ERROR_NOT_FOUND;
98 
99 		entry = container_of(resource, struct dbell_entry, resource);
100 		*priv_flags = entry->priv_flags;
101 		vmci_resource_put(resource);
102 	} else if (handle.context == VMCI_HYPERVISOR_CONTEXT_ID) {
103 		/*
104 		 * Hypervisor endpoints for notifications are not
105 		 * supported (yet).
106 		 */
107 		return VMCI_ERROR_INVALID_ARGS;
108 	} else {
109 		*priv_flags = vmci_context_get_priv_flags(handle.context);
110 	}
111 
112 	return VMCI_SUCCESS;
113 }
114 
115 /*
116  * Find doorbell entry by bitmap index.
117  */
118 static struct dbell_entry *dbell_index_table_find(u32 idx)
119 {
120 	u32 bucket = VMCI_DOORBELL_HASH(idx);
121 	struct dbell_entry *dbell;
122 
123 	hlist_for_each_entry(dbell, &vmci_doorbell_it.entries[bucket],
124 			     node) {
125 		if (idx == dbell->idx)
126 			return dbell;
127 	}
128 
129 	return NULL;
130 }
131 
132 /*
133  * Add the given entry to the index table.  This willi take a reference to the
134  * entry's resource so that the entry is not deleted before it is removed from
135  * the * table.
136  */
137 static void dbell_index_table_add(struct dbell_entry *entry)
138 {
139 	u32 bucket;
140 	u32 new_notify_idx;
141 
142 	vmci_resource_get(&entry->resource);
143 
144 	spin_lock_bh(&vmci_doorbell_it.lock);
145 
146 	/*
147 	 * Below we try to allocate an index in the notification
148 	 * bitmap with "not too much" sharing between resources. If we
149 	 * use less that the full bitmap, we either add to the end if
150 	 * there are no unused flags within the currently used area,
151 	 * or we search for unused ones. If we use the full bitmap, we
152 	 * allocate the index round robin.
153 	 */
154 	if (max_notify_idx < PAGE_SIZE || notify_idx_count < PAGE_SIZE) {
155 		if (last_notify_idx_released < max_notify_idx &&
156 		    !dbell_index_table_find(last_notify_idx_released)) {
157 			new_notify_idx = last_notify_idx_released;
158 			last_notify_idx_released = PAGE_SIZE;
159 		} else {
160 			bool reused = false;
161 			new_notify_idx = last_notify_idx_reserved;
162 			if (notify_idx_count + 1 < max_notify_idx) {
163 				do {
164 					if (!dbell_index_table_find
165 					    (new_notify_idx)) {
166 						reused = true;
167 						break;
168 					}
169 					new_notify_idx = (new_notify_idx + 1) %
170 					    max_notify_idx;
171 				} while (new_notify_idx !=
172 					 last_notify_idx_released);
173 			}
174 			if (!reused) {
175 				new_notify_idx = max_notify_idx;
176 				max_notify_idx++;
177 			}
178 		}
179 	} else {
180 		new_notify_idx = (last_notify_idx_reserved + 1) % PAGE_SIZE;
181 	}
182 
183 	last_notify_idx_reserved = new_notify_idx;
184 	notify_idx_count++;
185 
186 	entry->idx = new_notify_idx;
187 	bucket = VMCI_DOORBELL_HASH(entry->idx);
188 	hlist_add_head(&entry->node, &vmci_doorbell_it.entries[bucket]);
189 
190 	spin_unlock_bh(&vmci_doorbell_it.lock);
191 }
192 
193 /*
194  * Remove the given entry from the index table.  This will release() the
195  * entry's resource.
196  */
197 static void dbell_index_table_remove(struct dbell_entry *entry)
198 {
199 	spin_lock_bh(&vmci_doorbell_it.lock);
200 
201 	hlist_del_init(&entry->node);
202 
203 	notify_idx_count--;
204 	if (entry->idx == max_notify_idx - 1) {
205 		/*
206 		 * If we delete an entry with the maximum known
207 		 * notification index, we take the opportunity to
208 		 * prune the current max. As there might be other
209 		 * unused indices immediately below, we lower the
210 		 * maximum until we hit an index in use.
211 		 */
212 		while (max_notify_idx > 0 &&
213 		       !dbell_index_table_find(max_notify_idx - 1))
214 			max_notify_idx--;
215 	}
216 
217 	last_notify_idx_released = entry->idx;
218 
219 	spin_unlock_bh(&vmci_doorbell_it.lock);
220 
221 	vmci_resource_put(&entry->resource);
222 }
223 
224 /*
225  * Creates a link between the given doorbell handle and the given
226  * index in the bitmap in the device backend. A notification state
227  * is created in hypervisor.
228  */
229 static int dbell_link(struct vmci_handle handle, u32 notify_idx)
230 {
231 	struct vmci_doorbell_link_msg link_msg;
232 
233 	link_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
234 					    VMCI_DOORBELL_LINK);
235 	link_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
236 	link_msg.hdr.payload_size = sizeof(link_msg) - VMCI_DG_HEADERSIZE;
237 	link_msg.handle = handle;
238 	link_msg.notify_idx = notify_idx;
239 
240 	return vmci_send_datagram(&link_msg.hdr);
241 }
242 
243 /*
244  * Unlinks the given doorbell handle from an index in the bitmap in
245  * the device backend. The notification state is destroyed in hypervisor.
246  */
247 static int dbell_unlink(struct vmci_handle handle)
248 {
249 	struct vmci_doorbell_unlink_msg unlink_msg;
250 
251 	unlink_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
252 					      VMCI_DOORBELL_UNLINK);
253 	unlink_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
254 	unlink_msg.hdr.payload_size = sizeof(unlink_msg) - VMCI_DG_HEADERSIZE;
255 	unlink_msg.handle = handle;
256 
257 	return vmci_send_datagram(&unlink_msg.hdr);
258 }
259 
260 /*
261  * Notify another guest or the host.  We send a datagram down to the
262  * host via the hypervisor with the notification info.
263  */
264 static int dbell_notify_as_guest(struct vmci_handle handle, u32 priv_flags)
265 {
266 	struct vmci_doorbell_notify_msg notify_msg;
267 
268 	notify_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
269 					      VMCI_DOORBELL_NOTIFY);
270 	notify_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
271 	notify_msg.hdr.payload_size = sizeof(notify_msg) - VMCI_DG_HEADERSIZE;
272 	notify_msg.handle = handle;
273 
274 	return vmci_send_datagram(&notify_msg.hdr);
275 }
276 
277 /*
278  * Calls the specified callback in a delayed context.
279  */
280 static void dbell_delayed_dispatch(struct work_struct *work)
281 {
282 	struct dbell_entry *entry = container_of(work,
283 						 struct dbell_entry, work);
284 
285 	entry->notify_cb(entry->client_data);
286 	vmci_resource_put(&entry->resource);
287 }
288 
289 /*
290  * Dispatches a doorbell notification to the host context.
291  */
292 int vmci_dbell_host_context_notify(u32 src_cid, struct vmci_handle handle)
293 {
294 	struct dbell_entry *entry;
295 	struct vmci_resource *resource;
296 
297 	if (vmci_handle_is_invalid(handle)) {
298 		pr_devel("Notifying an invalid doorbell (handle=0x%x:0x%x)\n",
299 			 handle.context, handle.resource);
300 		return VMCI_ERROR_INVALID_ARGS;
301 	}
302 
303 	resource = vmci_resource_by_handle(handle,
304 					   VMCI_RESOURCE_TYPE_DOORBELL);
305 	if (!resource) {
306 		pr_devel("Notifying an unknown doorbell (handle=0x%x:0x%x)\n",
307 			 handle.context, handle.resource);
308 		return VMCI_ERROR_NOT_FOUND;
309 	}
310 
311 	entry = container_of(resource, struct dbell_entry, resource);
312 	if (entry->run_delayed) {
313 		if (!schedule_work(&entry->work))
314 			vmci_resource_put(resource);
315 	} else {
316 		entry->notify_cb(entry->client_data);
317 		vmci_resource_put(resource);
318 	}
319 
320 	return VMCI_SUCCESS;
321 }
322 
323 /*
324  * Register the notification bitmap with the host.
325  */
326 bool vmci_dbell_register_notification_bitmap(u64 bitmap_ppn)
327 {
328 	int result;
329 	struct vmci_notify_bm_set_msg bitmap_set_msg;
330 
331 	bitmap_set_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
332 						  VMCI_SET_NOTIFY_BITMAP);
333 	bitmap_set_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
334 	bitmap_set_msg.hdr.payload_size = sizeof(bitmap_set_msg) -
335 	    VMCI_DG_HEADERSIZE;
336 	if (vmci_use_ppn64())
337 		bitmap_set_msg.bitmap_ppn64 = bitmap_ppn;
338 	else
339 		bitmap_set_msg.bitmap_ppn32 = (u32) bitmap_ppn;
340 
341 	result = vmci_send_datagram(&bitmap_set_msg.hdr);
342 	if (result != VMCI_SUCCESS) {
343 		pr_devel("Failed to register (PPN=%llu) as notification bitmap (error=%d)\n",
344 			 bitmap_ppn, result);
345 		return false;
346 	}
347 	return true;
348 }
349 
350 /*
351  * Executes or schedules the handlers for a given notify index.
352  */
353 static void dbell_fire_entries(u32 notify_idx)
354 {
355 	u32 bucket = VMCI_DOORBELL_HASH(notify_idx);
356 	struct dbell_entry *dbell;
357 
358 	spin_lock_bh(&vmci_doorbell_it.lock);
359 
360 	hlist_for_each_entry(dbell, &vmci_doorbell_it.entries[bucket], node) {
361 		if (dbell->idx == notify_idx &&
362 		    atomic_read(&dbell->active) == 1) {
363 			if (dbell->run_delayed) {
364 				vmci_resource_get(&dbell->resource);
365 				if (!schedule_work(&dbell->work))
366 					vmci_resource_put(&dbell->resource);
367 			} else {
368 				dbell->notify_cb(dbell->client_data);
369 			}
370 		}
371 	}
372 
373 	spin_unlock_bh(&vmci_doorbell_it.lock);
374 }
375 
376 /*
377  * Scans the notification bitmap, collects pending notifications,
378  * resets the bitmap and invokes appropriate callbacks.
379  */
380 void vmci_dbell_scan_notification_entries(u8 *bitmap)
381 {
382 	u32 idx;
383 
384 	for (idx = 0; idx < max_notify_idx; idx++) {
385 		if (bitmap[idx] & 0x1) {
386 			bitmap[idx] &= ~1;
387 			dbell_fire_entries(idx);
388 		}
389 	}
390 }
391 
392 /*
393  * vmci_doorbell_create() - Creates a doorbell
394  * @handle:     A handle used to track the resource.  Can be invalid.
395  * @flags:      Flag that determines context of callback.
396  * @priv_flags: Privileges flags.
397  * @notify_cb:  The callback to be ivoked when the doorbell fires.
398  * @client_data:        A parameter to be passed to the callback.
399  *
400  * Creates a doorbell with the given callback. If the handle is
401  * VMCI_INVALID_HANDLE, a free handle will be assigned, if
402  * possible. The callback can be run immediately (potentially with
403  * locks held - the default) or delayed (in a kernel thread) by
404  * specifying the flag VMCI_FLAG_DELAYED_CB. If delayed execution
405  * is selected, a given callback may not be run if the kernel is
406  * unable to allocate memory for the delayed execution (highly
407  * unlikely).
408  */
409 int vmci_doorbell_create(struct vmci_handle *handle,
410 			 u32 flags,
411 			 u32 priv_flags,
412 			 vmci_callback notify_cb, void *client_data)
413 {
414 	struct dbell_entry *entry;
415 	struct vmci_handle new_handle;
416 	int result;
417 
418 	if (!handle || !notify_cb || flags & ~VMCI_FLAG_DELAYED_CB ||
419 	    priv_flags & ~VMCI_PRIVILEGE_ALL_FLAGS)
420 		return VMCI_ERROR_INVALID_ARGS;
421 
422 	entry = kmalloc(sizeof(*entry), GFP_KERNEL);
423 	if (entry == NULL) {
424 		pr_warn("Failed allocating memory for datagram entry\n");
425 		return VMCI_ERROR_NO_MEM;
426 	}
427 
428 	if (vmci_handle_is_invalid(*handle)) {
429 		u32 context_id = vmci_get_context_id();
430 
431 		if (context_id == VMCI_INVALID_ID) {
432 			pr_warn("Failed to get context ID\n");
433 			result = VMCI_ERROR_NO_RESOURCES;
434 			goto free_mem;
435 		}
436 
437 		/* Let resource code allocate a free ID for us */
438 		new_handle = vmci_make_handle(context_id, VMCI_INVALID_ID);
439 	} else {
440 		bool valid_context = false;
441 
442 		/*
443 		 * Validate the handle.  We must do both of the checks below
444 		 * because we can be acting as both a host and a guest at the
445 		 * same time. We always allow the host context ID, since the
446 		 * host functionality is in practice always there with the
447 		 * unified driver.
448 		 */
449 		if (handle->context == VMCI_HOST_CONTEXT_ID ||
450 		    (vmci_guest_code_active() &&
451 		     vmci_get_context_id() == handle->context)) {
452 			valid_context = true;
453 		}
454 
455 		if (!valid_context || handle->resource == VMCI_INVALID_ID) {
456 			pr_devel("Invalid argument (handle=0x%x:0x%x)\n",
457 				 handle->context, handle->resource);
458 			result = VMCI_ERROR_INVALID_ARGS;
459 			goto free_mem;
460 		}
461 
462 		new_handle = *handle;
463 	}
464 
465 	entry->idx = 0;
466 	INIT_HLIST_NODE(&entry->node);
467 	entry->priv_flags = priv_flags;
468 	INIT_WORK(&entry->work, dbell_delayed_dispatch);
469 	entry->run_delayed = flags & VMCI_FLAG_DELAYED_CB;
470 	entry->notify_cb = notify_cb;
471 	entry->client_data = client_data;
472 	atomic_set(&entry->active, 0);
473 
474 	result = vmci_resource_add(&entry->resource,
475 				   VMCI_RESOURCE_TYPE_DOORBELL,
476 				   new_handle);
477 	if (result != VMCI_SUCCESS) {
478 		pr_warn("Failed to add new resource (handle=0x%x:0x%x), error: %d\n",
479 			new_handle.context, new_handle.resource, result);
480 		goto free_mem;
481 	}
482 
483 	new_handle = vmci_resource_handle(&entry->resource);
484 	if (vmci_guest_code_active()) {
485 		dbell_index_table_add(entry);
486 		result = dbell_link(new_handle, entry->idx);
487 		if (VMCI_SUCCESS != result)
488 			goto destroy_resource;
489 
490 		atomic_set(&entry->active, 1);
491 	}
492 
493 	*handle = new_handle;
494 
495 	return result;
496 
497  destroy_resource:
498 	dbell_index_table_remove(entry);
499 	vmci_resource_remove(&entry->resource);
500  free_mem:
501 	kfree(entry);
502 	return result;
503 }
504 EXPORT_SYMBOL_GPL(vmci_doorbell_create);
505 
506 /*
507  * vmci_doorbell_destroy() - Destroy a doorbell.
508  * @handle:     The handle tracking the resource.
509  *
510  * Destroys a doorbell previously created with vmcii_doorbell_create. This
511  * operation may block waiting for a callback to finish.
512  */
513 int vmci_doorbell_destroy(struct vmci_handle handle)
514 {
515 	struct dbell_entry *entry;
516 	struct vmci_resource *resource;
517 
518 	if (vmci_handle_is_invalid(handle))
519 		return VMCI_ERROR_INVALID_ARGS;
520 
521 	resource = vmci_resource_by_handle(handle,
522 					   VMCI_RESOURCE_TYPE_DOORBELL);
523 	if (!resource) {
524 		pr_devel("Failed to destroy doorbell (handle=0x%x:0x%x)\n",
525 			 handle.context, handle.resource);
526 		return VMCI_ERROR_NOT_FOUND;
527 	}
528 
529 	entry = container_of(resource, struct dbell_entry, resource);
530 
531 	if (!hlist_unhashed(&entry->node)) {
532 		int result;
533 
534 		dbell_index_table_remove(entry);
535 
536 		result = dbell_unlink(handle);
537 		if (VMCI_SUCCESS != result) {
538 
539 			/*
540 			 * The only reason this should fail would be
541 			 * an inconsistency between guest and
542 			 * hypervisor state, where the guest believes
543 			 * it has an active registration whereas the
544 			 * hypervisor doesn't. One case where this may
545 			 * happen is if a doorbell is unregistered
546 			 * following a hibernation at a time where the
547 			 * doorbell state hasn't been restored on the
548 			 * hypervisor side yet. Since the handle has
549 			 * now been removed in the guest, we just
550 			 * print a warning and return success.
551 			 */
552 			pr_devel("Unlink of doorbell (handle=0x%x:0x%x) unknown by hypervisor (error=%d)\n",
553 				 handle.context, handle.resource, result);
554 		}
555 	}
556 
557 	/*
558 	 * Now remove the resource from the table.  It might still be in use
559 	 * after this, in a callback or still on the delayed work queue.
560 	 */
561 	vmci_resource_put(&entry->resource);
562 	vmci_resource_remove(&entry->resource);
563 
564 	kfree(entry);
565 
566 	return VMCI_SUCCESS;
567 }
568 EXPORT_SYMBOL_GPL(vmci_doorbell_destroy);
569 
570 /*
571  * vmci_doorbell_notify() - Ring the doorbell (and hide in the bushes).
572  * @dst:        The handlle identifying the doorbell resource
573  * @priv_flags: Priviledge flags.
574  *
575  * Generates a notification on the doorbell identified by the
576  * handle. For host side generation of notifications, the caller
577  * can specify what the privilege of the calling side is.
578  */
579 int vmci_doorbell_notify(struct vmci_handle dst, u32 priv_flags)
580 {
581 	int retval;
582 	enum vmci_route route;
583 	struct vmci_handle src;
584 
585 	if (vmci_handle_is_invalid(dst) ||
586 	    (priv_flags & ~VMCI_PRIVILEGE_ALL_FLAGS))
587 		return VMCI_ERROR_INVALID_ARGS;
588 
589 	src = VMCI_INVALID_HANDLE;
590 	retval = vmci_route(&src, &dst, false, &route);
591 	if (retval < VMCI_SUCCESS)
592 		return retval;
593 
594 	if (VMCI_ROUTE_AS_HOST == route)
595 		return vmci_ctx_notify_dbell(VMCI_HOST_CONTEXT_ID,
596 					     dst, priv_flags);
597 
598 	if (VMCI_ROUTE_AS_GUEST == route)
599 		return dbell_notify_as_guest(dst, priv_flags);
600 
601 	pr_warn("Unknown route (%d) for doorbell\n", route);
602 	return VMCI_ERROR_DST_UNREACHABLE;
603 }
604 EXPORT_SYMBOL_GPL(vmci_doorbell_notify);
605