1685a6bf8SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
283e2ec76SGeorge Zhang /*
383e2ec76SGeorge Zhang * VMware VMCI Driver
483e2ec76SGeorge Zhang *
583e2ec76SGeorge Zhang * Copyright (C) 2012 VMware, Inc. All rights reserved.
683e2ec76SGeorge Zhang */
783e2ec76SGeorge Zhang
883e2ec76SGeorge Zhang #include <linux/vmw_vmci_defs.h>
983e2ec76SGeorge Zhang #include <linux/vmw_vmci_api.h>
1083e2ec76SGeorge Zhang #include <linux/completion.h>
1183e2ec76SGeorge Zhang #include <linux/hash.h>
1283e2ec76SGeorge Zhang #include <linux/kernel.h>
1383e2ec76SGeorge Zhang #include <linux/list.h>
1483e2ec76SGeorge Zhang #include <linux/module.h>
1583e2ec76SGeorge Zhang #include <linux/sched.h>
1683e2ec76SGeorge Zhang #include <linux/slab.h>
1783e2ec76SGeorge Zhang
1883e2ec76SGeorge Zhang #include "vmci_datagram.h"
1983e2ec76SGeorge Zhang #include "vmci_doorbell.h"
2083e2ec76SGeorge Zhang #include "vmci_resource.h"
2183e2ec76SGeorge Zhang #include "vmci_driver.h"
2283e2ec76SGeorge Zhang #include "vmci_route.h"
2383e2ec76SGeorge Zhang
2483e2ec76SGeorge Zhang
2583e2ec76SGeorge Zhang #define VMCI_DOORBELL_INDEX_BITS 6
2683e2ec76SGeorge Zhang #define VMCI_DOORBELL_INDEX_TABLE_SIZE (1 << VMCI_DOORBELL_INDEX_BITS)
2783e2ec76SGeorge Zhang #define VMCI_DOORBELL_HASH(_idx) hash_32(_idx, VMCI_DOORBELL_INDEX_BITS)
2883e2ec76SGeorge Zhang
2983e2ec76SGeorge Zhang /*
3083e2ec76SGeorge Zhang * DoorbellEntry describes the a doorbell notification handle allocated by the
3183e2ec76SGeorge Zhang * host.
3283e2ec76SGeorge Zhang */
3383e2ec76SGeorge Zhang struct dbell_entry {
3483e2ec76SGeorge Zhang struct vmci_resource resource;
3583e2ec76SGeorge Zhang struct hlist_node node;
3683e2ec76SGeorge Zhang struct work_struct work;
3783e2ec76SGeorge Zhang vmci_callback notify_cb;
3883e2ec76SGeorge Zhang void *client_data;
3983e2ec76SGeorge Zhang u32 idx;
4083e2ec76SGeorge Zhang u32 priv_flags;
4183e2ec76SGeorge Zhang bool run_delayed;
4283e2ec76SGeorge Zhang atomic_t active; /* Only used by guest personality */
4383e2ec76SGeorge Zhang };
4483e2ec76SGeorge Zhang
4583e2ec76SGeorge Zhang /* The VMCI index table keeps track of currently registered doorbells. */
4683e2ec76SGeorge Zhang struct dbell_index_table {
4783e2ec76SGeorge Zhang spinlock_t lock; /* Index table lock */
4883e2ec76SGeorge Zhang struct hlist_head entries[VMCI_DOORBELL_INDEX_TABLE_SIZE];
4983e2ec76SGeorge Zhang };
5083e2ec76SGeorge Zhang
5183e2ec76SGeorge Zhang static struct dbell_index_table vmci_doorbell_it = {
5283e2ec76SGeorge Zhang .lock = __SPIN_LOCK_UNLOCKED(vmci_doorbell_it.lock),
5383e2ec76SGeorge Zhang };
5483e2ec76SGeorge Zhang
5583e2ec76SGeorge Zhang /*
5683e2ec76SGeorge Zhang * The max_notify_idx is one larger than the currently known bitmap index in
5783e2ec76SGeorge Zhang * use, and is used to determine how much of the bitmap needs to be scanned.
5883e2ec76SGeorge Zhang */
5983e2ec76SGeorge Zhang static u32 max_notify_idx;
6083e2ec76SGeorge Zhang
6183e2ec76SGeorge Zhang /*
6283e2ec76SGeorge Zhang * The notify_idx_count is used for determining whether there are free entries
6383e2ec76SGeorge Zhang * within the bitmap (if notify_idx_count + 1 < max_notify_idx).
6483e2ec76SGeorge Zhang */
6583e2ec76SGeorge Zhang static u32 notify_idx_count;
6683e2ec76SGeorge Zhang
6783e2ec76SGeorge Zhang /*
6883e2ec76SGeorge Zhang * The last_notify_idx_reserved is used to track the last index handed out - in
6983e2ec76SGeorge Zhang * the case where multiple handles share a notification index, we hand out
7083e2ec76SGeorge Zhang * indexes round robin based on last_notify_idx_reserved.
7183e2ec76SGeorge Zhang */
7283e2ec76SGeorge Zhang static u32 last_notify_idx_reserved;
7383e2ec76SGeorge Zhang
7483e2ec76SGeorge Zhang /* This is a one entry cache used to by the index allocation. */
7583e2ec76SGeorge Zhang static u32 last_notify_idx_released = PAGE_SIZE;
7683e2ec76SGeorge Zhang
7783e2ec76SGeorge Zhang
7883e2ec76SGeorge Zhang /*
7983e2ec76SGeorge Zhang * Utility function that retrieves the privilege flags associated
8083e2ec76SGeorge Zhang * with a given doorbell handle. For guest endpoints, the
8183e2ec76SGeorge Zhang * privileges are determined by the context ID, but for host
8283e2ec76SGeorge Zhang * endpoints privileges are associated with the complete
8383e2ec76SGeorge Zhang * handle. Hypervisor endpoints are not yet supported.
8483e2ec76SGeorge Zhang */
vmci_dbell_get_priv_flags(struct vmci_handle handle,u32 * priv_flags)8583e2ec76SGeorge Zhang int vmci_dbell_get_priv_flags(struct vmci_handle handle, u32 *priv_flags)
8683e2ec76SGeorge Zhang {
8783e2ec76SGeorge Zhang if (priv_flags == NULL || handle.context == VMCI_INVALID_ID)
8883e2ec76SGeorge Zhang return VMCI_ERROR_INVALID_ARGS;
8983e2ec76SGeorge Zhang
9083e2ec76SGeorge Zhang if (handle.context == VMCI_HOST_CONTEXT_ID) {
9183e2ec76SGeorge Zhang struct dbell_entry *entry;
9283e2ec76SGeorge Zhang struct vmci_resource *resource;
9383e2ec76SGeorge Zhang
9483e2ec76SGeorge Zhang resource = vmci_resource_by_handle(handle,
9583e2ec76SGeorge Zhang VMCI_RESOURCE_TYPE_DOORBELL);
9683e2ec76SGeorge Zhang if (!resource)
9783e2ec76SGeorge Zhang return VMCI_ERROR_NOT_FOUND;
9883e2ec76SGeorge Zhang
9983e2ec76SGeorge Zhang entry = container_of(resource, struct dbell_entry, resource);
10083e2ec76SGeorge Zhang *priv_flags = entry->priv_flags;
10183e2ec76SGeorge Zhang vmci_resource_put(resource);
10283e2ec76SGeorge Zhang } else if (handle.context == VMCI_HYPERVISOR_CONTEXT_ID) {
10383e2ec76SGeorge Zhang /*
10483e2ec76SGeorge Zhang * Hypervisor endpoints for notifications are not
10583e2ec76SGeorge Zhang * supported (yet).
10683e2ec76SGeorge Zhang */
10783e2ec76SGeorge Zhang return VMCI_ERROR_INVALID_ARGS;
10883e2ec76SGeorge Zhang } else {
10983e2ec76SGeorge Zhang *priv_flags = vmci_context_get_priv_flags(handle.context);
11083e2ec76SGeorge Zhang }
11183e2ec76SGeorge Zhang
11283e2ec76SGeorge Zhang return VMCI_SUCCESS;
11383e2ec76SGeorge Zhang }
11483e2ec76SGeorge Zhang
11583e2ec76SGeorge Zhang /*
11683e2ec76SGeorge Zhang * Find doorbell entry by bitmap index.
11783e2ec76SGeorge Zhang */
dbell_index_table_find(u32 idx)11883e2ec76SGeorge Zhang static struct dbell_entry *dbell_index_table_find(u32 idx)
11983e2ec76SGeorge Zhang {
12083e2ec76SGeorge Zhang u32 bucket = VMCI_DOORBELL_HASH(idx);
12183e2ec76SGeorge Zhang struct dbell_entry *dbell;
12283e2ec76SGeorge Zhang
123b67bfe0dSSasha Levin hlist_for_each_entry(dbell, &vmci_doorbell_it.entries[bucket],
12483e2ec76SGeorge Zhang node) {
12583e2ec76SGeorge Zhang if (idx == dbell->idx)
12683e2ec76SGeorge Zhang return dbell;
12783e2ec76SGeorge Zhang }
12883e2ec76SGeorge Zhang
12983e2ec76SGeorge Zhang return NULL;
13083e2ec76SGeorge Zhang }
13183e2ec76SGeorge Zhang
13283e2ec76SGeorge Zhang /*
13383e2ec76SGeorge Zhang * Add the given entry to the index table. This willi take a reference to the
13483e2ec76SGeorge Zhang * entry's resource so that the entry is not deleted before it is removed from
13583e2ec76SGeorge Zhang * the * table.
13683e2ec76SGeorge Zhang */
dbell_index_table_add(struct dbell_entry * entry)13783e2ec76SGeorge Zhang static void dbell_index_table_add(struct dbell_entry *entry)
13883e2ec76SGeorge Zhang {
13983e2ec76SGeorge Zhang u32 bucket;
14083e2ec76SGeorge Zhang u32 new_notify_idx;
14183e2ec76SGeorge Zhang
14283e2ec76SGeorge Zhang vmci_resource_get(&entry->resource);
14383e2ec76SGeorge Zhang
14483e2ec76SGeorge Zhang spin_lock_bh(&vmci_doorbell_it.lock);
14583e2ec76SGeorge Zhang
14683e2ec76SGeorge Zhang /*
14783e2ec76SGeorge Zhang * Below we try to allocate an index in the notification
14883e2ec76SGeorge Zhang * bitmap with "not too much" sharing between resources. If we
14983e2ec76SGeorge Zhang * use less that the full bitmap, we either add to the end if
15083e2ec76SGeorge Zhang * there are no unused flags within the currently used area,
15183e2ec76SGeorge Zhang * or we search for unused ones. If we use the full bitmap, we
15283e2ec76SGeorge Zhang * allocate the index round robin.
15383e2ec76SGeorge Zhang */
15483e2ec76SGeorge Zhang if (max_notify_idx < PAGE_SIZE || notify_idx_count < PAGE_SIZE) {
15583e2ec76SGeorge Zhang if (last_notify_idx_released < max_notify_idx &&
15683e2ec76SGeorge Zhang !dbell_index_table_find(last_notify_idx_released)) {
15783e2ec76SGeorge Zhang new_notify_idx = last_notify_idx_released;
15883e2ec76SGeorge Zhang last_notify_idx_released = PAGE_SIZE;
15983e2ec76SGeorge Zhang } else {
16083e2ec76SGeorge Zhang bool reused = false;
16183e2ec76SGeorge Zhang new_notify_idx = last_notify_idx_reserved;
16283e2ec76SGeorge Zhang if (notify_idx_count + 1 < max_notify_idx) {
16383e2ec76SGeorge Zhang do {
16483e2ec76SGeorge Zhang if (!dbell_index_table_find
16583e2ec76SGeorge Zhang (new_notify_idx)) {
16683e2ec76SGeorge Zhang reused = true;
16783e2ec76SGeorge Zhang break;
16883e2ec76SGeorge Zhang }
16983e2ec76SGeorge Zhang new_notify_idx = (new_notify_idx + 1) %
17083e2ec76SGeorge Zhang max_notify_idx;
17183e2ec76SGeorge Zhang } while (new_notify_idx !=
17283e2ec76SGeorge Zhang last_notify_idx_released);
17383e2ec76SGeorge Zhang }
17483e2ec76SGeorge Zhang if (!reused) {
17583e2ec76SGeorge Zhang new_notify_idx = max_notify_idx;
17683e2ec76SGeorge Zhang max_notify_idx++;
17783e2ec76SGeorge Zhang }
17883e2ec76SGeorge Zhang }
17983e2ec76SGeorge Zhang } else {
18083e2ec76SGeorge Zhang new_notify_idx = (last_notify_idx_reserved + 1) % PAGE_SIZE;
18183e2ec76SGeorge Zhang }
18283e2ec76SGeorge Zhang
18383e2ec76SGeorge Zhang last_notify_idx_reserved = new_notify_idx;
18483e2ec76SGeorge Zhang notify_idx_count++;
18583e2ec76SGeorge Zhang
18683e2ec76SGeorge Zhang entry->idx = new_notify_idx;
18783e2ec76SGeorge Zhang bucket = VMCI_DOORBELL_HASH(entry->idx);
18883e2ec76SGeorge Zhang hlist_add_head(&entry->node, &vmci_doorbell_it.entries[bucket]);
18983e2ec76SGeorge Zhang
19083e2ec76SGeorge Zhang spin_unlock_bh(&vmci_doorbell_it.lock);
19183e2ec76SGeorge Zhang }
19283e2ec76SGeorge Zhang
19383e2ec76SGeorge Zhang /*
19483e2ec76SGeorge Zhang * Remove the given entry from the index table. This will release() the
19583e2ec76SGeorge Zhang * entry's resource.
19683e2ec76SGeorge Zhang */
dbell_index_table_remove(struct dbell_entry * entry)19783e2ec76SGeorge Zhang static void dbell_index_table_remove(struct dbell_entry *entry)
19883e2ec76SGeorge Zhang {
19983e2ec76SGeorge Zhang spin_lock_bh(&vmci_doorbell_it.lock);
20083e2ec76SGeorge Zhang
20183e2ec76SGeorge Zhang hlist_del_init(&entry->node);
20283e2ec76SGeorge Zhang
20383e2ec76SGeorge Zhang notify_idx_count--;
20483e2ec76SGeorge Zhang if (entry->idx == max_notify_idx - 1) {
20583e2ec76SGeorge Zhang /*
20683e2ec76SGeorge Zhang * If we delete an entry with the maximum known
20783e2ec76SGeorge Zhang * notification index, we take the opportunity to
20883e2ec76SGeorge Zhang * prune the current max. As there might be other
20983e2ec76SGeorge Zhang * unused indices immediately below, we lower the
21083e2ec76SGeorge Zhang * maximum until we hit an index in use.
21183e2ec76SGeorge Zhang */
21283e2ec76SGeorge Zhang while (max_notify_idx > 0 &&
21383e2ec76SGeorge Zhang !dbell_index_table_find(max_notify_idx - 1))
21483e2ec76SGeorge Zhang max_notify_idx--;
21583e2ec76SGeorge Zhang }
21683e2ec76SGeorge Zhang
21783e2ec76SGeorge Zhang last_notify_idx_released = entry->idx;
21883e2ec76SGeorge Zhang
21983e2ec76SGeorge Zhang spin_unlock_bh(&vmci_doorbell_it.lock);
22083e2ec76SGeorge Zhang
22183e2ec76SGeorge Zhang vmci_resource_put(&entry->resource);
22283e2ec76SGeorge Zhang }
22383e2ec76SGeorge Zhang
22483e2ec76SGeorge Zhang /*
22583e2ec76SGeorge Zhang * Creates a link between the given doorbell handle and the given
22683e2ec76SGeorge Zhang * index in the bitmap in the device backend. A notification state
22783e2ec76SGeorge Zhang * is created in hypervisor.
22883e2ec76SGeorge Zhang */
dbell_link(struct vmci_handle handle,u32 notify_idx)22983e2ec76SGeorge Zhang static int dbell_link(struct vmci_handle handle, u32 notify_idx)
23083e2ec76SGeorge Zhang {
23183e2ec76SGeorge Zhang struct vmci_doorbell_link_msg link_msg;
23283e2ec76SGeorge Zhang
23383e2ec76SGeorge Zhang link_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
23483e2ec76SGeorge Zhang VMCI_DOORBELL_LINK);
23583e2ec76SGeorge Zhang link_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
23683e2ec76SGeorge Zhang link_msg.hdr.payload_size = sizeof(link_msg) - VMCI_DG_HEADERSIZE;
23783e2ec76SGeorge Zhang link_msg.handle = handle;
23883e2ec76SGeorge Zhang link_msg.notify_idx = notify_idx;
23983e2ec76SGeorge Zhang
24083e2ec76SGeorge Zhang return vmci_send_datagram(&link_msg.hdr);
24183e2ec76SGeorge Zhang }
24283e2ec76SGeorge Zhang
24383e2ec76SGeorge Zhang /*
24483e2ec76SGeorge Zhang * Unlinks the given doorbell handle from an index in the bitmap in
24583e2ec76SGeorge Zhang * the device backend. The notification state is destroyed in hypervisor.
24683e2ec76SGeorge Zhang */
dbell_unlink(struct vmci_handle handle)24783e2ec76SGeorge Zhang static int dbell_unlink(struct vmci_handle handle)
24883e2ec76SGeorge Zhang {
24983e2ec76SGeorge Zhang struct vmci_doorbell_unlink_msg unlink_msg;
25083e2ec76SGeorge Zhang
25183e2ec76SGeorge Zhang unlink_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
25283e2ec76SGeorge Zhang VMCI_DOORBELL_UNLINK);
25383e2ec76SGeorge Zhang unlink_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
25483e2ec76SGeorge Zhang unlink_msg.hdr.payload_size = sizeof(unlink_msg) - VMCI_DG_HEADERSIZE;
25583e2ec76SGeorge Zhang unlink_msg.handle = handle;
25683e2ec76SGeorge Zhang
25783e2ec76SGeorge Zhang return vmci_send_datagram(&unlink_msg.hdr);
25883e2ec76SGeorge Zhang }
25983e2ec76SGeorge Zhang
26083e2ec76SGeorge Zhang /*
26183e2ec76SGeorge Zhang * Notify another guest or the host. We send a datagram down to the
26283e2ec76SGeorge Zhang * host via the hypervisor with the notification info.
26383e2ec76SGeorge Zhang */
dbell_notify_as_guest(struct vmci_handle handle,u32 priv_flags)26483e2ec76SGeorge Zhang static int dbell_notify_as_guest(struct vmci_handle handle, u32 priv_flags)
26583e2ec76SGeorge Zhang {
26683e2ec76SGeorge Zhang struct vmci_doorbell_notify_msg notify_msg;
26783e2ec76SGeorge Zhang
26883e2ec76SGeorge Zhang notify_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
26983e2ec76SGeorge Zhang VMCI_DOORBELL_NOTIFY);
27083e2ec76SGeorge Zhang notify_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
27183e2ec76SGeorge Zhang notify_msg.hdr.payload_size = sizeof(notify_msg) - VMCI_DG_HEADERSIZE;
27283e2ec76SGeorge Zhang notify_msg.handle = handle;
27383e2ec76SGeorge Zhang
27483e2ec76SGeorge Zhang return vmci_send_datagram(¬ify_msg.hdr);
27583e2ec76SGeorge Zhang }
27683e2ec76SGeorge Zhang
27783e2ec76SGeorge Zhang /*
27883e2ec76SGeorge Zhang * Calls the specified callback in a delayed context.
27983e2ec76SGeorge Zhang */
dbell_delayed_dispatch(struct work_struct * work)28083e2ec76SGeorge Zhang static void dbell_delayed_dispatch(struct work_struct *work)
28183e2ec76SGeorge Zhang {
28283e2ec76SGeorge Zhang struct dbell_entry *entry = container_of(work,
28383e2ec76SGeorge Zhang struct dbell_entry, work);
28483e2ec76SGeorge Zhang
28583e2ec76SGeorge Zhang entry->notify_cb(entry->client_data);
28683e2ec76SGeorge Zhang vmci_resource_put(&entry->resource);
28783e2ec76SGeorge Zhang }
28883e2ec76SGeorge Zhang
28983e2ec76SGeorge Zhang /*
29083e2ec76SGeorge Zhang * Dispatches a doorbell notification to the host context.
29183e2ec76SGeorge Zhang */
vmci_dbell_host_context_notify(u32 src_cid,struct vmci_handle handle)29283e2ec76SGeorge Zhang int vmci_dbell_host_context_notify(u32 src_cid, struct vmci_handle handle)
29383e2ec76SGeorge Zhang {
29483e2ec76SGeorge Zhang struct dbell_entry *entry;
29583e2ec76SGeorge Zhang struct vmci_resource *resource;
29683e2ec76SGeorge Zhang
29783e2ec76SGeorge Zhang if (vmci_handle_is_invalid(handle)) {
29883e2ec76SGeorge Zhang pr_devel("Notifying an invalid doorbell (handle=0x%x:0x%x)\n",
29983e2ec76SGeorge Zhang handle.context, handle.resource);
30083e2ec76SGeorge Zhang return VMCI_ERROR_INVALID_ARGS;
30183e2ec76SGeorge Zhang }
30283e2ec76SGeorge Zhang
30383e2ec76SGeorge Zhang resource = vmci_resource_by_handle(handle,
30483e2ec76SGeorge Zhang VMCI_RESOURCE_TYPE_DOORBELL);
30583e2ec76SGeorge Zhang if (!resource) {
30683e2ec76SGeorge Zhang pr_devel("Notifying an unknown doorbell (handle=0x%x:0x%x)\n",
30783e2ec76SGeorge Zhang handle.context, handle.resource);
30883e2ec76SGeorge Zhang return VMCI_ERROR_NOT_FOUND;
30983e2ec76SGeorge Zhang }
31083e2ec76SGeorge Zhang
31183e2ec76SGeorge Zhang entry = container_of(resource, struct dbell_entry, resource);
31283e2ec76SGeorge Zhang if (entry->run_delayed) {
313ba03a9bbSNadav Amit if (!schedule_work(&entry->work))
314ba03a9bbSNadav Amit vmci_resource_put(resource);
31583e2ec76SGeorge Zhang } else {
31683e2ec76SGeorge Zhang entry->notify_cb(entry->client_data);
31783e2ec76SGeorge Zhang vmci_resource_put(resource);
31883e2ec76SGeorge Zhang }
31983e2ec76SGeorge Zhang
32083e2ec76SGeorge Zhang return VMCI_SUCCESS;
32183e2ec76SGeorge Zhang }
32283e2ec76SGeorge Zhang
32383e2ec76SGeorge Zhang /*
32483e2ec76SGeorge Zhang * Register the notification bitmap with the host.
32583e2ec76SGeorge Zhang */
vmci_dbell_register_notification_bitmap(u64 bitmap_ppn)326f2db7361SVishnu DASA bool vmci_dbell_register_notification_bitmap(u64 bitmap_ppn)
32783e2ec76SGeorge Zhang {
32883e2ec76SGeorge Zhang int result;
329*376565b9STetsuo Handa struct vmci_notify_bm_set_msg bitmap_set_msg = { };
33083e2ec76SGeorge Zhang
33183e2ec76SGeorge Zhang bitmap_set_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
33283e2ec76SGeorge Zhang VMCI_SET_NOTIFY_BITMAP);
33383e2ec76SGeorge Zhang bitmap_set_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
33483e2ec76SGeorge Zhang bitmap_set_msg.hdr.payload_size = sizeof(bitmap_set_msg) -
33583e2ec76SGeorge Zhang VMCI_DG_HEADERSIZE;
336f2db7361SVishnu DASA if (vmci_use_ppn64())
337f2db7361SVishnu DASA bitmap_set_msg.bitmap_ppn64 = bitmap_ppn;
338f2db7361SVishnu DASA else
339f2db7361SVishnu DASA bitmap_set_msg.bitmap_ppn32 = (u32) bitmap_ppn;
34083e2ec76SGeorge Zhang
34183e2ec76SGeorge Zhang result = vmci_send_datagram(&bitmap_set_msg.hdr);
34283e2ec76SGeorge Zhang if (result != VMCI_SUCCESS) {
343f2db7361SVishnu DASA pr_devel("Failed to register (PPN=%llu) as notification bitmap (error=%d)\n",
34483e2ec76SGeorge Zhang bitmap_ppn, result);
34583e2ec76SGeorge Zhang return false;
34683e2ec76SGeorge Zhang }
34783e2ec76SGeorge Zhang return true;
34883e2ec76SGeorge Zhang }
34983e2ec76SGeorge Zhang
35083e2ec76SGeorge Zhang /*
35183e2ec76SGeorge Zhang * Executes or schedules the handlers for a given notify index.
35283e2ec76SGeorge Zhang */
dbell_fire_entries(u32 notify_idx)35383e2ec76SGeorge Zhang static void dbell_fire_entries(u32 notify_idx)
35483e2ec76SGeorge Zhang {
35583e2ec76SGeorge Zhang u32 bucket = VMCI_DOORBELL_HASH(notify_idx);
35683e2ec76SGeorge Zhang struct dbell_entry *dbell;
35783e2ec76SGeorge Zhang
35883e2ec76SGeorge Zhang spin_lock_bh(&vmci_doorbell_it.lock);
35983e2ec76SGeorge Zhang
360b67bfe0dSSasha Levin hlist_for_each_entry(dbell, &vmci_doorbell_it.entries[bucket], node) {
36183e2ec76SGeorge Zhang if (dbell->idx == notify_idx &&
36283e2ec76SGeorge Zhang atomic_read(&dbell->active) == 1) {
36383e2ec76SGeorge Zhang if (dbell->run_delayed) {
36483e2ec76SGeorge Zhang vmci_resource_get(&dbell->resource);
365ba03a9bbSNadav Amit if (!schedule_work(&dbell->work))
366ba03a9bbSNadav Amit vmci_resource_put(&dbell->resource);
36783e2ec76SGeorge Zhang } else {
36883e2ec76SGeorge Zhang dbell->notify_cb(dbell->client_data);
36983e2ec76SGeorge Zhang }
37083e2ec76SGeorge Zhang }
37183e2ec76SGeorge Zhang }
37283e2ec76SGeorge Zhang
37383e2ec76SGeorge Zhang spin_unlock_bh(&vmci_doorbell_it.lock);
37483e2ec76SGeorge Zhang }
37583e2ec76SGeorge Zhang
37683e2ec76SGeorge Zhang /*
37783e2ec76SGeorge Zhang * Scans the notification bitmap, collects pending notifications,
37883e2ec76SGeorge Zhang * resets the bitmap and invokes appropriate callbacks.
37983e2ec76SGeorge Zhang */
vmci_dbell_scan_notification_entries(u8 * bitmap)38083e2ec76SGeorge Zhang void vmci_dbell_scan_notification_entries(u8 *bitmap)
38183e2ec76SGeorge Zhang {
38283e2ec76SGeorge Zhang u32 idx;
38383e2ec76SGeorge Zhang
38483e2ec76SGeorge Zhang for (idx = 0; idx < max_notify_idx; idx++) {
38583e2ec76SGeorge Zhang if (bitmap[idx] & 0x1) {
38683e2ec76SGeorge Zhang bitmap[idx] &= ~1;
38783e2ec76SGeorge Zhang dbell_fire_entries(idx);
38883e2ec76SGeorge Zhang }
38983e2ec76SGeorge Zhang }
39083e2ec76SGeorge Zhang }
39183e2ec76SGeorge Zhang
39283e2ec76SGeorge Zhang /*
39383e2ec76SGeorge Zhang * vmci_doorbell_create() - Creates a doorbell
39483e2ec76SGeorge Zhang * @handle: A handle used to track the resource. Can be invalid.
39583e2ec76SGeorge Zhang * @flags: Flag that determines context of callback.
39683e2ec76SGeorge Zhang * @priv_flags: Privileges flags.
39783e2ec76SGeorge Zhang * @notify_cb: The callback to be ivoked when the doorbell fires.
39883e2ec76SGeorge Zhang * @client_data: A parameter to be passed to the callback.
39983e2ec76SGeorge Zhang *
40083e2ec76SGeorge Zhang * Creates a doorbell with the given callback. If the handle is
40183e2ec76SGeorge Zhang * VMCI_INVALID_HANDLE, a free handle will be assigned, if
40283e2ec76SGeorge Zhang * possible. The callback can be run immediately (potentially with
40383e2ec76SGeorge Zhang * locks held - the default) or delayed (in a kernel thread) by
40483e2ec76SGeorge Zhang * specifying the flag VMCI_FLAG_DELAYED_CB. If delayed execution
40583e2ec76SGeorge Zhang * is selected, a given callback may not be run if the kernel is
40683e2ec76SGeorge Zhang * unable to allocate memory for the delayed execution (highly
40783e2ec76SGeorge Zhang * unlikely).
40883e2ec76SGeorge Zhang */
vmci_doorbell_create(struct vmci_handle * handle,u32 flags,u32 priv_flags,vmci_callback notify_cb,void * client_data)40983e2ec76SGeorge Zhang int vmci_doorbell_create(struct vmci_handle *handle,
41083e2ec76SGeorge Zhang u32 flags,
41183e2ec76SGeorge Zhang u32 priv_flags,
41283e2ec76SGeorge Zhang vmci_callback notify_cb, void *client_data)
41383e2ec76SGeorge Zhang {
41483e2ec76SGeorge Zhang struct dbell_entry *entry;
41583e2ec76SGeorge Zhang struct vmci_handle new_handle;
41683e2ec76SGeorge Zhang int result;
41783e2ec76SGeorge Zhang
41883e2ec76SGeorge Zhang if (!handle || !notify_cb || flags & ~VMCI_FLAG_DELAYED_CB ||
41983e2ec76SGeorge Zhang priv_flags & ~VMCI_PRIVILEGE_ALL_FLAGS)
42083e2ec76SGeorge Zhang return VMCI_ERROR_INVALID_ARGS;
42183e2ec76SGeorge Zhang
42283e2ec76SGeorge Zhang entry = kmalloc(sizeof(*entry), GFP_KERNEL);
42383e2ec76SGeorge Zhang if (entry == NULL) {
42483e2ec76SGeorge Zhang pr_warn("Failed allocating memory for datagram entry\n");
42583e2ec76SGeorge Zhang return VMCI_ERROR_NO_MEM;
42683e2ec76SGeorge Zhang }
42783e2ec76SGeorge Zhang
42883e2ec76SGeorge Zhang if (vmci_handle_is_invalid(*handle)) {
42983e2ec76SGeorge Zhang u32 context_id = vmci_get_context_id();
43083e2ec76SGeorge Zhang
431eb94cd68SJorgen Hansen if (context_id == VMCI_INVALID_ID) {
432eb94cd68SJorgen Hansen pr_warn("Failed to get context ID\n");
433eb94cd68SJorgen Hansen result = VMCI_ERROR_NO_RESOURCES;
434eb94cd68SJorgen Hansen goto free_mem;
435eb94cd68SJorgen Hansen }
436eb94cd68SJorgen Hansen
43783e2ec76SGeorge Zhang /* Let resource code allocate a free ID for us */
43883e2ec76SGeorge Zhang new_handle = vmci_make_handle(context_id, VMCI_INVALID_ID);
43983e2ec76SGeorge Zhang } else {
44083e2ec76SGeorge Zhang bool valid_context = false;
44183e2ec76SGeorge Zhang
44283e2ec76SGeorge Zhang /*
44383e2ec76SGeorge Zhang * Validate the handle. We must do both of the checks below
44483e2ec76SGeorge Zhang * because we can be acting as both a host and a guest at the
44583e2ec76SGeorge Zhang * same time. We always allow the host context ID, since the
44683e2ec76SGeorge Zhang * host functionality is in practice always there with the
44783e2ec76SGeorge Zhang * unified driver.
44883e2ec76SGeorge Zhang */
44983e2ec76SGeorge Zhang if (handle->context == VMCI_HOST_CONTEXT_ID ||
45083e2ec76SGeorge Zhang (vmci_guest_code_active() &&
45183e2ec76SGeorge Zhang vmci_get_context_id() == handle->context)) {
45283e2ec76SGeorge Zhang valid_context = true;
45383e2ec76SGeorge Zhang }
45483e2ec76SGeorge Zhang
45583e2ec76SGeorge Zhang if (!valid_context || handle->resource == VMCI_INVALID_ID) {
45683e2ec76SGeorge Zhang pr_devel("Invalid argument (handle=0x%x:0x%x)\n",
45783e2ec76SGeorge Zhang handle->context, handle->resource);
45883e2ec76SGeorge Zhang result = VMCI_ERROR_INVALID_ARGS;
45983e2ec76SGeorge Zhang goto free_mem;
46083e2ec76SGeorge Zhang }
46183e2ec76SGeorge Zhang
46283e2ec76SGeorge Zhang new_handle = *handle;
46383e2ec76SGeorge Zhang }
46483e2ec76SGeorge Zhang
46583e2ec76SGeorge Zhang entry->idx = 0;
46683e2ec76SGeorge Zhang INIT_HLIST_NODE(&entry->node);
46783e2ec76SGeorge Zhang entry->priv_flags = priv_flags;
46883e2ec76SGeorge Zhang INIT_WORK(&entry->work, dbell_delayed_dispatch);
46983e2ec76SGeorge Zhang entry->run_delayed = flags & VMCI_FLAG_DELAYED_CB;
47083e2ec76SGeorge Zhang entry->notify_cb = notify_cb;
47183e2ec76SGeorge Zhang entry->client_data = client_data;
47283e2ec76SGeorge Zhang atomic_set(&entry->active, 0);
47383e2ec76SGeorge Zhang
47483e2ec76SGeorge Zhang result = vmci_resource_add(&entry->resource,
47583e2ec76SGeorge Zhang VMCI_RESOURCE_TYPE_DOORBELL,
47683e2ec76SGeorge Zhang new_handle);
47783e2ec76SGeorge Zhang if (result != VMCI_SUCCESS) {
47883e2ec76SGeorge Zhang pr_warn("Failed to add new resource (handle=0x%x:0x%x), error: %d\n",
47983e2ec76SGeorge Zhang new_handle.context, new_handle.resource, result);
48083e2ec76SGeorge Zhang goto free_mem;
48183e2ec76SGeorge Zhang }
48283e2ec76SGeorge Zhang
48383e2ec76SGeorge Zhang new_handle = vmci_resource_handle(&entry->resource);
48483e2ec76SGeorge Zhang if (vmci_guest_code_active()) {
48583e2ec76SGeorge Zhang dbell_index_table_add(entry);
48683e2ec76SGeorge Zhang result = dbell_link(new_handle, entry->idx);
48783e2ec76SGeorge Zhang if (VMCI_SUCCESS != result)
48883e2ec76SGeorge Zhang goto destroy_resource;
48983e2ec76SGeorge Zhang
49083e2ec76SGeorge Zhang atomic_set(&entry->active, 1);
49183e2ec76SGeorge Zhang }
49283e2ec76SGeorge Zhang
49383e2ec76SGeorge Zhang *handle = new_handle;
49483e2ec76SGeorge Zhang
49583e2ec76SGeorge Zhang return result;
49683e2ec76SGeorge Zhang
49783e2ec76SGeorge Zhang destroy_resource:
49883e2ec76SGeorge Zhang dbell_index_table_remove(entry);
49983e2ec76SGeorge Zhang vmci_resource_remove(&entry->resource);
50083e2ec76SGeorge Zhang free_mem:
50183e2ec76SGeorge Zhang kfree(entry);
50283e2ec76SGeorge Zhang return result;
50383e2ec76SGeorge Zhang }
50483e2ec76SGeorge Zhang EXPORT_SYMBOL_GPL(vmci_doorbell_create);
50583e2ec76SGeorge Zhang
50683e2ec76SGeorge Zhang /*
50783e2ec76SGeorge Zhang * vmci_doorbell_destroy() - Destroy a doorbell.
50883e2ec76SGeorge Zhang * @handle: The handle tracking the resource.
50983e2ec76SGeorge Zhang *
51083e2ec76SGeorge Zhang * Destroys a doorbell previously created with vmcii_doorbell_create. This
51183e2ec76SGeorge Zhang * operation may block waiting for a callback to finish.
51283e2ec76SGeorge Zhang */
vmci_doorbell_destroy(struct vmci_handle handle)51383e2ec76SGeorge Zhang int vmci_doorbell_destroy(struct vmci_handle handle)
51483e2ec76SGeorge Zhang {
51583e2ec76SGeorge Zhang struct dbell_entry *entry;
51683e2ec76SGeorge Zhang struct vmci_resource *resource;
51783e2ec76SGeorge Zhang
51883e2ec76SGeorge Zhang if (vmci_handle_is_invalid(handle))
51983e2ec76SGeorge Zhang return VMCI_ERROR_INVALID_ARGS;
52083e2ec76SGeorge Zhang
52183e2ec76SGeorge Zhang resource = vmci_resource_by_handle(handle,
52283e2ec76SGeorge Zhang VMCI_RESOURCE_TYPE_DOORBELL);
52383e2ec76SGeorge Zhang if (!resource) {
52483e2ec76SGeorge Zhang pr_devel("Failed to destroy doorbell (handle=0x%x:0x%x)\n",
52583e2ec76SGeorge Zhang handle.context, handle.resource);
52683e2ec76SGeorge Zhang return VMCI_ERROR_NOT_FOUND;
52783e2ec76SGeorge Zhang }
52883e2ec76SGeorge Zhang
52983e2ec76SGeorge Zhang entry = container_of(resource, struct dbell_entry, resource);
53083e2ec76SGeorge Zhang
531eb94cd68SJorgen Hansen if (!hlist_unhashed(&entry->node)) {
53283e2ec76SGeorge Zhang int result;
53383e2ec76SGeorge Zhang
53483e2ec76SGeorge Zhang dbell_index_table_remove(entry);
53583e2ec76SGeorge Zhang
53683e2ec76SGeorge Zhang result = dbell_unlink(handle);
53783e2ec76SGeorge Zhang if (VMCI_SUCCESS != result) {
53883e2ec76SGeorge Zhang
53983e2ec76SGeorge Zhang /*
54083e2ec76SGeorge Zhang * The only reason this should fail would be
54183e2ec76SGeorge Zhang * an inconsistency between guest and
54283e2ec76SGeorge Zhang * hypervisor state, where the guest believes
54383e2ec76SGeorge Zhang * it has an active registration whereas the
54483e2ec76SGeorge Zhang * hypervisor doesn't. One case where this may
54583e2ec76SGeorge Zhang * happen is if a doorbell is unregistered
54683e2ec76SGeorge Zhang * following a hibernation at a time where the
54783e2ec76SGeorge Zhang * doorbell state hasn't been restored on the
54883e2ec76SGeorge Zhang * hypervisor side yet. Since the handle has
54983e2ec76SGeorge Zhang * now been removed in the guest, we just
55083e2ec76SGeorge Zhang * print a warning and return success.
55183e2ec76SGeorge Zhang */
55283e2ec76SGeorge Zhang pr_devel("Unlink of doorbell (handle=0x%x:0x%x) unknown by hypervisor (error=%d)\n",
55383e2ec76SGeorge Zhang handle.context, handle.resource, result);
55483e2ec76SGeorge Zhang }
55583e2ec76SGeorge Zhang }
55683e2ec76SGeorge Zhang
55783e2ec76SGeorge Zhang /*
55883e2ec76SGeorge Zhang * Now remove the resource from the table. It might still be in use
55983e2ec76SGeorge Zhang * after this, in a callback or still on the delayed work queue.
56083e2ec76SGeorge Zhang */
56183e2ec76SGeorge Zhang vmci_resource_put(&entry->resource);
56283e2ec76SGeorge Zhang vmci_resource_remove(&entry->resource);
56383e2ec76SGeorge Zhang
56483e2ec76SGeorge Zhang kfree(entry);
56583e2ec76SGeorge Zhang
56683e2ec76SGeorge Zhang return VMCI_SUCCESS;
56783e2ec76SGeorge Zhang }
56883e2ec76SGeorge Zhang EXPORT_SYMBOL_GPL(vmci_doorbell_destroy);
56983e2ec76SGeorge Zhang
57083e2ec76SGeorge Zhang /*
57183e2ec76SGeorge Zhang * vmci_doorbell_notify() - Ring the doorbell (and hide in the bushes).
57283e2ec76SGeorge Zhang * @dst: The handlle identifying the doorbell resource
57383e2ec76SGeorge Zhang * @priv_flags: Priviledge flags.
57483e2ec76SGeorge Zhang *
57583e2ec76SGeorge Zhang * Generates a notification on the doorbell identified by the
57683e2ec76SGeorge Zhang * handle. For host side generation of notifications, the caller
57783e2ec76SGeorge Zhang * can specify what the privilege of the calling side is.
57883e2ec76SGeorge Zhang */
vmci_doorbell_notify(struct vmci_handle dst,u32 priv_flags)57983e2ec76SGeorge Zhang int vmci_doorbell_notify(struct vmci_handle dst, u32 priv_flags)
58083e2ec76SGeorge Zhang {
58183e2ec76SGeorge Zhang int retval;
58283e2ec76SGeorge Zhang enum vmci_route route;
58383e2ec76SGeorge Zhang struct vmci_handle src;
58483e2ec76SGeorge Zhang
58583e2ec76SGeorge Zhang if (vmci_handle_is_invalid(dst) ||
58683e2ec76SGeorge Zhang (priv_flags & ~VMCI_PRIVILEGE_ALL_FLAGS))
58783e2ec76SGeorge Zhang return VMCI_ERROR_INVALID_ARGS;
58883e2ec76SGeorge Zhang
58983e2ec76SGeorge Zhang src = VMCI_INVALID_HANDLE;
59083e2ec76SGeorge Zhang retval = vmci_route(&src, &dst, false, &route);
59183e2ec76SGeorge Zhang if (retval < VMCI_SUCCESS)
59283e2ec76SGeorge Zhang return retval;
59383e2ec76SGeorge Zhang
59483e2ec76SGeorge Zhang if (VMCI_ROUTE_AS_HOST == route)
59583e2ec76SGeorge Zhang return vmci_ctx_notify_dbell(VMCI_HOST_CONTEXT_ID,
59683e2ec76SGeorge Zhang dst, priv_flags);
59783e2ec76SGeorge Zhang
59883e2ec76SGeorge Zhang if (VMCI_ROUTE_AS_GUEST == route)
59983e2ec76SGeorge Zhang return dbell_notify_as_guest(dst, priv_flags);
60083e2ec76SGeorge Zhang
60183e2ec76SGeorge Zhang pr_warn("Unknown route (%d) for doorbell\n", route);
60283e2ec76SGeorge Zhang return VMCI_ERROR_DST_UNREACHABLE;
60383e2ec76SGeorge Zhang }
60483e2ec76SGeorge Zhang EXPORT_SYMBOL_GPL(vmci_doorbell_notify);
605