183e2ec76SGeorge Zhang /* 283e2ec76SGeorge Zhang * VMware VMCI Driver 383e2ec76SGeorge Zhang * 483e2ec76SGeorge Zhang * Copyright (C) 2012 VMware, Inc. All rights reserved. 583e2ec76SGeorge Zhang * 683e2ec76SGeorge Zhang * This program is free software; you can redistribute it and/or modify it 783e2ec76SGeorge Zhang * under the terms of the GNU General Public License as published by the 883e2ec76SGeorge Zhang * Free Software Foundation version 2 and no later version. 983e2ec76SGeorge Zhang * 1083e2ec76SGeorge Zhang * This program is distributed in the hope that it will be useful, but 1183e2ec76SGeorge Zhang * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY 1283e2ec76SGeorge Zhang * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 1383e2ec76SGeorge Zhang * for more details. 1483e2ec76SGeorge Zhang */ 1583e2ec76SGeorge Zhang 1683e2ec76SGeorge Zhang #include <linux/vmw_vmci_defs.h> 1783e2ec76SGeorge Zhang #include <linux/vmw_vmci_api.h> 1883e2ec76SGeorge Zhang #include <linux/completion.h> 1983e2ec76SGeorge Zhang #include <linux/hash.h> 2083e2ec76SGeorge Zhang #include <linux/kernel.h> 2183e2ec76SGeorge Zhang #include <linux/list.h> 2283e2ec76SGeorge Zhang #include <linux/module.h> 2383e2ec76SGeorge Zhang #include <linux/sched.h> 2483e2ec76SGeorge Zhang #include <linux/slab.h> 2583e2ec76SGeorge Zhang 2683e2ec76SGeorge Zhang #include "vmci_datagram.h" 2783e2ec76SGeorge Zhang #include "vmci_doorbell.h" 2883e2ec76SGeorge Zhang #include "vmci_resource.h" 2983e2ec76SGeorge Zhang #include "vmci_driver.h" 3083e2ec76SGeorge Zhang #include "vmci_route.h" 3183e2ec76SGeorge Zhang 3283e2ec76SGeorge Zhang 3383e2ec76SGeorge Zhang #define VMCI_DOORBELL_INDEX_BITS 6 3483e2ec76SGeorge Zhang #define VMCI_DOORBELL_INDEX_TABLE_SIZE (1 << VMCI_DOORBELL_INDEX_BITS) 3583e2ec76SGeorge Zhang #define VMCI_DOORBELL_HASH(_idx) hash_32(_idx, VMCI_DOORBELL_INDEX_BITS) 3683e2ec76SGeorge Zhang 3783e2ec76SGeorge Zhang /* 3883e2ec76SGeorge Zhang * DoorbellEntry describes the a doorbell notification handle allocated by the 3983e2ec76SGeorge Zhang * host. 4083e2ec76SGeorge Zhang */ 4183e2ec76SGeorge Zhang struct dbell_entry { 4283e2ec76SGeorge Zhang struct vmci_resource resource; 4383e2ec76SGeorge Zhang struct hlist_node node; 4483e2ec76SGeorge Zhang struct work_struct work; 4583e2ec76SGeorge Zhang vmci_callback notify_cb; 4683e2ec76SGeorge Zhang void *client_data; 4783e2ec76SGeorge Zhang u32 idx; 4883e2ec76SGeorge Zhang u32 priv_flags; 4983e2ec76SGeorge Zhang bool run_delayed; 5083e2ec76SGeorge Zhang atomic_t active; /* Only used by guest personality */ 5183e2ec76SGeorge Zhang }; 5283e2ec76SGeorge Zhang 5383e2ec76SGeorge Zhang /* The VMCI index table keeps track of currently registered doorbells. */ 5483e2ec76SGeorge Zhang struct dbell_index_table { 5583e2ec76SGeorge Zhang spinlock_t lock; /* Index table lock */ 5683e2ec76SGeorge Zhang struct hlist_head entries[VMCI_DOORBELL_INDEX_TABLE_SIZE]; 5783e2ec76SGeorge Zhang }; 5883e2ec76SGeorge Zhang 5983e2ec76SGeorge Zhang static struct dbell_index_table vmci_doorbell_it = { 6083e2ec76SGeorge Zhang .lock = __SPIN_LOCK_UNLOCKED(vmci_doorbell_it.lock), 6183e2ec76SGeorge Zhang }; 6283e2ec76SGeorge Zhang 6383e2ec76SGeorge Zhang /* 6483e2ec76SGeorge Zhang * The max_notify_idx is one larger than the currently known bitmap index in 6583e2ec76SGeorge Zhang * use, and is used to determine how much of the bitmap needs to be scanned. 6683e2ec76SGeorge Zhang */ 6783e2ec76SGeorge Zhang static u32 max_notify_idx; 6883e2ec76SGeorge Zhang 6983e2ec76SGeorge Zhang /* 7083e2ec76SGeorge Zhang * The notify_idx_count is used for determining whether there are free entries 7183e2ec76SGeorge Zhang * within the bitmap (if notify_idx_count + 1 < max_notify_idx). 7283e2ec76SGeorge Zhang */ 7383e2ec76SGeorge Zhang static u32 notify_idx_count; 7483e2ec76SGeorge Zhang 7583e2ec76SGeorge Zhang /* 7683e2ec76SGeorge Zhang * The last_notify_idx_reserved is used to track the last index handed out - in 7783e2ec76SGeorge Zhang * the case where multiple handles share a notification index, we hand out 7883e2ec76SGeorge Zhang * indexes round robin based on last_notify_idx_reserved. 7983e2ec76SGeorge Zhang */ 8083e2ec76SGeorge Zhang static u32 last_notify_idx_reserved; 8183e2ec76SGeorge Zhang 8283e2ec76SGeorge Zhang /* This is a one entry cache used to by the index allocation. */ 8383e2ec76SGeorge Zhang static u32 last_notify_idx_released = PAGE_SIZE; 8483e2ec76SGeorge Zhang 8583e2ec76SGeorge Zhang 8683e2ec76SGeorge Zhang /* 8783e2ec76SGeorge Zhang * Utility function that retrieves the privilege flags associated 8883e2ec76SGeorge Zhang * with a given doorbell handle. For guest endpoints, the 8983e2ec76SGeorge Zhang * privileges are determined by the context ID, but for host 9083e2ec76SGeorge Zhang * endpoints privileges are associated with the complete 9183e2ec76SGeorge Zhang * handle. Hypervisor endpoints are not yet supported. 9283e2ec76SGeorge Zhang */ 9383e2ec76SGeorge Zhang int vmci_dbell_get_priv_flags(struct vmci_handle handle, u32 *priv_flags) 9483e2ec76SGeorge Zhang { 9583e2ec76SGeorge Zhang if (priv_flags == NULL || handle.context == VMCI_INVALID_ID) 9683e2ec76SGeorge Zhang return VMCI_ERROR_INVALID_ARGS; 9783e2ec76SGeorge Zhang 9883e2ec76SGeorge Zhang if (handle.context == VMCI_HOST_CONTEXT_ID) { 9983e2ec76SGeorge Zhang struct dbell_entry *entry; 10083e2ec76SGeorge Zhang struct vmci_resource *resource; 10183e2ec76SGeorge Zhang 10283e2ec76SGeorge Zhang resource = vmci_resource_by_handle(handle, 10383e2ec76SGeorge Zhang VMCI_RESOURCE_TYPE_DOORBELL); 10483e2ec76SGeorge Zhang if (!resource) 10583e2ec76SGeorge Zhang return VMCI_ERROR_NOT_FOUND; 10683e2ec76SGeorge Zhang 10783e2ec76SGeorge Zhang entry = container_of(resource, struct dbell_entry, resource); 10883e2ec76SGeorge Zhang *priv_flags = entry->priv_flags; 10983e2ec76SGeorge Zhang vmci_resource_put(resource); 11083e2ec76SGeorge Zhang } else if (handle.context == VMCI_HYPERVISOR_CONTEXT_ID) { 11183e2ec76SGeorge Zhang /* 11283e2ec76SGeorge Zhang * Hypervisor endpoints for notifications are not 11383e2ec76SGeorge Zhang * supported (yet). 11483e2ec76SGeorge Zhang */ 11583e2ec76SGeorge Zhang return VMCI_ERROR_INVALID_ARGS; 11683e2ec76SGeorge Zhang } else { 11783e2ec76SGeorge Zhang *priv_flags = vmci_context_get_priv_flags(handle.context); 11883e2ec76SGeorge Zhang } 11983e2ec76SGeorge Zhang 12083e2ec76SGeorge Zhang return VMCI_SUCCESS; 12183e2ec76SGeorge Zhang } 12283e2ec76SGeorge Zhang 12383e2ec76SGeorge Zhang /* 12483e2ec76SGeorge Zhang * Find doorbell entry by bitmap index. 12583e2ec76SGeorge Zhang */ 12683e2ec76SGeorge Zhang static struct dbell_entry *dbell_index_table_find(u32 idx) 12783e2ec76SGeorge Zhang { 12883e2ec76SGeorge Zhang u32 bucket = VMCI_DOORBELL_HASH(idx); 12983e2ec76SGeorge Zhang struct dbell_entry *dbell; 13083e2ec76SGeorge Zhang 131b67bfe0dSSasha Levin hlist_for_each_entry(dbell, &vmci_doorbell_it.entries[bucket], 13283e2ec76SGeorge Zhang node) { 13383e2ec76SGeorge Zhang if (idx == dbell->idx) 13483e2ec76SGeorge Zhang return dbell; 13583e2ec76SGeorge Zhang } 13683e2ec76SGeorge Zhang 13783e2ec76SGeorge Zhang return NULL; 13883e2ec76SGeorge Zhang } 13983e2ec76SGeorge Zhang 14083e2ec76SGeorge Zhang /* 14183e2ec76SGeorge Zhang * Add the given entry to the index table. This willi take a reference to the 14283e2ec76SGeorge Zhang * entry's resource so that the entry is not deleted before it is removed from 14383e2ec76SGeorge Zhang * the * table. 14483e2ec76SGeorge Zhang */ 14583e2ec76SGeorge Zhang static void dbell_index_table_add(struct dbell_entry *entry) 14683e2ec76SGeorge Zhang { 14783e2ec76SGeorge Zhang u32 bucket; 14883e2ec76SGeorge Zhang u32 new_notify_idx; 14983e2ec76SGeorge Zhang 15083e2ec76SGeorge Zhang vmci_resource_get(&entry->resource); 15183e2ec76SGeorge Zhang 15283e2ec76SGeorge Zhang spin_lock_bh(&vmci_doorbell_it.lock); 15383e2ec76SGeorge Zhang 15483e2ec76SGeorge Zhang /* 15583e2ec76SGeorge Zhang * Below we try to allocate an index in the notification 15683e2ec76SGeorge Zhang * bitmap with "not too much" sharing between resources. If we 15783e2ec76SGeorge Zhang * use less that the full bitmap, we either add to the end if 15883e2ec76SGeorge Zhang * there are no unused flags within the currently used area, 15983e2ec76SGeorge Zhang * or we search for unused ones. If we use the full bitmap, we 16083e2ec76SGeorge Zhang * allocate the index round robin. 16183e2ec76SGeorge Zhang */ 16283e2ec76SGeorge Zhang if (max_notify_idx < PAGE_SIZE || notify_idx_count < PAGE_SIZE) { 16383e2ec76SGeorge Zhang if (last_notify_idx_released < max_notify_idx && 16483e2ec76SGeorge Zhang !dbell_index_table_find(last_notify_idx_released)) { 16583e2ec76SGeorge Zhang new_notify_idx = last_notify_idx_released; 16683e2ec76SGeorge Zhang last_notify_idx_released = PAGE_SIZE; 16783e2ec76SGeorge Zhang } else { 16883e2ec76SGeorge Zhang bool reused = false; 16983e2ec76SGeorge Zhang new_notify_idx = last_notify_idx_reserved; 17083e2ec76SGeorge Zhang if (notify_idx_count + 1 < max_notify_idx) { 17183e2ec76SGeorge Zhang do { 17283e2ec76SGeorge Zhang if (!dbell_index_table_find 17383e2ec76SGeorge Zhang (new_notify_idx)) { 17483e2ec76SGeorge Zhang reused = true; 17583e2ec76SGeorge Zhang break; 17683e2ec76SGeorge Zhang } 17783e2ec76SGeorge Zhang new_notify_idx = (new_notify_idx + 1) % 17883e2ec76SGeorge Zhang max_notify_idx; 17983e2ec76SGeorge Zhang } while (new_notify_idx != 18083e2ec76SGeorge Zhang last_notify_idx_released); 18183e2ec76SGeorge Zhang } 18283e2ec76SGeorge Zhang if (!reused) { 18383e2ec76SGeorge Zhang new_notify_idx = max_notify_idx; 18483e2ec76SGeorge Zhang max_notify_idx++; 18583e2ec76SGeorge Zhang } 18683e2ec76SGeorge Zhang } 18783e2ec76SGeorge Zhang } else { 18883e2ec76SGeorge Zhang new_notify_idx = (last_notify_idx_reserved + 1) % PAGE_SIZE; 18983e2ec76SGeorge Zhang } 19083e2ec76SGeorge Zhang 19183e2ec76SGeorge Zhang last_notify_idx_reserved = new_notify_idx; 19283e2ec76SGeorge Zhang notify_idx_count++; 19383e2ec76SGeorge Zhang 19483e2ec76SGeorge Zhang entry->idx = new_notify_idx; 19583e2ec76SGeorge Zhang bucket = VMCI_DOORBELL_HASH(entry->idx); 19683e2ec76SGeorge Zhang hlist_add_head(&entry->node, &vmci_doorbell_it.entries[bucket]); 19783e2ec76SGeorge Zhang 19883e2ec76SGeorge Zhang spin_unlock_bh(&vmci_doorbell_it.lock); 19983e2ec76SGeorge Zhang } 20083e2ec76SGeorge Zhang 20183e2ec76SGeorge Zhang /* 20283e2ec76SGeorge Zhang * Remove the given entry from the index table. This will release() the 20383e2ec76SGeorge Zhang * entry's resource. 20483e2ec76SGeorge Zhang */ 20583e2ec76SGeorge Zhang static void dbell_index_table_remove(struct dbell_entry *entry) 20683e2ec76SGeorge Zhang { 20783e2ec76SGeorge Zhang spin_lock_bh(&vmci_doorbell_it.lock); 20883e2ec76SGeorge Zhang 20983e2ec76SGeorge Zhang hlist_del_init(&entry->node); 21083e2ec76SGeorge Zhang 21183e2ec76SGeorge Zhang notify_idx_count--; 21283e2ec76SGeorge Zhang if (entry->idx == max_notify_idx - 1) { 21383e2ec76SGeorge Zhang /* 21483e2ec76SGeorge Zhang * If we delete an entry with the maximum known 21583e2ec76SGeorge Zhang * notification index, we take the opportunity to 21683e2ec76SGeorge Zhang * prune the current max. As there might be other 21783e2ec76SGeorge Zhang * unused indices immediately below, we lower the 21883e2ec76SGeorge Zhang * maximum until we hit an index in use. 21983e2ec76SGeorge Zhang */ 22083e2ec76SGeorge Zhang while (max_notify_idx > 0 && 22183e2ec76SGeorge Zhang !dbell_index_table_find(max_notify_idx - 1)) 22283e2ec76SGeorge Zhang max_notify_idx--; 22383e2ec76SGeorge Zhang } 22483e2ec76SGeorge Zhang 22583e2ec76SGeorge Zhang last_notify_idx_released = entry->idx; 22683e2ec76SGeorge Zhang 22783e2ec76SGeorge Zhang spin_unlock_bh(&vmci_doorbell_it.lock); 22883e2ec76SGeorge Zhang 22983e2ec76SGeorge Zhang vmci_resource_put(&entry->resource); 23083e2ec76SGeorge Zhang } 23183e2ec76SGeorge Zhang 23283e2ec76SGeorge Zhang /* 23383e2ec76SGeorge Zhang * Creates a link between the given doorbell handle and the given 23483e2ec76SGeorge Zhang * index in the bitmap in the device backend. A notification state 23583e2ec76SGeorge Zhang * is created in hypervisor. 23683e2ec76SGeorge Zhang */ 23783e2ec76SGeorge Zhang static int dbell_link(struct vmci_handle handle, u32 notify_idx) 23883e2ec76SGeorge Zhang { 23983e2ec76SGeorge Zhang struct vmci_doorbell_link_msg link_msg; 24083e2ec76SGeorge Zhang 24183e2ec76SGeorge Zhang link_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID, 24283e2ec76SGeorge Zhang VMCI_DOORBELL_LINK); 24383e2ec76SGeorge Zhang link_msg.hdr.src = VMCI_ANON_SRC_HANDLE; 24483e2ec76SGeorge Zhang link_msg.hdr.payload_size = sizeof(link_msg) - VMCI_DG_HEADERSIZE; 24583e2ec76SGeorge Zhang link_msg.handle = handle; 24683e2ec76SGeorge Zhang link_msg.notify_idx = notify_idx; 24783e2ec76SGeorge Zhang 24883e2ec76SGeorge Zhang return vmci_send_datagram(&link_msg.hdr); 24983e2ec76SGeorge Zhang } 25083e2ec76SGeorge Zhang 25183e2ec76SGeorge Zhang /* 25283e2ec76SGeorge Zhang * Unlinks the given doorbell handle from an index in the bitmap in 25383e2ec76SGeorge Zhang * the device backend. The notification state is destroyed in hypervisor. 25483e2ec76SGeorge Zhang */ 25583e2ec76SGeorge Zhang static int dbell_unlink(struct vmci_handle handle) 25683e2ec76SGeorge Zhang { 25783e2ec76SGeorge Zhang struct vmci_doorbell_unlink_msg unlink_msg; 25883e2ec76SGeorge Zhang 25983e2ec76SGeorge Zhang unlink_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID, 26083e2ec76SGeorge Zhang VMCI_DOORBELL_UNLINK); 26183e2ec76SGeorge Zhang unlink_msg.hdr.src = VMCI_ANON_SRC_HANDLE; 26283e2ec76SGeorge Zhang unlink_msg.hdr.payload_size = sizeof(unlink_msg) - VMCI_DG_HEADERSIZE; 26383e2ec76SGeorge Zhang unlink_msg.handle = handle; 26483e2ec76SGeorge Zhang 26583e2ec76SGeorge Zhang return vmci_send_datagram(&unlink_msg.hdr); 26683e2ec76SGeorge Zhang } 26783e2ec76SGeorge Zhang 26883e2ec76SGeorge Zhang /* 26983e2ec76SGeorge Zhang * Notify another guest or the host. We send a datagram down to the 27083e2ec76SGeorge Zhang * host via the hypervisor with the notification info. 27183e2ec76SGeorge Zhang */ 27283e2ec76SGeorge Zhang static int dbell_notify_as_guest(struct vmci_handle handle, u32 priv_flags) 27383e2ec76SGeorge Zhang { 27483e2ec76SGeorge Zhang struct vmci_doorbell_notify_msg notify_msg; 27583e2ec76SGeorge Zhang 27683e2ec76SGeorge Zhang notify_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID, 27783e2ec76SGeorge Zhang VMCI_DOORBELL_NOTIFY); 27883e2ec76SGeorge Zhang notify_msg.hdr.src = VMCI_ANON_SRC_HANDLE; 27983e2ec76SGeorge Zhang notify_msg.hdr.payload_size = sizeof(notify_msg) - VMCI_DG_HEADERSIZE; 28083e2ec76SGeorge Zhang notify_msg.handle = handle; 28183e2ec76SGeorge Zhang 28283e2ec76SGeorge Zhang return vmci_send_datagram(¬ify_msg.hdr); 28383e2ec76SGeorge Zhang } 28483e2ec76SGeorge Zhang 28583e2ec76SGeorge Zhang /* 28683e2ec76SGeorge Zhang * Calls the specified callback in a delayed context. 28783e2ec76SGeorge Zhang */ 28883e2ec76SGeorge Zhang static void dbell_delayed_dispatch(struct work_struct *work) 28983e2ec76SGeorge Zhang { 29083e2ec76SGeorge Zhang struct dbell_entry *entry = container_of(work, 29183e2ec76SGeorge Zhang struct dbell_entry, work); 29283e2ec76SGeorge Zhang 29383e2ec76SGeorge Zhang entry->notify_cb(entry->client_data); 29483e2ec76SGeorge Zhang vmci_resource_put(&entry->resource); 29583e2ec76SGeorge Zhang } 29683e2ec76SGeorge Zhang 29783e2ec76SGeorge Zhang /* 29883e2ec76SGeorge Zhang * Dispatches a doorbell notification to the host context. 29983e2ec76SGeorge Zhang */ 30083e2ec76SGeorge Zhang int vmci_dbell_host_context_notify(u32 src_cid, struct vmci_handle handle) 30183e2ec76SGeorge Zhang { 30283e2ec76SGeorge Zhang struct dbell_entry *entry; 30383e2ec76SGeorge Zhang struct vmci_resource *resource; 30483e2ec76SGeorge Zhang 30583e2ec76SGeorge Zhang if (vmci_handle_is_invalid(handle)) { 30683e2ec76SGeorge Zhang pr_devel("Notifying an invalid doorbell (handle=0x%x:0x%x)\n", 30783e2ec76SGeorge Zhang handle.context, handle.resource); 30883e2ec76SGeorge Zhang return VMCI_ERROR_INVALID_ARGS; 30983e2ec76SGeorge Zhang } 31083e2ec76SGeorge Zhang 31183e2ec76SGeorge Zhang resource = vmci_resource_by_handle(handle, 31283e2ec76SGeorge Zhang VMCI_RESOURCE_TYPE_DOORBELL); 31383e2ec76SGeorge Zhang if (!resource) { 31483e2ec76SGeorge Zhang pr_devel("Notifying an unknown doorbell (handle=0x%x:0x%x)\n", 31583e2ec76SGeorge Zhang handle.context, handle.resource); 31683e2ec76SGeorge Zhang return VMCI_ERROR_NOT_FOUND; 31783e2ec76SGeorge Zhang } 31883e2ec76SGeorge Zhang 31983e2ec76SGeorge Zhang entry = container_of(resource, struct dbell_entry, resource); 32083e2ec76SGeorge Zhang if (entry->run_delayed) { 32183e2ec76SGeorge Zhang schedule_work(&entry->work); 32283e2ec76SGeorge Zhang } else { 32383e2ec76SGeorge Zhang entry->notify_cb(entry->client_data); 32483e2ec76SGeorge Zhang vmci_resource_put(resource); 32583e2ec76SGeorge Zhang } 32683e2ec76SGeorge Zhang 32783e2ec76SGeorge Zhang return VMCI_SUCCESS; 32883e2ec76SGeorge Zhang } 32983e2ec76SGeorge Zhang 33083e2ec76SGeorge Zhang /* 33183e2ec76SGeorge Zhang * Register the notification bitmap with the host. 33283e2ec76SGeorge Zhang */ 333f2db7361SVishnu DASA bool vmci_dbell_register_notification_bitmap(u64 bitmap_ppn) 33483e2ec76SGeorge Zhang { 33583e2ec76SGeorge Zhang int result; 33683e2ec76SGeorge Zhang struct vmci_notify_bm_set_msg bitmap_set_msg; 33783e2ec76SGeorge Zhang 33883e2ec76SGeorge Zhang bitmap_set_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID, 33983e2ec76SGeorge Zhang VMCI_SET_NOTIFY_BITMAP); 34083e2ec76SGeorge Zhang bitmap_set_msg.hdr.src = VMCI_ANON_SRC_HANDLE; 34183e2ec76SGeorge Zhang bitmap_set_msg.hdr.payload_size = sizeof(bitmap_set_msg) - 34283e2ec76SGeorge Zhang VMCI_DG_HEADERSIZE; 343f2db7361SVishnu DASA if (vmci_use_ppn64()) 344f2db7361SVishnu DASA bitmap_set_msg.bitmap_ppn64 = bitmap_ppn; 345f2db7361SVishnu DASA else 346f2db7361SVishnu DASA bitmap_set_msg.bitmap_ppn32 = (u32) bitmap_ppn; 34783e2ec76SGeorge Zhang 34883e2ec76SGeorge Zhang result = vmci_send_datagram(&bitmap_set_msg.hdr); 34983e2ec76SGeorge Zhang if (result != VMCI_SUCCESS) { 350f2db7361SVishnu DASA pr_devel("Failed to register (PPN=%llu) as notification bitmap (error=%d)\n", 35183e2ec76SGeorge Zhang bitmap_ppn, result); 35283e2ec76SGeorge Zhang return false; 35383e2ec76SGeorge Zhang } 35483e2ec76SGeorge Zhang return true; 35583e2ec76SGeorge Zhang } 35683e2ec76SGeorge Zhang 35783e2ec76SGeorge Zhang /* 35883e2ec76SGeorge Zhang * Executes or schedules the handlers for a given notify index. 35983e2ec76SGeorge Zhang */ 36083e2ec76SGeorge Zhang static void dbell_fire_entries(u32 notify_idx) 36183e2ec76SGeorge Zhang { 36283e2ec76SGeorge Zhang u32 bucket = VMCI_DOORBELL_HASH(notify_idx); 36383e2ec76SGeorge Zhang struct dbell_entry *dbell; 36483e2ec76SGeorge Zhang 36583e2ec76SGeorge Zhang spin_lock_bh(&vmci_doorbell_it.lock); 36683e2ec76SGeorge Zhang 367b67bfe0dSSasha Levin hlist_for_each_entry(dbell, &vmci_doorbell_it.entries[bucket], node) { 36883e2ec76SGeorge Zhang if (dbell->idx == notify_idx && 36983e2ec76SGeorge Zhang atomic_read(&dbell->active) == 1) { 37083e2ec76SGeorge Zhang if (dbell->run_delayed) { 37183e2ec76SGeorge Zhang vmci_resource_get(&dbell->resource); 37283e2ec76SGeorge Zhang schedule_work(&dbell->work); 37383e2ec76SGeorge Zhang } else { 37483e2ec76SGeorge Zhang dbell->notify_cb(dbell->client_data); 37583e2ec76SGeorge Zhang } 37683e2ec76SGeorge Zhang } 37783e2ec76SGeorge Zhang } 37883e2ec76SGeorge Zhang 37983e2ec76SGeorge Zhang spin_unlock_bh(&vmci_doorbell_it.lock); 38083e2ec76SGeorge Zhang } 38183e2ec76SGeorge Zhang 38283e2ec76SGeorge Zhang /* 38383e2ec76SGeorge Zhang * Scans the notification bitmap, collects pending notifications, 38483e2ec76SGeorge Zhang * resets the bitmap and invokes appropriate callbacks. 38583e2ec76SGeorge Zhang */ 38683e2ec76SGeorge Zhang void vmci_dbell_scan_notification_entries(u8 *bitmap) 38783e2ec76SGeorge Zhang { 38883e2ec76SGeorge Zhang u32 idx; 38983e2ec76SGeorge Zhang 39083e2ec76SGeorge Zhang for (idx = 0; idx < max_notify_idx; idx++) { 39183e2ec76SGeorge Zhang if (bitmap[idx] & 0x1) { 39283e2ec76SGeorge Zhang bitmap[idx] &= ~1; 39383e2ec76SGeorge Zhang dbell_fire_entries(idx); 39483e2ec76SGeorge Zhang } 39583e2ec76SGeorge Zhang } 39683e2ec76SGeorge Zhang } 39783e2ec76SGeorge Zhang 39883e2ec76SGeorge Zhang /* 39983e2ec76SGeorge Zhang * vmci_doorbell_create() - Creates a doorbell 40083e2ec76SGeorge Zhang * @handle: A handle used to track the resource. Can be invalid. 40183e2ec76SGeorge Zhang * @flags: Flag that determines context of callback. 40283e2ec76SGeorge Zhang * @priv_flags: Privileges flags. 40383e2ec76SGeorge Zhang * @notify_cb: The callback to be ivoked when the doorbell fires. 40483e2ec76SGeorge Zhang * @client_data: A parameter to be passed to the callback. 40583e2ec76SGeorge Zhang * 40683e2ec76SGeorge Zhang * Creates a doorbell with the given callback. If the handle is 40783e2ec76SGeorge Zhang * VMCI_INVALID_HANDLE, a free handle will be assigned, if 40883e2ec76SGeorge Zhang * possible. The callback can be run immediately (potentially with 40983e2ec76SGeorge Zhang * locks held - the default) or delayed (in a kernel thread) by 41083e2ec76SGeorge Zhang * specifying the flag VMCI_FLAG_DELAYED_CB. If delayed execution 41183e2ec76SGeorge Zhang * is selected, a given callback may not be run if the kernel is 41283e2ec76SGeorge Zhang * unable to allocate memory for the delayed execution (highly 41383e2ec76SGeorge Zhang * unlikely). 41483e2ec76SGeorge Zhang */ 41583e2ec76SGeorge Zhang int vmci_doorbell_create(struct vmci_handle *handle, 41683e2ec76SGeorge Zhang u32 flags, 41783e2ec76SGeorge Zhang u32 priv_flags, 41883e2ec76SGeorge Zhang vmci_callback notify_cb, void *client_data) 41983e2ec76SGeorge Zhang { 42083e2ec76SGeorge Zhang struct dbell_entry *entry; 42183e2ec76SGeorge Zhang struct vmci_handle new_handle; 42283e2ec76SGeorge Zhang int result; 42383e2ec76SGeorge Zhang 42483e2ec76SGeorge Zhang if (!handle || !notify_cb || flags & ~VMCI_FLAG_DELAYED_CB || 42583e2ec76SGeorge Zhang priv_flags & ~VMCI_PRIVILEGE_ALL_FLAGS) 42683e2ec76SGeorge Zhang return VMCI_ERROR_INVALID_ARGS; 42783e2ec76SGeorge Zhang 42883e2ec76SGeorge Zhang entry = kmalloc(sizeof(*entry), GFP_KERNEL); 42983e2ec76SGeorge Zhang if (entry == NULL) { 43083e2ec76SGeorge Zhang pr_warn("Failed allocating memory for datagram entry\n"); 43183e2ec76SGeorge Zhang return VMCI_ERROR_NO_MEM; 43283e2ec76SGeorge Zhang } 43383e2ec76SGeorge Zhang 43483e2ec76SGeorge Zhang if (vmci_handle_is_invalid(*handle)) { 43583e2ec76SGeorge Zhang u32 context_id = vmci_get_context_id(); 43683e2ec76SGeorge Zhang 437eb94cd68SJorgen Hansen if (context_id == VMCI_INVALID_ID) { 438eb94cd68SJorgen Hansen pr_warn("Failed to get context ID\n"); 439eb94cd68SJorgen Hansen result = VMCI_ERROR_NO_RESOURCES; 440eb94cd68SJorgen Hansen goto free_mem; 441eb94cd68SJorgen Hansen } 442eb94cd68SJorgen Hansen 44383e2ec76SGeorge Zhang /* Let resource code allocate a free ID for us */ 44483e2ec76SGeorge Zhang new_handle = vmci_make_handle(context_id, VMCI_INVALID_ID); 44583e2ec76SGeorge Zhang } else { 44683e2ec76SGeorge Zhang bool valid_context = false; 44783e2ec76SGeorge Zhang 44883e2ec76SGeorge Zhang /* 44983e2ec76SGeorge Zhang * Validate the handle. We must do both of the checks below 45083e2ec76SGeorge Zhang * because we can be acting as both a host and a guest at the 45183e2ec76SGeorge Zhang * same time. We always allow the host context ID, since the 45283e2ec76SGeorge Zhang * host functionality is in practice always there with the 45383e2ec76SGeorge Zhang * unified driver. 45483e2ec76SGeorge Zhang */ 45583e2ec76SGeorge Zhang if (handle->context == VMCI_HOST_CONTEXT_ID || 45683e2ec76SGeorge Zhang (vmci_guest_code_active() && 45783e2ec76SGeorge Zhang vmci_get_context_id() == handle->context)) { 45883e2ec76SGeorge Zhang valid_context = true; 45983e2ec76SGeorge Zhang } 46083e2ec76SGeorge Zhang 46183e2ec76SGeorge Zhang if (!valid_context || handle->resource == VMCI_INVALID_ID) { 46283e2ec76SGeorge Zhang pr_devel("Invalid argument (handle=0x%x:0x%x)\n", 46383e2ec76SGeorge Zhang handle->context, handle->resource); 46483e2ec76SGeorge Zhang result = VMCI_ERROR_INVALID_ARGS; 46583e2ec76SGeorge Zhang goto free_mem; 46683e2ec76SGeorge Zhang } 46783e2ec76SGeorge Zhang 46883e2ec76SGeorge Zhang new_handle = *handle; 46983e2ec76SGeorge Zhang } 47083e2ec76SGeorge Zhang 47183e2ec76SGeorge Zhang entry->idx = 0; 47283e2ec76SGeorge Zhang INIT_HLIST_NODE(&entry->node); 47383e2ec76SGeorge Zhang entry->priv_flags = priv_flags; 47483e2ec76SGeorge Zhang INIT_WORK(&entry->work, dbell_delayed_dispatch); 47583e2ec76SGeorge Zhang entry->run_delayed = flags & VMCI_FLAG_DELAYED_CB; 47683e2ec76SGeorge Zhang entry->notify_cb = notify_cb; 47783e2ec76SGeorge Zhang entry->client_data = client_data; 47883e2ec76SGeorge Zhang atomic_set(&entry->active, 0); 47983e2ec76SGeorge Zhang 48083e2ec76SGeorge Zhang result = vmci_resource_add(&entry->resource, 48183e2ec76SGeorge Zhang VMCI_RESOURCE_TYPE_DOORBELL, 48283e2ec76SGeorge Zhang new_handle); 48383e2ec76SGeorge Zhang if (result != VMCI_SUCCESS) { 48483e2ec76SGeorge Zhang pr_warn("Failed to add new resource (handle=0x%x:0x%x), error: %d\n", 48583e2ec76SGeorge Zhang new_handle.context, new_handle.resource, result); 48683e2ec76SGeorge Zhang goto free_mem; 48783e2ec76SGeorge Zhang } 48883e2ec76SGeorge Zhang 48983e2ec76SGeorge Zhang new_handle = vmci_resource_handle(&entry->resource); 49083e2ec76SGeorge Zhang if (vmci_guest_code_active()) { 49183e2ec76SGeorge Zhang dbell_index_table_add(entry); 49283e2ec76SGeorge Zhang result = dbell_link(new_handle, entry->idx); 49383e2ec76SGeorge Zhang if (VMCI_SUCCESS != result) 49483e2ec76SGeorge Zhang goto destroy_resource; 49583e2ec76SGeorge Zhang 49683e2ec76SGeorge Zhang atomic_set(&entry->active, 1); 49783e2ec76SGeorge Zhang } 49883e2ec76SGeorge Zhang 49983e2ec76SGeorge Zhang *handle = new_handle; 50083e2ec76SGeorge Zhang 50183e2ec76SGeorge Zhang return result; 50283e2ec76SGeorge Zhang 50383e2ec76SGeorge Zhang destroy_resource: 50483e2ec76SGeorge Zhang dbell_index_table_remove(entry); 50583e2ec76SGeorge Zhang vmci_resource_remove(&entry->resource); 50683e2ec76SGeorge Zhang free_mem: 50783e2ec76SGeorge Zhang kfree(entry); 50883e2ec76SGeorge Zhang return result; 50983e2ec76SGeorge Zhang } 51083e2ec76SGeorge Zhang EXPORT_SYMBOL_GPL(vmci_doorbell_create); 51183e2ec76SGeorge Zhang 51283e2ec76SGeorge Zhang /* 51383e2ec76SGeorge Zhang * vmci_doorbell_destroy() - Destroy a doorbell. 51483e2ec76SGeorge Zhang * @handle: The handle tracking the resource. 51583e2ec76SGeorge Zhang * 51683e2ec76SGeorge Zhang * Destroys a doorbell previously created with vmcii_doorbell_create. This 51783e2ec76SGeorge Zhang * operation may block waiting for a callback to finish. 51883e2ec76SGeorge Zhang */ 51983e2ec76SGeorge Zhang int vmci_doorbell_destroy(struct vmci_handle handle) 52083e2ec76SGeorge Zhang { 52183e2ec76SGeorge Zhang struct dbell_entry *entry; 52283e2ec76SGeorge Zhang struct vmci_resource *resource; 52383e2ec76SGeorge Zhang 52483e2ec76SGeorge Zhang if (vmci_handle_is_invalid(handle)) 52583e2ec76SGeorge Zhang return VMCI_ERROR_INVALID_ARGS; 52683e2ec76SGeorge Zhang 52783e2ec76SGeorge Zhang resource = vmci_resource_by_handle(handle, 52883e2ec76SGeorge Zhang VMCI_RESOURCE_TYPE_DOORBELL); 52983e2ec76SGeorge Zhang if (!resource) { 53083e2ec76SGeorge Zhang pr_devel("Failed to destroy doorbell (handle=0x%x:0x%x)\n", 53183e2ec76SGeorge Zhang handle.context, handle.resource); 53283e2ec76SGeorge Zhang return VMCI_ERROR_NOT_FOUND; 53383e2ec76SGeorge Zhang } 53483e2ec76SGeorge Zhang 53583e2ec76SGeorge Zhang entry = container_of(resource, struct dbell_entry, resource); 53683e2ec76SGeorge Zhang 537eb94cd68SJorgen Hansen if (!hlist_unhashed(&entry->node)) { 53883e2ec76SGeorge Zhang int result; 53983e2ec76SGeorge Zhang 54083e2ec76SGeorge Zhang dbell_index_table_remove(entry); 54183e2ec76SGeorge Zhang 54283e2ec76SGeorge Zhang result = dbell_unlink(handle); 54383e2ec76SGeorge Zhang if (VMCI_SUCCESS != result) { 54483e2ec76SGeorge Zhang 54583e2ec76SGeorge Zhang /* 54683e2ec76SGeorge Zhang * The only reason this should fail would be 54783e2ec76SGeorge Zhang * an inconsistency between guest and 54883e2ec76SGeorge Zhang * hypervisor state, where the guest believes 54983e2ec76SGeorge Zhang * it has an active registration whereas the 55083e2ec76SGeorge Zhang * hypervisor doesn't. One case where this may 55183e2ec76SGeorge Zhang * happen is if a doorbell is unregistered 55283e2ec76SGeorge Zhang * following a hibernation at a time where the 55383e2ec76SGeorge Zhang * doorbell state hasn't been restored on the 55483e2ec76SGeorge Zhang * hypervisor side yet. Since the handle has 55583e2ec76SGeorge Zhang * now been removed in the guest, we just 55683e2ec76SGeorge Zhang * print a warning and return success. 55783e2ec76SGeorge Zhang */ 55883e2ec76SGeorge Zhang pr_devel("Unlink of doorbell (handle=0x%x:0x%x) unknown by hypervisor (error=%d)\n", 55983e2ec76SGeorge Zhang handle.context, handle.resource, result); 56083e2ec76SGeorge Zhang } 56183e2ec76SGeorge Zhang } 56283e2ec76SGeorge Zhang 56383e2ec76SGeorge Zhang /* 56483e2ec76SGeorge Zhang * Now remove the resource from the table. It might still be in use 56583e2ec76SGeorge Zhang * after this, in a callback or still on the delayed work queue. 56683e2ec76SGeorge Zhang */ 56783e2ec76SGeorge Zhang vmci_resource_put(&entry->resource); 56883e2ec76SGeorge Zhang vmci_resource_remove(&entry->resource); 56983e2ec76SGeorge Zhang 57083e2ec76SGeorge Zhang kfree(entry); 57183e2ec76SGeorge Zhang 57283e2ec76SGeorge Zhang return VMCI_SUCCESS; 57383e2ec76SGeorge Zhang } 57483e2ec76SGeorge Zhang EXPORT_SYMBOL_GPL(vmci_doorbell_destroy); 57583e2ec76SGeorge Zhang 57683e2ec76SGeorge Zhang /* 57783e2ec76SGeorge Zhang * vmci_doorbell_notify() - Ring the doorbell (and hide in the bushes). 57883e2ec76SGeorge Zhang * @dst: The handlle identifying the doorbell resource 57983e2ec76SGeorge Zhang * @priv_flags: Priviledge flags. 58083e2ec76SGeorge Zhang * 58183e2ec76SGeorge Zhang * Generates a notification on the doorbell identified by the 58283e2ec76SGeorge Zhang * handle. For host side generation of notifications, the caller 58383e2ec76SGeorge Zhang * can specify what the privilege of the calling side is. 58483e2ec76SGeorge Zhang */ 58583e2ec76SGeorge Zhang int vmci_doorbell_notify(struct vmci_handle dst, u32 priv_flags) 58683e2ec76SGeorge Zhang { 58783e2ec76SGeorge Zhang int retval; 58883e2ec76SGeorge Zhang enum vmci_route route; 58983e2ec76SGeorge Zhang struct vmci_handle src; 59083e2ec76SGeorge Zhang 59183e2ec76SGeorge Zhang if (vmci_handle_is_invalid(dst) || 59283e2ec76SGeorge Zhang (priv_flags & ~VMCI_PRIVILEGE_ALL_FLAGS)) 59383e2ec76SGeorge Zhang return VMCI_ERROR_INVALID_ARGS; 59483e2ec76SGeorge Zhang 59583e2ec76SGeorge Zhang src = VMCI_INVALID_HANDLE; 59683e2ec76SGeorge Zhang retval = vmci_route(&src, &dst, false, &route); 59783e2ec76SGeorge Zhang if (retval < VMCI_SUCCESS) 59883e2ec76SGeorge Zhang return retval; 59983e2ec76SGeorge Zhang 60083e2ec76SGeorge Zhang if (VMCI_ROUTE_AS_HOST == route) 60183e2ec76SGeorge Zhang return vmci_ctx_notify_dbell(VMCI_HOST_CONTEXT_ID, 60283e2ec76SGeorge Zhang dst, priv_flags); 60383e2ec76SGeorge Zhang 60483e2ec76SGeorge Zhang if (VMCI_ROUTE_AS_GUEST == route) 60583e2ec76SGeorge Zhang return dbell_notify_as_guest(dst, priv_flags); 60683e2ec76SGeorge Zhang 60783e2ec76SGeorge Zhang pr_warn("Unknown route (%d) for doorbell\n", route); 60883e2ec76SGeorge Zhang return VMCI_ERROR_DST_UNREACHABLE; 60983e2ec76SGeorge Zhang } 61083e2ec76SGeorge Zhang EXPORT_SYMBOL_GPL(vmci_doorbell_notify); 611