17f30db1eSPaul Blakey // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
27f30db1eSPaul Blakey /* Copyright (c) 2018 Mellanox Technologies */
37f30db1eSPaul Blakey
47f30db1eSPaul Blakey #include <linux/jhash.h>
57f30db1eSPaul Blakey #include <linux/slab.h>
67f30db1eSPaul Blakey #include <linux/xarray.h>
77f30db1eSPaul Blakey #include <linux/hashtable.h>
8*5d5defd6SRoi Dayan #include <linux/refcount.h>
97f30db1eSPaul Blakey
107f30db1eSPaul Blakey #include "mapping.h"
117f30db1eSPaul Blakey
127f30db1eSPaul Blakey #define MAPPING_GRACE_PERIOD 2000
137f30db1eSPaul Blakey
14*5d5defd6SRoi Dayan static LIST_HEAD(shared_ctx_list);
15*5d5defd6SRoi Dayan static DEFINE_MUTEX(shared_ctx_lock);
16*5d5defd6SRoi Dayan
177f30db1eSPaul Blakey struct mapping_ctx {
187f30db1eSPaul Blakey struct xarray xarray;
197f30db1eSPaul Blakey DECLARE_HASHTABLE(ht, 8);
207f30db1eSPaul Blakey struct mutex lock; /* Guards hashtable and xarray */
217f30db1eSPaul Blakey unsigned long max_id;
227f30db1eSPaul Blakey size_t data_size;
237f30db1eSPaul Blakey bool delayed_removal;
247f30db1eSPaul Blakey struct delayed_work dwork;
257f30db1eSPaul Blakey struct list_head pending_list;
267f30db1eSPaul Blakey spinlock_t pending_list_lock; /* Guards pending list */
27*5d5defd6SRoi Dayan u64 id;
28*5d5defd6SRoi Dayan u8 type;
29*5d5defd6SRoi Dayan struct list_head list;
30*5d5defd6SRoi Dayan refcount_t refcount;
317f30db1eSPaul Blakey };
327f30db1eSPaul Blakey
337f30db1eSPaul Blakey struct mapping_item {
347f30db1eSPaul Blakey struct rcu_head rcu;
357f30db1eSPaul Blakey struct list_head list;
367f30db1eSPaul Blakey unsigned long timeout;
377f30db1eSPaul Blakey struct hlist_node node;
387f30db1eSPaul Blakey int cnt;
397f30db1eSPaul Blakey u32 id;
407f30db1eSPaul Blakey char data[];
417f30db1eSPaul Blakey };
427f30db1eSPaul Blakey
mapping_add(struct mapping_ctx * ctx,void * data,u32 * id)437f30db1eSPaul Blakey int mapping_add(struct mapping_ctx *ctx, void *data, u32 *id)
447f30db1eSPaul Blakey {
457f30db1eSPaul Blakey struct mapping_item *mi;
467f30db1eSPaul Blakey int err = -ENOMEM;
477f30db1eSPaul Blakey u32 hash_key;
487f30db1eSPaul Blakey
497f30db1eSPaul Blakey mutex_lock(&ctx->lock);
507f30db1eSPaul Blakey
517f30db1eSPaul Blakey hash_key = jhash(data, ctx->data_size, 0);
527f30db1eSPaul Blakey hash_for_each_possible(ctx->ht, mi, node, hash_key) {
537f30db1eSPaul Blakey if (!memcmp(data, mi->data, ctx->data_size))
547f30db1eSPaul Blakey goto attach;
557f30db1eSPaul Blakey }
567f30db1eSPaul Blakey
577f30db1eSPaul Blakey mi = kzalloc(sizeof(*mi) + ctx->data_size, GFP_KERNEL);
587f30db1eSPaul Blakey if (!mi)
597f30db1eSPaul Blakey goto err_alloc;
607f30db1eSPaul Blakey
617f30db1eSPaul Blakey memcpy(mi->data, data, ctx->data_size);
627f30db1eSPaul Blakey hash_add(ctx->ht, &mi->node, hash_key);
637f30db1eSPaul Blakey
647f30db1eSPaul Blakey err = xa_alloc(&ctx->xarray, &mi->id, mi, XA_LIMIT(1, ctx->max_id),
657f30db1eSPaul Blakey GFP_KERNEL);
667f30db1eSPaul Blakey if (err)
677f30db1eSPaul Blakey goto err_assign;
687f30db1eSPaul Blakey attach:
697f30db1eSPaul Blakey ++mi->cnt;
707f30db1eSPaul Blakey *id = mi->id;
717f30db1eSPaul Blakey
727f30db1eSPaul Blakey mutex_unlock(&ctx->lock);
737f30db1eSPaul Blakey
747f30db1eSPaul Blakey return 0;
757f30db1eSPaul Blakey
767f30db1eSPaul Blakey err_assign:
777f30db1eSPaul Blakey hash_del(&mi->node);
787f30db1eSPaul Blakey kfree(mi);
797f30db1eSPaul Blakey err_alloc:
807f30db1eSPaul Blakey mutex_unlock(&ctx->lock);
817f30db1eSPaul Blakey
827f30db1eSPaul Blakey return err;
837f30db1eSPaul Blakey }
847f30db1eSPaul Blakey
mapping_remove_and_free(struct mapping_ctx * ctx,struct mapping_item * mi)857f30db1eSPaul Blakey static void mapping_remove_and_free(struct mapping_ctx *ctx,
867f30db1eSPaul Blakey struct mapping_item *mi)
877f30db1eSPaul Blakey {
887f30db1eSPaul Blakey xa_erase(&ctx->xarray, mi->id);
897f30db1eSPaul Blakey kfree_rcu(mi, rcu);
907f30db1eSPaul Blakey }
917f30db1eSPaul Blakey
mapping_free_item(struct mapping_ctx * ctx,struct mapping_item * mi)927f30db1eSPaul Blakey static void mapping_free_item(struct mapping_ctx *ctx,
937f30db1eSPaul Blakey struct mapping_item *mi)
947f30db1eSPaul Blakey {
957f30db1eSPaul Blakey if (!ctx->delayed_removal) {
967f30db1eSPaul Blakey mapping_remove_and_free(ctx, mi);
977f30db1eSPaul Blakey return;
987f30db1eSPaul Blakey }
997f30db1eSPaul Blakey
1007f30db1eSPaul Blakey mi->timeout = jiffies + msecs_to_jiffies(MAPPING_GRACE_PERIOD);
1017f30db1eSPaul Blakey
1027f30db1eSPaul Blakey spin_lock(&ctx->pending_list_lock);
1037f30db1eSPaul Blakey list_add_tail(&mi->list, &ctx->pending_list);
1047f30db1eSPaul Blakey spin_unlock(&ctx->pending_list_lock);
1057f30db1eSPaul Blakey
1067f30db1eSPaul Blakey schedule_delayed_work(&ctx->dwork, MAPPING_GRACE_PERIOD);
1077f30db1eSPaul Blakey }
1087f30db1eSPaul Blakey
mapping_remove(struct mapping_ctx * ctx,u32 id)1097f30db1eSPaul Blakey int mapping_remove(struct mapping_ctx *ctx, u32 id)
1107f30db1eSPaul Blakey {
1117f30db1eSPaul Blakey unsigned long index = id;
1127f30db1eSPaul Blakey struct mapping_item *mi;
1137f30db1eSPaul Blakey int err = -ENOENT;
1147f30db1eSPaul Blakey
1157f30db1eSPaul Blakey mutex_lock(&ctx->lock);
1167f30db1eSPaul Blakey mi = xa_load(&ctx->xarray, index);
1177f30db1eSPaul Blakey if (!mi)
1187f30db1eSPaul Blakey goto out;
1197f30db1eSPaul Blakey err = 0;
1207f30db1eSPaul Blakey
1217f30db1eSPaul Blakey if (--mi->cnt > 0)
1227f30db1eSPaul Blakey goto out;
1237f30db1eSPaul Blakey
1247f30db1eSPaul Blakey hash_del(&mi->node);
1257f30db1eSPaul Blakey mapping_free_item(ctx, mi);
1267f30db1eSPaul Blakey out:
1277f30db1eSPaul Blakey mutex_unlock(&ctx->lock);
1287f30db1eSPaul Blakey
1297f30db1eSPaul Blakey return err;
1307f30db1eSPaul Blakey }
1317f30db1eSPaul Blakey
mapping_find(struct mapping_ctx * ctx,u32 id,void * data)1327f30db1eSPaul Blakey int mapping_find(struct mapping_ctx *ctx, u32 id, void *data)
1337f30db1eSPaul Blakey {
1347f30db1eSPaul Blakey unsigned long index = id;
1357f30db1eSPaul Blakey struct mapping_item *mi;
1367f30db1eSPaul Blakey int err = -ENOENT;
1377f30db1eSPaul Blakey
1387f30db1eSPaul Blakey rcu_read_lock();
1397f30db1eSPaul Blakey mi = xa_load(&ctx->xarray, index);
1407f30db1eSPaul Blakey if (!mi)
1417f30db1eSPaul Blakey goto err_find;
1427f30db1eSPaul Blakey
1437f30db1eSPaul Blakey memcpy(data, mi->data, ctx->data_size);
1447f30db1eSPaul Blakey err = 0;
1457f30db1eSPaul Blakey
1467f30db1eSPaul Blakey err_find:
1477f30db1eSPaul Blakey rcu_read_unlock();
1487f30db1eSPaul Blakey return err;
1497f30db1eSPaul Blakey }
1507f30db1eSPaul Blakey
1517f30db1eSPaul Blakey static void
mapping_remove_and_free_list(struct mapping_ctx * ctx,struct list_head * list)1527f30db1eSPaul Blakey mapping_remove_and_free_list(struct mapping_ctx *ctx, struct list_head *list)
1537f30db1eSPaul Blakey {
1547f30db1eSPaul Blakey struct mapping_item *mi;
1557f30db1eSPaul Blakey
1567f30db1eSPaul Blakey list_for_each_entry(mi, list, list)
1577f30db1eSPaul Blakey mapping_remove_and_free(ctx, mi);
1587f30db1eSPaul Blakey }
1597f30db1eSPaul Blakey
mapping_work_handler(struct work_struct * work)1607f30db1eSPaul Blakey static void mapping_work_handler(struct work_struct *work)
1617f30db1eSPaul Blakey {
1627f30db1eSPaul Blakey unsigned long min_timeout = 0, now = jiffies;
1637f30db1eSPaul Blakey struct mapping_item *mi, *next;
1647f30db1eSPaul Blakey LIST_HEAD(pending_items);
1657f30db1eSPaul Blakey struct mapping_ctx *ctx;
1667f30db1eSPaul Blakey
1677f30db1eSPaul Blakey ctx = container_of(work, struct mapping_ctx, dwork.work);
1687f30db1eSPaul Blakey
1697f30db1eSPaul Blakey spin_lock(&ctx->pending_list_lock);
1707f30db1eSPaul Blakey list_for_each_entry_safe(mi, next, &ctx->pending_list, list) {
1717f30db1eSPaul Blakey if (time_after(now, mi->timeout))
1727f30db1eSPaul Blakey list_move(&mi->list, &pending_items);
1737f30db1eSPaul Blakey else if (!min_timeout ||
1747f30db1eSPaul Blakey time_before(mi->timeout, min_timeout))
1757f30db1eSPaul Blakey min_timeout = mi->timeout;
1767f30db1eSPaul Blakey }
1777f30db1eSPaul Blakey spin_unlock(&ctx->pending_list_lock);
1787f30db1eSPaul Blakey
1797f30db1eSPaul Blakey mapping_remove_and_free_list(ctx, &pending_items);
1807f30db1eSPaul Blakey
1817f30db1eSPaul Blakey if (min_timeout)
1827f30db1eSPaul Blakey schedule_delayed_work(&ctx->dwork, abs(min_timeout - now));
1837f30db1eSPaul Blakey }
1847f30db1eSPaul Blakey
mapping_flush_work(struct mapping_ctx * ctx)1857f30db1eSPaul Blakey static void mapping_flush_work(struct mapping_ctx *ctx)
1867f30db1eSPaul Blakey {
1877f30db1eSPaul Blakey if (!ctx->delayed_removal)
1887f30db1eSPaul Blakey return;
1897f30db1eSPaul Blakey
1907f30db1eSPaul Blakey cancel_delayed_work_sync(&ctx->dwork);
1917f30db1eSPaul Blakey mapping_remove_and_free_list(ctx, &ctx->pending_list);
1927f30db1eSPaul Blakey }
1937f30db1eSPaul Blakey
1947f30db1eSPaul Blakey struct mapping_ctx *
mapping_create(size_t data_size,u32 max_id,bool delayed_removal)1957f30db1eSPaul Blakey mapping_create(size_t data_size, u32 max_id, bool delayed_removal)
1967f30db1eSPaul Blakey {
1977f30db1eSPaul Blakey struct mapping_ctx *ctx;
1987f30db1eSPaul Blakey
1997f30db1eSPaul Blakey ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
2007f30db1eSPaul Blakey if (!ctx)
2017f30db1eSPaul Blakey return ERR_PTR(-ENOMEM);
2027f30db1eSPaul Blakey
2037f30db1eSPaul Blakey ctx->max_id = max_id ? max_id : UINT_MAX;
2047f30db1eSPaul Blakey ctx->data_size = data_size;
2057f30db1eSPaul Blakey
2067f30db1eSPaul Blakey if (delayed_removal) {
2077f30db1eSPaul Blakey INIT_DELAYED_WORK(&ctx->dwork, mapping_work_handler);
2087f30db1eSPaul Blakey INIT_LIST_HEAD(&ctx->pending_list);
2097f30db1eSPaul Blakey spin_lock_init(&ctx->pending_list_lock);
2107f30db1eSPaul Blakey ctx->delayed_removal = true;
2117f30db1eSPaul Blakey }
2127f30db1eSPaul Blakey
2137f30db1eSPaul Blakey mutex_init(&ctx->lock);
2147f30db1eSPaul Blakey xa_init_flags(&ctx->xarray, XA_FLAGS_ALLOC1);
2157f30db1eSPaul Blakey
216*5d5defd6SRoi Dayan refcount_set(&ctx->refcount, 1);
217*5d5defd6SRoi Dayan INIT_LIST_HEAD(&ctx->list);
218*5d5defd6SRoi Dayan
219*5d5defd6SRoi Dayan return ctx;
220*5d5defd6SRoi Dayan }
221*5d5defd6SRoi Dayan
222*5d5defd6SRoi Dayan struct mapping_ctx *
mapping_create_for_id(u64 id,u8 type,size_t data_size,u32 max_id,bool delayed_removal)223*5d5defd6SRoi Dayan mapping_create_for_id(u64 id, u8 type, size_t data_size, u32 max_id, bool delayed_removal)
224*5d5defd6SRoi Dayan {
225*5d5defd6SRoi Dayan struct mapping_ctx *ctx;
226*5d5defd6SRoi Dayan
227*5d5defd6SRoi Dayan mutex_lock(&shared_ctx_lock);
228*5d5defd6SRoi Dayan list_for_each_entry(ctx, &shared_ctx_list, list) {
229*5d5defd6SRoi Dayan if (ctx->id == id && ctx->type == type) {
230*5d5defd6SRoi Dayan if (refcount_inc_not_zero(&ctx->refcount))
231*5d5defd6SRoi Dayan goto unlock;
232*5d5defd6SRoi Dayan break;
233*5d5defd6SRoi Dayan }
234*5d5defd6SRoi Dayan }
235*5d5defd6SRoi Dayan
236*5d5defd6SRoi Dayan ctx = mapping_create(data_size, max_id, delayed_removal);
237*5d5defd6SRoi Dayan if (IS_ERR(ctx))
238*5d5defd6SRoi Dayan goto unlock;
239*5d5defd6SRoi Dayan
240*5d5defd6SRoi Dayan ctx->id = id;
241*5d5defd6SRoi Dayan ctx->type = type;
242*5d5defd6SRoi Dayan list_add(&ctx->list, &shared_ctx_list);
243*5d5defd6SRoi Dayan
244*5d5defd6SRoi Dayan unlock:
245*5d5defd6SRoi Dayan mutex_unlock(&shared_ctx_lock);
2467f30db1eSPaul Blakey return ctx;
2477f30db1eSPaul Blakey }
2487f30db1eSPaul Blakey
mapping_destroy(struct mapping_ctx * ctx)2497f30db1eSPaul Blakey void mapping_destroy(struct mapping_ctx *ctx)
2507f30db1eSPaul Blakey {
251*5d5defd6SRoi Dayan if (!refcount_dec_and_test(&ctx->refcount))
252*5d5defd6SRoi Dayan return;
253*5d5defd6SRoi Dayan
254*5d5defd6SRoi Dayan mutex_lock(&shared_ctx_lock);
255*5d5defd6SRoi Dayan list_del(&ctx->list);
256*5d5defd6SRoi Dayan mutex_unlock(&shared_ctx_lock);
257*5d5defd6SRoi Dayan
2587f30db1eSPaul Blakey mapping_flush_work(ctx);
2597f30db1eSPaul Blakey xa_destroy(&ctx->xarray);
2607f30db1eSPaul Blakey mutex_destroy(&ctx->lock);
2617f30db1eSPaul Blakey
2627f30db1eSPaul Blakey kfree(ctx);
2637f30db1eSPaul Blakey }
264