194abbccdSEli Cohen // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
294abbccdSEli Cohen /* Copyright (c) 2020 Mellanox Technologies Ltd. */
394abbccdSEli Cohen
45262912eSEli Cohen #include <linux/vhost_types.h>
594abbccdSEli Cohen #include <linux/vdpa.h>
694abbccdSEli Cohen #include <linux/gcd.h>
794abbccdSEli Cohen #include <linux/string.h>
894abbccdSEli Cohen #include <linux/mlx5/qp.h>
994abbccdSEli Cohen #include "mlx5_vdpa.h"
1094abbccdSEli Cohen
1194abbccdSEli Cohen /* DIV_ROUND_UP where the divider is a power of 2 give by its log base 2 value */
1294abbccdSEli Cohen #define MLX5_DIV_ROUND_UP_POW2(_n, _s) \
1394abbccdSEli Cohen ({ \
1494abbccdSEli Cohen u64 __s = _s; \
1594abbccdSEli Cohen u64 _res; \
1694abbccdSEli Cohen _res = (((_n) + (1 << (__s)) - 1) >> (__s)); \
1794abbccdSEli Cohen _res; \
1894abbccdSEli Cohen })
1994abbccdSEli Cohen
get_octo_len(u64 len,int page_shift)2094abbccdSEli Cohen static int get_octo_len(u64 len, int page_shift)
2194abbccdSEli Cohen {
2294abbccdSEli Cohen u64 page_size = 1ULL << page_shift;
2394abbccdSEli Cohen int npages;
2494abbccdSEli Cohen
2594abbccdSEli Cohen npages = ALIGN(len, page_size) >> page_shift;
2694abbccdSEli Cohen return (npages + 1) / 2;
2794abbccdSEli Cohen }
2894abbccdSEli Cohen
mlx5_set_access_mode(void * mkc,int mode)2994abbccdSEli Cohen static void mlx5_set_access_mode(void *mkc, int mode)
3094abbccdSEli Cohen {
3194abbccdSEli Cohen MLX5_SET(mkc, mkc, access_mode_1_0, mode & 0x3);
3294abbccdSEli Cohen MLX5_SET(mkc, mkc, access_mode_4_2, mode >> 2);
3394abbccdSEli Cohen }
3494abbccdSEli Cohen
populate_mtts(struct mlx5_vdpa_direct_mr * mr,__be64 * mtt)3594abbccdSEli Cohen static void populate_mtts(struct mlx5_vdpa_direct_mr *mr, __be64 *mtt)
3694abbccdSEli Cohen {
3794abbccdSEli Cohen struct scatterlist *sg;
38710eb8e3SEli Cohen int nsg = mr->nsg;
39710eb8e3SEli Cohen u64 dma_addr;
40710eb8e3SEli Cohen u64 dma_len;
41710eb8e3SEli Cohen int j = 0;
4294abbccdSEli Cohen int i;
4394abbccdSEli Cohen
44710eb8e3SEli Cohen for_each_sg(mr->sg_head.sgl, sg, mr->nent, i) {
45710eb8e3SEli Cohen for (dma_addr = sg_dma_address(sg), dma_len = sg_dma_len(sg);
46710eb8e3SEli Cohen nsg && dma_len;
47710eb8e3SEli Cohen nsg--, dma_addr += BIT(mr->log_size), dma_len -= BIT(mr->log_size))
48710eb8e3SEli Cohen mtt[j++] = cpu_to_be64(dma_addr);
49710eb8e3SEli Cohen }
5094abbccdSEli Cohen }
5194abbccdSEli Cohen
create_direct_mr(struct mlx5_vdpa_dev * mvdev,struct mlx5_vdpa_direct_mr * mr)5294abbccdSEli Cohen static int create_direct_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_direct_mr *mr)
5394abbccdSEli Cohen {
5494abbccdSEli Cohen int inlen;
5594abbccdSEli Cohen void *mkc;
5694abbccdSEli Cohen void *in;
5794abbccdSEli Cohen int err;
5894abbccdSEli Cohen
5994abbccdSEli Cohen inlen = MLX5_ST_SZ_BYTES(create_mkey_in) + roundup(MLX5_ST_SZ_BYTES(mtt) * mr->nsg, 16);
6094abbccdSEli Cohen in = kvzalloc(inlen, GFP_KERNEL);
6194abbccdSEli Cohen if (!in)
6294abbccdSEli Cohen return -ENOMEM;
6394abbccdSEli Cohen
6494abbccdSEli Cohen MLX5_SET(create_mkey_in, in, uid, mvdev->res.uid);
6594abbccdSEli Cohen mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
6694abbccdSEli Cohen MLX5_SET(mkc, mkc, lw, !!(mr->perm & VHOST_MAP_WO));
6794abbccdSEli Cohen MLX5_SET(mkc, mkc, lr, !!(mr->perm & VHOST_MAP_RO));
6894abbccdSEli Cohen mlx5_set_access_mode(mkc, MLX5_MKC_ACCESS_MODE_MTT);
6994abbccdSEli Cohen MLX5_SET(mkc, mkc, qpn, 0xffffff);
7094abbccdSEli Cohen MLX5_SET(mkc, mkc, pd, mvdev->res.pdn);
7194abbccdSEli Cohen MLX5_SET64(mkc, mkc, start_addr, mr->offset);
7294abbccdSEli Cohen MLX5_SET64(mkc, mkc, len, mr->end - mr->start);
7394abbccdSEli Cohen MLX5_SET(mkc, mkc, log_page_size, mr->log_size);
7494abbccdSEli Cohen MLX5_SET(mkc, mkc, translations_octword_size,
7594abbccdSEli Cohen get_octo_len(mr->end - mr->start, mr->log_size));
7694abbccdSEli Cohen MLX5_SET(create_mkey_in, in, translations_octword_actual_size,
7794abbccdSEli Cohen get_octo_len(mr->end - mr->start, mr->log_size));
7894abbccdSEli Cohen populate_mtts(mr, MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt));
7994abbccdSEli Cohen err = mlx5_vdpa_create_mkey(mvdev, &mr->mr, in, inlen);
8094abbccdSEli Cohen kvfree(in);
8194abbccdSEli Cohen if (err) {
8294abbccdSEli Cohen mlx5_vdpa_warn(mvdev, "Failed to create direct MR\n");
8394abbccdSEli Cohen return err;
8494abbccdSEli Cohen }
8594abbccdSEli Cohen
8694abbccdSEli Cohen return 0;
8794abbccdSEli Cohen }
8894abbccdSEli Cohen
destroy_direct_mr(struct mlx5_vdpa_dev * mvdev,struct mlx5_vdpa_direct_mr * mr)8994abbccdSEli Cohen static void destroy_direct_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_direct_mr *mr)
9094abbccdSEli Cohen {
9183fec3f1SAharon Landau mlx5_vdpa_destroy_mkey(mvdev, mr->mr);
9294abbccdSEli Cohen }
9394abbccdSEli Cohen
map_start(struct vhost_iotlb_map * map,struct mlx5_vdpa_direct_mr * mr)9494abbccdSEli Cohen static u64 map_start(struct vhost_iotlb_map *map, struct mlx5_vdpa_direct_mr *mr)
9594abbccdSEli Cohen {
9694abbccdSEli Cohen return max_t(u64, map->start, mr->start);
9794abbccdSEli Cohen }
9894abbccdSEli Cohen
map_end(struct vhost_iotlb_map * map,struct mlx5_vdpa_direct_mr * mr)9994abbccdSEli Cohen static u64 map_end(struct vhost_iotlb_map *map, struct mlx5_vdpa_direct_mr *mr)
10094abbccdSEli Cohen {
10194abbccdSEli Cohen return min_t(u64, map->last + 1, mr->end);
10294abbccdSEli Cohen }
10394abbccdSEli Cohen
maplen(struct vhost_iotlb_map * map,struct mlx5_vdpa_direct_mr * mr)10494abbccdSEli Cohen static u64 maplen(struct vhost_iotlb_map *map, struct mlx5_vdpa_direct_mr *mr)
10594abbccdSEli Cohen {
10694abbccdSEli Cohen return map_end(map, mr) - map_start(map, mr);
10794abbccdSEli Cohen }
10894abbccdSEli Cohen
10994abbccdSEli Cohen #define MLX5_VDPA_INVALID_START_ADDR ((u64)-1)
11094abbccdSEli Cohen #define MLX5_VDPA_INVALID_LEN ((u64)-1)
11194abbccdSEli Cohen
indir_start_addr(struct mlx5_vdpa_mr * mkey)11294abbccdSEli Cohen static u64 indir_start_addr(struct mlx5_vdpa_mr *mkey)
11394abbccdSEli Cohen {
11494abbccdSEli Cohen struct mlx5_vdpa_direct_mr *s;
11594abbccdSEli Cohen
11694abbccdSEli Cohen s = list_first_entry_or_null(&mkey->head, struct mlx5_vdpa_direct_mr, list);
11794abbccdSEli Cohen if (!s)
11894abbccdSEli Cohen return MLX5_VDPA_INVALID_START_ADDR;
11994abbccdSEli Cohen
12094abbccdSEli Cohen return s->start;
12194abbccdSEli Cohen }
12294abbccdSEli Cohen
indir_len(struct mlx5_vdpa_mr * mkey)12394abbccdSEli Cohen static u64 indir_len(struct mlx5_vdpa_mr *mkey)
12494abbccdSEli Cohen {
12594abbccdSEli Cohen struct mlx5_vdpa_direct_mr *s;
12694abbccdSEli Cohen struct mlx5_vdpa_direct_mr *e;
12794abbccdSEli Cohen
12894abbccdSEli Cohen s = list_first_entry_or_null(&mkey->head, struct mlx5_vdpa_direct_mr, list);
12994abbccdSEli Cohen if (!s)
13094abbccdSEli Cohen return MLX5_VDPA_INVALID_LEN;
13194abbccdSEli Cohen
13294abbccdSEli Cohen e = list_last_entry(&mkey->head, struct mlx5_vdpa_direct_mr, list);
13394abbccdSEli Cohen
13494abbccdSEli Cohen return e->end - s->start;
13594abbccdSEli Cohen }
13694abbccdSEli Cohen
13794abbccdSEli Cohen #define LOG_MAX_KLM_SIZE 30
13894abbccdSEli Cohen #define MAX_KLM_SIZE BIT(LOG_MAX_KLM_SIZE)
13994abbccdSEli Cohen
klm_bcount(u64 size)14094abbccdSEli Cohen static u32 klm_bcount(u64 size)
14194abbccdSEli Cohen {
14294abbccdSEli Cohen return (u32)size;
14394abbccdSEli Cohen }
14494abbccdSEli Cohen
fill_indir(struct mlx5_vdpa_dev * mvdev,struct mlx5_vdpa_mr * mkey,void * in)14594abbccdSEli Cohen static void fill_indir(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mkey, void *in)
14694abbccdSEli Cohen {
14794abbccdSEli Cohen struct mlx5_vdpa_direct_mr *dmr;
14894abbccdSEli Cohen struct mlx5_klm *klmarr;
14994abbccdSEli Cohen struct mlx5_klm *klm;
15094abbccdSEli Cohen bool first = true;
15194abbccdSEli Cohen u64 preve;
15294abbccdSEli Cohen int i;
15394abbccdSEli Cohen
15494abbccdSEli Cohen klmarr = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
15594abbccdSEli Cohen i = 0;
15694abbccdSEli Cohen list_for_each_entry(dmr, &mkey->head, list) {
15794abbccdSEli Cohen again:
15894abbccdSEli Cohen klm = &klmarr[i++];
15994abbccdSEli Cohen if (first) {
16094abbccdSEli Cohen preve = dmr->start;
16194abbccdSEli Cohen first = false;
16294abbccdSEli Cohen }
16394abbccdSEli Cohen
16494abbccdSEli Cohen if (preve == dmr->start) {
16583fec3f1SAharon Landau klm->key = cpu_to_be32(dmr->mr);
16694abbccdSEli Cohen klm->bcount = cpu_to_be32(klm_bcount(dmr->end - dmr->start));
16794abbccdSEli Cohen preve = dmr->end;
16894abbccdSEli Cohen } else {
16994abbccdSEli Cohen klm->key = cpu_to_be32(mvdev->res.null_mkey);
17094abbccdSEli Cohen klm->bcount = cpu_to_be32(klm_bcount(dmr->start - preve));
17194abbccdSEli Cohen preve = dmr->start;
17294abbccdSEli Cohen goto again;
17394abbccdSEli Cohen }
17494abbccdSEli Cohen }
17594abbccdSEli Cohen }
17694abbccdSEli Cohen
klm_byte_size(int nklms)17794abbccdSEli Cohen static int klm_byte_size(int nklms)
17894abbccdSEli Cohen {
17994abbccdSEli Cohen return 16 * ALIGN(nklms, 4);
18094abbccdSEli Cohen }
18194abbccdSEli Cohen
create_indirect_key(struct mlx5_vdpa_dev * mvdev,struct mlx5_vdpa_mr * mr)18294abbccdSEli Cohen static int create_indirect_key(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mr)
18394abbccdSEli Cohen {
18494abbccdSEli Cohen int inlen;
18594abbccdSEli Cohen void *mkc;
18694abbccdSEli Cohen void *in;
18794abbccdSEli Cohen int err;
18894abbccdSEli Cohen u64 start;
18994abbccdSEli Cohen u64 len;
19094abbccdSEli Cohen
19194abbccdSEli Cohen start = indir_start_addr(mr);
19294abbccdSEli Cohen len = indir_len(mr);
19394abbccdSEli Cohen if (start == MLX5_VDPA_INVALID_START_ADDR || len == MLX5_VDPA_INVALID_LEN)
19494abbccdSEli Cohen return -EINVAL;
19594abbccdSEli Cohen
19694abbccdSEli Cohen inlen = MLX5_ST_SZ_BYTES(create_mkey_in) + klm_byte_size(mr->num_klms);
19794abbccdSEli Cohen in = kzalloc(inlen, GFP_KERNEL);
19894abbccdSEli Cohen if (!in)
19994abbccdSEli Cohen return -ENOMEM;
20094abbccdSEli Cohen
20194abbccdSEli Cohen MLX5_SET(create_mkey_in, in, uid, mvdev->res.uid);
20294abbccdSEli Cohen mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
20394abbccdSEli Cohen MLX5_SET(mkc, mkc, lw, 1);
20494abbccdSEli Cohen MLX5_SET(mkc, mkc, lr, 1);
20594abbccdSEli Cohen mlx5_set_access_mode(mkc, MLX5_MKC_ACCESS_MODE_KLMS);
20694abbccdSEli Cohen MLX5_SET(mkc, mkc, qpn, 0xffffff);
20794abbccdSEli Cohen MLX5_SET(mkc, mkc, pd, mvdev->res.pdn);
20894abbccdSEli Cohen MLX5_SET64(mkc, mkc, start_addr, start);
20994abbccdSEli Cohen MLX5_SET64(mkc, mkc, len, len);
21094abbccdSEli Cohen MLX5_SET(mkc, mkc, translations_octword_size, klm_byte_size(mr->num_klms) / 16);
21194abbccdSEli Cohen MLX5_SET(create_mkey_in, in, translations_octword_actual_size, mr->num_klms);
21294abbccdSEli Cohen fill_indir(mvdev, mr, in);
21394abbccdSEli Cohen err = mlx5_vdpa_create_mkey(mvdev, &mr->mkey, in, inlen);
21494abbccdSEli Cohen kfree(in);
21594abbccdSEli Cohen return err;
21694abbccdSEli Cohen }
21794abbccdSEli Cohen
destroy_indirect_key(struct mlx5_vdpa_dev * mvdev,struct mlx5_vdpa_mr * mkey)21894abbccdSEli Cohen static void destroy_indirect_key(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mkey)
21994abbccdSEli Cohen {
22083fec3f1SAharon Landau mlx5_vdpa_destroy_mkey(mvdev, mkey->mkey);
22194abbccdSEli Cohen }
22294abbccdSEli Cohen
map_direct_mr(struct mlx5_vdpa_dev * mvdev,struct mlx5_vdpa_direct_mr * mr,struct vhost_iotlb * iotlb)22394abbccdSEli Cohen static int map_direct_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_direct_mr *mr,
22494abbccdSEli Cohen struct vhost_iotlb *iotlb)
22594abbccdSEli Cohen {
22694abbccdSEli Cohen struct vhost_iotlb_map *map;
22794abbccdSEli Cohen unsigned long lgcd = 0;
22894abbccdSEli Cohen int log_entity_size;
22994abbccdSEli Cohen unsigned long size;
23094abbccdSEli Cohen int err;
23194abbccdSEli Cohen struct page *pg;
23294abbccdSEli Cohen unsigned int nsg;
23394abbccdSEli Cohen int sglen;
23415c49f91SSi-Wei Liu u64 pa, offset;
23594abbccdSEli Cohen u64 paend;
23694abbccdSEli Cohen struct scatterlist *sg;
2377d23dcdfSEli Cohen struct device *dma = mvdev->vdev.dma_dev;
23894abbccdSEli Cohen
23994abbccdSEli Cohen for (map = vhost_iotlb_itree_first(iotlb, mr->start, mr->end - 1);
240*6c5ddc2bSSi-Wei Liu map; map = vhost_iotlb_itree_next(map, mr->start, mr->end - 1)) {
24194abbccdSEli Cohen size = maplen(map, mr);
24294abbccdSEli Cohen lgcd = gcd(lgcd, size);
24394abbccdSEli Cohen }
24494abbccdSEli Cohen log_entity_size = ilog2(lgcd);
24594abbccdSEli Cohen
24694abbccdSEli Cohen sglen = 1 << log_entity_size;
24794abbccdSEli Cohen nsg = MLX5_DIV_ROUND_UP_POW2(mr->end - mr->start, log_entity_size);
24894abbccdSEli Cohen
24994abbccdSEli Cohen err = sg_alloc_table(&mr->sg_head, nsg, GFP_KERNEL);
25094abbccdSEli Cohen if (err)
25194abbccdSEli Cohen return err;
25294abbccdSEli Cohen
25394abbccdSEli Cohen sg = mr->sg_head.sgl;
25494abbccdSEli Cohen for (map = vhost_iotlb_itree_first(iotlb, mr->start, mr->end - 1);
25594abbccdSEli Cohen map; map = vhost_iotlb_itree_next(map, mr->start, mr->end - 1)) {
25615c49f91SSi-Wei Liu offset = mr->start > map->start ? mr->start - map->start : 0;
25715c49f91SSi-Wei Liu pa = map->addr + offset;
25815c49f91SSi-Wei Liu paend = map->addr + offset + maplen(map, mr);
25915c49f91SSi-Wei Liu for (; pa < paend; pa += sglen) {
26094abbccdSEli Cohen pg = pfn_to_page(__phys_to_pfn(pa));
26194abbccdSEli Cohen if (!sg) {
26294abbccdSEli Cohen mlx5_vdpa_warn(mvdev, "sg null. start 0x%llx, end 0x%llx\n",
26394abbccdSEli Cohen map->start, map->last + 1);
26494abbccdSEli Cohen err = -ENOMEM;
26594abbccdSEli Cohen goto err_map;
26694abbccdSEli Cohen }
26794abbccdSEli Cohen sg_set_page(sg, pg, sglen, 0);
26894abbccdSEli Cohen sg = sg_next(sg);
26994abbccdSEli Cohen if (!sg)
27094abbccdSEli Cohen goto done;
27194abbccdSEli Cohen }
27294abbccdSEli Cohen }
27394abbccdSEli Cohen done:
27494abbccdSEli Cohen mr->log_size = log_entity_size;
27594abbccdSEli Cohen mr->nsg = nsg;
276710eb8e3SEli Cohen mr->nent = dma_map_sg_attrs(dma, mr->sg_head.sgl, mr->nsg, DMA_BIDIRECTIONAL, 0);
277be286f84SEli Cohen if (!mr->nent) {
278be286f84SEli Cohen err = -ENOMEM;
27994abbccdSEli Cohen goto err_map;
280be286f84SEli Cohen }
28194abbccdSEli Cohen
28294abbccdSEli Cohen err = create_direct_mr(mvdev, mr);
28394abbccdSEli Cohen if (err)
28494abbccdSEli Cohen goto err_direct;
28594abbccdSEli Cohen
28694abbccdSEli Cohen return 0;
28794abbccdSEli Cohen
28894abbccdSEli Cohen err_direct:
28994abbccdSEli Cohen dma_unmap_sg_attrs(dma, mr->sg_head.sgl, mr->nsg, DMA_BIDIRECTIONAL, 0);
29094abbccdSEli Cohen err_map:
29194abbccdSEli Cohen sg_free_table(&mr->sg_head);
29294abbccdSEli Cohen return err;
29394abbccdSEli Cohen }
29494abbccdSEli Cohen
unmap_direct_mr(struct mlx5_vdpa_dev * mvdev,struct mlx5_vdpa_direct_mr * mr)29594abbccdSEli Cohen static void unmap_direct_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_direct_mr *mr)
29694abbccdSEli Cohen {
2977d23dcdfSEli Cohen struct device *dma = mvdev->vdev.dma_dev;
29894abbccdSEli Cohen
29994abbccdSEli Cohen destroy_direct_mr(mvdev, mr);
30094abbccdSEli Cohen dma_unmap_sg_attrs(dma, mr->sg_head.sgl, mr->nsg, DMA_BIDIRECTIONAL, 0);
30194abbccdSEli Cohen sg_free_table(&mr->sg_head);
30294abbccdSEli Cohen }
30394abbccdSEli Cohen
add_direct_chain(struct mlx5_vdpa_dev * mvdev,u64 start,u64 size,u8 perm,struct vhost_iotlb * iotlb)30494abbccdSEli Cohen static int add_direct_chain(struct mlx5_vdpa_dev *mvdev, u64 start, u64 size, u8 perm,
30594abbccdSEli Cohen struct vhost_iotlb *iotlb)
30694abbccdSEli Cohen {
30794abbccdSEli Cohen struct mlx5_vdpa_mr *mr = &mvdev->mr;
30894abbccdSEli Cohen struct mlx5_vdpa_direct_mr *dmr;
30994abbccdSEli Cohen struct mlx5_vdpa_direct_mr *n;
31094abbccdSEli Cohen LIST_HEAD(tmp);
31194abbccdSEli Cohen u64 st;
31294abbccdSEli Cohen u64 sz;
31394abbccdSEli Cohen int err;
31494abbccdSEli Cohen
31594abbccdSEli Cohen st = start;
31694abbccdSEli Cohen while (size) {
31794abbccdSEli Cohen sz = (u32)min_t(u64, MAX_KLM_SIZE, size);
31894abbccdSEli Cohen dmr = kzalloc(sizeof(*dmr), GFP_KERNEL);
31905acc4beSAlex Dewar if (!dmr) {
32005acc4beSAlex Dewar err = -ENOMEM;
32194abbccdSEli Cohen goto err_alloc;
32205acc4beSAlex Dewar }
32394abbccdSEli Cohen
32494abbccdSEli Cohen dmr->start = st;
32594abbccdSEli Cohen dmr->end = st + sz;
32694abbccdSEli Cohen dmr->perm = perm;
32794abbccdSEli Cohen err = map_direct_mr(mvdev, dmr, iotlb);
32894abbccdSEli Cohen if (err) {
32994abbccdSEli Cohen kfree(dmr);
33094abbccdSEli Cohen goto err_alloc;
33194abbccdSEli Cohen }
33294abbccdSEli Cohen
33394abbccdSEli Cohen list_add_tail(&dmr->list, &tmp);
33494abbccdSEli Cohen size -= sz;
33594abbccdSEli Cohen mr->num_directs++;
33694abbccdSEli Cohen mr->num_klms++;
33794abbccdSEli Cohen st += sz;
33894abbccdSEli Cohen }
33994abbccdSEli Cohen list_splice_tail(&tmp, &mr->head);
34094abbccdSEli Cohen return 0;
34194abbccdSEli Cohen
34294abbccdSEli Cohen err_alloc:
34394abbccdSEli Cohen list_for_each_entry_safe(dmr, n, &mr->head, list) {
34494abbccdSEli Cohen list_del_init(&dmr->list);
34594abbccdSEli Cohen unmap_direct_mr(mvdev, dmr);
34694abbccdSEli Cohen kfree(dmr);
34794abbccdSEli Cohen }
34894abbccdSEli Cohen return err;
34994abbccdSEli Cohen }
35094abbccdSEli Cohen
35194abbccdSEli Cohen /* The iotlb pointer contains a list of maps. Go over the maps, possibly
35294abbccdSEli Cohen * merging mergeable maps, and create direct memory keys that provide the
35394abbccdSEli Cohen * device access to memory. The direct mkeys are then referred to by the
35494abbccdSEli Cohen * indirect memory key that provides access to the enitre address space given
35594abbccdSEli Cohen * by iotlb.
35694abbccdSEli Cohen */
create_user_mr(struct mlx5_vdpa_dev * mvdev,struct vhost_iotlb * iotlb)3576f5312f8SEli Cohen static int create_user_mr(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *iotlb)
35894abbccdSEli Cohen {
35994abbccdSEli Cohen struct mlx5_vdpa_mr *mr = &mvdev->mr;
36094abbccdSEli Cohen struct mlx5_vdpa_direct_mr *dmr;
36194abbccdSEli Cohen struct mlx5_vdpa_direct_mr *n;
36294abbccdSEli Cohen struct vhost_iotlb_map *map;
36394abbccdSEli Cohen u32 pperm = U16_MAX;
36494abbccdSEli Cohen u64 last = U64_MAX;
36594abbccdSEli Cohen u64 ps = U64_MAX;
36694abbccdSEli Cohen u64 pe = U64_MAX;
36794abbccdSEli Cohen u64 start = 0;
36894abbccdSEli Cohen int err = 0;
36994abbccdSEli Cohen int nnuls;
37094abbccdSEli Cohen
37194abbccdSEli Cohen INIT_LIST_HEAD(&mr->head);
37294abbccdSEli Cohen for (map = vhost_iotlb_itree_first(iotlb, start, last); map;
37394abbccdSEli Cohen map = vhost_iotlb_itree_next(map, start, last)) {
37494abbccdSEli Cohen start = map->start;
37594abbccdSEli Cohen if (pe == map->start && pperm == map->perm) {
37694abbccdSEli Cohen pe = map->last + 1;
37794abbccdSEli Cohen } else {
37894abbccdSEli Cohen if (ps != U64_MAX) {
37994abbccdSEli Cohen if (pe < map->start) {
38094abbccdSEli Cohen /* We have a hole in the map. Check how
38194abbccdSEli Cohen * many null keys are required to fill it.
38294abbccdSEli Cohen */
38394abbccdSEli Cohen nnuls = MLX5_DIV_ROUND_UP_POW2(map->start - pe,
38494abbccdSEli Cohen LOG_MAX_KLM_SIZE);
38594abbccdSEli Cohen mr->num_klms += nnuls;
38694abbccdSEli Cohen }
38794abbccdSEli Cohen err = add_direct_chain(mvdev, ps, pe - ps, pperm, iotlb);
38894abbccdSEli Cohen if (err)
38994abbccdSEli Cohen goto err_chain;
39094abbccdSEli Cohen }
39194abbccdSEli Cohen ps = map->start;
39294abbccdSEli Cohen pe = map->last + 1;
39394abbccdSEli Cohen pperm = map->perm;
39494abbccdSEli Cohen }
39594abbccdSEli Cohen }
39694abbccdSEli Cohen err = add_direct_chain(mvdev, ps, pe - ps, pperm, iotlb);
39794abbccdSEli Cohen if (err)
39894abbccdSEli Cohen goto err_chain;
39994abbccdSEli Cohen
40094abbccdSEli Cohen /* Create the memory key that defines the guests's address space. This
40194abbccdSEli Cohen * memory key refers to the direct keys that contain the MTT
40294abbccdSEli Cohen * translations
40394abbccdSEli Cohen */
40494abbccdSEli Cohen err = create_indirect_key(mvdev, mr);
40594abbccdSEli Cohen if (err)
40694abbccdSEli Cohen goto err_chain;
40794abbccdSEli Cohen
4086f5312f8SEli Cohen mr->user_mr = true;
40994abbccdSEli Cohen return 0;
41094abbccdSEli Cohen
41194abbccdSEli Cohen err_chain:
41294abbccdSEli Cohen list_for_each_entry_safe_reverse(dmr, n, &mr->head, list) {
41394abbccdSEli Cohen list_del_init(&dmr->list);
41494abbccdSEli Cohen unmap_direct_mr(mvdev, dmr);
41594abbccdSEli Cohen kfree(dmr);
41694abbccdSEli Cohen }
41794abbccdSEli Cohen return err;
41894abbccdSEli Cohen }
41994abbccdSEli Cohen
create_dma_mr(struct mlx5_vdpa_dev * mvdev,struct mlx5_vdpa_mr * mr)4206f5312f8SEli Cohen static int create_dma_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mr)
4216f5312f8SEli Cohen {
4226f5312f8SEli Cohen int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
4236f5312f8SEli Cohen void *mkc;
4246f5312f8SEli Cohen u32 *in;
4256f5312f8SEli Cohen int err;
4266f5312f8SEli Cohen
4276f5312f8SEli Cohen in = kzalloc(inlen, GFP_KERNEL);
4286f5312f8SEli Cohen if (!in)
4296f5312f8SEli Cohen return -ENOMEM;
4306f5312f8SEli Cohen
4316f5312f8SEli Cohen mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
4326f5312f8SEli Cohen
4336f5312f8SEli Cohen MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_PA);
4346f5312f8SEli Cohen MLX5_SET(mkc, mkc, length64, 1);
4356f5312f8SEli Cohen MLX5_SET(mkc, mkc, lw, 1);
4366f5312f8SEli Cohen MLX5_SET(mkc, mkc, lr, 1);
4376f5312f8SEli Cohen MLX5_SET(mkc, mkc, pd, mvdev->res.pdn);
4386f5312f8SEli Cohen MLX5_SET(mkc, mkc, qpn, 0xffffff);
4396f5312f8SEli Cohen
4406f5312f8SEli Cohen err = mlx5_vdpa_create_mkey(mvdev, &mr->mkey, in, inlen);
4416f5312f8SEli Cohen if (!err)
4426f5312f8SEli Cohen mr->user_mr = false;
4436f5312f8SEli Cohen
4446f5312f8SEli Cohen kfree(in);
4456f5312f8SEli Cohen return err;
4466f5312f8SEli Cohen }
4476f5312f8SEli Cohen
destroy_dma_mr(struct mlx5_vdpa_dev * mvdev,struct mlx5_vdpa_mr * mr)4486f5312f8SEli Cohen static void destroy_dma_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mr)
4496f5312f8SEli Cohen {
45083fec3f1SAharon Landau mlx5_vdpa_destroy_mkey(mvdev, mr->mkey);
4516f5312f8SEli Cohen }
4526f5312f8SEli Cohen
dup_iotlb(struct mlx5_vdpa_dev * mvdev,struct vhost_iotlb * src)4535262912eSEli Cohen static int dup_iotlb(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *src)
45494abbccdSEli Cohen {
4555262912eSEli Cohen struct vhost_iotlb_map *map;
4565262912eSEli Cohen u64 start = 0, last = ULLONG_MAX;
45794abbccdSEli Cohen int err;
45894abbccdSEli Cohen
4595262912eSEli Cohen if (!src) {
4605262912eSEli Cohen err = vhost_iotlb_add_range(mvdev->cvq.iotlb, start, last, start, VHOST_ACCESS_RW);
46194abbccdSEli Cohen return err;
46294abbccdSEli Cohen }
46394abbccdSEli Cohen
4645262912eSEli Cohen for (map = vhost_iotlb_itree_first(src, start, last); map;
4655262912eSEli Cohen map = vhost_iotlb_itree_next(map, start, last)) {
4665262912eSEli Cohen err = vhost_iotlb_add_range(mvdev->cvq.iotlb, map->start, map->last,
4675262912eSEli Cohen map->addr, map->perm);
4685262912eSEli Cohen if (err)
4696f5312f8SEli Cohen return err;
4706f5312f8SEli Cohen }
4715262912eSEli Cohen return 0;
4725262912eSEli Cohen }
4735262912eSEli Cohen
prune_iotlb(struct mlx5_vdpa_dev * mvdev)4745262912eSEli Cohen static void prune_iotlb(struct mlx5_vdpa_dev *mvdev)
4755262912eSEli Cohen {
4765262912eSEli Cohen vhost_iotlb_del_range(mvdev->cvq.iotlb, 0, ULLONG_MAX);
4775262912eSEli Cohen }
4786f5312f8SEli Cohen
destroy_user_mr(struct mlx5_vdpa_dev * mvdev,struct mlx5_vdpa_mr * mr)4796f5312f8SEli Cohen static void destroy_user_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mr)
4806f5312f8SEli Cohen {
48194abbccdSEli Cohen struct mlx5_vdpa_direct_mr *dmr;
48294abbccdSEli Cohen struct mlx5_vdpa_direct_mr *n;
48394abbccdSEli Cohen
48494abbccdSEli Cohen destroy_indirect_key(mvdev, mr);
48594abbccdSEli Cohen list_for_each_entry_safe_reverse(dmr, n, &mr->head, list) {
48694abbccdSEli Cohen list_del_init(&dmr->list);
48794abbccdSEli Cohen unmap_direct_mr(mvdev, dmr);
48894abbccdSEli Cohen kfree(dmr);
48994abbccdSEli Cohen }
4906f5312f8SEli Cohen }
4916f5312f8SEli Cohen
_mlx5_vdpa_destroy_cvq_mr(struct mlx5_vdpa_dev * mvdev,unsigned int asid)4929ee81100SDragos Tatulea static void _mlx5_vdpa_destroy_cvq_mr(struct mlx5_vdpa_dev *mvdev, unsigned int asid)
4939ee81100SDragos Tatulea {
4949ee81100SDragos Tatulea if (mvdev->group2asid[MLX5_VDPA_CVQ_GROUP] != asid)
4959ee81100SDragos Tatulea return;
4969ee81100SDragos Tatulea
4979ee81100SDragos Tatulea prune_iotlb(mvdev);
4989ee81100SDragos Tatulea }
4999ee81100SDragos Tatulea
_mlx5_vdpa_destroy_dvq_mr(struct mlx5_vdpa_dev * mvdev,unsigned int asid)5009ee81100SDragos Tatulea static void _mlx5_vdpa_destroy_dvq_mr(struct mlx5_vdpa_dev *mvdev, unsigned int asid)
5016f5312f8SEli Cohen {
5026f5312f8SEli Cohen struct mlx5_vdpa_mr *mr = &mvdev->mr;
5036f5312f8SEli Cohen
5049ee81100SDragos Tatulea if (mvdev->group2asid[MLX5_VDPA_DATAVQ_GROUP] != asid)
5059ee81100SDragos Tatulea return;
5066f5312f8SEli Cohen
5079ee81100SDragos Tatulea if (!mr->initialized)
5089ee81100SDragos Tatulea return;
5099ee81100SDragos Tatulea
5106f5312f8SEli Cohen if (mr->user_mr)
5116f5312f8SEli Cohen destroy_user_mr(mvdev, mr);
5126f5312f8SEli Cohen else
5136f5312f8SEli Cohen destroy_dma_mr(mvdev, mr);
5146f5312f8SEli Cohen
51594abbccdSEli Cohen mr->initialized = false;
5169ee81100SDragos Tatulea }
5179ee81100SDragos Tatulea
mlx5_vdpa_destroy_mr_asid(struct mlx5_vdpa_dev * mvdev,unsigned int asid)518ad03a0f4SEugenio Pérez void mlx5_vdpa_destroy_mr_asid(struct mlx5_vdpa_dev *mvdev, unsigned int asid)
5199ee81100SDragos Tatulea {
5209ee81100SDragos Tatulea struct mlx5_vdpa_mr *mr = &mvdev->mr;
5219ee81100SDragos Tatulea
5229ee81100SDragos Tatulea mutex_lock(&mr->mkey_mtx);
5239ee81100SDragos Tatulea
5249ee81100SDragos Tatulea _mlx5_vdpa_destroy_dvq_mr(mvdev, asid);
5259ee81100SDragos Tatulea _mlx5_vdpa_destroy_cvq_mr(mvdev, asid);
5269ee81100SDragos Tatulea
52794abbccdSEli Cohen mutex_unlock(&mr->mkey_mtx);
52894abbccdSEli Cohen }
52994abbccdSEli Cohen
mlx5_vdpa_destroy_mr(struct mlx5_vdpa_dev * mvdev)5309ee81100SDragos Tatulea void mlx5_vdpa_destroy_mr(struct mlx5_vdpa_dev *mvdev)
5319ee81100SDragos Tatulea {
5329ee81100SDragos Tatulea mlx5_vdpa_destroy_mr_asid(mvdev, mvdev->group2asid[MLX5_VDPA_CVQ_GROUP]);
5339ee81100SDragos Tatulea mlx5_vdpa_destroy_mr_asid(mvdev, mvdev->group2asid[MLX5_VDPA_DATAVQ_GROUP]);
5349ee81100SDragos Tatulea }
5359ee81100SDragos Tatulea
_mlx5_vdpa_create_cvq_mr(struct mlx5_vdpa_dev * mvdev,struct vhost_iotlb * iotlb,unsigned int asid)5369ee81100SDragos Tatulea static int _mlx5_vdpa_create_cvq_mr(struct mlx5_vdpa_dev *mvdev,
5379ee81100SDragos Tatulea struct vhost_iotlb *iotlb,
5389ee81100SDragos Tatulea unsigned int asid)
5399ee81100SDragos Tatulea {
5409ee81100SDragos Tatulea if (mvdev->group2asid[MLX5_VDPA_CVQ_GROUP] != asid)
5419ee81100SDragos Tatulea return 0;
5429ee81100SDragos Tatulea
5439ee81100SDragos Tatulea return dup_iotlb(mvdev, iotlb);
5449ee81100SDragos Tatulea }
5459ee81100SDragos Tatulea
_mlx5_vdpa_create_dvq_mr(struct mlx5_vdpa_dev * mvdev,struct vhost_iotlb * iotlb,unsigned int asid)5469ee81100SDragos Tatulea static int _mlx5_vdpa_create_dvq_mr(struct mlx5_vdpa_dev *mvdev,
5479ee81100SDragos Tatulea struct vhost_iotlb *iotlb,
5489ee81100SDragos Tatulea unsigned int asid)
5495262912eSEli Cohen {
5505262912eSEli Cohen struct mlx5_vdpa_mr *mr = &mvdev->mr;
5515262912eSEli Cohen int err;
5525262912eSEli Cohen
5539ee81100SDragos Tatulea if (mvdev->group2asid[MLX5_VDPA_DATAVQ_GROUP] != asid)
5549ee81100SDragos Tatulea return 0;
5559ee81100SDragos Tatulea
5565262912eSEli Cohen if (mr->initialized)
5575262912eSEli Cohen return 0;
5585262912eSEli Cohen
5595262912eSEli Cohen if (iotlb)
5605262912eSEli Cohen err = create_user_mr(mvdev, iotlb);
5615262912eSEli Cohen else
5625262912eSEli Cohen err = create_dma_mr(mvdev, mr);
5635262912eSEli Cohen
5645262912eSEli Cohen if (err)
5655262912eSEli Cohen return err;
5665262912eSEli Cohen
5675262912eSEli Cohen mr->initialized = true;
5689ee81100SDragos Tatulea
5699ee81100SDragos Tatulea return 0;
5709ee81100SDragos Tatulea }
5719ee81100SDragos Tatulea
_mlx5_vdpa_create_mr(struct mlx5_vdpa_dev * mvdev,struct vhost_iotlb * iotlb,unsigned int asid)5729ee81100SDragos Tatulea static int _mlx5_vdpa_create_mr(struct mlx5_vdpa_dev *mvdev,
5739ee81100SDragos Tatulea struct vhost_iotlb *iotlb, unsigned int asid)
5749ee81100SDragos Tatulea {
5759ee81100SDragos Tatulea int err;
5769ee81100SDragos Tatulea
5779ee81100SDragos Tatulea err = _mlx5_vdpa_create_dvq_mr(mvdev, iotlb, asid);
5789ee81100SDragos Tatulea if (err)
5799ee81100SDragos Tatulea return err;
5809ee81100SDragos Tatulea
5819ee81100SDragos Tatulea err = _mlx5_vdpa_create_cvq_mr(mvdev, iotlb, asid);
5829ee81100SDragos Tatulea if (err)
5839ee81100SDragos Tatulea goto out_err;
5849ee81100SDragos Tatulea
5855262912eSEli Cohen return 0;
5865262912eSEli Cohen
5875262912eSEli Cohen out_err:
5889ee81100SDragos Tatulea _mlx5_vdpa_destroy_dvq_mr(mvdev, asid);
5895262912eSEli Cohen
5905262912eSEli Cohen return err;
5915262912eSEli Cohen }
5925262912eSEli Cohen
mlx5_vdpa_create_mr(struct mlx5_vdpa_dev * mvdev,struct vhost_iotlb * iotlb,unsigned int asid)59338fc462fSEli Cohen int mlx5_vdpa_create_mr(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *iotlb,
59438fc462fSEli Cohen unsigned int asid)
5955262912eSEli Cohen {
5965262912eSEli Cohen int err;
5975262912eSEli Cohen
5985262912eSEli Cohen mutex_lock(&mvdev->mr.mkey_mtx);
59938fc462fSEli Cohen err = _mlx5_vdpa_create_mr(mvdev, iotlb, asid);
6005262912eSEli Cohen mutex_unlock(&mvdev->mr.mkey_mtx);
6015262912eSEli Cohen return err;
6025262912eSEli Cohen }
6035262912eSEli Cohen
mlx5_vdpa_handle_set_map(struct mlx5_vdpa_dev * mvdev,struct vhost_iotlb * iotlb,bool * change_map,unsigned int asid)60494abbccdSEli Cohen int mlx5_vdpa_handle_set_map(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *iotlb,
60538fc462fSEli Cohen bool *change_map, unsigned int asid)
60694abbccdSEli Cohen {
60794abbccdSEli Cohen struct mlx5_vdpa_mr *mr = &mvdev->mr;
6081a86b377SEli Cohen int err = 0;
60994abbccdSEli Cohen
61094abbccdSEli Cohen *change_map = false;
61194abbccdSEli Cohen mutex_lock(&mr->mkey_mtx);
61294abbccdSEli Cohen if (mr->initialized) {
61394abbccdSEli Cohen mlx5_vdpa_info(mvdev, "memory map update\n");
61494abbccdSEli Cohen *change_map = true;
61594abbccdSEli Cohen }
61694abbccdSEli Cohen if (!*change_map)
61738fc462fSEli Cohen err = _mlx5_vdpa_create_mr(mvdev, iotlb, asid);
61894abbccdSEli Cohen mutex_unlock(&mr->mkey_mtx);
61994abbccdSEli Cohen
62094abbccdSEli Cohen return err;
62194abbccdSEli Cohen }
622