1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 // Copyright (c) 2019 Mellanox Technologies
3 
4 #include <linux/mlx5/driver.h>
5 #include <linux/mlx5/device.h>
6 
7 #include "mlx5_core.h"
8 #include "lib/mlx5.h"
9 
10 struct mlx5_dm {
11 	/* protect access to icm bitmask */
12 	spinlock_t lock;
13 	unsigned long *steering_sw_icm_alloc_blocks;
14 	unsigned long *header_modify_sw_icm_alloc_blocks;
15 	unsigned long *header_modify_pattern_sw_icm_alloc_blocks;
16 };
17 
mlx5_dm_create(struct mlx5_core_dev * dev)18 struct mlx5_dm *mlx5_dm_create(struct mlx5_core_dev *dev)
19 {
20 	u64 header_modify_pattern_icm_blocks = 0;
21 	u64 header_modify_icm_blocks = 0;
22 	u64 steering_icm_blocks = 0;
23 	struct mlx5_dm *dm;
24 	bool support_v2;
25 
26 	if (!(MLX5_CAP_GEN_64(dev, general_obj_types) & MLX5_GENERAL_OBJ_TYPES_CAP_SW_ICM))
27 		return NULL;
28 
29 	dm = kzalloc(sizeof(*dm), GFP_KERNEL);
30 	if (!dm)
31 		return ERR_PTR(-ENOMEM);
32 
33 	spin_lock_init(&dm->lock);
34 
35 	if (MLX5_CAP64_DEV_MEM(dev, steering_sw_icm_start_address)) {
36 		steering_icm_blocks =
37 			BIT(MLX5_CAP_DEV_MEM(dev, log_steering_sw_icm_size) -
38 			    MLX5_LOG_SW_ICM_BLOCK_SIZE(dev));
39 
40 		dm->steering_sw_icm_alloc_blocks =
41 			bitmap_zalloc(steering_icm_blocks, GFP_KERNEL);
42 		if (!dm->steering_sw_icm_alloc_blocks)
43 			goto err_steering;
44 	}
45 
46 	if (MLX5_CAP64_DEV_MEM(dev, header_modify_sw_icm_start_address)) {
47 		header_modify_icm_blocks =
48 			BIT(MLX5_CAP_DEV_MEM(dev, log_header_modify_sw_icm_size) -
49 			    MLX5_LOG_SW_ICM_BLOCK_SIZE(dev));
50 
51 		dm->header_modify_sw_icm_alloc_blocks =
52 			bitmap_zalloc(header_modify_icm_blocks, GFP_KERNEL);
53 		if (!dm->header_modify_sw_icm_alloc_blocks)
54 			goto err_modify_hdr;
55 	}
56 
57 	support_v2 = MLX5_CAP_FLOWTABLE_NIC_RX(dev, sw_owner_v2) &&
58 		     MLX5_CAP_FLOWTABLE_NIC_TX(dev, sw_owner_v2) &&
59 		     MLX5_CAP64_DEV_MEM(dev, header_modify_pattern_sw_icm_start_address);
60 
61 	if (support_v2) {
62 		header_modify_pattern_icm_blocks =
63 			BIT(MLX5_CAP_DEV_MEM(dev, log_header_modify_pattern_sw_icm_size) -
64 			    MLX5_LOG_SW_ICM_BLOCK_SIZE(dev));
65 
66 		dm->header_modify_pattern_sw_icm_alloc_blocks =
67 			bitmap_zalloc(header_modify_pattern_icm_blocks, GFP_KERNEL);
68 		if (!dm->header_modify_pattern_sw_icm_alloc_blocks)
69 			goto err_pattern;
70 	}
71 
72 	return dm;
73 
74 err_pattern:
75 	bitmap_free(dm->header_modify_sw_icm_alloc_blocks);
76 
77 err_modify_hdr:
78 	bitmap_free(dm->steering_sw_icm_alloc_blocks);
79 
80 err_steering:
81 	kfree(dm);
82 
83 	return ERR_PTR(-ENOMEM);
84 }
85 
mlx5_dm_cleanup(struct mlx5_core_dev * dev)86 void mlx5_dm_cleanup(struct mlx5_core_dev *dev)
87 {
88 	struct mlx5_dm *dm = dev->dm;
89 
90 	if (!dev->dm)
91 		return;
92 
93 	if (dm->steering_sw_icm_alloc_blocks) {
94 		WARN_ON(!bitmap_empty(dm->steering_sw_icm_alloc_blocks,
95 				      BIT(MLX5_CAP_DEV_MEM(dev, log_steering_sw_icm_size) -
96 					  MLX5_LOG_SW_ICM_BLOCK_SIZE(dev))));
97 		bitmap_free(dm->steering_sw_icm_alloc_blocks);
98 	}
99 
100 	if (dm->header_modify_sw_icm_alloc_blocks) {
101 		WARN_ON(!bitmap_empty(dm->header_modify_sw_icm_alloc_blocks,
102 				      BIT(MLX5_CAP_DEV_MEM(dev,
103 							   log_header_modify_sw_icm_size) -
104 				      MLX5_LOG_SW_ICM_BLOCK_SIZE(dev))));
105 		bitmap_free(dm->header_modify_sw_icm_alloc_blocks);
106 	}
107 
108 	if (dm->header_modify_pattern_sw_icm_alloc_blocks) {
109 		WARN_ON(!bitmap_empty(dm->header_modify_pattern_sw_icm_alloc_blocks,
110 				      BIT(MLX5_CAP_DEV_MEM(dev,
111 							   log_header_modify_pattern_sw_icm_size) -
112 					  MLX5_LOG_SW_ICM_BLOCK_SIZE(dev))));
113 		bitmap_free(dm->header_modify_pattern_sw_icm_alloc_blocks);
114 	}
115 
116 	kfree(dm);
117 }
118 
mlx5_dm_sw_icm_alloc(struct mlx5_core_dev * dev,enum mlx5_sw_icm_type type,u64 length,u32 log_alignment,u16 uid,phys_addr_t * addr,u32 * obj_id)119 int mlx5_dm_sw_icm_alloc(struct mlx5_core_dev *dev, enum mlx5_sw_icm_type type,
120 			 u64 length, u32 log_alignment, u16 uid,
121 			 phys_addr_t *addr, u32 *obj_id)
122 {
123 	u32 num_blocks = DIV_ROUND_UP_ULL(length, MLX5_SW_ICM_BLOCK_SIZE(dev));
124 	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {};
125 	u32 in[MLX5_ST_SZ_DW(create_sw_icm_in)] = {};
126 	struct mlx5_dm *dm = dev->dm;
127 	unsigned long *block_map;
128 	u64 icm_start_addr;
129 	u32 log_icm_size;
130 	u64 align_mask;
131 	u32 max_blocks;
132 	u64 block_idx;
133 	void *sw_icm;
134 	int ret;
135 
136 	if (!dev->dm)
137 		return -EOPNOTSUPP;
138 
139 	if (!length || (length & (length - 1)) ||
140 	    length & (MLX5_SW_ICM_BLOCK_SIZE(dev) - 1))
141 		return -EINVAL;
142 
143 	MLX5_SET(general_obj_in_cmd_hdr, in, opcode,
144 		 MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
145 	MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_OBJ_TYPE_SW_ICM);
146 	MLX5_SET(general_obj_in_cmd_hdr, in, uid, uid);
147 
148 	switch (type) {
149 	case MLX5_SW_ICM_TYPE_STEERING:
150 		icm_start_addr = MLX5_CAP64_DEV_MEM(dev, steering_sw_icm_start_address);
151 		log_icm_size = MLX5_CAP_DEV_MEM(dev, log_steering_sw_icm_size);
152 		block_map = dm->steering_sw_icm_alloc_blocks;
153 		break;
154 	case MLX5_SW_ICM_TYPE_HEADER_MODIFY:
155 		icm_start_addr = MLX5_CAP64_DEV_MEM(dev, header_modify_sw_icm_start_address);
156 		log_icm_size = MLX5_CAP_DEV_MEM(dev,
157 						log_header_modify_sw_icm_size);
158 		block_map = dm->header_modify_sw_icm_alloc_blocks;
159 		break;
160 	case MLX5_SW_ICM_TYPE_HEADER_MODIFY_PATTERN:
161 		icm_start_addr = MLX5_CAP64_DEV_MEM(dev,
162 						    header_modify_pattern_sw_icm_start_address);
163 		log_icm_size = MLX5_CAP_DEV_MEM(dev,
164 						log_header_modify_pattern_sw_icm_size);
165 		block_map = dm->header_modify_pattern_sw_icm_alloc_blocks;
166 		break;
167 	default:
168 		return -EINVAL;
169 	}
170 
171 	if (!block_map)
172 		return -EOPNOTSUPP;
173 
174 	max_blocks = BIT(log_icm_size - MLX5_LOG_SW_ICM_BLOCK_SIZE(dev));
175 
176 	if (log_alignment < MLX5_LOG_SW_ICM_BLOCK_SIZE(dev))
177 		log_alignment = MLX5_LOG_SW_ICM_BLOCK_SIZE(dev);
178 	align_mask = BIT(log_alignment - MLX5_LOG_SW_ICM_BLOCK_SIZE(dev)) - 1;
179 
180 	spin_lock(&dm->lock);
181 	block_idx = bitmap_find_next_zero_area(block_map, max_blocks, 0,
182 					       num_blocks, align_mask);
183 
184 	if (block_idx < max_blocks)
185 		bitmap_set(block_map,
186 			   block_idx, num_blocks);
187 
188 	spin_unlock(&dm->lock);
189 
190 	if (block_idx >= max_blocks)
191 		return -ENOMEM;
192 
193 	sw_icm = MLX5_ADDR_OF(create_sw_icm_in, in, sw_icm);
194 	icm_start_addr += block_idx << MLX5_LOG_SW_ICM_BLOCK_SIZE(dev);
195 	MLX5_SET64(sw_icm, sw_icm, sw_icm_start_addr,
196 		   icm_start_addr);
197 	MLX5_SET(sw_icm, sw_icm, log_sw_icm_size, ilog2(length));
198 
199 	ret = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
200 	if (ret) {
201 		spin_lock(&dm->lock);
202 		bitmap_clear(block_map,
203 			     block_idx, num_blocks);
204 		spin_unlock(&dm->lock);
205 
206 		return ret;
207 	}
208 
209 	*addr = icm_start_addr;
210 	*obj_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
211 
212 	return 0;
213 }
214 EXPORT_SYMBOL_GPL(mlx5_dm_sw_icm_alloc);
215 
mlx5_dm_sw_icm_dealloc(struct mlx5_core_dev * dev,enum mlx5_sw_icm_type type,u64 length,u16 uid,phys_addr_t addr,u32 obj_id)216 int mlx5_dm_sw_icm_dealloc(struct mlx5_core_dev *dev, enum mlx5_sw_icm_type type,
217 			   u64 length, u16 uid, phys_addr_t addr, u32 obj_id)
218 {
219 	u32 num_blocks = DIV_ROUND_UP_ULL(length, MLX5_SW_ICM_BLOCK_SIZE(dev));
220 	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {};
221 	u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {};
222 	struct mlx5_dm *dm = dev->dm;
223 	unsigned long *block_map;
224 	u64 icm_start_addr;
225 	u64 start_idx;
226 	int err;
227 
228 	if (!dev->dm)
229 		return -EOPNOTSUPP;
230 
231 	switch (type) {
232 	case MLX5_SW_ICM_TYPE_STEERING:
233 		icm_start_addr = MLX5_CAP64_DEV_MEM(dev, steering_sw_icm_start_address);
234 		block_map = dm->steering_sw_icm_alloc_blocks;
235 		break;
236 	case MLX5_SW_ICM_TYPE_HEADER_MODIFY:
237 		icm_start_addr = MLX5_CAP64_DEV_MEM(dev, header_modify_sw_icm_start_address);
238 		block_map = dm->header_modify_sw_icm_alloc_blocks;
239 		break;
240 	case MLX5_SW_ICM_TYPE_HEADER_MODIFY_PATTERN:
241 		icm_start_addr = MLX5_CAP64_DEV_MEM(dev,
242 						    header_modify_pattern_sw_icm_start_address);
243 		block_map = dm->header_modify_pattern_sw_icm_alloc_blocks;
244 		break;
245 	default:
246 		return -EINVAL;
247 	}
248 
249 	MLX5_SET(general_obj_in_cmd_hdr, in, opcode,
250 		 MLX5_CMD_OP_DESTROY_GENERAL_OBJECT);
251 	MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_OBJ_TYPE_SW_ICM);
252 	MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, obj_id);
253 	MLX5_SET(general_obj_in_cmd_hdr, in, uid, uid);
254 
255 	err =  mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
256 	if (err)
257 		return err;
258 
259 	start_idx = (addr - icm_start_addr) >> MLX5_LOG_SW_ICM_BLOCK_SIZE(dev);
260 	spin_lock(&dm->lock);
261 	bitmap_clear(block_map,
262 		     start_idx, num_blocks);
263 	spin_unlock(&dm->lock);
264 
265 	return 0;
266 }
267 EXPORT_SYMBOL_GPL(mlx5_dm_sw_icm_dealloc);
268