1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2 /* Copyright (c) 2021, Microsoft Corporation. */
3 
4 #include <linux/module.h>
5 #include <linux/pci.h>
6 #include <linux/utsname.h>
7 #include <linux/version.h>
8 
9 #include <net/mana/mana.h>
10 
11 static u32 mana_gd_r32(struct gdma_context *g, u64 offset)
12 {
13 	return readl(g->bar0_va + offset);
14 }
15 
16 static u64 mana_gd_r64(struct gdma_context *g, u64 offset)
17 {
18 	return readq(g->bar0_va + offset);
19 }
20 
21 static void mana_gd_init_pf_regs(struct pci_dev *pdev)
22 {
23 	struct gdma_context *gc = pci_get_drvdata(pdev);
24 	void __iomem *sriov_base_va;
25 	u64 sriov_base_off;
26 
27 	gc->db_page_size = mana_gd_r32(gc, GDMA_PF_REG_DB_PAGE_SIZE) & 0xFFFF;
28 	gc->db_page_base = gc->bar0_va +
29 				mana_gd_r64(gc, GDMA_PF_REG_DB_PAGE_OFF);
30 
31 	sriov_base_off = mana_gd_r64(gc, GDMA_SRIOV_REG_CFG_BASE_OFF);
32 
33 	sriov_base_va = gc->bar0_va + sriov_base_off;
34 	gc->shm_base = sriov_base_va +
35 			mana_gd_r64(gc, sriov_base_off + GDMA_PF_REG_SHM_OFF);
36 }
37 
38 static void mana_gd_init_vf_regs(struct pci_dev *pdev)
39 {
40 	struct gdma_context *gc = pci_get_drvdata(pdev);
41 
42 	gc->db_page_size = mana_gd_r32(gc, GDMA_REG_DB_PAGE_SIZE) & 0xFFFF;
43 
44 	gc->db_page_base = gc->bar0_va +
45 				mana_gd_r64(gc, GDMA_REG_DB_PAGE_OFFSET);
46 
47 	gc->phys_db_page_base = gc->bar0_pa +
48 				mana_gd_r64(gc, GDMA_REG_DB_PAGE_OFFSET);
49 
50 	gc->shm_base = gc->bar0_va + mana_gd_r64(gc, GDMA_REG_SHM_OFFSET);
51 }
52 
53 static void mana_gd_init_registers(struct pci_dev *pdev)
54 {
55 	struct gdma_context *gc = pci_get_drvdata(pdev);
56 
57 	if (gc->is_pf)
58 		mana_gd_init_pf_regs(pdev);
59 	else
60 		mana_gd_init_vf_regs(pdev);
61 }
62 
63 static int mana_gd_query_max_resources(struct pci_dev *pdev)
64 {
65 	struct gdma_context *gc = pci_get_drvdata(pdev);
66 	struct gdma_query_max_resources_resp resp = {};
67 	struct gdma_general_req req = {};
68 	int err;
69 
70 	mana_gd_init_req_hdr(&req.hdr, GDMA_QUERY_MAX_RESOURCES,
71 			     sizeof(req), sizeof(resp));
72 
73 	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
74 	if (err || resp.hdr.status) {
75 		dev_err(gc->dev, "Failed to query resource info: %d, 0x%x\n",
76 			err, resp.hdr.status);
77 		return err ? err : -EPROTO;
78 	}
79 
80 	if (gc->num_msix_usable > resp.max_msix)
81 		gc->num_msix_usable = resp.max_msix;
82 
83 	if (gc->num_msix_usable <= 1)
84 		return -ENOSPC;
85 
86 	gc->max_num_queues = num_online_cpus();
87 	if (gc->max_num_queues > MANA_MAX_NUM_QUEUES)
88 		gc->max_num_queues = MANA_MAX_NUM_QUEUES;
89 
90 	if (gc->max_num_queues > resp.max_eq)
91 		gc->max_num_queues = resp.max_eq;
92 
93 	if (gc->max_num_queues > resp.max_cq)
94 		gc->max_num_queues = resp.max_cq;
95 
96 	if (gc->max_num_queues > resp.max_sq)
97 		gc->max_num_queues = resp.max_sq;
98 
99 	if (gc->max_num_queues > resp.max_rq)
100 		gc->max_num_queues = resp.max_rq;
101 
102 	/* The Hardware Channel (HWC) used 1 MSI-X */
103 	if (gc->max_num_queues > gc->num_msix_usable - 1)
104 		gc->max_num_queues = gc->num_msix_usable - 1;
105 
106 	return 0;
107 }
108 
109 static int mana_gd_detect_devices(struct pci_dev *pdev)
110 {
111 	struct gdma_context *gc = pci_get_drvdata(pdev);
112 	struct gdma_list_devices_resp resp = {};
113 	struct gdma_general_req req = {};
114 	struct gdma_dev_id dev;
115 	u32 i, max_num_devs;
116 	u16 dev_type;
117 	int err;
118 
119 	mana_gd_init_req_hdr(&req.hdr, GDMA_LIST_DEVICES, sizeof(req),
120 			     sizeof(resp));
121 
122 	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
123 	if (err || resp.hdr.status) {
124 		dev_err(gc->dev, "Failed to detect devices: %d, 0x%x\n", err,
125 			resp.hdr.status);
126 		return err ? err : -EPROTO;
127 	}
128 
129 	max_num_devs = min_t(u32, MAX_NUM_GDMA_DEVICES, resp.num_of_devs);
130 
131 	for (i = 0; i < max_num_devs; i++) {
132 		dev = resp.devs[i];
133 		dev_type = dev.type;
134 
135 		/* HWC is already detected in mana_hwc_create_channel(). */
136 		if (dev_type == GDMA_DEVICE_HWC)
137 			continue;
138 
139 		if (dev_type == GDMA_DEVICE_MANA) {
140 			gc->mana.gdma_context = gc;
141 			gc->mana.dev_id = dev;
142 		}
143 	}
144 
145 	return gc->mana.dev_id.type == 0 ? -ENODEV : 0;
146 }
147 
148 int mana_gd_send_request(struct gdma_context *gc, u32 req_len, const void *req,
149 			 u32 resp_len, void *resp)
150 {
151 	struct hw_channel_context *hwc = gc->hwc.driver_data;
152 
153 	return mana_hwc_send_request(hwc, req_len, req, resp_len, resp);
154 }
155 EXPORT_SYMBOL_NS(mana_gd_send_request, NET_MANA);
156 
157 int mana_gd_alloc_memory(struct gdma_context *gc, unsigned int length,
158 			 struct gdma_mem_info *gmi)
159 {
160 	dma_addr_t dma_handle;
161 	void *buf;
162 
163 	if (length < PAGE_SIZE || !is_power_of_2(length))
164 		return -EINVAL;
165 
166 	gmi->dev = gc->dev;
167 	buf = dma_alloc_coherent(gmi->dev, length, &dma_handle, GFP_KERNEL);
168 	if (!buf)
169 		return -ENOMEM;
170 
171 	gmi->dma_handle = dma_handle;
172 	gmi->virt_addr = buf;
173 	gmi->length = length;
174 
175 	return 0;
176 }
177 
178 void mana_gd_free_memory(struct gdma_mem_info *gmi)
179 {
180 	dma_free_coherent(gmi->dev, gmi->length, gmi->virt_addr,
181 			  gmi->dma_handle);
182 }
183 
184 static int mana_gd_create_hw_eq(struct gdma_context *gc,
185 				struct gdma_queue *queue)
186 {
187 	struct gdma_create_queue_resp resp = {};
188 	struct gdma_create_queue_req req = {};
189 	int err;
190 
191 	if (queue->type != GDMA_EQ)
192 		return -EINVAL;
193 
194 	mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_QUEUE,
195 			     sizeof(req), sizeof(resp));
196 
197 	req.hdr.dev_id = queue->gdma_dev->dev_id;
198 	req.type = queue->type;
199 	req.pdid = queue->gdma_dev->pdid;
200 	req.doolbell_id = queue->gdma_dev->doorbell;
201 	req.gdma_region = queue->mem_info.dma_region_handle;
202 	req.queue_size = queue->queue_size;
203 	req.log2_throttle_limit = queue->eq.log2_throttle_limit;
204 	req.eq_pci_msix_index = queue->eq.msix_index;
205 
206 	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
207 	if (err || resp.hdr.status) {
208 		dev_err(gc->dev, "Failed to create queue: %d, 0x%x\n", err,
209 			resp.hdr.status);
210 		return err ? err : -EPROTO;
211 	}
212 
213 	queue->id = resp.queue_index;
214 	queue->eq.disable_needed = true;
215 	queue->mem_info.dma_region_handle = GDMA_INVALID_DMA_REGION;
216 	return 0;
217 }
218 
219 static int mana_gd_disable_queue(struct gdma_queue *queue)
220 {
221 	struct gdma_context *gc = queue->gdma_dev->gdma_context;
222 	struct gdma_disable_queue_req req = {};
223 	struct gdma_general_resp resp = {};
224 	int err;
225 
226 	WARN_ON(queue->type != GDMA_EQ);
227 
228 	mana_gd_init_req_hdr(&req.hdr, GDMA_DISABLE_QUEUE,
229 			     sizeof(req), sizeof(resp));
230 
231 	req.hdr.dev_id = queue->gdma_dev->dev_id;
232 	req.type = queue->type;
233 	req.queue_index =  queue->id;
234 	req.alloc_res_id_on_creation = 1;
235 
236 	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
237 	if (err || resp.hdr.status) {
238 		dev_err(gc->dev, "Failed to disable queue: %d, 0x%x\n", err,
239 			resp.hdr.status);
240 		return err ? err : -EPROTO;
241 	}
242 
243 	return 0;
244 }
245 
246 #define DOORBELL_OFFSET_SQ	0x0
247 #define DOORBELL_OFFSET_RQ	0x400
248 #define DOORBELL_OFFSET_CQ	0x800
249 #define DOORBELL_OFFSET_EQ	0xFF8
250 
251 static void mana_gd_ring_doorbell(struct gdma_context *gc, u32 db_index,
252 				  enum gdma_queue_type q_type, u32 qid,
253 				  u32 tail_ptr, u8 num_req)
254 {
255 	void __iomem *addr = gc->db_page_base + gc->db_page_size * db_index;
256 	union gdma_doorbell_entry e = {};
257 
258 	switch (q_type) {
259 	case GDMA_EQ:
260 		e.eq.id = qid;
261 		e.eq.tail_ptr = tail_ptr;
262 		e.eq.arm = num_req;
263 
264 		addr += DOORBELL_OFFSET_EQ;
265 		break;
266 
267 	case GDMA_CQ:
268 		e.cq.id = qid;
269 		e.cq.tail_ptr = tail_ptr;
270 		e.cq.arm = num_req;
271 
272 		addr += DOORBELL_OFFSET_CQ;
273 		break;
274 
275 	case GDMA_RQ:
276 		e.rq.id = qid;
277 		e.rq.tail_ptr = tail_ptr;
278 		e.rq.wqe_cnt = num_req;
279 
280 		addr += DOORBELL_OFFSET_RQ;
281 		break;
282 
283 	case GDMA_SQ:
284 		e.sq.id = qid;
285 		e.sq.tail_ptr = tail_ptr;
286 
287 		addr += DOORBELL_OFFSET_SQ;
288 		break;
289 
290 	default:
291 		WARN_ON(1);
292 		return;
293 	}
294 
295 	/* Ensure all writes are done before ring doorbell */
296 	wmb();
297 
298 	writeq(e.as_uint64, addr);
299 }
300 
301 void mana_gd_wq_ring_doorbell(struct gdma_context *gc, struct gdma_queue *queue)
302 {
303 	mana_gd_ring_doorbell(gc, queue->gdma_dev->doorbell, queue->type,
304 			      queue->id, queue->head * GDMA_WQE_BU_SIZE, 1);
305 }
306 
307 void mana_gd_ring_cq(struct gdma_queue *cq, u8 arm_bit)
308 {
309 	struct gdma_context *gc = cq->gdma_dev->gdma_context;
310 
311 	u32 num_cqe = cq->queue_size / GDMA_CQE_SIZE;
312 
313 	u32 head = cq->head % (num_cqe << GDMA_CQE_OWNER_BITS);
314 
315 	mana_gd_ring_doorbell(gc, cq->gdma_dev->doorbell, cq->type, cq->id,
316 			      head, arm_bit);
317 }
318 
319 static void mana_gd_process_eqe(struct gdma_queue *eq)
320 {
321 	u32 head = eq->head % (eq->queue_size / GDMA_EQE_SIZE);
322 	struct gdma_context *gc = eq->gdma_dev->gdma_context;
323 	struct gdma_eqe *eq_eqe_ptr = eq->queue_mem_ptr;
324 	union gdma_eqe_info eqe_info;
325 	enum gdma_eqe_type type;
326 	struct gdma_event event;
327 	struct gdma_queue *cq;
328 	struct gdma_eqe *eqe;
329 	u32 cq_id;
330 
331 	eqe = &eq_eqe_ptr[head];
332 	eqe_info.as_uint32 = eqe->eqe_info;
333 	type = eqe_info.type;
334 
335 	switch (type) {
336 	case GDMA_EQE_COMPLETION:
337 		cq_id = eqe->details[0] & 0xFFFFFF;
338 		if (WARN_ON_ONCE(cq_id >= gc->max_num_cqs))
339 			break;
340 
341 		cq = gc->cq_table[cq_id];
342 		if (WARN_ON_ONCE(!cq || cq->type != GDMA_CQ || cq->id != cq_id))
343 			break;
344 
345 		if (cq->cq.callback)
346 			cq->cq.callback(cq->cq.context, cq);
347 
348 		break;
349 
350 	case GDMA_EQE_TEST_EVENT:
351 		gc->test_event_eq_id = eq->id;
352 		complete(&gc->eq_test_event);
353 		break;
354 
355 	case GDMA_EQE_HWC_INIT_EQ_ID_DB:
356 	case GDMA_EQE_HWC_INIT_DATA:
357 	case GDMA_EQE_HWC_INIT_DONE:
358 		if (!eq->eq.callback)
359 			break;
360 
361 		event.type = type;
362 		memcpy(&event.details, &eqe->details, GDMA_EVENT_DATA_SIZE);
363 		eq->eq.callback(eq->eq.context, eq, &event);
364 		break;
365 
366 	default:
367 		break;
368 	}
369 }
370 
371 static void mana_gd_process_eq_events(void *arg)
372 {
373 	u32 owner_bits, new_bits, old_bits;
374 	union gdma_eqe_info eqe_info;
375 	struct gdma_eqe *eq_eqe_ptr;
376 	struct gdma_queue *eq = arg;
377 	struct gdma_context *gc;
378 	struct gdma_eqe *eqe;
379 	u32 head, num_eqe;
380 	int i;
381 
382 	gc = eq->gdma_dev->gdma_context;
383 
384 	num_eqe = eq->queue_size / GDMA_EQE_SIZE;
385 	eq_eqe_ptr = eq->queue_mem_ptr;
386 
387 	/* Process up to 5 EQEs at a time, and update the HW head. */
388 	for (i = 0; i < 5; i++) {
389 		eqe = &eq_eqe_ptr[eq->head % num_eqe];
390 		eqe_info.as_uint32 = eqe->eqe_info;
391 		owner_bits = eqe_info.owner_bits;
392 
393 		old_bits = (eq->head / num_eqe - 1) & GDMA_EQE_OWNER_MASK;
394 		/* No more entries */
395 		if (owner_bits == old_bits)
396 			break;
397 
398 		new_bits = (eq->head / num_eqe) & GDMA_EQE_OWNER_MASK;
399 		if (owner_bits != new_bits) {
400 			dev_err(gc->dev, "EQ %d: overflow detected\n", eq->id);
401 			break;
402 		}
403 
404 		/* Per GDMA spec, rmb is necessary after checking owner_bits, before
405 		 * reading eqe.
406 		 */
407 		rmb();
408 
409 		mana_gd_process_eqe(eq);
410 
411 		eq->head++;
412 	}
413 
414 	head = eq->head % (num_eqe << GDMA_EQE_OWNER_BITS);
415 
416 	mana_gd_ring_doorbell(gc, eq->gdma_dev->doorbell, eq->type, eq->id,
417 			      head, SET_ARM_BIT);
418 }
419 
420 static int mana_gd_register_irq(struct gdma_queue *queue,
421 				const struct gdma_queue_spec *spec)
422 {
423 	struct gdma_dev *gd = queue->gdma_dev;
424 	struct gdma_irq_context *gic;
425 	struct gdma_context *gc;
426 	struct gdma_resource *r;
427 	unsigned int msi_index;
428 	unsigned long flags;
429 	struct device *dev;
430 	int err = 0;
431 
432 	gc = gd->gdma_context;
433 	r = &gc->msix_resource;
434 	dev = gc->dev;
435 
436 	spin_lock_irqsave(&r->lock, flags);
437 
438 	msi_index = find_first_zero_bit(r->map, r->size);
439 	if (msi_index >= r->size || msi_index >= gc->num_msix_usable) {
440 		err = -ENOSPC;
441 	} else {
442 		bitmap_set(r->map, msi_index, 1);
443 		queue->eq.msix_index = msi_index;
444 	}
445 
446 	spin_unlock_irqrestore(&r->lock, flags);
447 
448 	if (err) {
449 		dev_err(dev, "Register IRQ err:%d, msi:%u rsize:%u, nMSI:%u",
450 			err, msi_index, r->size, gc->num_msix_usable);
451 
452 		return err;
453 	}
454 
455 	gic = &gc->irq_contexts[msi_index];
456 
457 	WARN_ON(gic->handler || gic->arg);
458 
459 	gic->arg = queue;
460 
461 	gic->handler = mana_gd_process_eq_events;
462 
463 	return 0;
464 }
465 
466 static void mana_gd_deregiser_irq(struct gdma_queue *queue)
467 {
468 	struct gdma_dev *gd = queue->gdma_dev;
469 	struct gdma_irq_context *gic;
470 	struct gdma_context *gc;
471 	struct gdma_resource *r;
472 	unsigned int msix_index;
473 	unsigned long flags;
474 
475 	gc = gd->gdma_context;
476 	r = &gc->msix_resource;
477 
478 	/* At most num_online_cpus() + 1 interrupts are used. */
479 	msix_index = queue->eq.msix_index;
480 	if (WARN_ON(msix_index >= gc->num_msix_usable))
481 		return;
482 
483 	gic = &gc->irq_contexts[msix_index];
484 	gic->handler = NULL;
485 	gic->arg = NULL;
486 
487 	spin_lock_irqsave(&r->lock, flags);
488 	bitmap_clear(r->map, msix_index, 1);
489 	spin_unlock_irqrestore(&r->lock, flags);
490 
491 	queue->eq.msix_index = INVALID_PCI_MSIX_INDEX;
492 }
493 
494 int mana_gd_test_eq(struct gdma_context *gc, struct gdma_queue *eq)
495 {
496 	struct gdma_generate_test_event_req req = {};
497 	struct gdma_general_resp resp = {};
498 	struct device *dev = gc->dev;
499 	int err;
500 
501 	mutex_lock(&gc->eq_test_event_mutex);
502 
503 	init_completion(&gc->eq_test_event);
504 	gc->test_event_eq_id = INVALID_QUEUE_ID;
505 
506 	mana_gd_init_req_hdr(&req.hdr, GDMA_GENERATE_TEST_EQE,
507 			     sizeof(req), sizeof(resp));
508 
509 	req.hdr.dev_id = eq->gdma_dev->dev_id;
510 	req.queue_index = eq->id;
511 
512 	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
513 	if (err) {
514 		dev_err(dev, "test_eq failed: %d\n", err);
515 		goto out;
516 	}
517 
518 	err = -EPROTO;
519 
520 	if (resp.hdr.status) {
521 		dev_err(dev, "test_eq failed: 0x%x\n", resp.hdr.status);
522 		goto out;
523 	}
524 
525 	if (!wait_for_completion_timeout(&gc->eq_test_event, 30 * HZ)) {
526 		dev_err(dev, "test_eq timed out on queue %d\n", eq->id);
527 		goto out;
528 	}
529 
530 	if (eq->id != gc->test_event_eq_id) {
531 		dev_err(dev, "test_eq got an event on wrong queue %d (%d)\n",
532 			gc->test_event_eq_id, eq->id);
533 		goto out;
534 	}
535 
536 	err = 0;
537 out:
538 	mutex_unlock(&gc->eq_test_event_mutex);
539 	return err;
540 }
541 
542 static void mana_gd_destroy_eq(struct gdma_context *gc, bool flush_evenets,
543 			       struct gdma_queue *queue)
544 {
545 	int err;
546 
547 	if (flush_evenets) {
548 		err = mana_gd_test_eq(gc, queue);
549 		if (err)
550 			dev_warn(gc->dev, "Failed to flush EQ: %d\n", err);
551 	}
552 
553 	mana_gd_deregiser_irq(queue);
554 
555 	if (queue->eq.disable_needed)
556 		mana_gd_disable_queue(queue);
557 }
558 
559 static int mana_gd_create_eq(struct gdma_dev *gd,
560 			     const struct gdma_queue_spec *spec,
561 			     bool create_hwq, struct gdma_queue *queue)
562 {
563 	struct gdma_context *gc = gd->gdma_context;
564 	struct device *dev = gc->dev;
565 	u32 log2_num_entries;
566 	int err;
567 
568 	queue->eq.msix_index = INVALID_PCI_MSIX_INDEX;
569 
570 	log2_num_entries = ilog2(queue->queue_size / GDMA_EQE_SIZE);
571 
572 	if (spec->eq.log2_throttle_limit > log2_num_entries) {
573 		dev_err(dev, "EQ throttling limit (%lu) > maximum EQE (%u)\n",
574 			spec->eq.log2_throttle_limit, log2_num_entries);
575 		return -EINVAL;
576 	}
577 
578 	err = mana_gd_register_irq(queue, spec);
579 	if (err) {
580 		dev_err(dev, "Failed to register irq: %d\n", err);
581 		return err;
582 	}
583 
584 	queue->eq.callback = spec->eq.callback;
585 	queue->eq.context = spec->eq.context;
586 	queue->head |= INITIALIZED_OWNER_BIT(log2_num_entries);
587 	queue->eq.log2_throttle_limit = spec->eq.log2_throttle_limit ?: 1;
588 
589 	if (create_hwq) {
590 		err = mana_gd_create_hw_eq(gc, queue);
591 		if (err)
592 			goto out;
593 
594 		err = mana_gd_test_eq(gc, queue);
595 		if (err)
596 			goto out;
597 	}
598 
599 	return 0;
600 out:
601 	dev_err(dev, "Failed to create EQ: %d\n", err);
602 	mana_gd_destroy_eq(gc, false, queue);
603 	return err;
604 }
605 
606 static void mana_gd_create_cq(const struct gdma_queue_spec *spec,
607 			      struct gdma_queue *queue)
608 {
609 	u32 log2_num_entries = ilog2(spec->queue_size / GDMA_CQE_SIZE);
610 
611 	queue->head |= INITIALIZED_OWNER_BIT(log2_num_entries);
612 	queue->cq.parent = spec->cq.parent_eq;
613 	queue->cq.context = spec->cq.context;
614 	queue->cq.callback = spec->cq.callback;
615 }
616 
617 static void mana_gd_destroy_cq(struct gdma_context *gc,
618 			       struct gdma_queue *queue)
619 {
620 	u32 id = queue->id;
621 
622 	if (id >= gc->max_num_cqs)
623 		return;
624 
625 	if (!gc->cq_table[id])
626 		return;
627 
628 	gc->cq_table[id] = NULL;
629 }
630 
631 int mana_gd_create_hwc_queue(struct gdma_dev *gd,
632 			     const struct gdma_queue_spec *spec,
633 			     struct gdma_queue **queue_ptr)
634 {
635 	struct gdma_context *gc = gd->gdma_context;
636 	struct gdma_mem_info *gmi;
637 	struct gdma_queue *queue;
638 	int err;
639 
640 	queue = kzalloc(sizeof(*queue), GFP_KERNEL);
641 	if (!queue)
642 		return -ENOMEM;
643 
644 	gmi = &queue->mem_info;
645 	err = mana_gd_alloc_memory(gc, spec->queue_size, gmi);
646 	if (err)
647 		goto free_q;
648 
649 	queue->head = 0;
650 	queue->tail = 0;
651 	queue->queue_mem_ptr = gmi->virt_addr;
652 	queue->queue_size = spec->queue_size;
653 	queue->monitor_avl_buf = spec->monitor_avl_buf;
654 	queue->type = spec->type;
655 	queue->gdma_dev = gd;
656 
657 	if (spec->type == GDMA_EQ)
658 		err = mana_gd_create_eq(gd, spec, false, queue);
659 	else if (spec->type == GDMA_CQ)
660 		mana_gd_create_cq(spec, queue);
661 
662 	if (err)
663 		goto out;
664 
665 	*queue_ptr = queue;
666 	return 0;
667 out:
668 	mana_gd_free_memory(gmi);
669 free_q:
670 	kfree(queue);
671 	return err;
672 }
673 
674 int mana_gd_destroy_dma_region(struct gdma_context *gc, u64 dma_region_handle)
675 {
676 	struct gdma_destroy_dma_region_req req = {};
677 	struct gdma_general_resp resp = {};
678 	int err;
679 
680 	if (dma_region_handle == GDMA_INVALID_DMA_REGION)
681 		return 0;
682 
683 	mana_gd_init_req_hdr(&req.hdr, GDMA_DESTROY_DMA_REGION, sizeof(req),
684 			     sizeof(resp));
685 	req.dma_region_handle = dma_region_handle;
686 
687 	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
688 	if (err || resp.hdr.status) {
689 		dev_err(gc->dev, "Failed to destroy DMA region: %d, 0x%x\n",
690 			err, resp.hdr.status);
691 		return -EPROTO;
692 	}
693 
694 	return 0;
695 }
696 EXPORT_SYMBOL_NS(mana_gd_destroy_dma_region, NET_MANA);
697 
698 static int mana_gd_create_dma_region(struct gdma_dev *gd,
699 				     struct gdma_mem_info *gmi)
700 {
701 	unsigned int num_page = gmi->length / PAGE_SIZE;
702 	struct gdma_create_dma_region_req *req = NULL;
703 	struct gdma_create_dma_region_resp resp = {};
704 	struct gdma_context *gc = gd->gdma_context;
705 	struct hw_channel_context *hwc;
706 	u32 length = gmi->length;
707 	size_t req_msg_size;
708 	int err;
709 	int i;
710 
711 	if (length < PAGE_SIZE || !is_power_of_2(length))
712 		return -EINVAL;
713 
714 	if (offset_in_page(gmi->virt_addr) != 0)
715 		return -EINVAL;
716 
717 	hwc = gc->hwc.driver_data;
718 	req_msg_size = struct_size(req, page_addr_list, num_page);
719 	if (req_msg_size > hwc->max_req_msg_size)
720 		return -EINVAL;
721 
722 	req = kzalloc(req_msg_size, GFP_KERNEL);
723 	if (!req)
724 		return -ENOMEM;
725 
726 	mana_gd_init_req_hdr(&req->hdr, GDMA_CREATE_DMA_REGION,
727 			     req_msg_size, sizeof(resp));
728 	req->length = length;
729 	req->offset_in_page = 0;
730 	req->gdma_page_type = GDMA_PAGE_TYPE_4K;
731 	req->page_count = num_page;
732 	req->page_addr_list_len = num_page;
733 
734 	for (i = 0; i < num_page; i++)
735 		req->page_addr_list[i] = gmi->dma_handle +  i * PAGE_SIZE;
736 
737 	err = mana_gd_send_request(gc, req_msg_size, req, sizeof(resp), &resp);
738 	if (err)
739 		goto out;
740 
741 	if (resp.hdr.status ||
742 	    resp.dma_region_handle == GDMA_INVALID_DMA_REGION) {
743 		dev_err(gc->dev, "Failed to create DMA region: 0x%x\n",
744 			resp.hdr.status);
745 		err = -EPROTO;
746 		goto out;
747 	}
748 
749 	gmi->dma_region_handle = resp.dma_region_handle;
750 out:
751 	kfree(req);
752 	return err;
753 }
754 
755 int mana_gd_create_mana_eq(struct gdma_dev *gd,
756 			   const struct gdma_queue_spec *spec,
757 			   struct gdma_queue **queue_ptr)
758 {
759 	struct gdma_context *gc = gd->gdma_context;
760 	struct gdma_mem_info *gmi;
761 	struct gdma_queue *queue;
762 	int err;
763 
764 	if (spec->type != GDMA_EQ)
765 		return -EINVAL;
766 
767 	queue = kzalloc(sizeof(*queue), GFP_KERNEL);
768 	if (!queue)
769 		return -ENOMEM;
770 
771 	gmi = &queue->mem_info;
772 	err = mana_gd_alloc_memory(gc, spec->queue_size, gmi);
773 	if (err)
774 		goto free_q;
775 
776 	err = mana_gd_create_dma_region(gd, gmi);
777 	if (err)
778 		goto out;
779 
780 	queue->head = 0;
781 	queue->tail = 0;
782 	queue->queue_mem_ptr = gmi->virt_addr;
783 	queue->queue_size = spec->queue_size;
784 	queue->monitor_avl_buf = spec->monitor_avl_buf;
785 	queue->type = spec->type;
786 	queue->gdma_dev = gd;
787 
788 	err = mana_gd_create_eq(gd, spec, true, queue);
789 	if (err)
790 		goto out;
791 
792 	*queue_ptr = queue;
793 	return 0;
794 out:
795 	mana_gd_free_memory(gmi);
796 free_q:
797 	kfree(queue);
798 	return err;
799 }
800 
801 int mana_gd_create_mana_wq_cq(struct gdma_dev *gd,
802 			      const struct gdma_queue_spec *spec,
803 			      struct gdma_queue **queue_ptr)
804 {
805 	struct gdma_context *gc = gd->gdma_context;
806 	struct gdma_mem_info *gmi;
807 	struct gdma_queue *queue;
808 	int err;
809 
810 	if (spec->type != GDMA_CQ && spec->type != GDMA_SQ &&
811 	    spec->type != GDMA_RQ)
812 		return -EINVAL;
813 
814 	queue = kzalloc(sizeof(*queue), GFP_KERNEL);
815 	if (!queue)
816 		return -ENOMEM;
817 
818 	gmi = &queue->mem_info;
819 	err = mana_gd_alloc_memory(gc, spec->queue_size, gmi);
820 	if (err)
821 		goto free_q;
822 
823 	err = mana_gd_create_dma_region(gd, gmi);
824 	if (err)
825 		goto out;
826 
827 	queue->head = 0;
828 	queue->tail = 0;
829 	queue->queue_mem_ptr = gmi->virt_addr;
830 	queue->queue_size = spec->queue_size;
831 	queue->monitor_avl_buf = spec->monitor_avl_buf;
832 	queue->type = spec->type;
833 	queue->gdma_dev = gd;
834 
835 	if (spec->type == GDMA_CQ)
836 		mana_gd_create_cq(spec, queue);
837 
838 	*queue_ptr = queue;
839 	return 0;
840 out:
841 	mana_gd_free_memory(gmi);
842 free_q:
843 	kfree(queue);
844 	return err;
845 }
846 
847 void mana_gd_destroy_queue(struct gdma_context *gc, struct gdma_queue *queue)
848 {
849 	struct gdma_mem_info *gmi = &queue->mem_info;
850 
851 	switch (queue->type) {
852 	case GDMA_EQ:
853 		mana_gd_destroy_eq(gc, queue->eq.disable_needed, queue);
854 		break;
855 
856 	case GDMA_CQ:
857 		mana_gd_destroy_cq(gc, queue);
858 		break;
859 
860 	case GDMA_RQ:
861 		break;
862 
863 	case GDMA_SQ:
864 		break;
865 
866 	default:
867 		dev_err(gc->dev, "Can't destroy unknown queue: type=%d\n",
868 			queue->type);
869 		return;
870 	}
871 
872 	mana_gd_destroy_dma_region(gc, gmi->dma_region_handle);
873 	mana_gd_free_memory(gmi);
874 	kfree(queue);
875 }
876 
877 int mana_gd_verify_vf_version(struct pci_dev *pdev)
878 {
879 	struct gdma_context *gc = pci_get_drvdata(pdev);
880 	struct gdma_verify_ver_resp resp = {};
881 	struct gdma_verify_ver_req req = {};
882 	int err;
883 
884 	mana_gd_init_req_hdr(&req.hdr, GDMA_VERIFY_VF_DRIVER_VERSION,
885 			     sizeof(req), sizeof(resp));
886 
887 	req.protocol_ver_min = GDMA_PROTOCOL_FIRST;
888 	req.protocol_ver_max = GDMA_PROTOCOL_LAST;
889 
890 	req.gd_drv_cap_flags1 = GDMA_DRV_CAP_FLAGS1;
891 	req.gd_drv_cap_flags2 = GDMA_DRV_CAP_FLAGS2;
892 	req.gd_drv_cap_flags3 = GDMA_DRV_CAP_FLAGS3;
893 	req.gd_drv_cap_flags4 = GDMA_DRV_CAP_FLAGS4;
894 
895 	req.drv_ver = 0;	/* Unused*/
896 	req.os_type = 0x10;	/* Linux */
897 	req.os_ver_major = LINUX_VERSION_MAJOR;
898 	req.os_ver_minor = LINUX_VERSION_PATCHLEVEL;
899 	req.os_ver_build = LINUX_VERSION_SUBLEVEL;
900 	strscpy(req.os_ver_str1, utsname()->sysname, sizeof(req.os_ver_str1));
901 	strscpy(req.os_ver_str2, utsname()->release, sizeof(req.os_ver_str2));
902 	strscpy(req.os_ver_str3, utsname()->version, sizeof(req.os_ver_str3));
903 
904 	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
905 	if (err || resp.hdr.status) {
906 		dev_err(gc->dev, "VfVerifyVersionOutput: %d, status=0x%x\n",
907 			err, resp.hdr.status);
908 		return err ? err : -EPROTO;
909 	}
910 
911 	return 0;
912 }
913 
914 int mana_gd_register_device(struct gdma_dev *gd)
915 {
916 	struct gdma_context *gc = gd->gdma_context;
917 	struct gdma_register_device_resp resp = {};
918 	struct gdma_general_req req = {};
919 	int err;
920 
921 	gd->pdid = INVALID_PDID;
922 	gd->doorbell = INVALID_DOORBELL;
923 	gd->gpa_mkey = INVALID_MEM_KEY;
924 
925 	mana_gd_init_req_hdr(&req.hdr, GDMA_REGISTER_DEVICE, sizeof(req),
926 			     sizeof(resp));
927 
928 	req.hdr.dev_id = gd->dev_id;
929 
930 	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
931 	if (err || resp.hdr.status) {
932 		dev_err(gc->dev, "gdma_register_device_resp failed: %d, 0x%x\n",
933 			err, resp.hdr.status);
934 		return err ? err : -EPROTO;
935 	}
936 
937 	gd->pdid = resp.pdid;
938 	gd->gpa_mkey = resp.gpa_mkey;
939 	gd->doorbell = resp.db_id;
940 
941 	return 0;
942 }
943 
944 int mana_gd_deregister_device(struct gdma_dev *gd)
945 {
946 	struct gdma_context *gc = gd->gdma_context;
947 	struct gdma_general_resp resp = {};
948 	struct gdma_general_req req = {};
949 	int err;
950 
951 	if (gd->pdid == INVALID_PDID)
952 		return -EINVAL;
953 
954 	mana_gd_init_req_hdr(&req.hdr, GDMA_DEREGISTER_DEVICE, sizeof(req),
955 			     sizeof(resp));
956 
957 	req.hdr.dev_id = gd->dev_id;
958 
959 	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
960 	if (err || resp.hdr.status) {
961 		dev_err(gc->dev, "Failed to deregister device: %d, 0x%x\n",
962 			err, resp.hdr.status);
963 		if (!err)
964 			err = -EPROTO;
965 	}
966 
967 	gd->pdid = INVALID_PDID;
968 	gd->doorbell = INVALID_DOORBELL;
969 	gd->gpa_mkey = INVALID_MEM_KEY;
970 
971 	return err;
972 }
973 
974 u32 mana_gd_wq_avail_space(struct gdma_queue *wq)
975 {
976 	u32 used_space = (wq->head - wq->tail) * GDMA_WQE_BU_SIZE;
977 	u32 wq_size = wq->queue_size;
978 
979 	WARN_ON_ONCE(used_space > wq_size);
980 
981 	return wq_size - used_space;
982 }
983 
984 u8 *mana_gd_get_wqe_ptr(const struct gdma_queue *wq, u32 wqe_offset)
985 {
986 	u32 offset = (wqe_offset * GDMA_WQE_BU_SIZE) & (wq->queue_size - 1);
987 
988 	WARN_ON_ONCE((offset + GDMA_WQE_BU_SIZE) > wq->queue_size);
989 
990 	return wq->queue_mem_ptr + offset;
991 }
992 
993 static u32 mana_gd_write_client_oob(const struct gdma_wqe_request *wqe_req,
994 				    enum gdma_queue_type q_type,
995 				    u32 client_oob_size, u32 sgl_data_size,
996 				    u8 *wqe_ptr)
997 {
998 	bool oob_in_sgl = !!(wqe_req->flags & GDMA_WR_OOB_IN_SGL);
999 	bool pad_data = !!(wqe_req->flags & GDMA_WR_PAD_BY_SGE0);
1000 	struct gdma_wqe *header = (struct gdma_wqe *)wqe_ptr;
1001 	u8 *ptr;
1002 
1003 	memset(header, 0, sizeof(struct gdma_wqe));
1004 	header->num_sge = wqe_req->num_sge;
1005 	header->inline_oob_size_div4 = client_oob_size / sizeof(u32);
1006 
1007 	if (oob_in_sgl) {
1008 		WARN_ON_ONCE(!pad_data || wqe_req->num_sge < 2);
1009 
1010 		header->client_oob_in_sgl = 1;
1011 
1012 		if (pad_data)
1013 			header->last_vbytes = wqe_req->sgl[0].size;
1014 	}
1015 
1016 	if (q_type == GDMA_SQ)
1017 		header->client_data_unit = wqe_req->client_data_unit;
1018 
1019 	/* The size of gdma_wqe + client_oob_size must be less than or equal
1020 	 * to one Basic Unit (i.e. 32 bytes), so the pointer can't go beyond
1021 	 * the queue memory buffer boundary.
1022 	 */
1023 	ptr = wqe_ptr + sizeof(header);
1024 
1025 	if (wqe_req->inline_oob_data && wqe_req->inline_oob_size > 0) {
1026 		memcpy(ptr, wqe_req->inline_oob_data, wqe_req->inline_oob_size);
1027 
1028 		if (client_oob_size > wqe_req->inline_oob_size)
1029 			memset(ptr + wqe_req->inline_oob_size, 0,
1030 			       client_oob_size - wqe_req->inline_oob_size);
1031 	}
1032 
1033 	return sizeof(header) + client_oob_size;
1034 }
1035 
1036 static void mana_gd_write_sgl(struct gdma_queue *wq, u8 *wqe_ptr,
1037 			      const struct gdma_wqe_request *wqe_req)
1038 {
1039 	u32 sgl_size = sizeof(struct gdma_sge) * wqe_req->num_sge;
1040 	const u8 *address = (u8 *)wqe_req->sgl;
1041 	u8 *base_ptr, *end_ptr;
1042 	u32 size_to_end;
1043 
1044 	base_ptr = wq->queue_mem_ptr;
1045 	end_ptr = base_ptr + wq->queue_size;
1046 	size_to_end = (u32)(end_ptr - wqe_ptr);
1047 
1048 	if (size_to_end < sgl_size) {
1049 		memcpy(wqe_ptr, address, size_to_end);
1050 
1051 		wqe_ptr = base_ptr;
1052 		address += size_to_end;
1053 		sgl_size -= size_to_end;
1054 	}
1055 
1056 	memcpy(wqe_ptr, address, sgl_size);
1057 }
1058 
1059 int mana_gd_post_work_request(struct gdma_queue *wq,
1060 			      const struct gdma_wqe_request *wqe_req,
1061 			      struct gdma_posted_wqe_info *wqe_info)
1062 {
1063 	u32 client_oob_size = wqe_req->inline_oob_size;
1064 	struct gdma_context *gc;
1065 	u32 sgl_data_size;
1066 	u32 max_wqe_size;
1067 	u32 wqe_size;
1068 	u8 *wqe_ptr;
1069 
1070 	if (wqe_req->num_sge == 0)
1071 		return -EINVAL;
1072 
1073 	if (wq->type == GDMA_RQ) {
1074 		if (client_oob_size != 0)
1075 			return -EINVAL;
1076 
1077 		client_oob_size = INLINE_OOB_SMALL_SIZE;
1078 
1079 		max_wqe_size = GDMA_MAX_RQE_SIZE;
1080 	} else {
1081 		if (client_oob_size != INLINE_OOB_SMALL_SIZE &&
1082 		    client_oob_size != INLINE_OOB_LARGE_SIZE)
1083 			return -EINVAL;
1084 
1085 		max_wqe_size = GDMA_MAX_SQE_SIZE;
1086 	}
1087 
1088 	sgl_data_size = sizeof(struct gdma_sge) * wqe_req->num_sge;
1089 	wqe_size = ALIGN(sizeof(struct gdma_wqe) + client_oob_size +
1090 			 sgl_data_size, GDMA_WQE_BU_SIZE);
1091 	if (wqe_size > max_wqe_size)
1092 		return -EINVAL;
1093 
1094 	if (wq->monitor_avl_buf && wqe_size > mana_gd_wq_avail_space(wq)) {
1095 		gc = wq->gdma_dev->gdma_context;
1096 		dev_err(gc->dev, "unsuccessful flow control!\n");
1097 		return -ENOSPC;
1098 	}
1099 
1100 	if (wqe_info)
1101 		wqe_info->wqe_size_in_bu = wqe_size / GDMA_WQE_BU_SIZE;
1102 
1103 	wqe_ptr = mana_gd_get_wqe_ptr(wq, wq->head);
1104 	wqe_ptr += mana_gd_write_client_oob(wqe_req, wq->type, client_oob_size,
1105 					    sgl_data_size, wqe_ptr);
1106 	if (wqe_ptr >= (u8 *)wq->queue_mem_ptr + wq->queue_size)
1107 		wqe_ptr -= wq->queue_size;
1108 
1109 	mana_gd_write_sgl(wq, wqe_ptr, wqe_req);
1110 
1111 	wq->head += wqe_size / GDMA_WQE_BU_SIZE;
1112 
1113 	return 0;
1114 }
1115 
1116 int mana_gd_post_and_ring(struct gdma_queue *queue,
1117 			  const struct gdma_wqe_request *wqe_req,
1118 			  struct gdma_posted_wqe_info *wqe_info)
1119 {
1120 	struct gdma_context *gc = queue->gdma_dev->gdma_context;
1121 	int err;
1122 
1123 	err = mana_gd_post_work_request(queue, wqe_req, wqe_info);
1124 	if (err)
1125 		return err;
1126 
1127 	mana_gd_wq_ring_doorbell(gc, queue);
1128 
1129 	return 0;
1130 }
1131 
1132 static int mana_gd_read_cqe(struct gdma_queue *cq, struct gdma_comp *comp)
1133 {
1134 	unsigned int num_cqe = cq->queue_size / sizeof(struct gdma_cqe);
1135 	struct gdma_cqe *cq_cqe = cq->queue_mem_ptr;
1136 	u32 owner_bits, new_bits, old_bits;
1137 	struct gdma_cqe *cqe;
1138 
1139 	cqe = &cq_cqe[cq->head % num_cqe];
1140 	owner_bits = cqe->cqe_info.owner_bits;
1141 
1142 	old_bits = (cq->head / num_cqe - 1) & GDMA_CQE_OWNER_MASK;
1143 	/* Return 0 if no more entries. */
1144 	if (owner_bits == old_bits)
1145 		return 0;
1146 
1147 	new_bits = (cq->head / num_cqe) & GDMA_CQE_OWNER_MASK;
1148 	/* Return -1 if overflow detected. */
1149 	if (WARN_ON_ONCE(owner_bits != new_bits))
1150 		return -1;
1151 
1152 	/* Per GDMA spec, rmb is necessary after checking owner_bits, before
1153 	 * reading completion info
1154 	 */
1155 	rmb();
1156 
1157 	comp->wq_num = cqe->cqe_info.wq_num;
1158 	comp->is_sq = cqe->cqe_info.is_sq;
1159 	memcpy(comp->cqe_data, cqe->cqe_data, GDMA_COMP_DATA_SIZE);
1160 
1161 	return 1;
1162 }
1163 
1164 int mana_gd_poll_cq(struct gdma_queue *cq, struct gdma_comp *comp, int num_cqe)
1165 {
1166 	int cqe_idx;
1167 	int ret;
1168 
1169 	for (cqe_idx = 0; cqe_idx < num_cqe; cqe_idx++) {
1170 		ret = mana_gd_read_cqe(cq, &comp[cqe_idx]);
1171 
1172 		if (ret < 0) {
1173 			cq->head -= cqe_idx;
1174 			return ret;
1175 		}
1176 
1177 		if (ret == 0)
1178 			break;
1179 
1180 		cq->head++;
1181 	}
1182 
1183 	return cqe_idx;
1184 }
1185 
1186 static irqreturn_t mana_gd_intr(int irq, void *arg)
1187 {
1188 	struct gdma_irq_context *gic = arg;
1189 
1190 	if (gic->handler)
1191 		gic->handler(gic->arg);
1192 
1193 	return IRQ_HANDLED;
1194 }
1195 
1196 int mana_gd_alloc_res_map(u32 res_avail, struct gdma_resource *r)
1197 {
1198 	r->map = bitmap_zalloc(res_avail, GFP_KERNEL);
1199 	if (!r->map)
1200 		return -ENOMEM;
1201 
1202 	r->size = res_avail;
1203 	spin_lock_init(&r->lock);
1204 
1205 	return 0;
1206 }
1207 
1208 void mana_gd_free_res_map(struct gdma_resource *r)
1209 {
1210 	bitmap_free(r->map);
1211 	r->map = NULL;
1212 	r->size = 0;
1213 }
1214 
1215 static int mana_gd_setup_irqs(struct pci_dev *pdev)
1216 {
1217 	unsigned int max_queues_per_port = num_online_cpus();
1218 	struct gdma_context *gc = pci_get_drvdata(pdev);
1219 	struct gdma_irq_context *gic;
1220 	unsigned int max_irqs, cpu;
1221 	int nvec, irq;
1222 	int err, i = 0, j;
1223 
1224 	if (max_queues_per_port > MANA_MAX_NUM_QUEUES)
1225 		max_queues_per_port = MANA_MAX_NUM_QUEUES;
1226 
1227 	/* Need 1 interrupt for the Hardware communication Channel (HWC) */
1228 	max_irqs = max_queues_per_port + 1;
1229 
1230 	nvec = pci_alloc_irq_vectors(pdev, 2, max_irqs, PCI_IRQ_MSIX);
1231 	if (nvec < 0)
1232 		return nvec;
1233 
1234 	gc->irq_contexts = kcalloc(nvec, sizeof(struct gdma_irq_context),
1235 				   GFP_KERNEL);
1236 	if (!gc->irq_contexts) {
1237 		err = -ENOMEM;
1238 		goto free_irq_vector;
1239 	}
1240 
1241 	for (i = 0; i < nvec; i++) {
1242 		gic = &gc->irq_contexts[i];
1243 		gic->handler = NULL;
1244 		gic->arg = NULL;
1245 
1246 		if (!i)
1247 			snprintf(gic->name, MANA_IRQ_NAME_SZ, "mana_hwc@pci:%s",
1248 				 pci_name(pdev));
1249 		else
1250 			snprintf(gic->name, MANA_IRQ_NAME_SZ, "mana_q%d@pci:%s",
1251 				 i - 1, pci_name(pdev));
1252 
1253 		irq = pci_irq_vector(pdev, i);
1254 		if (irq < 0) {
1255 			err = irq;
1256 			goto free_irq;
1257 		}
1258 
1259 		err = request_irq(irq, mana_gd_intr, 0, gic->name, gic);
1260 		if (err)
1261 			goto free_irq;
1262 
1263 		cpu = cpumask_local_spread(i, gc->numa_node);
1264 		irq_set_affinity_and_hint(irq, cpumask_of(cpu));
1265 	}
1266 
1267 	err = mana_gd_alloc_res_map(nvec, &gc->msix_resource);
1268 	if (err)
1269 		goto free_irq;
1270 
1271 	gc->max_num_msix = nvec;
1272 	gc->num_msix_usable = nvec;
1273 
1274 	return 0;
1275 
1276 free_irq:
1277 	for (j = i - 1; j >= 0; j--) {
1278 		irq = pci_irq_vector(pdev, j);
1279 		gic = &gc->irq_contexts[j];
1280 
1281 		irq_update_affinity_hint(irq, NULL);
1282 		free_irq(irq, gic);
1283 	}
1284 
1285 	kfree(gc->irq_contexts);
1286 	gc->irq_contexts = NULL;
1287 free_irq_vector:
1288 	pci_free_irq_vectors(pdev);
1289 	return err;
1290 }
1291 
1292 static void mana_gd_remove_irqs(struct pci_dev *pdev)
1293 {
1294 	struct gdma_context *gc = pci_get_drvdata(pdev);
1295 	struct gdma_irq_context *gic;
1296 	int irq, i;
1297 
1298 	if (gc->max_num_msix < 1)
1299 		return;
1300 
1301 	mana_gd_free_res_map(&gc->msix_resource);
1302 
1303 	for (i = 0; i < gc->max_num_msix; i++) {
1304 		irq = pci_irq_vector(pdev, i);
1305 		if (irq < 0)
1306 			continue;
1307 
1308 		gic = &gc->irq_contexts[i];
1309 
1310 		/* Need to clear the hint before free_irq */
1311 		irq_update_affinity_hint(irq, NULL);
1312 		free_irq(irq, gic);
1313 	}
1314 
1315 	pci_free_irq_vectors(pdev);
1316 
1317 	gc->max_num_msix = 0;
1318 	gc->num_msix_usable = 0;
1319 	kfree(gc->irq_contexts);
1320 	gc->irq_contexts = NULL;
1321 }
1322 
1323 static int mana_gd_setup(struct pci_dev *pdev)
1324 {
1325 	struct gdma_context *gc = pci_get_drvdata(pdev);
1326 	int err;
1327 
1328 	mana_gd_init_registers(pdev);
1329 	mana_smc_init(&gc->shm_channel, gc->dev, gc->shm_base);
1330 
1331 	err = mana_gd_setup_irqs(pdev);
1332 	if (err)
1333 		return err;
1334 
1335 	err = mana_hwc_create_channel(gc);
1336 	if (err)
1337 		goto remove_irq;
1338 
1339 	err = mana_gd_verify_vf_version(pdev);
1340 	if (err)
1341 		goto destroy_hwc;
1342 
1343 	err = mana_gd_query_max_resources(pdev);
1344 	if (err)
1345 		goto destroy_hwc;
1346 
1347 	err = mana_gd_detect_devices(pdev);
1348 	if (err)
1349 		goto destroy_hwc;
1350 
1351 	return 0;
1352 
1353 destroy_hwc:
1354 	mana_hwc_destroy_channel(gc);
1355 remove_irq:
1356 	mana_gd_remove_irqs(pdev);
1357 	return err;
1358 }
1359 
1360 static void mana_gd_cleanup(struct pci_dev *pdev)
1361 {
1362 	struct gdma_context *gc = pci_get_drvdata(pdev);
1363 
1364 	mana_hwc_destroy_channel(gc);
1365 
1366 	mana_gd_remove_irqs(pdev);
1367 }
1368 
1369 static bool mana_is_pf(unsigned short dev_id)
1370 {
1371 	return dev_id == MANA_PF_DEVICE_ID;
1372 }
1373 
1374 static int mana_gd_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
1375 {
1376 	struct gdma_context *gc;
1377 	void __iomem *bar0_va;
1378 	int bar = 0;
1379 	int err;
1380 
1381 	/* Each port has 2 CQs, each CQ has at most 1 EQE at a time */
1382 	BUILD_BUG_ON(2 * MAX_PORTS_IN_MANA_DEV * GDMA_EQE_SIZE > EQ_SIZE);
1383 
1384 	err = pci_enable_device(pdev);
1385 	if (err)
1386 		return -ENXIO;
1387 
1388 	pci_set_master(pdev);
1389 
1390 	err = pci_request_regions(pdev, "mana");
1391 	if (err)
1392 		goto disable_dev;
1393 
1394 	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
1395 	if (err)
1396 		goto release_region;
1397 
1398 	err = dma_set_max_seg_size(&pdev->dev, UINT_MAX);
1399 	if (err) {
1400 		dev_err(&pdev->dev, "Failed to set dma device segment size\n");
1401 		goto release_region;
1402 	}
1403 
1404 	err = -ENOMEM;
1405 	gc = vzalloc(sizeof(*gc));
1406 	if (!gc)
1407 		goto release_region;
1408 
1409 	mutex_init(&gc->eq_test_event_mutex);
1410 	pci_set_drvdata(pdev, gc);
1411 	gc->bar0_pa = pci_resource_start(pdev, 0);
1412 
1413 	bar0_va = pci_iomap(pdev, bar, 0);
1414 	if (!bar0_va)
1415 		goto free_gc;
1416 
1417 	gc->numa_node = dev_to_node(&pdev->dev);
1418 	gc->is_pf = mana_is_pf(pdev->device);
1419 	gc->bar0_va = bar0_va;
1420 	gc->dev = &pdev->dev;
1421 
1422 	err = mana_gd_setup(pdev);
1423 	if (err)
1424 		goto unmap_bar;
1425 
1426 	err = mana_probe(&gc->mana, false);
1427 	if (err)
1428 		goto cleanup_gd;
1429 
1430 	return 0;
1431 
1432 cleanup_gd:
1433 	mana_gd_cleanup(pdev);
1434 unmap_bar:
1435 	pci_iounmap(pdev, bar0_va);
1436 free_gc:
1437 	pci_set_drvdata(pdev, NULL);
1438 	vfree(gc);
1439 release_region:
1440 	pci_release_regions(pdev);
1441 disable_dev:
1442 	pci_clear_master(pdev);
1443 	pci_disable_device(pdev);
1444 	dev_err(&pdev->dev, "gdma probe failed: err = %d\n", err);
1445 	return err;
1446 }
1447 
1448 static void mana_gd_remove(struct pci_dev *pdev)
1449 {
1450 	struct gdma_context *gc = pci_get_drvdata(pdev);
1451 
1452 	mana_remove(&gc->mana, false);
1453 
1454 	mana_gd_cleanup(pdev);
1455 
1456 	pci_iounmap(pdev, gc->bar0_va);
1457 
1458 	vfree(gc);
1459 
1460 	pci_release_regions(pdev);
1461 	pci_clear_master(pdev);
1462 	pci_disable_device(pdev);
1463 }
1464 
1465 /* The 'state' parameter is not used. */
1466 static int mana_gd_suspend(struct pci_dev *pdev, pm_message_t state)
1467 {
1468 	struct gdma_context *gc = pci_get_drvdata(pdev);
1469 
1470 	mana_remove(&gc->mana, true);
1471 
1472 	mana_gd_cleanup(pdev);
1473 
1474 	return 0;
1475 }
1476 
1477 /* In case the NIC hardware stops working, the suspend and resume callbacks will
1478  * fail -- if this happens, it's safer to just report an error than try to undo
1479  * what has been done.
1480  */
1481 static int mana_gd_resume(struct pci_dev *pdev)
1482 {
1483 	struct gdma_context *gc = pci_get_drvdata(pdev);
1484 	int err;
1485 
1486 	err = mana_gd_setup(pdev);
1487 	if (err)
1488 		return err;
1489 
1490 	err = mana_probe(&gc->mana, true);
1491 	if (err)
1492 		return err;
1493 
1494 	return 0;
1495 }
1496 
1497 /* Quiesce the device for kexec. This is also called upon reboot/shutdown. */
1498 static void mana_gd_shutdown(struct pci_dev *pdev)
1499 {
1500 	struct gdma_context *gc = pci_get_drvdata(pdev);
1501 
1502 	dev_info(&pdev->dev, "Shutdown was called\n");
1503 
1504 	mana_remove(&gc->mana, true);
1505 
1506 	mana_gd_cleanup(pdev);
1507 
1508 	pci_disable_device(pdev);
1509 }
1510 
1511 static const struct pci_device_id mana_id_table[] = {
1512 	{ PCI_DEVICE(PCI_VENDOR_ID_MICROSOFT, MANA_PF_DEVICE_ID) },
1513 	{ PCI_DEVICE(PCI_VENDOR_ID_MICROSOFT, MANA_VF_DEVICE_ID) },
1514 	{ }
1515 };
1516 
1517 static struct pci_driver mana_driver = {
1518 	.name		= "mana",
1519 	.id_table	= mana_id_table,
1520 	.probe		= mana_gd_probe,
1521 	.remove		= mana_gd_remove,
1522 	.suspend	= mana_gd_suspend,
1523 	.resume		= mana_gd_resume,
1524 	.shutdown	= mana_gd_shutdown,
1525 };
1526 
1527 module_pci_driver(mana_driver);
1528 
1529 MODULE_DEVICE_TABLE(pci, mana_id_table);
1530 
1531 MODULE_LICENSE("Dual BSD/GPL");
1532 MODULE_DESCRIPTION("Microsoft Azure Network Adapter driver");
1533