23 	struct erdma_dev *dev = container_of(nb, struct erdma_dev, netdev_nb);  in erdma_netdev_event()  local
25 	if (dev->netdev == NULL || dev->netdev != netdev)  in erdma_netdev_event()
30 		dev->state = IB_PORT_ACTIVE;  in erdma_netdev_event()
31 		erdma_port_event(dev, IB_EVENT_PORT_ACTIVE);  in erdma_netdev_event()
34 		dev->state = IB_PORT_DOWN;  in erdma_netdev_event()
35 		erdma_port_event(dev, IB_EVENT_PORT_ERR);  in erdma_netdev_event()
38 		if (dev->mtu != netdev->mtu) {  in erdma_netdev_event()
39 			erdma_set_mtu(dev, netdev->mtu);  in erdma_netdev_event()
40 			dev->mtu = netdev->mtu;  in erdma_netdev_event()
56 static int erdma_enum_and_get_netdev(struct erdma_dev *dev)  in erdma_enum_and_get_netdev()  argument
62 	if (dev->netdev)  in erdma_enum_and_get_netdev()
75 					       dev->attrs.peer_addr)) {  in erdma_enum_and_get_netdev()
76 			ret = ib_device_set_netdev(&dev->ibdev, netdev, 1);  in erdma_enum_and_get_netdev()
79 				ibdev_warn(&dev->ibdev,  in erdma_enum_and_get_netdev()
84 			dev->netdev = netdev;  in erdma_enum_and_get_netdev()
94 static int erdma_device_register(struct erdma_dev *dev)  in erdma_device_register()  argument
96 	struct ib_device *ibdev = &dev->ibdev;  in erdma_device_register()
99 	ret = erdma_enum_and_get_netdev(dev);  in erdma_device_register()
103 	dev->mtu = dev->netdev->mtu;  in erdma_device_register()
104 	addrconf_addr_eui48((u8 *)&ibdev->node_guid, dev->netdev->dev_addr);  in erdma_device_register()
106 	ret = ib_register_device(ibdev, "erdma_%d", &dev->pdev->dev);  in erdma_device_register()
108 		dev_err(&dev->pdev->dev,  in erdma_device_register()
113 	dev->netdev_nb.notifier_call = erdma_netdev_event;  in erdma_device_register()
114 	ret = register_netdevice_notifier(&dev->netdev_nb);  in erdma_device_register()
116 		ibdev_err(&dev->ibdev, "failed to register notifier.\n");  in erdma_device_register()
125 	struct erdma_dev *dev = data;  in erdma_comm_irq_handler()  local
127 	erdma_cmdq_completion_handler(&dev->cmdq);  in erdma_comm_irq_handler()
128 	erdma_aeq_event_handler(dev);  in erdma_comm_irq_handler()
133 static int erdma_request_vectors(struct erdma_dev *dev)  in erdma_request_vectors()  argument
138 	ret = pci_alloc_irq_vectors(dev->pdev, 1, expect_irq_num, PCI_IRQ_MSIX);  in erdma_request_vectors()
140 		dev_err(&dev->pdev->dev, "request irq vectors failed(%d)\n",  in erdma_request_vectors()
144 	dev->attrs.irq_num = ret;  in erdma_request_vectors()
149 static int erdma_comm_irq_init(struct erdma_dev *dev)  in erdma_comm_irq_init()  argument
151 	snprintf(dev->comm_irq.name, ERDMA_IRQNAME_SIZE, "erdma-common@pci:%s",  in erdma_comm_irq_init()
152 		 pci_name(dev->pdev));  in erdma_comm_irq_init()
153 	dev->comm_irq.msix_vector =  in erdma_comm_irq_init()
154 		pci_irq_vector(dev->pdev, ERDMA_MSIX_VECTOR_CMDQ);  in erdma_comm_irq_init()
156 	cpumask_set_cpu(cpumask_first(cpumask_of_pcibus(dev->pdev->bus)),  in erdma_comm_irq_init()
157 			&dev->comm_irq.affinity_hint_mask);  in erdma_comm_irq_init()
158 	irq_set_affinity_hint(dev->comm_irq.msix_vector,  in erdma_comm_irq_init()
159 			      &dev->comm_irq.affinity_hint_mask);  in erdma_comm_irq_init()
161 	return request_irq(dev->comm_irq.msix_vector, erdma_comm_irq_handler, 0,  in erdma_comm_irq_init()
162 			   dev->comm_irq.name, dev);  in erdma_comm_irq_init()
165 static void erdma_comm_irq_uninit(struct erdma_dev *dev)  in erdma_comm_irq_uninit()  argument
167 	irq_set_affinity_hint(dev->comm_irq.msix_vector, NULL);  in erdma_comm_irq_uninit()
168 	free_irq(dev->comm_irq.msix_vector, dev);  in erdma_comm_irq_uninit()
171 static int erdma_device_init(struct erdma_dev *dev, struct pci_dev *pdev)  in erdma_device_init()  argument
175 	ret = dma_set_mask_and_coherent(&pdev->dev,  in erdma_device_init()
180 	dma_set_max_seg_size(&pdev->dev, UINT_MAX);  in erdma_device_init()
185 static void erdma_hw_reset(struct erdma_dev *dev)  in erdma_hw_reset()  argument
189 	erdma_reg_write32(dev, ERDMA_REGS_DEV_CTRL_REG, ctrl);  in erdma_hw_reset()
192 static int erdma_wait_hw_init_done(struct erdma_dev *dev)  in erdma_wait_hw_init_done()  argument
196 	erdma_reg_write32(dev, ERDMA_REGS_DEV_CTRL_REG,  in erdma_wait_hw_init_done()
200 		if (erdma_reg_read32_filed(dev, ERDMA_REGS_DEV_ST_REG,  in erdma_wait_hw_init_done()
208 		dev_err(&dev->pdev->dev, "wait init done failed.\n");  in erdma_wait_hw_init_done()
222 	struct erdma_dev *dev;  in erdma_probe_dev()  local
228 		dev_err(&pdev->dev, "pci_enable_device failed(%d)\n", err);  in erdma_probe_dev()
234 	dev = ib_alloc_device(erdma_dev, ibdev);  in erdma_probe_dev()
235 	if (!dev) {  in erdma_probe_dev()
236 		dev_err(&pdev->dev, "ib_alloc_device failed\n");  in erdma_probe_dev()
241 	pci_set_drvdata(pdev, dev);  in erdma_probe_dev()
242 	dev->pdev = pdev;  in erdma_probe_dev()
243 	dev->attrs.numa_node = dev_to_node(&pdev->dev);  in erdma_probe_dev()
252 	dev->func_bar_addr = pci_resource_start(pdev, ERDMA_FUNC_BAR);  in erdma_probe_dev()
253 	dev->func_bar_len = pci_resource_len(pdev, ERDMA_FUNC_BAR);  in erdma_probe_dev()
255 	dev->func_bar =  in erdma_probe_dev()
256 		devm_ioremap(&pdev->dev, dev->func_bar_addr, dev->func_bar_len);  in erdma_probe_dev()
257 	if (!dev->func_bar) {  in erdma_probe_dev()
258 		dev_err(&pdev->dev, "devm_ioremap failed.\n");  in erdma_probe_dev()
263 	version = erdma_reg_read32(dev, ERDMA_REGS_VERSION_REG);  in erdma_probe_dev()
270 	err = erdma_device_init(dev, pdev);  in erdma_probe_dev()
274 	err = erdma_request_vectors(dev);  in erdma_probe_dev()
278 	err = erdma_comm_irq_init(dev);  in erdma_probe_dev()
282 	err = erdma_aeq_init(dev);  in erdma_probe_dev()
286 	err = erdma_cmdq_init(dev);  in erdma_probe_dev()
290 	err = erdma_wait_hw_init_done(dev);  in erdma_probe_dev()
294 	err = erdma_ceqs_init(dev);  in erdma_probe_dev()
298 	erdma_finish_cmdq_init(dev);  in erdma_probe_dev()
303 	erdma_hw_reset(dev);  in erdma_probe_dev()
306 	erdma_cmdq_destroy(dev);  in erdma_probe_dev()
309 	erdma_aeq_destroy(dev);  in erdma_probe_dev()
312 	erdma_comm_irq_uninit(dev);  in erdma_probe_dev()
315 	pci_free_irq_vectors(dev->pdev);  in erdma_probe_dev()
318 	devm_iounmap(&pdev->dev, dev->func_bar);  in erdma_probe_dev()
324 	ib_dealloc_device(&dev->ibdev);  in erdma_probe_dev()
334 	struct erdma_dev *dev = pci_get_drvdata(pdev);  in erdma_remove_dev()  local
336 	erdma_ceqs_uninit(dev);  in erdma_remove_dev()
337 	erdma_hw_reset(dev);  in erdma_remove_dev()
338 	erdma_cmdq_destroy(dev);  in erdma_remove_dev()
339 	erdma_aeq_destroy(dev);  in erdma_remove_dev()
340 	erdma_comm_irq_uninit(dev);  in erdma_remove_dev()
341 	pci_free_irq_vectors(dev->pdev);  in erdma_remove_dev()
343 	devm_iounmap(&pdev->dev, dev->func_bar);  in erdma_remove_dev()
346 	ib_dealloc_device(&dev->ibdev);  in erdma_remove_dev()
353 static int erdma_dev_attrs_init(struct erdma_dev *dev)  in erdma_dev_attrs_init()  argument
361 	err = erdma_post_cmd_wait(&dev->cmdq, &req_hdr, sizeof(req_hdr), &cap0,  in erdma_dev_attrs_init()
366 	dev->attrs.max_cqe = 1 << ERDMA_GET_CAP(MAX_CQE, cap0);  in erdma_dev_attrs_init()
367 	dev->attrs.max_mr_size = 1ULL << ERDMA_GET_CAP(MAX_MR_SIZE, cap0);  in erdma_dev_attrs_init()
368 	dev->attrs.max_mw = 1 << ERDMA_GET_CAP(MAX_MW, cap1);  in erdma_dev_attrs_init()
369 	dev->attrs.max_recv_wr = 1 << ERDMA_GET_CAP(MAX_RECV_WR, cap0);  in erdma_dev_attrs_init()
370 	dev->attrs.local_dma_key = ERDMA_GET_CAP(DMA_LOCAL_KEY, cap1);  in erdma_dev_attrs_init()
371 	dev->attrs.cc = ERDMA_GET_CAP(DEFAULT_CC, cap1);  in erdma_dev_attrs_init()
372 	dev->attrs.max_qp = ERDMA_NQP_PER_QBLOCK * ERDMA_GET_CAP(QBLOCK, cap1);  in erdma_dev_attrs_init()
373 	dev->attrs.max_mr = dev->attrs.max_qp << 1;  in erdma_dev_attrs_init()
374 	dev->attrs.max_cq = dev->attrs.max_qp << 1;  in erdma_dev_attrs_init()
375 	dev->attrs.cap_flags = ERDMA_GET_CAP(FLAGS, cap0);  in erdma_dev_attrs_init()
377 	dev->attrs.max_send_wr = ERDMA_MAX_SEND_WR;  in erdma_dev_attrs_init()
378 	dev->attrs.max_ord = ERDMA_MAX_ORD;  in erdma_dev_attrs_init()
379 	dev->attrs.max_ird = ERDMA_MAX_IRD;  in erdma_dev_attrs_init()
380 	dev->attrs.max_send_sge = ERDMA_MAX_SEND_SGE;  in erdma_dev_attrs_init()
381 	dev->attrs.max_recv_sge = ERDMA_MAX_RECV_SGE;  in erdma_dev_attrs_init()
382 	dev->attrs.max_sge_rd = ERDMA_MAX_SGE_RD;  in erdma_dev_attrs_init()
383 	dev->attrs.max_pd = ERDMA_MAX_PD;  in erdma_dev_attrs_init()
385 	dev->res_cb[ERDMA_RES_TYPE_PD].max_cap = ERDMA_MAX_PD;  in erdma_dev_attrs_init()
386 	dev->res_cb[ERDMA_RES_TYPE_STAG_IDX].max_cap = dev->attrs.max_mr;  in erdma_dev_attrs_init()
391 	err = erdma_post_cmd_wait(&dev->cmdq, &req_hdr, sizeof(req_hdr), &cap0,  in erdma_dev_attrs_init()
394 		dev->attrs.fw_version =  in erdma_dev_attrs_init()
400 static int erdma_device_config(struct erdma_dev *dev)  in erdma_device_config()  argument
404 	if (!(dev->attrs.cap_flags & ERDMA_DEV_CAP_FLAGS_EXTEND_DB))  in erdma_device_config()
413 	return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);  in erdma_device_config()
416 static int erdma_res_cb_init(struct erdma_dev *dev)  in erdma_res_cb_init()  argument
421 		dev->res_cb[i].next_alloc_idx = 1;  in erdma_res_cb_init()
422 		spin_lock_init(&dev->res_cb[i].lock);  in erdma_res_cb_init()
423 		dev->res_cb[i].bitmap =  in erdma_res_cb_init()
424 			bitmap_zalloc(dev->res_cb[i].max_cap, GFP_KERNEL);  in erdma_res_cb_init()
425 		if (!dev->res_cb[i].bitmap)  in erdma_res_cb_init()
433 		bitmap_free(dev->res_cb[j].bitmap);  in erdma_res_cb_init()
438 static void erdma_res_cb_free(struct erdma_dev *dev)  in erdma_res_cb_free()  argument
443 		bitmap_free(dev->res_cb[i].bitmap);  in erdma_res_cb_free()
493 	struct erdma_dev *dev = pci_get_drvdata(pdev);  in erdma_ib_device_add()  local
494 	struct ib_device *ibdev = &dev->ibdev;  in erdma_ib_device_add()
498 	ret = erdma_dev_attrs_init(dev);  in erdma_ib_device_add()
502 	ret = erdma_device_config(dev);  in erdma_ib_device_add()
515 	ibdev->num_comp_vectors = dev->attrs.irq_num - 1;  in erdma_ib_device_add()
519 	INIT_LIST_HEAD(&dev->cep_list);  in erdma_ib_device_add()
521 	spin_lock_init(&dev->lock);  in erdma_ib_device_add()
522 	xa_init_flags(&dev->qp_xa, XA_FLAGS_ALLOC1);  in erdma_ib_device_add()
523 	xa_init_flags(&dev->cq_xa, XA_FLAGS_ALLOC1);  in erdma_ib_device_add()
524 	dev->next_alloc_cqn = 1;  in erdma_ib_device_add()
525 	dev->next_alloc_qpn = 1;  in erdma_ib_device_add()
527 	ret = erdma_res_cb_init(dev);  in erdma_ib_device_add()
531 	atomic_set(&dev->num_ctx, 0);  in erdma_ib_device_add()
533 	mac = erdma_reg_read32(dev, ERDMA_REGS_NETDEV_MAC_L_REG);  in erdma_ib_device_add()
534 	mac |= (u64)erdma_reg_read32(dev, ERDMA_REGS_NETDEV_MAC_H_REG) << 32;  in erdma_ib_device_add()
536 	u64_to_ether_addr(mac, dev->attrs.peer_addr);  in erdma_ib_device_add()
538 	dev->reflush_wq = alloc_workqueue("erdma-reflush-wq", WQ_UNBOUND,  in erdma_ib_device_add()
540 	if (!dev->reflush_wq) {  in erdma_ib_device_add()
545 	ret = erdma_device_register(dev);  in erdma_ib_device_add()
552 	destroy_workqueue(dev->reflush_wq);  in erdma_ib_device_add()
554 	xa_destroy(&dev->qp_xa);  in erdma_ib_device_add()
555 	xa_destroy(&dev->cq_xa);  in erdma_ib_device_add()
557 	erdma_res_cb_free(dev);  in erdma_ib_device_add()
564 	struct erdma_dev *dev = pci_get_drvdata(pdev);  in erdma_ib_device_remove()  local
566 	unregister_netdevice_notifier(&dev->netdev_nb);  in erdma_ib_device_remove()
567 	ib_unregister_device(&dev->ibdev);  in erdma_ib_device_remove()
569 	destroy_workqueue(dev->reflush_wq);  in erdma_ib_device_remove()
570 	erdma_res_cb_free(dev);  in erdma_ib_device_remove()
571 	xa_destroy(&dev->qp_xa);  in erdma_ib_device_remove()
572 	xa_destroy(&dev->cq_xa);  in erdma_ib_device_remove()