qp.c (5fe9dec0d045437e48f112b8fa705197bd7bc3c0) qp.c (b037c29a8056b8e896c4e084ba7cc30d6a1f165f)
1/*
2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:

--- 466 unchanged lines hidden (view full) ---

475 return 1;
476}
477
478static int first_med_bfreg(void)
479{
480 return 1;
481}
482
1/*
2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:

--- 466 unchanged lines hidden (view full) ---

475 return 1;
476}
477
478static int first_med_bfreg(void)
479{
480 return 1;
481}
482
483static int next_bfreg(int n)
484{
485 n++;
486
487 while (((n % 4) & 2))
488 n++;
489
490 return n;
491}
492
493enum {
494 /* this is the first blue flame register in the array of bfregs assigned
495 * to a processes. Since we do not use it for blue flame but rather
496 * regular 64 bit doorbells, we do not need a lock for maintaiing
497 * "odd/even" order
498 */
499 NUM_NON_BLUE_FLAME_BFREGS = 1,
500};
501
483enum {
484 /* this is the first blue flame register in the array of bfregs assigned
485 * to a processes. Since we do not use it for blue flame but rather
486 * regular 64 bit doorbells, we do not need a lock for maintaiing
487 * "odd/even" order
488 */
489 NUM_NON_BLUE_FLAME_BFREGS = 1,
490};
491
502static int num_med_bfreg(struct mlx5_bfreg_info *bfregi)
492static int max_bfregs(struct mlx5_ib_dev *dev, struct mlx5_bfreg_info *bfregi)
503{
493{
494 return get_num_uars(dev, bfregi) * MLX5_NON_FP_BFREGS_PER_UAR;
495}
496
497static int num_med_bfreg(struct mlx5_ib_dev *dev,
498 struct mlx5_bfreg_info *bfregi)
499{
504 int n;
505
500 int n;
501
506 n = bfregi->num_uars * MLX5_NON_FP_BFREGS_PER_UAR -
507 bfregi->num_low_latency_bfregs - NUM_NON_BLUE_FLAME_BFREGS;
502 n = max_bfregs(dev, bfregi) - bfregi->num_low_latency_bfregs -
503 NUM_NON_BLUE_FLAME_BFREGS;
508
509 return n >= 0 ? n : 0;
510}
511
504
505 return n >= 0 ? n : 0;
506}
507
512static int max_bfregi(struct mlx5_bfreg_info *bfregi)
508static int first_hi_bfreg(struct mlx5_ib_dev *dev,
509 struct mlx5_bfreg_info *bfregi)
513{
510{
514 return bfregi->num_uars * 4;
515}
516
517static int first_hi_bfreg(struct mlx5_bfreg_info *bfregi)
518{
519 int med;
520
511 int med;
512
521 med = num_med_bfreg(bfregi);
522 return next_bfreg(med);
513 med = num_med_bfreg(dev, bfregi);
514 return ++med;
523}
524
515}
516
525static int alloc_high_class_bfreg(struct mlx5_bfreg_info *bfregi)
517static int alloc_high_class_bfreg(struct mlx5_ib_dev *dev,
518 struct mlx5_bfreg_info *bfregi)
526{
527 int i;
528
519{
520 int i;
521
529 for (i = first_hi_bfreg(bfregi); i < max_bfregi(bfregi); i = next_bfreg(i)) {
530 if (!test_bit(i, bfregi->bitmap)) {
531 set_bit(i, bfregi->bitmap);
522 for (i = first_hi_bfreg(dev, bfregi); i < max_bfregs(dev, bfregi); i++) {
523 if (!bfregi->count[i]) {
532 bfregi->count[i]++;
533 return i;
534 }
535 }
536
537 return -ENOMEM;
538}
539
524 bfregi->count[i]++;
525 return i;
526 }
527 }
528
529 return -ENOMEM;
530}
531
540static int alloc_med_class_bfreg(struct mlx5_bfreg_info *bfregi)
532static int alloc_med_class_bfreg(struct mlx5_ib_dev *dev,
533 struct mlx5_bfreg_info *bfregi)
541{
542 int minidx = first_med_bfreg();
543 int i;
544
534{
535 int minidx = first_med_bfreg();
536 int i;
537
545 for (i = first_med_bfreg(); i < first_hi_bfreg(bfregi); i = next_bfreg(i)) {
538 for (i = first_med_bfreg(); i < first_hi_bfreg(dev, bfregi); i++) {
546 if (bfregi->count[i] < bfregi->count[minidx])
547 minidx = i;
548 if (!bfregi->count[minidx])
549 break;
550 }
551
552 bfregi->count[minidx]++;
553 return minidx;
554}
555
539 if (bfregi->count[i] < bfregi->count[minidx])
540 minidx = i;
541 if (!bfregi->count[minidx])
542 break;
543 }
544
545 bfregi->count[minidx]++;
546 return minidx;
547}
548
556static int alloc_bfreg(struct mlx5_bfreg_info *bfregi,
549static int alloc_bfreg(struct mlx5_ib_dev *dev,
550 struct mlx5_bfreg_info *bfregi,
557 enum mlx5_ib_latency_class lat)
558{
559 int bfregn = -EINVAL;
560
561 mutex_lock(&bfregi->lock);
562 switch (lat) {
563 case MLX5_IB_LATENCY_CLASS_LOW:
564 BUILD_BUG_ON(NUM_NON_BLUE_FLAME_BFREGS != 1);
565 bfregn = 0;
566 bfregi->count[bfregn]++;
567 break;
568
569 case MLX5_IB_LATENCY_CLASS_MEDIUM:
570 if (bfregi->ver < 2)
571 bfregn = -ENOMEM;
572 else
551 enum mlx5_ib_latency_class lat)
552{
553 int bfregn = -EINVAL;
554
555 mutex_lock(&bfregi->lock);
556 switch (lat) {
557 case MLX5_IB_LATENCY_CLASS_LOW:
558 BUILD_BUG_ON(NUM_NON_BLUE_FLAME_BFREGS != 1);
559 bfregn = 0;
560 bfregi->count[bfregn]++;
561 break;
562
563 case MLX5_IB_LATENCY_CLASS_MEDIUM:
564 if (bfregi->ver < 2)
565 bfregn = -ENOMEM;
566 else
573 bfregn = alloc_med_class_bfreg(bfregi);
567 bfregn = alloc_med_class_bfreg(dev, bfregi);
574 break;
575
576 case MLX5_IB_LATENCY_CLASS_HIGH:
577 if (bfregi->ver < 2)
578 bfregn = -ENOMEM;
579 else
568 break;
569
570 case MLX5_IB_LATENCY_CLASS_HIGH:
571 if (bfregi->ver < 2)
572 bfregn = -ENOMEM;
573 else
580 bfregn = alloc_high_class_bfreg(bfregi);
574 bfregn = alloc_high_class_bfreg(dev, bfregi);
581 break;
575 break;
582
583 case MLX5_IB_LATENCY_CLASS_FAST_PATH:
584 bfregn = 2;
585 break;
586 }
587 mutex_unlock(&bfregi->lock);
588
589 return bfregn;
590}
591
576 }
577 mutex_unlock(&bfregi->lock);
578
579 return bfregn;
580}
581
592static void free_med_class_bfreg(struct mlx5_bfreg_info *bfregi, int bfregn)
582static void free_bfreg(struct mlx5_ib_dev *dev, struct mlx5_bfreg_info *bfregi, int bfregn)
593{
583{
594 clear_bit(bfregn, bfregi->bitmap);
595 --bfregi->count[bfregn];
596}
597
598static void free_high_class_bfreg(struct mlx5_bfreg_info *bfregi, int bfregn)
599{
600 clear_bit(bfregn, bfregi->bitmap);
601 --bfregi->count[bfregn];
602}
603
604static void free_bfreg(struct mlx5_bfreg_info *bfregi, int bfregn)
605{
606 int nbfregs = bfregi->num_uars * MLX5_BFREGS_PER_UAR;
607 int high_bfreg = nbfregs - bfregi->num_low_latency_bfregs;
608
609 mutex_lock(&bfregi->lock);
584 mutex_lock(&bfregi->lock);
610 if (bfregn == 0) {
611 --bfregi->count[bfregn];
612 goto out;
613 }
614
615 if (bfregn < high_bfreg) {
616 free_med_class_bfreg(bfregi, bfregn);
617 goto out;
618 }
619
620 free_high_class_bfreg(bfregi, bfregn);
621
622out:
585 bfregi->count[bfregn]--;
623 mutex_unlock(&bfregi->lock);
624}
625
626static enum mlx5_qp_state to_mlx5_state(enum ib_qp_state state)
627{
628 switch (state) {
629 case IB_QPS_RESET: return MLX5_QP_STATE_RST;
630 case IB_QPS_INIT: return MLX5_QP_STATE_INIT;

--- 25 unchanged lines hidden (view full) ---

656 }
657}
658
659static void mlx5_ib_lock_cqs(struct mlx5_ib_cq *send_cq,
660 struct mlx5_ib_cq *recv_cq);
661static void mlx5_ib_unlock_cqs(struct mlx5_ib_cq *send_cq,
662 struct mlx5_ib_cq *recv_cq);
663
586 mutex_unlock(&bfregi->lock);
587}
588
589static enum mlx5_qp_state to_mlx5_state(enum ib_qp_state state)
590{
591 switch (state) {
592 case IB_QPS_RESET: return MLX5_QP_STATE_RST;
593 case IB_QPS_INIT: return MLX5_QP_STATE_INIT;

--- 25 unchanged lines hidden (view full) ---

619 }
620}
621
622static void mlx5_ib_lock_cqs(struct mlx5_ib_cq *send_cq,
623 struct mlx5_ib_cq *recv_cq);
624static void mlx5_ib_unlock_cqs(struct mlx5_ib_cq *send_cq,
625 struct mlx5_ib_cq *recv_cq);
626
664static int bfregn_to_uar_index(struct mlx5_bfreg_info *bfregi, int bfregn)
627static int bfregn_to_uar_index(struct mlx5_ib_dev *dev,
628 struct mlx5_bfreg_info *bfregi, int bfregn)
665{
629{
666 return bfregi->uars[bfregn / MLX5_BFREGS_PER_UAR].index;
630 int bfregs_per_sys_page;
631 int index_of_sys_page;
632 int offset;
633
634 bfregs_per_sys_page = get_uars_per_sys_page(dev, bfregi->lib_uar_4k) *
635 MLX5_NON_FP_BFREGS_PER_UAR;
636 index_of_sys_page = bfregn / bfregs_per_sys_page;
637
638 offset = bfregn % bfregs_per_sys_page / MLX5_NON_FP_BFREGS_PER_UAR;
639
640 return bfregi->sys_pages[index_of_sys_page] + offset;
667}
668
669static int mlx5_ib_umem_get(struct mlx5_ib_dev *dev,
670 struct ib_pd *pd,
671 unsigned long addr, size_t size,
672 struct ib_umem **umem,
673 int *npages, int *page_shift, int *ncont,
674 u32 *offset)

--- 86 unchanged lines hidden (view full) ---

761 rwq->create_type = MLX5_WQ_USER;
762 return 0;
763
764err_umem:
765 ib_umem_release(rwq->umem);
766 return err;
767}
768
641}
642
643static int mlx5_ib_umem_get(struct mlx5_ib_dev *dev,
644 struct ib_pd *pd,
645 unsigned long addr, size_t size,
646 struct ib_umem **umem,
647 int *npages, int *page_shift, int *ncont,
648 u32 *offset)

--- 86 unchanged lines hidden (view full) ---

735 rwq->create_type = MLX5_WQ_USER;
736 return 0;
737
738err_umem:
739 ib_umem_release(rwq->umem);
740 return err;
741}
742
743static int adjust_bfregn(struct mlx5_ib_dev *dev,
744 struct mlx5_bfreg_info *bfregi, int bfregn)
745{
746 return bfregn / MLX5_NON_FP_BFREGS_PER_UAR * MLX5_BFREGS_PER_UAR +
747 bfregn % MLX5_NON_FP_BFREGS_PER_UAR;
748}
749
769static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
770 struct mlx5_ib_qp *qp, struct ib_udata *udata,
771 struct ib_qp_init_attr *attr,
772 u32 **in,
773 struct mlx5_ib_create_qp_resp *resp, int *inlen,
774 struct mlx5_ib_qp_base *base)
775{
776 struct mlx5_ib_ucontext *context;

--- 18 unchanged lines hidden (view full) ---

795 context = to_mucontext(pd->uobject->context);
796 /*
797 * TBD: should come from the verbs when we have the API
798 */
799 if (qp->flags & MLX5_IB_QP_CROSS_CHANNEL)
800 /* In CROSS_CHANNEL CQ and QP must use the same UAR */
801 bfregn = MLX5_CROSS_CHANNEL_BFREG;
802 else {
750static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
751 struct mlx5_ib_qp *qp, struct ib_udata *udata,
752 struct ib_qp_init_attr *attr,
753 u32 **in,
754 struct mlx5_ib_create_qp_resp *resp, int *inlen,
755 struct mlx5_ib_qp_base *base)
756{
757 struct mlx5_ib_ucontext *context;

--- 18 unchanged lines hidden (view full) ---

776 context = to_mucontext(pd->uobject->context);
777 /*
778 * TBD: should come from the verbs when we have the API
779 */
780 if (qp->flags & MLX5_IB_QP_CROSS_CHANNEL)
781 /* In CROSS_CHANNEL CQ and QP must use the same UAR */
782 bfregn = MLX5_CROSS_CHANNEL_BFREG;
783 else {
803 bfregn = alloc_bfreg(&context->bfregi, MLX5_IB_LATENCY_CLASS_HIGH);
784 bfregn = alloc_bfreg(dev, &context->bfregi, MLX5_IB_LATENCY_CLASS_HIGH);
804 if (bfregn < 0) {
805 mlx5_ib_dbg(dev, "failed to allocate low latency BFREG\n");
806 mlx5_ib_dbg(dev, "reverting to medium latency\n");
785 if (bfregn < 0) {
786 mlx5_ib_dbg(dev, "failed to allocate low latency BFREG\n");
787 mlx5_ib_dbg(dev, "reverting to medium latency\n");
807 bfregn = alloc_bfreg(&context->bfregi, MLX5_IB_LATENCY_CLASS_MEDIUM);
788 bfregn = alloc_bfreg(dev, &context->bfregi, MLX5_IB_LATENCY_CLASS_MEDIUM);
808 if (bfregn < 0) {
809 mlx5_ib_dbg(dev, "failed to allocate medium latency BFREG\n");
810 mlx5_ib_dbg(dev, "reverting to high latency\n");
789 if (bfregn < 0) {
790 mlx5_ib_dbg(dev, "failed to allocate medium latency BFREG\n");
791 mlx5_ib_dbg(dev, "reverting to high latency\n");
811 bfregn = alloc_bfreg(&context->bfregi, MLX5_IB_LATENCY_CLASS_LOW);
792 bfregn = alloc_bfreg(dev, &context->bfregi, MLX5_IB_LATENCY_CLASS_LOW);
812 if (bfregn < 0) {
813 mlx5_ib_warn(dev, "bfreg allocation failed\n");
814 return bfregn;
815 }
816 }
817 }
818 }
819
793 if (bfregn < 0) {
794 mlx5_ib_warn(dev, "bfreg allocation failed\n");
795 return bfregn;
796 }
797 }
798 }
799 }
800
820 uar_index = bfregn_to_uar_index(&context->bfregi, bfregn);
801 uar_index = bfregn_to_uar_index(dev, &context->bfregi, bfregn);
821 mlx5_ib_dbg(dev, "bfregn 0x%x, uar_index 0x%x\n", bfregn, uar_index);
822
823 qp->rq.offset = 0;
824 qp->sq.wqe_shift = ilog2(MLX5_SEND_WQE_BB);
825 qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift;
826
827 err = set_user_buf_size(dev, qp, &ucmd, base, attr);
828 if (err)

--- 24 unchanged lines hidden (view full) ---

853 mlx5_ib_populate_pas(dev, ubuffer->umem, page_shift, pas, 0);
854
855 qpc = MLX5_ADDR_OF(create_qp_in, *in, qpc);
856
857 MLX5_SET(qpc, qpc, log_page_size, page_shift - MLX5_ADAPTER_PAGE_SHIFT);
858 MLX5_SET(qpc, qpc, page_offset, offset);
859
860 MLX5_SET(qpc, qpc, uar_page, uar_index);
802 mlx5_ib_dbg(dev, "bfregn 0x%x, uar_index 0x%x\n", bfregn, uar_index);
803
804 qp->rq.offset = 0;
805 qp->sq.wqe_shift = ilog2(MLX5_SEND_WQE_BB);
806 qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift;
807
808 err = set_user_buf_size(dev, qp, &ucmd, base, attr);
809 if (err)

--- 24 unchanged lines hidden (view full) ---

834 mlx5_ib_populate_pas(dev, ubuffer->umem, page_shift, pas, 0);
835
836 qpc = MLX5_ADDR_OF(create_qp_in, *in, qpc);
837
838 MLX5_SET(qpc, qpc, log_page_size, page_shift - MLX5_ADAPTER_PAGE_SHIFT);
839 MLX5_SET(qpc, qpc, page_offset, offset);
840
841 MLX5_SET(qpc, qpc, uar_page, uar_index);
861 resp->bfreg_index = bfregn;
842 resp->bfreg_index = adjust_bfregn(dev, &context->bfregi, bfregn);
862 qp->bfregn = bfregn;
863
864 err = mlx5_ib_db_map_user(context, ucmd.db_addr, &qp->db);
865 if (err) {
866 mlx5_ib_dbg(dev, "map failed\n");
867 goto err_free;
868 }
869

--- 12 unchanged lines hidden (view full) ---

882err_free:
883 kvfree(*in);
884
885err_umem:
886 if (ubuffer->umem)
887 ib_umem_release(ubuffer->umem);
888
889err_bfreg:
843 qp->bfregn = bfregn;
844
845 err = mlx5_ib_db_map_user(context, ucmd.db_addr, &qp->db);
846 if (err) {
847 mlx5_ib_dbg(dev, "map failed\n");
848 goto err_free;
849 }
850

--- 12 unchanged lines hidden (view full) ---

863err_free:
864 kvfree(*in);
865
866err_umem:
867 if (ubuffer->umem)
868 ib_umem_release(ubuffer->umem);
869
870err_bfreg:
890 free_bfreg(&context->bfregi, bfregn);
871 free_bfreg(dev, &context->bfregi, bfregn);
891 return err;
892}
893
872 return err;
873}
874
894static void destroy_qp_user(struct ib_pd *pd, struct mlx5_ib_qp *qp,
895 struct mlx5_ib_qp_base *base)
875static void destroy_qp_user(struct mlx5_ib_dev *dev, struct ib_pd *pd,
876 struct mlx5_ib_qp *qp, struct mlx5_ib_qp_base *base)
896{
897 struct mlx5_ib_ucontext *context;
898
899 context = to_mucontext(pd->uobject->context);
900 mlx5_ib_db_unmap_user(context, &qp->db);
901 if (base->ubuffer.umem)
902 ib_umem_release(base->ubuffer.umem);
877{
878 struct mlx5_ib_ucontext *context;
879
880 context = to_mucontext(pd->uobject->context);
881 mlx5_ib_db_unmap_user(context, &qp->db);
882 if (base->ubuffer.umem)
883 ib_umem_release(base->ubuffer.umem);
903 free_bfreg(&context->bfregi, qp->bfregn);
884 free_bfreg(dev, &context->bfregi, qp->bfregn);
904}
905
906static int create_kernel_qp(struct mlx5_ib_dev *dev,
907 struct ib_qp_init_attr *init_attr,
908 struct mlx5_ib_qp *qp,
909 u32 **in, int *inlen,
910 struct mlx5_ib_qp_base *base)
911{

--- 867 unchanged lines hidden (view full) ---

1779 list_add_tail(&qp->cq_recv_list, &recv_cq->list_recv_qp);
1780 mlx5_ib_unlock_cqs(send_cq, recv_cq);
1781 spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
1782
1783 return 0;
1784
1785err_create:
1786 if (qp->create_type == MLX5_QP_USER)
885}
886
887static int create_kernel_qp(struct mlx5_ib_dev *dev,
888 struct ib_qp_init_attr *init_attr,
889 struct mlx5_ib_qp *qp,
890 u32 **in, int *inlen,
891 struct mlx5_ib_qp_base *base)
892{

--- 867 unchanged lines hidden (view full) ---

1760 list_add_tail(&qp->cq_recv_list, &recv_cq->list_recv_qp);
1761 mlx5_ib_unlock_cqs(send_cq, recv_cq);
1762 spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
1763
1764 return 0;
1765
1766err_create:
1767 if (qp->create_type == MLX5_QP_USER)
1787 destroy_qp_user(pd, qp, base);
1768 destroy_qp_user(dev, pd, qp, base);
1788 else if (qp->create_type == MLX5_QP_KERNEL)
1789 destroy_qp_kernel(dev, qp);
1790
1791 kvfree(in);
1792 return err;
1793}
1794
1795static void mlx5_ib_lock_cqs(struct mlx5_ib_cq *send_cq, struct mlx5_ib_cq *recv_cq)

--- 161 unchanged lines hidden (view full) ---

1957 if (err)
1958 mlx5_ib_warn(dev, "failed to destroy QP 0x%x\n",
1959 base->mqp.qpn);
1960 }
1961
1962 if (qp->create_type == MLX5_QP_KERNEL)
1963 destroy_qp_kernel(dev, qp);
1964 else if (qp->create_type == MLX5_QP_USER)
1769 else if (qp->create_type == MLX5_QP_KERNEL)
1770 destroy_qp_kernel(dev, qp);
1771
1772 kvfree(in);
1773 return err;
1774}
1775
1776static void mlx5_ib_lock_cqs(struct mlx5_ib_cq *send_cq, struct mlx5_ib_cq *recv_cq)

--- 161 unchanged lines hidden (view full) ---

1938 if (err)
1939 mlx5_ib_warn(dev, "failed to destroy QP 0x%x\n",
1940 base->mqp.qpn);
1941 }
1942
1943 if (qp->create_type == MLX5_QP_KERNEL)
1944 destroy_qp_kernel(dev, qp);
1945 else if (qp->create_type == MLX5_QP_USER)
1965 destroy_qp_user(&get_pd(qp)->ibpd, qp, base);
1946 destroy_qp_user(dev, &get_pd(qp)->ibpd, qp, base);
1966}
1967
1968static const char *ib_qp_type_str(enum ib_qp_type type)
1969{
1970 switch (type) {
1971 case IB_QPT_SMI:
1972 return "IB_QPT_SMI";
1973 case IB_QPT_GSI:

--- 2951 unchanged lines hidden --- 		1947}
1948
1949static const char *ib_qp_type_str(enum ib_qp_type type)
1950{
1951 switch (type) {
1952 case IB_QPT_SMI:
1953 return "IB_QPT_SMI";
1954 case IB_QPT_GSI:

--- 2951 unchanged lines hidden ---