init.c (eeea8b40cd2866ca24f25e5ef09225edb076ae45) init.c (60368186fd853899c1819bcefa47f85fe8d5e5ad)
1/*
2 * Copyright(c) 2015, 2016 Intel Corporation.
3 *
4 * This file is provided under a dual BSD/GPLv2 license. When using or
5 * redistributing this file, you may do so under either license.
6 *
7 * GPL LICENSE SUMMARY
8 *

--- 695 unchanged lines hidden (view full) ---

704 else
705 ret = loadtime_init(dd);
706 if (ret)
707 goto done;
708
709 /* allocate dummy tail memory for all receive contexts */
710 dd->rcvhdrtail_dummy_kvaddr = dma_zalloc_coherent(
711 &dd->pcidev->dev, sizeof(u64),
1/*
2 * Copyright(c) 2015, 2016 Intel Corporation.
3 *
4 * This file is provided under a dual BSD/GPLv2 license. When using or
5 * redistributing this file, you may do so under either license.
6 *
7 * GPL LICENSE SUMMARY
8 *

--- 695 unchanged lines hidden (view full) ---

704 else
705 ret = loadtime_init(dd);
706 if (ret)
707 goto done;
708
709 /* allocate dummy tail memory for all receive contexts */
710 dd->rcvhdrtail_dummy_kvaddr = dma_zalloc_coherent(
711 &dd->pcidev->dev, sizeof(u64),
712 &dd->rcvhdrtail_dummy_physaddr,
712 &dd->rcvhdrtail_dummy_dma,
713 GFP_KERNEL);
714
715 if (!dd->rcvhdrtail_dummy_kvaddr) {
716 dd_dev_err(dd, "cannot allocate dummy tail memory\n");
717 ret = -ENOMEM;
718 goto done;
719 }
720

--- 216 unchanged lines hidden (view full) ---

937{
938 unsigned e;
939
940 if (!rcd)
941 return;
942
943 if (rcd->rcvhdrq) {
944 dma_free_coherent(&dd->pcidev->dev, rcd->rcvhdrq_size,
713 GFP_KERNEL);
714
715 if (!dd->rcvhdrtail_dummy_kvaddr) {
716 dd_dev_err(dd, "cannot allocate dummy tail memory\n");
717 ret = -ENOMEM;
718 goto done;
719 }
720

--- 216 unchanged lines hidden (view full) ---

937{
938 unsigned e;
939
940 if (!rcd)
941 return;
942
943 if (rcd->rcvhdrq) {
944 dma_free_coherent(&dd->pcidev->dev, rcd->rcvhdrq_size,
945 rcd->rcvhdrq, rcd->rcvhdrq_phys);
945 rcd->rcvhdrq, rcd->rcvhdrq_dma);
946 rcd->rcvhdrq = NULL;
947 if (rcd->rcvhdrtail_kvaddr) {
948 dma_free_coherent(&dd->pcidev->dev, PAGE_SIZE,
949 (void *)rcd->rcvhdrtail_kvaddr,
946 rcd->rcvhdrq = NULL;
947 if (rcd->rcvhdrtail_kvaddr) {
948 dma_free_coherent(&dd->pcidev->dev, PAGE_SIZE,
949 (void *)rcd->rcvhdrtail_kvaddr,
950 rcd->rcvhdrqtailaddr_phys);
950 rcd->rcvhdrqtailaddr_dma);
951 rcd->rcvhdrtail_kvaddr = NULL;
952 }
953 }
954
955 /* all the RcvArray entries should have been cleared by now */
956 kfree(rcd->egrbufs.rcvtids);
957
958 for (e = 0; e < rcd->egrbufs.alloced; e++) {
951 rcd->rcvhdrtail_kvaddr = NULL;
952 }
953 }
954
955 /* all the RcvArray entries should have been cleared by now */
956 kfree(rcd->egrbufs.rcvtids);
957
958 for (e = 0; e < rcd->egrbufs.alloced; e++) {
959 if (rcd->egrbufs.buffers[e].phys)
959 if (rcd->egrbufs.buffers[e].dma)
960 dma_free_coherent(&dd->pcidev->dev,
961 rcd->egrbufs.buffers[e].len,
962 rcd->egrbufs.buffers[e].addr,
960 dma_free_coherent(&dd->pcidev->dev,
961 rcd->egrbufs.buffers[e].len,
962 rcd->egrbufs.buffers[e].addr,
963 rcd->egrbufs.buffers[e].phys);
963 rcd->egrbufs.buffers[e].dma);
964 }
965 kfree(rcd->egrbufs.buffers);
966
967 sc_free(rcd->sc);
968 vfree(rcd->user_event_mask);
969 vfree(rcd->subctxt_uregbase);
970 vfree(rcd->subctxt_rcvegrbuf);
971 vfree(rcd->subctxt_rcvhdr_base);

--- 377 unchanged lines hidden (view full) ---

1349 spin_lock_irqsave(&dd->uctxt_lock, flags);
1350 tmp = dd->rcd;
1351 dd->rcd = NULL;
1352 spin_unlock_irqrestore(&dd->uctxt_lock, flags);
1353
1354 if (dd->rcvhdrtail_dummy_kvaddr) {
1355 dma_free_coherent(&dd->pcidev->dev, sizeof(u64),
1356 (void *)dd->rcvhdrtail_dummy_kvaddr,
964 }
965 kfree(rcd->egrbufs.buffers);
966
967 sc_free(rcd->sc);
968 vfree(rcd->user_event_mask);
969 vfree(rcd->subctxt_uregbase);
970 vfree(rcd->subctxt_rcvegrbuf);
971 vfree(rcd->subctxt_rcvhdr_base);

--- 377 unchanged lines hidden (view full) ---

1349 spin_lock_irqsave(&dd->uctxt_lock, flags);
1350 tmp = dd->rcd;
1351 dd->rcd = NULL;
1352 spin_unlock_irqrestore(&dd->uctxt_lock, flags);
1353
1354 if (dd->rcvhdrtail_dummy_kvaddr) {
1355 dma_free_coherent(&dd->pcidev->dev, sizeof(u64),
1356 (void *)dd->rcvhdrtail_dummy_kvaddr,
1357 dd->rcvhdrtail_dummy_physaddr);
1357 dd->rcvhdrtail_dummy_dma);
1358 dd->rcvhdrtail_dummy_kvaddr = NULL;
1359 }
1360
1361 for (ctxt = 0; tmp && ctxt < dd->num_rcv_contexts; ctxt++) {
1362 struct hfi1_ctxtdata *rcd = tmp[ctxt];
1363
1364 tmp[ctxt] = NULL; /* debugging paranoia */
1365 if (rcd) {

--- 206 unchanged lines hidden (view full) ---

1572 * or be forced into a low address range).
1573 */
1574int hfi1_create_rcvhdrq(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd)
1575{
1576 unsigned amt;
1577 u64 reg;
1578
1579 if (!rcd->rcvhdrq) {
1358 dd->rcvhdrtail_dummy_kvaddr = NULL;
1359 }
1360
1361 for (ctxt = 0; tmp && ctxt < dd->num_rcv_contexts; ctxt++) {
1362 struct hfi1_ctxtdata *rcd = tmp[ctxt];
1363
1364 tmp[ctxt] = NULL; /* debugging paranoia */
1365 if (rcd) {

--- 206 unchanged lines hidden (view full) ---

1572 * or be forced into a low address range).
1573 */
1574int hfi1_create_rcvhdrq(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd)
1575{
1576 unsigned amt;
1577 u64 reg;
1578
1579 if (!rcd->rcvhdrq) {
1580 dma_addr_t phys_hdrqtail;
1580 dma_addr_t dma_hdrqtail;
1581 gfp_t gfp_flags;
1582
1583 /*
1584 * rcvhdrqentsize is in DWs, so we have to convert to bytes
1585 * (* sizeof(u32)).
1586 */
1587 amt = PAGE_ALIGN(rcd->rcvhdrq_cnt * rcd->rcvhdrqentsize *
1588 sizeof(u32));
1589
1590 gfp_flags = (rcd->ctxt >= dd->first_user_ctxt) ?
1591 GFP_USER : GFP_KERNEL;
1592 rcd->rcvhdrq = dma_zalloc_coherent(
1581 gfp_t gfp_flags;
1582
1583 /*
1584 * rcvhdrqentsize is in DWs, so we have to convert to bytes
1585 * (* sizeof(u32)).
1586 */
1587 amt = PAGE_ALIGN(rcd->rcvhdrq_cnt * rcd->rcvhdrqentsize *
1588 sizeof(u32));
1589
1590 gfp_flags = (rcd->ctxt >= dd->first_user_ctxt) ?
1591 GFP_USER : GFP_KERNEL;
1592 rcd->rcvhdrq = dma_zalloc_coherent(
1593 &dd->pcidev->dev, amt, &rcd->rcvhdrq_phys,
1593 &dd->pcidev->dev, amt, &rcd->rcvhdrq_dma,
1594 gfp_flags | __GFP_COMP);
1595
1596 if (!rcd->rcvhdrq) {
1597 dd_dev_err(dd,
1598 "attempt to allocate %d bytes for ctxt %u rcvhdrq failed\n",
1599 amt, rcd->ctxt);
1600 goto bail;
1601 }
1602
1603 if (HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) {
1604 rcd->rcvhdrtail_kvaddr = dma_zalloc_coherent(
1594 gfp_flags | __GFP_COMP);
1595
1596 if (!rcd->rcvhdrq) {
1597 dd_dev_err(dd,
1598 "attempt to allocate %d bytes for ctxt %u rcvhdrq failed\n",
1599 amt, rcd->ctxt);
1600 goto bail;
1601 }
1602
1603 if (HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) {
1604 rcd->rcvhdrtail_kvaddr = dma_zalloc_coherent(
1605 &dd->pcidev->dev, PAGE_SIZE, &phys_hdrqtail,
1605 &dd->pcidev->dev, PAGE_SIZE, &dma_hdrqtail,
1606 gfp_flags);
1607 if (!rcd->rcvhdrtail_kvaddr)
1608 goto bail_free;
1606 gfp_flags);
1607 if (!rcd->rcvhdrtail_kvaddr)
1608 goto bail_free;
1609 rcd->rcvhdrqtailaddr_phys = phys_hdrqtail;
1609 rcd->rcvhdrqtailaddr_dma = dma_hdrqtail;
1610 }
1611
1612 rcd->rcvhdrq_size = amt;
1613 }
1614 /*
1615 * These values are per-context:
1616 * RcvHdrCnt
1617 * RcvHdrEntSize

--- 11 unchanged lines hidden (view full) ---

1629 << RCV_HDR_SIZE_HDR_SIZE_SHIFT;
1630 write_kctxt_csr(dd, rcd->ctxt, RCV_HDR_SIZE, reg);
1631
1632 /*
1633 * Program dummy tail address for every receive context
1634 * before enabling any receive context
1635 */
1636 write_kctxt_csr(dd, rcd->ctxt, RCV_HDR_TAIL_ADDR,
1610 }
1611
1612 rcd->rcvhdrq_size = amt;
1613 }
1614 /*
1615 * These values are per-context:
1616 * RcvHdrCnt
1617 * RcvHdrEntSize

--- 11 unchanged lines hidden (view full) ---

1629 << RCV_HDR_SIZE_HDR_SIZE_SHIFT;
1630 write_kctxt_csr(dd, rcd->ctxt, RCV_HDR_SIZE, reg);
1631
1632 /*
1633 * Program dummy tail address for every receive context
1634 * before enabling any receive context
1635 */
1636 write_kctxt_csr(dd, rcd->ctxt, RCV_HDR_TAIL_ADDR,
1637 dd->rcvhdrtail_dummy_physaddr);
1637 dd->rcvhdrtail_dummy_dma);
1638
1639 return 0;
1640
1641bail_free:
1642 dd_dev_err(dd,
1643 "attempt to allocate 1 page for ctxt %u rcvhdrqtailaddr failed\n",
1644 rcd->ctxt);
1645 vfree(rcd->user_event_mask);
1646 rcd->user_event_mask = NULL;
1647 dma_free_coherent(&dd->pcidev->dev, amt, rcd->rcvhdrq,
1638
1639 return 0;
1640
1641bail_free:
1642 dd_dev_err(dd,
1643 "attempt to allocate 1 page for ctxt %u rcvhdrqtailaddr failed\n",
1644 rcd->ctxt);
1645 vfree(rcd->user_event_mask);
1646 rcd->user_event_mask = NULL;
1647 dma_free_coherent(&dd->pcidev->dev, amt, rcd->rcvhdrq,
1648 rcd->rcvhdrq_phys);
1648 rcd->rcvhdrq_dma);
1649 rcd->rcvhdrq = NULL;
1650bail:
1651 return -ENOMEM;
1652}
1653
1654/**
1655 * allocate eager buffers, both kernel and user contexts.
1656 * @rcd: the context we are setting up.

--- 44 unchanged lines hidden (view full) ---

1701 rcd->egrbufs.rcvtid_size = max((unsigned long)round_mtu,
1702 rounddown_pow_of_two(rcd->egrbufs.size / 8));
1703
1704 while (alloced_bytes < rcd->egrbufs.size &&
1705 rcd->egrbufs.alloced < rcd->egrbufs.count) {
1706 rcd->egrbufs.buffers[idx].addr =
1707 dma_zalloc_coherent(&dd->pcidev->dev,
1708 rcd->egrbufs.rcvtid_size,
1649 rcd->rcvhdrq = NULL;
1650bail:
1651 return -ENOMEM;
1652}
1653
1654/**
1655 * allocate eager buffers, both kernel and user contexts.
1656 * @rcd: the context we are setting up.

--- 44 unchanged lines hidden (view full) ---

1701 rcd->egrbufs.rcvtid_size = max((unsigned long)round_mtu,
1702 rounddown_pow_of_two(rcd->egrbufs.size / 8));
1703
1704 while (alloced_bytes < rcd->egrbufs.size &&
1705 rcd->egrbufs.alloced < rcd->egrbufs.count) {
1706 rcd->egrbufs.buffers[idx].addr =
1707 dma_zalloc_coherent(&dd->pcidev->dev,
1708 rcd->egrbufs.rcvtid_size,
1709 &rcd->egrbufs.buffers[idx].phys,
1709 &rcd->egrbufs.buffers[idx].dma,
1710 gfp_flags);
1711 if (rcd->egrbufs.buffers[idx].addr) {
1712 rcd->egrbufs.buffers[idx].len =
1713 rcd->egrbufs.rcvtid_size;
1714 rcd->egrbufs.rcvtids[rcd->egrbufs.alloced].addr =
1715 rcd->egrbufs.buffers[idx].addr;
1710 gfp_flags);
1711 if (rcd->egrbufs.buffers[idx].addr) {
1712 rcd->egrbufs.buffers[idx].len =
1713 rcd->egrbufs.rcvtid_size;
1714 rcd->egrbufs.rcvtids[rcd->egrbufs.alloced].addr =
1715 rcd->egrbufs.buffers[idx].addr;
1716 rcd->egrbufs.rcvtids[rcd->egrbufs.alloced].phys =
1717 rcd->egrbufs.buffers[idx].phys;
1716 rcd->egrbufs.rcvtids[rcd->egrbufs.alloced].dma =
1717 rcd->egrbufs.buffers[idx].dma;
1718 rcd->egrbufs.alloced++;
1719 alloced_bytes += rcd->egrbufs.rcvtid_size;
1720 idx++;
1721 } else {
1722 u32 new_size, i, j;
1723 u64 offset = 0;
1724
1725 /*

--- 24 unchanged lines hidden (view full) ---

1750 /*
1751 * Re-partition already allocated buffers to a smaller
1752 * size.
1753 */
1754 rcd->egrbufs.alloced = 0;
1755 for (i = 0, j = 0, offset = 0; j < idx; i++) {
1756 if (i >= rcd->egrbufs.count)
1757 break;
1718 rcd->egrbufs.alloced++;
1719 alloced_bytes += rcd->egrbufs.rcvtid_size;
1720 idx++;
1721 } else {
1722 u32 new_size, i, j;
1723 u64 offset = 0;
1724
1725 /*

--- 24 unchanged lines hidden (view full) ---

1750 /*
1751 * Re-partition already allocated buffers to a smaller
1752 * size.
1753 */
1754 rcd->egrbufs.alloced = 0;
1755 for (i = 0, j = 0, offset = 0; j < idx; i++) {
1756 if (i >= rcd->egrbufs.count)
1757 break;
1758 rcd->egrbufs.rcvtids[i].phys =
1759 rcd->egrbufs.buffers[j].phys + offset;
1758 rcd->egrbufs.rcvtids[i].dma =
1759 rcd->egrbufs.buffers[j].dma + offset;
1760 rcd->egrbufs.rcvtids[i].addr =
1761 rcd->egrbufs.buffers[j].addr + offset;
1762 rcd->egrbufs.alloced++;
1760 rcd->egrbufs.rcvtids[i].addr =
1761 rcd->egrbufs.buffers[j].addr + offset;
1762 rcd->egrbufs.alloced++;
1763 if ((rcd->egrbufs.buffers[j].phys + offset +
1763 if ((rcd->egrbufs.buffers[j].dma + offset +
1764 new_size) ==
1764 new_size) ==
1765 (rcd->egrbufs.buffers[j].phys +
1765 (rcd->egrbufs.buffers[j].dma +
1766 rcd->egrbufs.buffers[j].len)) {
1767 j++;
1768 offset = 0;
1769 } else {
1770 offset += new_size;
1771 }
1772 }
1773 rcd->egrbufs.rcvtid_size = new_size;

--- 35 unchanged lines hidden (view full) ---

1809 "ctxt%u: current Eager buffer size is invalid %u\n",
1810 rcd->ctxt, rcd->egrbufs.rcvtid_size);
1811 ret = -EINVAL;
1812 goto bail;
1813 }
1814
1815 for (idx = 0; idx < rcd->egrbufs.alloced; idx++) {
1816 hfi1_put_tid(dd, rcd->eager_base + idx, PT_EAGER,
1766 rcd->egrbufs.buffers[j].len)) {
1767 j++;
1768 offset = 0;
1769 } else {
1770 offset += new_size;
1771 }
1772 }
1773 rcd->egrbufs.rcvtid_size = new_size;

--- 35 unchanged lines hidden (view full) ---

1809 "ctxt%u: current Eager buffer size is invalid %u\n",
1810 rcd->ctxt, rcd->egrbufs.rcvtid_size);
1811 ret = -EINVAL;
1812 goto bail;
1813 }
1814
1815 for (idx = 0; idx < rcd->egrbufs.alloced; idx++) {
1816 hfi1_put_tid(dd, rcd->eager_base + idx, PT_EAGER,
1817 rcd->egrbufs.rcvtids[idx].phys, order);
1817 rcd->egrbufs.rcvtids[idx].dma, order);
1818 cond_resched();
1819 }
1820 goto bail;
1821
1822bail_rcvegrbuf_phys:
1823 for (idx = 0; idx < rcd->egrbufs.alloced &&
1824 rcd->egrbufs.buffers[idx].addr;
1825 idx++) {
1826 dma_free_coherent(&dd->pcidev->dev,
1827 rcd->egrbufs.buffers[idx].len,
1828 rcd->egrbufs.buffers[idx].addr,
1818 cond_resched();
1819 }
1820 goto bail;
1821
1822bail_rcvegrbuf_phys:
1823 for (idx = 0; idx < rcd->egrbufs.alloced &&
1824 rcd->egrbufs.buffers[idx].addr;
1825 idx++) {
1826 dma_free_coherent(&dd->pcidev->dev,
1827 rcd->egrbufs.buffers[idx].len,
1828 rcd->egrbufs.buffers[idx].addr,
1829 rcd->egrbufs.buffers[idx].phys);
1829 rcd->egrbufs.buffers[idx].dma);
1830 rcd->egrbufs.buffers[idx].addr = NULL;
1830 rcd->egrbufs.buffers[idx].addr = NULL;
1831 rcd->egrbufs.buffers[idx].phys = 0;
1831 rcd->egrbufs.buffers[idx].dma = 0;
1832 rcd->egrbufs.buffers[idx].len = 0;
1833 }
1834bail:
1835 return ret;
1836}
1832 rcd->egrbufs.buffers[idx].len = 0;
1833 }
1834bail:
1835 return ret;
1836}