1a61127c2SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
28c8f1c95SAlan Cox /*
38c8f1c95SAlan Cox * Copyright (c) 2007, Intel Corporation.
48c8f1c95SAlan Cox * All Rights Reserved.
58c8f1c95SAlan Cox *
68c8f1c95SAlan Cox * Authors: Thomas Hellstrom <thomas-at-tungstengraphics.com>
78c8f1c95SAlan Cox * Alan Cox <alan@linux.intel.com>
88c8f1c95SAlan Cox */
98c8f1c95SAlan Cox
10f2d061edSThomas Zimmermann #include "gem.h" /* TODO: for struct psb_gem_object, see psb_gtt_restore() */
110c7b178aSSam Ravnborg #include "psb_drv.h"
128c8f1c95SAlan Cox
138c8f1c95SAlan Cox
148c8f1c95SAlan Cox /*
158c8f1c95SAlan Cox * GTT resource allocator - manage page mappings in GTT space
168c8f1c95SAlan Cox */
178c8f1c95SAlan Cox
psb_gtt_allocate_resource(struct drm_psb_private * pdev,struct resource * res,const char * name,resource_size_t size,resource_size_t align,bool stolen,u32 * offset)183c101135SThomas Zimmermann int psb_gtt_allocate_resource(struct drm_psb_private *pdev, struct resource *res,
193c101135SThomas Zimmermann const char *name, resource_size_t size, resource_size_t align,
203c101135SThomas Zimmermann bool stolen, u32 *offset)
213c101135SThomas Zimmermann {
223c101135SThomas Zimmermann struct resource *root = pdev->gtt_mem;
233c101135SThomas Zimmermann resource_size_t start, end;
243c101135SThomas Zimmermann int ret;
253c101135SThomas Zimmermann
263c101135SThomas Zimmermann if (stolen) {
273c101135SThomas Zimmermann /* The start of the GTT is backed by stolen pages. */
283c101135SThomas Zimmermann start = root->start;
293c101135SThomas Zimmermann end = root->start + pdev->gtt.stolen_size - 1;
303c101135SThomas Zimmermann } else {
313c101135SThomas Zimmermann /* The rest is backed by system pages. */
323c101135SThomas Zimmermann start = root->start + pdev->gtt.stolen_size;
333c101135SThomas Zimmermann end = root->end;
343c101135SThomas Zimmermann }
353c101135SThomas Zimmermann
363c101135SThomas Zimmermann res->name = name;
373c101135SThomas Zimmermann ret = allocate_resource(root, res, size, start, end, align, NULL, NULL);
383c101135SThomas Zimmermann if (ret)
393c101135SThomas Zimmermann return ret;
403c101135SThomas Zimmermann *offset = res->start - root->start;
413c101135SThomas Zimmermann
423c101135SThomas Zimmermann return 0;
433c101135SThomas Zimmermann }
443c101135SThomas Zimmermann
458c8f1c95SAlan Cox /**
468c8f1c95SAlan Cox * psb_gtt_mask_pte - generate GTT pte entry
478c8f1c95SAlan Cox * @pfn: page number to encode
488c8f1c95SAlan Cox * @type: type of memory in the GTT
498c8f1c95SAlan Cox *
508c8f1c95SAlan Cox * Set the GTT entry for the appropriate memory type.
518c8f1c95SAlan Cox */
psb_gtt_mask_pte(uint32_t pfn,int type)52d339386cSThomas Zimmermann uint32_t psb_gtt_mask_pte(uint32_t pfn, int type)
538c8f1c95SAlan Cox {
548c8f1c95SAlan Cox uint32_t mask = PSB_PTE_VALID;
558c8f1c95SAlan Cox
56398b4706SAlan Cox /* Ensure we explode rather than put an invalid low mapping of
57398b4706SAlan Cox a high mapping page into the gtt */
58398b4706SAlan Cox BUG_ON(pfn & ~(0xFFFFFFFF >> PAGE_SHIFT));
59398b4706SAlan Cox
608c8f1c95SAlan Cox if (type & PSB_MMU_CACHED_MEMORY)
618c8f1c95SAlan Cox mask |= PSB_PTE_CACHED;
628c8f1c95SAlan Cox if (type & PSB_MMU_RO_MEMORY)
638c8f1c95SAlan Cox mask |= PSB_PTE_RO;
648c8f1c95SAlan Cox if (type & PSB_MMU_WO_MEMORY)
658c8f1c95SAlan Cox mask |= PSB_PTE_WO;
668c8f1c95SAlan Cox
678c8f1c95SAlan Cox return (pfn << PAGE_SHIFT) | mask;
688c8f1c95SAlan Cox }
698c8f1c95SAlan Cox
psb_gtt_entry(struct drm_psb_private * pdev,const struct resource * res)70e1f80341SThomas Zimmermann static u32 __iomem *psb_gtt_entry(struct drm_psb_private *pdev, const struct resource *res)
718c8f1c95SAlan Cox {
72e1f80341SThomas Zimmermann unsigned long offset = res->start - pdev->gtt_mem->start;
738c8f1c95SAlan Cox
74e1f80341SThomas Zimmermann return pdev->gtt_map + (offset >> PAGE_SHIFT);
758c8f1c95SAlan Cox }
768c8f1c95SAlan Cox
7714e92dd1SThomas Zimmermann /* Acquires GTT mutex internally. */
psb_gtt_insert_pages(struct drm_psb_private * pdev,const struct resource * res,struct page ** pages)78e1f80341SThomas Zimmermann void psb_gtt_insert_pages(struct drm_psb_private *pdev, const struct resource *res,
79e1f80341SThomas Zimmermann struct page **pages)
808c8f1c95SAlan Cox {
81e1f80341SThomas Zimmermann resource_size_t npages, i;
82eab37607SKirill A. Shutemov u32 __iomem *gtt_slot;
83eab37607SKirill A. Shutemov u32 pte;
848c8f1c95SAlan Cox
8514e92dd1SThomas Zimmermann mutex_lock(&pdev->gtt_mutex);
8614e92dd1SThomas Zimmermann
878c8f1c95SAlan Cox /* Write our page entries into the GTT itself */
88e1f80341SThomas Zimmermann
89e1f80341SThomas Zimmermann npages = resource_size(res) >> PAGE_SHIFT;
90e1f80341SThomas Zimmermann gtt_slot = psb_gtt_entry(pdev, res);
91e1f80341SThomas Zimmermann
92e1f80341SThomas Zimmermann for (i = 0; i < npages; ++i, ++gtt_slot) {
93e1f80341SThomas Zimmermann pte = psb_gtt_mask_pte(page_to_pfn(pages[i]), PSB_MMU_CACHED_MEMORY);
94e1f80341SThomas Zimmermann iowrite32(pte, gtt_slot);
95a6ba582dSAlan Cox }
96ebc7d647SPatrik Jakobsson
978c8f1c95SAlan Cox /* Make sure all the entries are set before we return */
988c8f1c95SAlan Cox ioread32(gtt_slot - 1);
9914e92dd1SThomas Zimmermann
10014e92dd1SThomas Zimmermann mutex_unlock(&pdev->gtt_mutex);
1018c8f1c95SAlan Cox }
1028c8f1c95SAlan Cox
10314e92dd1SThomas Zimmermann /* Acquires GTT mutex internally. */
psb_gtt_remove_pages(struct drm_psb_private * pdev,const struct resource * res)104e1f80341SThomas Zimmermann void psb_gtt_remove_pages(struct drm_psb_private *pdev, const struct resource *res)
1058c8f1c95SAlan Cox {
106e1f80341SThomas Zimmermann resource_size_t npages, i;
107eab37607SKirill A. Shutemov u32 __iomem *gtt_slot;
108eab37607SKirill A. Shutemov u32 pte;
1098c8f1c95SAlan Cox
11014e92dd1SThomas Zimmermann mutex_lock(&pdev->gtt_mutex);
11114e92dd1SThomas Zimmermann
112e1f80341SThomas Zimmermann /* Install scratch page for the resource */
1138c8f1c95SAlan Cox
114e1f80341SThomas Zimmermann pte = psb_gtt_mask_pte(page_to_pfn(pdev->scratch_page), PSB_MMU_CACHED_MEMORY);
1158c8f1c95SAlan Cox
116e1f80341SThomas Zimmermann npages = resource_size(res) >> PAGE_SHIFT;
117e1f80341SThomas Zimmermann gtt_slot = psb_gtt_entry(pdev, res);
118e1f80341SThomas Zimmermann
119e1f80341SThomas Zimmermann for (i = 0; i < npages; ++i, ++gtt_slot)
120e1f80341SThomas Zimmermann iowrite32(pte, gtt_slot);
121e1f80341SThomas Zimmermann
122e1f80341SThomas Zimmermann /* Make sure all the entries are set before we return */
1238c8f1c95SAlan Cox ioread32(gtt_slot - 1);
12414e92dd1SThomas Zimmermann
12514e92dd1SThomas Zimmermann mutex_unlock(&pdev->gtt_mutex);
1268c8f1c95SAlan Cox }
1278c8f1c95SAlan Cox
psb_gtt_enable(struct drm_psb_private * dev_priv)1285169f359SThomas Zimmermann static int psb_gtt_enable(struct drm_psb_private *dev_priv)
1298c8f1c95SAlan Cox {
1305169f359SThomas Zimmermann struct drm_device *dev = &dev_priv->dev;
131a2c68495SThomas Zimmermann struct pci_dev *pdev = to_pci_dev(dev->dev);
1325169f359SThomas Zimmermann int ret;
1338c8f1c95SAlan Cox
1345169f359SThomas Zimmermann ret = pci_read_config_word(pdev, PSB_GMCH_CTRL, &dev_priv->gmch_ctrl);
1355169f359SThomas Zimmermann if (ret)
1365169f359SThomas Zimmermann return pcibios_err_to_errno(ret);
1375169f359SThomas Zimmermann ret = pci_write_config_word(pdev, PSB_GMCH_CTRL, dev_priv->gmch_ctrl | _PSB_GMCH_ENABLED);
1385169f359SThomas Zimmermann if (ret)
1395169f359SThomas Zimmermann return pcibios_err_to_errno(ret);
1405169f359SThomas Zimmermann
1415169f359SThomas Zimmermann dev_priv->pge_ctl = PSB_RVDC32(PSB_PGETBL_CTL);
1425169f359SThomas Zimmermann PSB_WVDC32(dev_priv->pge_ctl | _PSB_PGETBL_ENABLED, PSB_PGETBL_CTL);
1435169f359SThomas Zimmermann
1445169f359SThomas Zimmermann (void)PSB_RVDC32(PSB_PGETBL_CTL);
1455169f359SThomas Zimmermann
1465169f359SThomas Zimmermann return 0;
1475169f359SThomas Zimmermann }
1485169f359SThomas Zimmermann
psb_gtt_disable(struct drm_psb_private * dev_priv)1495169f359SThomas Zimmermann static void psb_gtt_disable(struct drm_psb_private *dev_priv)
1505169f359SThomas Zimmermann {
1515169f359SThomas Zimmermann struct drm_device *dev = &dev_priv->dev;
1525169f359SThomas Zimmermann struct pci_dev *pdev = to_pci_dev(dev->dev);
1536069fd81SThomas Zimmermann
1546069fd81SThomas Zimmermann pci_write_config_word(pdev, PSB_GMCH_CTRL, dev_priv->gmch_ctrl);
1558c8f1c95SAlan Cox PSB_WVDC32(dev_priv->pge_ctl, PSB_PGETBL_CTL);
1566069fd81SThomas Zimmermann
1575169f359SThomas Zimmermann (void)PSB_RVDC32(PSB_PGETBL_CTL);
1585169f359SThomas Zimmermann }
1595169f359SThomas Zimmermann
psb_gtt_fini(struct drm_device * dev)1605169f359SThomas Zimmermann void psb_gtt_fini(struct drm_device *dev)
1615169f359SThomas Zimmermann {
1625169f359SThomas Zimmermann struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
1635169f359SThomas Zimmermann
1645169f359SThomas Zimmermann iounmap(dev_priv->gtt_map);
1655169f359SThomas Zimmermann psb_gtt_disable(dev_priv);
1666069fd81SThomas Zimmermann mutex_destroy(&dev_priv->gtt_mutex);
1678c8f1c95SAlan Cox }
1688c8f1c95SAlan Cox
16960a78f9eSThomas Zimmermann /* Clear GTT. Use a scratch page to avoid accidents or scribbles. */
psb_gtt_clear(struct drm_psb_private * pdev)17060a78f9eSThomas Zimmermann static void psb_gtt_clear(struct drm_psb_private *pdev)
17160a78f9eSThomas Zimmermann {
17260a78f9eSThomas Zimmermann resource_size_t pfn_base;
17360a78f9eSThomas Zimmermann unsigned long i;
17460a78f9eSThomas Zimmermann uint32_t pte;
17560a78f9eSThomas Zimmermann
17660a78f9eSThomas Zimmermann pfn_base = page_to_pfn(pdev->scratch_page);
17760a78f9eSThomas Zimmermann pte = psb_gtt_mask_pte(pfn_base, PSB_MMU_CACHED_MEMORY);
17860a78f9eSThomas Zimmermann
17960a78f9eSThomas Zimmermann for (i = 0; i < pdev->gtt.gtt_pages; ++i)
18060a78f9eSThomas Zimmermann iowrite32(pte, pdev->gtt_map + i);
18160a78f9eSThomas Zimmermann
18260a78f9eSThomas Zimmermann (void)ioread32(pdev->gtt_map + i - 1);
18360a78f9eSThomas Zimmermann }
18460a78f9eSThomas Zimmermann
psb_gtt_init_ranges(struct drm_psb_private * dev_priv)185*07739597SThomas Zimmermann static void psb_gtt_init_ranges(struct drm_psb_private *dev_priv)
186*07739597SThomas Zimmermann {
187*07739597SThomas Zimmermann struct drm_device *dev = &dev_priv->dev;
188*07739597SThomas Zimmermann struct pci_dev *pdev = to_pci_dev(dev->dev);
189*07739597SThomas Zimmermann struct psb_gtt *pg = &dev_priv->gtt;
190*07739597SThomas Zimmermann resource_size_t gtt_phys_start, mmu_gatt_start, gtt_start, gtt_pages,
191*07739597SThomas Zimmermann gatt_start, gatt_pages;
192*07739597SThomas Zimmermann struct resource *gtt_mem;
193*07739597SThomas Zimmermann
194*07739597SThomas Zimmermann /* The root resource we allocate address space from */
195*07739597SThomas Zimmermann gtt_phys_start = dev_priv->pge_ctl & PAGE_MASK;
196*07739597SThomas Zimmermann
197*07739597SThomas Zimmermann /*
198*07739597SThomas Zimmermann * The video MMU has a HW bug when accessing 0x0d0000000. Make
199*07739597SThomas Zimmermann * GATT start at 0x0e0000000. This doesn't actually matter for
200*07739597SThomas Zimmermann * us now, but maybe will if the video acceleration ever gets
201*07739597SThomas Zimmermann * opened up.
202*07739597SThomas Zimmermann */
203*07739597SThomas Zimmermann mmu_gatt_start = 0xe0000000;
204*07739597SThomas Zimmermann
205*07739597SThomas Zimmermann gtt_start = pci_resource_start(pdev, PSB_GTT_RESOURCE);
206*07739597SThomas Zimmermann gtt_pages = pci_resource_len(pdev, PSB_GTT_RESOURCE) >> PAGE_SHIFT;
207*07739597SThomas Zimmermann
208*07739597SThomas Zimmermann /* CDV doesn't report this. In which case the system has 64 gtt pages */
209*07739597SThomas Zimmermann if (!gtt_start || !gtt_pages) {
210*07739597SThomas Zimmermann dev_dbg(dev->dev, "GTT PCI BAR not initialized.\n");
211*07739597SThomas Zimmermann gtt_pages = 64;
212*07739597SThomas Zimmermann gtt_start = dev_priv->pge_ctl;
213*07739597SThomas Zimmermann }
214*07739597SThomas Zimmermann
215*07739597SThomas Zimmermann gatt_start = pci_resource_start(pdev, PSB_GATT_RESOURCE);
216*07739597SThomas Zimmermann gatt_pages = pci_resource_len(pdev, PSB_GATT_RESOURCE) >> PAGE_SHIFT;
217*07739597SThomas Zimmermann
218*07739597SThomas Zimmermann if (!gatt_pages || !gatt_start) {
219*07739597SThomas Zimmermann static struct resource fudge; /* Preferably peppermint */
220*07739597SThomas Zimmermann
221*07739597SThomas Zimmermann /*
222*07739597SThomas Zimmermann * This can occur on CDV systems. Fudge it in this case. We
223*07739597SThomas Zimmermann * really don't care what imaginary space is being allocated
224*07739597SThomas Zimmermann * at this point.
225*07739597SThomas Zimmermann */
226*07739597SThomas Zimmermann dev_dbg(dev->dev, "GATT PCI BAR not initialized.\n");
227*07739597SThomas Zimmermann gatt_start = 0x40000000;
228*07739597SThomas Zimmermann gatt_pages = (128 * 1024 * 1024) >> PAGE_SHIFT;
229*07739597SThomas Zimmermann
230*07739597SThomas Zimmermann /*
231*07739597SThomas Zimmermann * This is a little confusing but in fact the GTT is providing
232*07739597SThomas Zimmermann * a view from the GPU into memory and not vice versa. As such
233*07739597SThomas Zimmermann * this is really allocating space that is not the same as the
234*07739597SThomas Zimmermann * CPU address space on CDV.
235*07739597SThomas Zimmermann */
236*07739597SThomas Zimmermann fudge.start = 0x40000000;
237*07739597SThomas Zimmermann fudge.end = 0x40000000 + 128 * 1024 * 1024 - 1;
238*07739597SThomas Zimmermann fudge.name = "fudge";
239*07739597SThomas Zimmermann fudge.flags = IORESOURCE_MEM;
240*07739597SThomas Zimmermann
241*07739597SThomas Zimmermann gtt_mem = &fudge;
242*07739597SThomas Zimmermann } else {
243*07739597SThomas Zimmermann gtt_mem = &pdev->resource[PSB_GATT_RESOURCE];
244*07739597SThomas Zimmermann }
245*07739597SThomas Zimmermann
246*07739597SThomas Zimmermann pg->gtt_phys_start = gtt_phys_start;
247*07739597SThomas Zimmermann pg->mmu_gatt_start = mmu_gatt_start;
248*07739597SThomas Zimmermann pg->gtt_start = gtt_start;
249*07739597SThomas Zimmermann pg->gtt_pages = gtt_pages;
250*07739597SThomas Zimmermann pg->gatt_start = gatt_start;
251*07739597SThomas Zimmermann pg->gatt_pages = gatt_pages;
252*07739597SThomas Zimmermann dev_priv->gtt_mem = gtt_mem;
253*07739597SThomas Zimmermann }
254*07739597SThomas Zimmermann
psb_gtt_init(struct drm_device * dev)25597bd66c4SThomas Zimmermann int psb_gtt_init(struct drm_device *dev)
2568c8f1c95SAlan Cox {
257f71635e8SThomas Zimmermann struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
2585169f359SThomas Zimmermann struct psb_gtt *pg = &dev_priv->gtt;
2595169f359SThomas Zimmermann int ret;
2608c8f1c95SAlan Cox
2618c8f1c95SAlan Cox mutex_init(&dev_priv->gtt_mutex);
262070839eaSPatrik Jakobsson
2635169f359SThomas Zimmermann ret = psb_gtt_enable(dev_priv);
2645169f359SThomas Zimmermann if (ret)
2655169f359SThomas Zimmermann goto err_mutex_destroy;
2668c8f1c95SAlan Cox
267*07739597SThomas Zimmermann psb_gtt_init_ranges(dev_priv);
2688c8f1c95SAlan Cox
269*07739597SThomas Zimmermann dev_priv->gtt_map = ioremap(pg->gtt_phys_start, pg->gtt_pages << PAGE_SHIFT);
2708c8f1c95SAlan Cox if (!dev_priv->gtt_map) {
2718c8f1c95SAlan Cox dev_err(dev->dev, "Failure to map gtt.\n");
2728c8f1c95SAlan Cox ret = -ENOMEM;
2735169f359SThomas Zimmermann goto err_psb_gtt_disable;
2748c8f1c95SAlan Cox }
2758c8f1c95SAlan Cox
27660a78f9eSThomas Zimmermann psb_gtt_clear(dev_priv);
2778c8f1c95SAlan Cox
2788c8f1c95SAlan Cox return 0;
2798c8f1c95SAlan Cox
2805169f359SThomas Zimmermann err_psb_gtt_disable:
2815169f359SThomas Zimmermann psb_gtt_disable(dev_priv);
2825169f359SThomas Zimmermann err_mutex_destroy:
2836069fd81SThomas Zimmermann mutex_destroy(&dev_priv->gtt_mutex);
2848c8f1c95SAlan Cox return ret;
2858c8f1c95SAlan Cox }
286070839eaSPatrik Jakobsson
psb_gtt_resume(struct drm_device * dev)28742ceddb6SThomas Zimmermann int psb_gtt_resume(struct drm_device *dev)
28897bd66c4SThomas Zimmermann {
28997bd66c4SThomas Zimmermann struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
2905169f359SThomas Zimmermann struct psb_gtt *pg = &dev_priv->gtt;
291*07739597SThomas Zimmermann unsigned int old_gtt_pages = pg->gtt_pages;
292d00f44ddSThomas Zimmermann int ret;
29397bd66c4SThomas Zimmermann
29497bd66c4SThomas Zimmermann /* Enable the GTT */
2955169f359SThomas Zimmermann ret = psb_gtt_enable(dev_priv);
2965169f359SThomas Zimmermann if (ret)
2975169f359SThomas Zimmermann return ret;
29897bd66c4SThomas Zimmermann
299*07739597SThomas Zimmermann psb_gtt_init_ranges(dev_priv);
30097bd66c4SThomas Zimmermann
301*07739597SThomas Zimmermann if (old_gtt_pages != pg->gtt_pages) {
30297bd66c4SThomas Zimmermann dev_err(dev->dev, "GTT resume error.\n");
303*07739597SThomas Zimmermann ret = -ENODEV;
3045169f359SThomas Zimmermann goto err_psb_gtt_disable;
30597bd66c4SThomas Zimmermann }
30697bd66c4SThomas Zimmermann
30797bd66c4SThomas Zimmermann psb_gtt_clear(dev_priv);
30897bd66c4SThomas Zimmermann
3095169f359SThomas Zimmermann err_psb_gtt_disable:
3105169f359SThomas Zimmermann psb_gtt_disable(dev_priv);
31197bd66c4SThomas Zimmermann return ret;
31297bd66c4SThomas Zimmermann }
313