1f033428dSChris Wilson /* 2f033428dSChris Wilson * SPDX-License-Identifier: MIT 3f033428dSChris Wilson * 4f033428dSChris Wilson * Copyright © 2014-2016 Intel Corporation 5f033428dSChris Wilson */ 6f033428dSChris Wilson 7f033428dSChris Wilson #include <linux/highmem.h> 8f033428dSChris Wilson #include <linux/shmem_fs.h> 9f033428dSChris Wilson #include <linux/swap.h> 10f033428dSChris Wilson 11f033428dSChris Wilson #include <drm/drm.h> /* for drm_legacy.h! */ 12f033428dSChris Wilson #include <drm/drm_cache.h> 13f033428dSChris Wilson #include <drm/drm_legacy.h> /* for drm_pci.h! */ 14f033428dSChris Wilson #include <drm/drm_pci.h> 15f033428dSChris Wilson 16f033428dSChris Wilson #include "i915_drv.h" 17f033428dSChris Wilson #include "i915_gem_object.h" 18f033428dSChris Wilson 19f033428dSChris Wilson static int i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj) 20f033428dSChris Wilson { 21f033428dSChris Wilson struct address_space *mapping = obj->base.filp->f_mapping; 22f033428dSChris Wilson struct drm_dma_handle *phys; 23f033428dSChris Wilson struct sg_table *st; 24f033428dSChris Wilson struct scatterlist *sg; 25f033428dSChris Wilson char *vaddr; 26f033428dSChris Wilson int i; 27f033428dSChris Wilson int err; 28f033428dSChris Wilson 29f033428dSChris Wilson if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj))) 30f033428dSChris Wilson return -EINVAL; 31f033428dSChris Wilson 32f033428dSChris Wilson /* Always aligning to the object size, allows a single allocation 33f033428dSChris Wilson * to handle all possible callers, and given typical object sizes, 34f033428dSChris Wilson * the alignment of the buddy allocation will naturally match. 35f033428dSChris Wilson */ 36f033428dSChris Wilson phys = drm_pci_alloc(obj->base.dev, 37f033428dSChris Wilson roundup_pow_of_two(obj->base.size), 38f033428dSChris Wilson roundup_pow_of_two(obj->base.size)); 39f033428dSChris Wilson if (!phys) 40f033428dSChris Wilson return -ENOMEM; 41f033428dSChris Wilson 42f033428dSChris Wilson vaddr = phys->vaddr; 43f033428dSChris Wilson for (i = 0; i < obj->base.size / PAGE_SIZE; i++) { 44f033428dSChris Wilson struct page *page; 45f033428dSChris Wilson char *src; 46f033428dSChris Wilson 47f033428dSChris Wilson page = shmem_read_mapping_page(mapping, i); 48f033428dSChris Wilson if (IS_ERR(page)) { 49f033428dSChris Wilson err = PTR_ERR(page); 50f033428dSChris Wilson goto err_phys; 51f033428dSChris Wilson } 52f033428dSChris Wilson 53f033428dSChris Wilson src = kmap_atomic(page); 54f033428dSChris Wilson memcpy(vaddr, src, PAGE_SIZE); 55f033428dSChris Wilson drm_clflush_virt_range(vaddr, PAGE_SIZE); 56f033428dSChris Wilson kunmap_atomic(src); 57f033428dSChris Wilson 58f033428dSChris Wilson put_page(page); 59f033428dSChris Wilson vaddr += PAGE_SIZE; 60f033428dSChris Wilson } 61f033428dSChris Wilson 62f033428dSChris Wilson i915_gem_chipset_flush(to_i915(obj->base.dev)); 63f033428dSChris Wilson 64f033428dSChris Wilson st = kmalloc(sizeof(*st), GFP_KERNEL); 65f033428dSChris Wilson if (!st) { 66f033428dSChris Wilson err = -ENOMEM; 67f033428dSChris Wilson goto err_phys; 68f033428dSChris Wilson } 69f033428dSChris Wilson 70f033428dSChris Wilson if (sg_alloc_table(st, 1, GFP_KERNEL)) { 71f033428dSChris Wilson kfree(st); 72f033428dSChris Wilson err = -ENOMEM; 73f033428dSChris Wilson goto err_phys; 74f033428dSChris Wilson } 75f033428dSChris Wilson 76f033428dSChris Wilson sg = st->sgl; 77f033428dSChris Wilson sg->offset = 0; 78f033428dSChris Wilson sg->length = obj->base.size; 79f033428dSChris Wilson 80f033428dSChris Wilson sg_dma_address(sg) = phys->busaddr; 81f033428dSChris Wilson sg_dma_len(sg) = obj->base.size; 82f033428dSChris Wilson 83f033428dSChris Wilson obj->phys_handle = phys; 84f033428dSChris Wilson 85f033428dSChris Wilson __i915_gem_object_set_pages(obj, st, sg->length); 86f033428dSChris Wilson 87f033428dSChris Wilson return 0; 88f033428dSChris Wilson 89f033428dSChris Wilson err_phys: 90f033428dSChris Wilson drm_pci_free(obj->base.dev, phys); 91f033428dSChris Wilson 92f033428dSChris Wilson return err; 93f033428dSChris Wilson } 94f033428dSChris Wilson 95f033428dSChris Wilson static void 96f033428dSChris Wilson i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj, 97f033428dSChris Wilson struct sg_table *pages) 98f033428dSChris Wilson { 99f033428dSChris Wilson __i915_gem_object_release_shmem(obj, pages, false); 100f033428dSChris Wilson 101f033428dSChris Wilson if (obj->mm.dirty) { 102f033428dSChris Wilson struct address_space *mapping = obj->base.filp->f_mapping; 103f033428dSChris Wilson char *vaddr = obj->phys_handle->vaddr; 104f033428dSChris Wilson int i; 105f033428dSChris Wilson 106f033428dSChris Wilson for (i = 0; i < obj->base.size / PAGE_SIZE; i++) { 107f033428dSChris Wilson struct page *page; 108f033428dSChris Wilson char *dst; 109f033428dSChris Wilson 110f033428dSChris Wilson page = shmem_read_mapping_page(mapping, i); 111f033428dSChris Wilson if (IS_ERR(page)) 112f033428dSChris Wilson continue; 113f033428dSChris Wilson 114f033428dSChris Wilson dst = kmap_atomic(page); 115f033428dSChris Wilson drm_clflush_virt_range(vaddr, PAGE_SIZE); 116f033428dSChris Wilson memcpy(dst, vaddr, PAGE_SIZE); 117f033428dSChris Wilson kunmap_atomic(dst); 118f033428dSChris Wilson 119f033428dSChris Wilson set_page_dirty(page); 120f033428dSChris Wilson if (obj->mm.madv == I915_MADV_WILLNEED) 121f033428dSChris Wilson mark_page_accessed(page); 122f033428dSChris Wilson put_page(page); 123f033428dSChris Wilson vaddr += PAGE_SIZE; 124f033428dSChris Wilson } 125f033428dSChris Wilson obj->mm.dirty = false; 126f033428dSChris Wilson } 127f033428dSChris Wilson 128f033428dSChris Wilson sg_free_table(pages); 129f033428dSChris Wilson kfree(pages); 130f033428dSChris Wilson 131f033428dSChris Wilson drm_pci_free(obj->base.dev, obj->phys_handle); 132f033428dSChris Wilson } 133f033428dSChris Wilson 134f033428dSChris Wilson static void 135f033428dSChris Wilson i915_gem_object_release_phys(struct drm_i915_gem_object *obj) 136f033428dSChris Wilson { 137f033428dSChris Wilson i915_gem_object_unpin_pages(obj); 138f033428dSChris Wilson } 139f033428dSChris Wilson 140f033428dSChris Wilson static const struct drm_i915_gem_object_ops i915_gem_phys_ops = { 141f033428dSChris Wilson .get_pages = i915_gem_object_get_pages_phys, 142f033428dSChris Wilson .put_pages = i915_gem_object_put_pages_phys, 143f033428dSChris Wilson .release = i915_gem_object_release_phys, 144f033428dSChris Wilson }; 145f033428dSChris Wilson 146f033428dSChris Wilson int i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, int align) 147f033428dSChris Wilson { 148f033428dSChris Wilson struct sg_table *pages; 149f033428dSChris Wilson int err; 150f033428dSChris Wilson 151f033428dSChris Wilson if (align > obj->base.size) 152f033428dSChris Wilson return -EINVAL; 153f033428dSChris Wilson 154f033428dSChris Wilson if (obj->ops == &i915_gem_phys_ops) 155f033428dSChris Wilson return 0; 156f033428dSChris Wilson 157f033428dSChris Wilson if (obj->ops != &i915_gem_shmem_ops) 158f033428dSChris Wilson return -EINVAL; 159f033428dSChris Wilson 160f033428dSChris Wilson err = i915_gem_object_unbind(obj); 161f033428dSChris Wilson if (err) 162f033428dSChris Wilson return err; 163f033428dSChris Wilson 164f033428dSChris Wilson mutex_lock(&obj->mm.lock); 165f033428dSChris Wilson 166f033428dSChris Wilson if (obj->mm.madv != I915_MADV_WILLNEED) { 167f033428dSChris Wilson err = -EFAULT; 168f033428dSChris Wilson goto err_unlock; 169f033428dSChris Wilson } 170f033428dSChris Wilson 171f033428dSChris Wilson if (obj->mm.quirked) { 172f033428dSChris Wilson err = -EFAULT; 173f033428dSChris Wilson goto err_unlock; 174f033428dSChris Wilson } 175f033428dSChris Wilson 176f033428dSChris Wilson if (obj->mm.mapping) { 177f033428dSChris Wilson err = -EBUSY; 178f033428dSChris Wilson goto err_unlock; 179f033428dSChris Wilson } 180f033428dSChris Wilson 181f033428dSChris Wilson pages = __i915_gem_object_unset_pages(obj); 182f033428dSChris Wilson 183f033428dSChris Wilson obj->ops = &i915_gem_phys_ops; 184f033428dSChris Wilson 185f033428dSChris Wilson err = ____i915_gem_object_get_pages(obj); 186f033428dSChris Wilson if (err) 187f033428dSChris Wilson goto err_xfer; 188f033428dSChris Wilson 189f033428dSChris Wilson /* Perma-pin (until release) the physical set of pages */ 190f033428dSChris Wilson __i915_gem_object_pin_pages(obj); 191f033428dSChris Wilson 192f033428dSChris Wilson if (!IS_ERR_OR_NULL(pages)) 193f033428dSChris Wilson i915_gem_shmem_ops.put_pages(obj, pages); 194f033428dSChris Wilson mutex_unlock(&obj->mm.lock); 195f033428dSChris Wilson return 0; 196f033428dSChris Wilson 197f033428dSChris Wilson err_xfer: 198f033428dSChris Wilson obj->ops = &i915_gem_shmem_ops; 199f033428dSChris Wilson if (!IS_ERR_OR_NULL(pages)) { 200f033428dSChris Wilson unsigned int sg_page_sizes = i915_sg_page_sizes(pages->sgl); 201f033428dSChris Wilson 202f033428dSChris Wilson __i915_gem_object_set_pages(obj, pages, sg_page_sizes); 203f033428dSChris Wilson } 204f033428dSChris Wilson err_unlock: 205f033428dSChris Wilson mutex_unlock(&obj->mm.lock); 206f033428dSChris Wilson return err; 207f033428dSChris Wilson } 208f033428dSChris Wilson 209f033428dSChris Wilson #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 210f033428dSChris Wilson #include "selftests/i915_gem_phys.c" 211f033428dSChris Wilson #endif 212