110be98a7SChris Wilson /*
210be98a7SChris Wilson * SPDX-License-Identifier: MIT
310be98a7SChris Wilson *
410be98a7SChris Wilson * Copyright © 2016 Intel Corporation
510be98a7SChris Wilson */
610be98a7SChris Wilson
737d63f8fSChris Wilson #include "i915_scatterlist.h"
837d63f8fSChris Wilson
910be98a7SChris Wilson #include "huge_gem_object.h"
1010be98a7SChris Wilson
huge_free_pages(struct drm_i915_gem_object * obj,struct sg_table * pages)1110be98a7SChris Wilson static void huge_free_pages(struct drm_i915_gem_object *obj,
1210be98a7SChris Wilson struct sg_table *pages)
1310be98a7SChris Wilson {
1410be98a7SChris Wilson unsigned long nreal = obj->scratch / PAGE_SIZE;
1569305732SChris Wilson struct sgt_iter sgt_iter;
1669305732SChris Wilson struct page *page;
1710be98a7SChris Wilson
1869305732SChris Wilson for_each_sgt_page(page, sgt_iter, pages) {
1969305732SChris Wilson __free_page(page);
2069305732SChris Wilson if (!--nreal)
2169305732SChris Wilson break;
2269305732SChris Wilson }
2310be98a7SChris Wilson
2410be98a7SChris Wilson sg_free_table(pages);
2510be98a7SChris Wilson kfree(pages);
2610be98a7SChris Wilson }
2710be98a7SChris Wilson
huge_get_pages(struct drm_i915_gem_object * obj)2810be98a7SChris Wilson static int huge_get_pages(struct drm_i915_gem_object *obj)
2910be98a7SChris Wilson {
308c1f2127SChris Wilson #define GFP (GFP_KERNEL | __GFP_NOWARN | __GFP_RETRY_MAYFAIL)
3110be98a7SChris Wilson const unsigned long nreal = obj->scratch / PAGE_SIZE;
32*c3bfba9aSChris Wilson unsigned int npages; /* restricted by sg_alloc_table */
3310be98a7SChris Wilson struct scatterlist *sg, *src, *end;
3410be98a7SChris Wilson struct sg_table *pages;
3510be98a7SChris Wilson unsigned long n;
3610be98a7SChris Wilson
37*c3bfba9aSChris Wilson if (overflows_type(obj->base.size / PAGE_SIZE, npages))
38*c3bfba9aSChris Wilson return -E2BIG;
39*c3bfba9aSChris Wilson
40*c3bfba9aSChris Wilson npages = obj->base.size / PAGE_SIZE;
4110be98a7SChris Wilson pages = kmalloc(sizeof(*pages), GFP);
4210be98a7SChris Wilson if (!pages)
4310be98a7SChris Wilson return -ENOMEM;
4410be98a7SChris Wilson
4510be98a7SChris Wilson if (sg_alloc_table(pages, npages, GFP)) {
4610be98a7SChris Wilson kfree(pages);
4710be98a7SChris Wilson return -ENOMEM;
4810be98a7SChris Wilson }
4910be98a7SChris Wilson
5010be98a7SChris Wilson sg = pages->sgl;
5110be98a7SChris Wilson for (n = 0; n < nreal; n++) {
5210be98a7SChris Wilson struct page *page;
5310be98a7SChris Wilson
5410be98a7SChris Wilson page = alloc_page(GFP | __GFP_HIGHMEM);
5510be98a7SChris Wilson if (!page) {
5610be98a7SChris Wilson sg_mark_end(sg);
5710be98a7SChris Wilson goto err;
5810be98a7SChris Wilson }
5910be98a7SChris Wilson
6010be98a7SChris Wilson sg_set_page(sg, page, PAGE_SIZE, 0);
6110be98a7SChris Wilson sg = __sg_next(sg);
6210be98a7SChris Wilson }
6310be98a7SChris Wilson if (nreal < npages) {
6410be98a7SChris Wilson for (end = sg, src = pages->sgl; sg; sg = __sg_next(sg)) {
6510be98a7SChris Wilson sg_set_page(sg, sg_page(src), PAGE_SIZE, 0);
6610be98a7SChris Wilson src = __sg_next(src);
6710be98a7SChris Wilson if (src == end)
6810be98a7SChris Wilson src = pages->sgl;
6910be98a7SChris Wilson }
7010be98a7SChris Wilson }
7110be98a7SChris Wilson
7210be98a7SChris Wilson if (i915_gem_gtt_prepare_pages(obj, pages))
7310be98a7SChris Wilson goto err;
7410be98a7SChris Wilson
758c949515SMatthew Auld __i915_gem_object_set_pages(obj, pages);
7610be98a7SChris Wilson
7710be98a7SChris Wilson return 0;
7810be98a7SChris Wilson
7910be98a7SChris Wilson err:
8010be98a7SChris Wilson huge_free_pages(obj, pages);
8110be98a7SChris Wilson return -ENOMEM;
8210be98a7SChris Wilson #undef GFP
8310be98a7SChris Wilson }
8410be98a7SChris Wilson
huge_put_pages(struct drm_i915_gem_object * obj,struct sg_table * pages)8510be98a7SChris Wilson static void huge_put_pages(struct drm_i915_gem_object *obj,
8610be98a7SChris Wilson struct sg_table *pages)
8710be98a7SChris Wilson {
8810be98a7SChris Wilson i915_gem_gtt_finish_pages(obj, pages);
8910be98a7SChris Wilson huge_free_pages(obj, pages);
9010be98a7SChris Wilson
9110be98a7SChris Wilson obj->mm.dirty = false;
9210be98a7SChris Wilson }
9310be98a7SChris Wilson
9410be98a7SChris Wilson static const struct drm_i915_gem_object_ops huge_ops = {
957d192daaSChris Wilson .name = "huge-gem",
9610be98a7SChris Wilson .get_pages = huge_get_pages,
9710be98a7SChris Wilson .put_pages = huge_put_pages,
9810be98a7SChris Wilson };
9910be98a7SChris Wilson
10010be98a7SChris Wilson struct drm_i915_gem_object *
huge_gem_object(struct drm_i915_private * i915,phys_addr_t phys_size,dma_addr_t dma_size)10110be98a7SChris Wilson huge_gem_object(struct drm_i915_private *i915,
10210be98a7SChris Wilson phys_addr_t phys_size,
10310be98a7SChris Wilson dma_addr_t dma_size)
10410be98a7SChris Wilson {
1057867d709SChris Wilson static struct lock_class_key lock_class;
10610be98a7SChris Wilson struct drm_i915_gem_object *obj;
10710be98a7SChris Wilson unsigned int cache_level;
10810be98a7SChris Wilson
10910be98a7SChris Wilson GEM_BUG_ON(!phys_size || phys_size > dma_size);
11010be98a7SChris Wilson GEM_BUG_ON(!IS_ALIGNED(phys_size, PAGE_SIZE));
11110be98a7SChris Wilson GEM_BUG_ON(!IS_ALIGNED(dma_size, I915_GTT_PAGE_SIZE));
11210be98a7SChris Wilson
11310be98a7SChris Wilson if (overflows_type(dma_size, obj->base.size))
11410be98a7SChris Wilson return ERR_PTR(-E2BIG);
11510be98a7SChris Wilson
11610be98a7SChris Wilson obj = i915_gem_object_alloc();
11710be98a7SChris Wilson if (!obj)
11810be98a7SChris Wilson return ERR_PTR(-ENOMEM);
11910be98a7SChris Wilson
12010be98a7SChris Wilson drm_gem_private_object_init(&i915->drm, &obj->base, dma_size);
1210ff37575SThomas Hellström i915_gem_object_init(obj, &huge_ops, &lock_class, 0);
1220ff37575SThomas Hellström obj->mem_flags |= I915_BO_FLAG_STRUCT_PAGE;
12310be98a7SChris Wilson
12410be98a7SChris Wilson obj->read_domains = I915_GEM_DOMAIN_CPU;
12510be98a7SChris Wilson obj->write_domain = I915_GEM_DOMAIN_CPU;
12610be98a7SChris Wilson cache_level = HAS_LLC(i915) ? I915_CACHE_LLC : I915_CACHE_NONE;
12710be98a7SChris Wilson i915_gem_object_set_cache_coherency(obj, cache_level);
12810be98a7SChris Wilson obj->scratch = phys_size;
12910be98a7SChris Wilson
13010be98a7SChris Wilson return obj;
13110be98a7SChris Wilson }
132