1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2019 Intel Corporation
4  */
5 
6 #include "intel_memory_region.h"
7 #include "intel_region_ttm.h"
8 #include "gem/i915_gem_region.h"
9 #include "gem/i915_gem_lmem.h"
10 #include "i915_drv.h"
11 
12 static void lmem_put_pages(struct drm_i915_gem_object *obj,
13 			   struct sg_table *pages)
14 {
15 	intel_region_ttm_node_free(obj->mm.region, obj->mm.st_mm_node);
16 	obj->mm.dirty = false;
17 	sg_free_table(pages);
18 	kfree(pages);
19 }
20 
21 static int lmem_get_pages(struct drm_i915_gem_object *obj)
22 {
23 	unsigned int flags;
24 	struct sg_table *pages;
25 
26 	flags = I915_ALLOC_MIN_PAGE_SIZE;
27 	if (obj->flags & I915_BO_ALLOC_CONTIGUOUS)
28 		flags |= I915_ALLOC_CONTIGUOUS;
29 
30 	obj->mm.st_mm_node = intel_region_ttm_node_alloc(obj->mm.region,
31 							 obj->base.size,
32 							 flags);
33 	if (IS_ERR(obj->mm.st_mm_node))
34 		return PTR_ERR(obj->mm.st_mm_node);
35 
36 	/* Range manager is always contigous */
37 	if (obj->mm.region->is_range_manager)
38 		obj->flags |= I915_BO_ALLOC_CONTIGUOUS;
39 	pages = intel_region_ttm_node_to_st(obj->mm.region, obj->mm.st_mm_node);
40 	if (IS_ERR(pages)) {
41 		intel_region_ttm_node_free(obj->mm.region, obj->mm.st_mm_node);
42 		return PTR_ERR(pages);
43 	}
44 
45 	__i915_gem_object_set_pages(obj, pages, i915_sg_dma_sizes(pages->sgl));
46 
47 	if (obj->flags & I915_BO_ALLOC_CPU_CLEAR) {
48 		void __iomem *vaddr =
49 			i915_gem_object_lmem_io_map(obj, 0, obj->base.size);
50 
51 		if (!vaddr) {
52 			struct sg_table *pages =
53 				__i915_gem_object_unset_pages(obj);
54 
55 			if (!IS_ERR_OR_NULL(pages))
56 				lmem_put_pages(obj, pages);
57 		}
58 
59 		memset_io(vaddr, 0, obj->base.size);
60 		io_mapping_unmap(vaddr);
61 	}
62 
63 	return 0;
64 }
65 
66 const struct drm_i915_gem_object_ops i915_gem_lmem_obj_ops = {
67 	.name = "i915_gem_object_lmem",
68 	.flags = I915_GEM_OBJECT_HAS_IOMEM,
69 
70 	.get_pages = lmem_get_pages,
71 	.put_pages = lmem_put_pages,
72 	.release = i915_gem_object_release_memory_region,
73 };
74 
75 void __iomem *
76 i915_gem_object_lmem_io_map(struct drm_i915_gem_object *obj,
77 			    unsigned long n,
78 			    unsigned long size)
79 {
80 	resource_size_t offset;
81 
82 	GEM_BUG_ON(!i915_gem_object_is_contiguous(obj));
83 
84 	offset = i915_gem_object_get_dma_address(obj, n);
85 	offset -= obj->mm.region->region.start;
86 
87 	return io_mapping_map_wc(&obj->mm.region->iomap, offset, size);
88 }
89 
90 bool i915_gem_object_is_lmem(struct drm_i915_gem_object *obj)
91 {
92 	struct intel_memory_region *mr = obj->mm.region;
93 
94 	return mr && (mr->type == INTEL_MEMORY_LOCAL ||
95 		      mr->type == INTEL_MEMORY_STOLEN_LOCAL);
96 }
97 
98 struct drm_i915_gem_object *
99 i915_gem_object_create_lmem(struct drm_i915_private *i915,
100 			    resource_size_t size,
101 			    unsigned int flags)
102 {
103 	return i915_gem_object_create_region(i915->mm.regions[INTEL_REGION_LMEM],
104 					     size, flags);
105 }
106 
107 int __i915_gem_lmem_object_init(struct intel_memory_region *mem,
108 				struct drm_i915_gem_object *obj,
109 				resource_size_t size,
110 				unsigned int flags)
111 {
112 	static struct lock_class_key lock_class;
113 	struct drm_i915_private *i915 = mem->i915;
114 
115 	drm_gem_private_object_init(&i915->drm, &obj->base, size);
116 	i915_gem_object_init(obj, &i915_gem_lmem_obj_ops, &lock_class, flags);
117 
118 	obj->read_domains = I915_GEM_DOMAIN_WC | I915_GEM_DOMAIN_GTT;
119 
120 	i915_gem_object_set_cache_coherency(obj, I915_CACHE_NONE);
121 
122 	i915_gem_object_init_memory_region(obj, mem);
123 
124 	return 0;
125 }
126