1 /*
2  * SPDX-License-Identifier: MIT
3  *
4  * Copyright © 2016 Intel Corporation
5  */
6 
7 #include "i915_scatterlist.h"
8 #include "i915_ttm_buddy_manager.h"
9 
10 #include <drm/drm_buddy.h>
11 #include <drm/drm_mm.h>
12 
13 #include <linux/slab.h>
14 
15 bool i915_sg_trim(struct sg_table *orig_st)
16 {
17 	struct sg_table new_st;
18 	struct scatterlist *sg, *new_sg;
19 	unsigned int i;
20 
21 	if (orig_st->nents == orig_st->orig_nents)
22 		return false;
23 
24 	if (sg_alloc_table(&new_st, orig_st->nents, GFP_KERNEL | __GFP_NOWARN))
25 		return false;
26 
27 	new_sg = new_st.sgl;
28 	for_each_sg(orig_st->sgl, sg, orig_st->nents, i) {
29 		sg_set_page(new_sg, sg_page(sg), sg->length, 0);
30 		sg_dma_address(new_sg) = sg_dma_address(sg);
31 		sg_dma_len(new_sg) = sg_dma_len(sg);
32 
33 		new_sg = sg_next(new_sg);
34 	}
35 	GEM_BUG_ON(new_sg); /* Should walk exactly nents and hit the end */
36 
37 	sg_free_table(orig_st);
38 
39 	*orig_st = new_st;
40 	return true;
41 }
42 
43 static void i915_refct_sgt_release(struct kref *ref)
44 {
45 	struct i915_refct_sgt *rsgt =
46 		container_of(ref, typeof(*rsgt), kref);
47 
48 	sg_free_table(&rsgt->table);
49 	kfree(rsgt);
50 }
51 
52 static const struct i915_refct_sgt_ops rsgt_ops = {
53 	.release = i915_refct_sgt_release
54 };
55 
56 /**
57  * i915_refct_sgt_init - Initialize a struct i915_refct_sgt with default ops
58  * @rsgt: The struct i915_refct_sgt to initialize.
59  * size: The size of the underlying memory buffer.
60  */
61 void i915_refct_sgt_init(struct i915_refct_sgt *rsgt, size_t size)
62 {
63 	__i915_refct_sgt_init(rsgt, size, &rsgt_ops);
64 }
65 
66 /**
67  * i915_rsgt_from_mm_node - Create a refcounted sg_table from a struct
68  * drm_mm_node
69  * @node: The drm_mm_node.
70  * @region_start: An offset to add to the dma addresses of the sg list.
71  *
72  * Create a struct sg_table, initializing it from a struct drm_mm_node,
73  * taking a maximum segment length into account, splitting into segments
74  * if necessary.
75  *
76  * Return: A pointer to a kmalloced struct i915_refct_sgt on success, negative
77  * error code cast to an error pointer on failure.
78  */
79 struct i915_refct_sgt *i915_rsgt_from_mm_node(const struct drm_mm_node *node,
80 					      u64 region_start)
81 {
82 	const u64 max_segment = SZ_1G; /* Do we have a limit on this? */
83 	u64 segment_pages = max_segment >> PAGE_SHIFT;
84 	u64 block_size, offset, prev_end;
85 	struct i915_refct_sgt *rsgt;
86 	struct sg_table *st;
87 	struct scatterlist *sg;
88 
89 	rsgt = kmalloc(sizeof(*rsgt), GFP_KERNEL);
90 	if (!rsgt)
91 		return ERR_PTR(-ENOMEM);
92 
93 	i915_refct_sgt_init(rsgt, node->size << PAGE_SHIFT);
94 	st = &rsgt->table;
95 	if (sg_alloc_table(st, DIV_ROUND_UP(node->size, segment_pages),
96 			   GFP_KERNEL)) {
97 		i915_refct_sgt_put(rsgt);
98 		return ERR_PTR(-ENOMEM);
99 	}
100 
101 	sg = st->sgl;
102 	st->nents = 0;
103 	prev_end = (resource_size_t)-1;
104 	block_size = node->size << PAGE_SHIFT;
105 	offset = node->start << PAGE_SHIFT;
106 
107 	while (block_size) {
108 		u64 len;
109 
110 		if (offset != prev_end || sg->length >= max_segment) {
111 			if (st->nents)
112 				sg = __sg_next(sg);
113 
114 			sg_dma_address(sg) = region_start + offset;
115 			sg_dma_len(sg) = 0;
116 			sg->length = 0;
117 			st->nents++;
118 		}
119 
120 		len = min(block_size, max_segment - sg->length);
121 		sg->length += len;
122 		sg_dma_len(sg) += len;
123 
124 		offset += len;
125 		block_size -= len;
126 
127 		prev_end = offset;
128 	}
129 
130 	sg_mark_end(sg);
131 	i915_sg_trim(st);
132 
133 	return rsgt;
134 }
135 
136 /**
137  * i915_rsgt_from_buddy_resource - Create a refcounted sg_table from a struct
138  * i915_buddy_block list
139  * @res: The struct i915_ttm_buddy_resource.
140  * @region_start: An offset to add to the dma addresses of the sg list.
141  *
142  * Create a struct sg_table, initializing it from struct i915_buddy_block list,
143  * taking a maximum segment length into account, splitting into segments
144  * if necessary.
145  *
146  * Return: A pointer to a kmalloced struct i915_refct_sgts on success, negative
147  * error code cast to an error pointer on failure.
148  */
149 struct i915_refct_sgt *i915_rsgt_from_buddy_resource(struct ttm_resource *res,
150 						     u64 region_start)
151 {
152 	struct i915_ttm_buddy_resource *bman_res = to_ttm_buddy_resource(res);
153 	const u64 size = res->num_pages << PAGE_SHIFT;
154 	const u64 max_segment = rounddown(UINT_MAX, PAGE_SIZE);
155 	struct drm_buddy *mm = bman_res->mm;
156 	struct list_head *blocks = &bman_res->blocks;
157 	struct drm_buddy_block *block;
158 	struct i915_refct_sgt *rsgt;
159 	struct scatterlist *sg;
160 	struct sg_table *st;
161 	resource_size_t prev_end;
162 
163 	GEM_BUG_ON(list_empty(blocks));
164 
165 	rsgt = kmalloc(sizeof(*rsgt), GFP_KERNEL);
166 	if (!rsgt)
167 		return ERR_PTR(-ENOMEM);
168 
169 	i915_refct_sgt_init(rsgt, size);
170 	st = &rsgt->table;
171 	if (sg_alloc_table(st, res->num_pages, GFP_KERNEL)) {
172 		i915_refct_sgt_put(rsgt);
173 		return ERR_PTR(-ENOMEM);
174 	}
175 
176 	sg = st->sgl;
177 	st->nents = 0;
178 	prev_end = (resource_size_t)-1;
179 
180 	list_for_each_entry(block, blocks, link) {
181 		u64 block_size, offset;
182 
183 		block_size = min_t(u64, size, drm_buddy_block_size(mm, block));
184 		offset = drm_buddy_block_offset(block);
185 
186 		while (block_size) {
187 			u64 len;
188 
189 			if (offset != prev_end || sg->length >= max_segment) {
190 				if (st->nents)
191 					sg = __sg_next(sg);
192 
193 				sg_dma_address(sg) = region_start + offset;
194 				sg_dma_len(sg) = 0;
195 				sg->length = 0;
196 				st->nents++;
197 			}
198 
199 			len = min(block_size, max_segment - sg->length);
200 			sg->length += len;
201 			sg_dma_len(sg) += len;
202 
203 			offset += len;
204 			block_size -= len;
205 
206 			prev_end = offset;
207 		}
208 	}
209 
210 	sg_mark_end(sg);
211 	i915_sg_trim(st);
212 
213 	return rsgt;
214 }
215 
216 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
217 #include "selftests/scatterlist.c"
218 #endif
219