1 /*
2  * SPDX-License-Identifier: MIT
3  *
4  * Copyright © 2014-2016 Intel Corporation
5  */
6 
7 #include <drm/drm_cache.h>
8 
9 #include "gt/intel_gt.h"
10 #include "gt/intel_gt_pm.h"
11 
12 #include "i915_drv.h"
13 #include "i915_gem_object.h"
14 #include "i915_scatterlist.h"
15 #include "i915_gem_lmem.h"
16 #include "i915_gem_mman.h"
17 
18 void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj,
19 				 struct sg_table *pages,
20 				 unsigned int sg_page_sizes)
21 {
22 	struct drm_i915_private *i915 = to_i915(obj->base.dev);
23 	unsigned long supported = RUNTIME_INFO(i915)->page_sizes;
24 	bool shrinkable;
25 	int i;
26 
27 	assert_object_held_shared(obj);
28 
29 	if (i915_gem_object_is_volatile(obj))
30 		obj->mm.madv = I915_MADV_DONTNEED;
31 
32 	/* Make the pages coherent with the GPU (flushing any swapin). */
33 	if (obj->cache_dirty) {
34 		WARN_ON_ONCE(IS_DGFX(i915));
35 		obj->write_domain = 0;
36 		if (i915_gem_object_has_struct_page(obj))
37 			drm_clflush_sg(pages);
38 		obj->cache_dirty = false;
39 	}
40 
41 	obj->mm.get_page.sg_pos = pages->sgl;
42 	obj->mm.get_page.sg_idx = 0;
43 	obj->mm.get_dma_page.sg_pos = pages->sgl;
44 	obj->mm.get_dma_page.sg_idx = 0;
45 
46 	obj->mm.pages = pages;
47 
48 	GEM_BUG_ON(!sg_page_sizes);
49 	obj->mm.page_sizes.phys = sg_page_sizes;
50 
51 	/*
52 	 * Calculate the supported page-sizes which fit into the given
53 	 * sg_page_sizes. This will give us the page-sizes which we may be able
54 	 * to use opportunistically when later inserting into the GTT. For
55 	 * example if phys=2G, then in theory we should be able to use 1G, 2M,
56 	 * 64K or 4K pages, although in practice this will depend on a number of
57 	 * other factors.
58 	 */
59 	obj->mm.page_sizes.sg = 0;
60 	for_each_set_bit(i, &supported, ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) {
61 		if (obj->mm.page_sizes.phys & ~0u << i)
62 			obj->mm.page_sizes.sg |= BIT(i);
63 	}
64 	GEM_BUG_ON(!HAS_PAGE_SIZES(i915, obj->mm.page_sizes.sg));
65 
66 	shrinkable = i915_gem_object_is_shrinkable(obj);
67 
68 	if (i915_gem_object_is_tiled(obj) &&
69 	    i915->gem_quirks & GEM_QUIRK_PIN_SWIZZLED_PAGES) {
70 		GEM_BUG_ON(i915_gem_object_has_tiling_quirk(obj));
71 		i915_gem_object_set_tiling_quirk(obj);
72 		GEM_BUG_ON(!list_empty(&obj->mm.link));
73 		atomic_inc(&obj->mm.shrink_pin);
74 		shrinkable = false;
75 	}
76 
77 	if (shrinkable && !i915_gem_object_has_self_managed_shrink_list(obj)) {
78 		struct list_head *list;
79 		unsigned long flags;
80 
81 		assert_object_held(obj);
82 		spin_lock_irqsave(&i915->mm.obj_lock, flags);
83 
84 		i915->mm.shrink_count++;
85 		i915->mm.shrink_memory += obj->base.size;
86 
87 		if (obj->mm.madv != I915_MADV_WILLNEED)
88 			list = &i915->mm.purge_list;
89 		else
90 			list = &i915->mm.shrink_list;
91 		list_add_tail(&obj->mm.link, list);
92 
93 		atomic_set(&obj->mm.shrink_pin, 0);
94 		spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
95 	}
96 }
97 
98 int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
99 {
100 	struct drm_i915_private *i915 = to_i915(obj->base.dev);
101 	int err;
102 
103 	assert_object_held_shared(obj);
104 
105 	if (unlikely(obj->mm.madv != I915_MADV_WILLNEED)) {
106 		drm_dbg(&i915->drm,
107 			"Attempting to obtain a purgeable object\n");
108 		return -EFAULT;
109 	}
110 
111 	err = obj->ops->get_pages(obj);
112 	GEM_BUG_ON(!err && !i915_gem_object_has_pages(obj));
113 
114 	return err;
115 }
116 
117 /* Ensure that the associated pages are gathered from the backing storage
118  * and pinned into our object. i915_gem_object_pin_pages() may be called
119  * multiple times before they are released by a single call to
120  * i915_gem_object_unpin_pages() - once the pages are no longer referenced
121  * either as a result of memory pressure (reaping pages under the shrinker)
122  * or as the object is itself released.
123  */
124 int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
125 {
126 	int err;
127 
128 	assert_object_held(obj);
129 
130 	assert_object_held_shared(obj);
131 
132 	if (unlikely(!i915_gem_object_has_pages(obj))) {
133 		GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj));
134 
135 		err = ____i915_gem_object_get_pages(obj);
136 		if (err)
137 			return err;
138 
139 		smp_mb__before_atomic();
140 	}
141 	atomic_inc(&obj->mm.pages_pin_count);
142 
143 	return 0;
144 }
145 
146 int i915_gem_object_pin_pages_unlocked(struct drm_i915_gem_object *obj)
147 {
148 	struct i915_gem_ww_ctx ww;
149 	int err;
150 
151 	i915_gem_ww_ctx_init(&ww, true);
152 retry:
153 	err = i915_gem_object_lock(obj, &ww);
154 	if (!err)
155 		err = i915_gem_object_pin_pages(obj);
156 
157 	if (err == -EDEADLK) {
158 		err = i915_gem_ww_ctx_backoff(&ww);
159 		if (!err)
160 			goto retry;
161 	}
162 	i915_gem_ww_ctx_fini(&ww);
163 	return err;
164 }
165 
166 /* Immediately discard the backing storage */
167 int i915_gem_object_truncate(struct drm_i915_gem_object *obj)
168 {
169 	if (obj->ops->truncate)
170 		return obj->ops->truncate(obj);
171 
172 	return 0;
173 }
174 
175 static void __i915_gem_object_reset_page_iter(struct drm_i915_gem_object *obj)
176 {
177 	struct radix_tree_iter iter;
178 	void __rcu **slot;
179 
180 	rcu_read_lock();
181 	radix_tree_for_each_slot(slot, &obj->mm.get_page.radix, &iter, 0)
182 		radix_tree_delete(&obj->mm.get_page.radix, iter.index);
183 	radix_tree_for_each_slot(slot, &obj->mm.get_dma_page.radix, &iter, 0)
184 		radix_tree_delete(&obj->mm.get_dma_page.radix, iter.index);
185 	rcu_read_unlock();
186 }
187 
188 static void unmap_object(struct drm_i915_gem_object *obj, void *ptr)
189 {
190 	if (is_vmalloc_addr(ptr))
191 		vunmap(ptr);
192 }
193 
194 static void flush_tlb_invalidate(struct drm_i915_gem_object *obj)
195 {
196 	struct drm_i915_private *i915 = to_i915(obj->base.dev);
197 	struct intel_gt *gt = to_gt(i915);
198 
199 	if (!obj->mm.tlb)
200 		return;
201 
202 	intel_gt_invalidate_tlb(gt, obj->mm.tlb);
203 	obj->mm.tlb = 0;
204 }
205 
206 struct sg_table *
207 __i915_gem_object_unset_pages(struct drm_i915_gem_object *obj)
208 {
209 	struct sg_table *pages;
210 
211 	assert_object_held_shared(obj);
212 
213 	pages = fetch_and_zero(&obj->mm.pages);
214 	if (IS_ERR_OR_NULL(pages))
215 		return pages;
216 
217 	if (i915_gem_object_is_volatile(obj))
218 		obj->mm.madv = I915_MADV_WILLNEED;
219 
220 	if (!i915_gem_object_has_self_managed_shrink_list(obj))
221 		i915_gem_object_make_unshrinkable(obj);
222 
223 	if (obj->mm.mapping) {
224 		unmap_object(obj, page_mask_bits(obj->mm.mapping));
225 		obj->mm.mapping = NULL;
226 	}
227 
228 	__i915_gem_object_reset_page_iter(obj);
229 	obj->mm.page_sizes.phys = obj->mm.page_sizes.sg = 0;
230 
231 	flush_tlb_invalidate(obj);
232 
233 	return pages;
234 }
235 
236 int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
237 {
238 	struct sg_table *pages;
239 
240 	if (i915_gem_object_has_pinned_pages(obj))
241 		return -EBUSY;
242 
243 	/* May be called by shrinker from within get_pages() (on another bo) */
244 	assert_object_held_shared(obj);
245 
246 	i915_gem_object_release_mmap_offset(obj);
247 
248 	/*
249 	 * ->put_pages might need to allocate memory for the bit17 swizzle
250 	 * array, hence protect them from being reaped by removing them from gtt
251 	 * lists early.
252 	 */
253 	pages = __i915_gem_object_unset_pages(obj);
254 
255 	/*
256 	 * XXX Temporary hijinx to avoid updating all backends to handle
257 	 * NULL pages. In the future, when we have more asynchronous
258 	 * get_pages backends we should be better able to handle the
259 	 * cancellation of the async task in a more uniform manner.
260 	 */
261 	if (!IS_ERR_OR_NULL(pages))
262 		obj->ops->put_pages(obj, pages);
263 
264 	return 0;
265 }
266 
267 /* The 'mapping' part of i915_gem_object_pin_map() below */
268 static void *i915_gem_object_map_page(struct drm_i915_gem_object *obj,
269 				      enum i915_map_type type)
270 {
271 	unsigned long n_pages = obj->base.size >> PAGE_SHIFT, i;
272 	struct page *stack[32], **pages = stack, *page;
273 	struct sgt_iter iter;
274 	pgprot_t pgprot;
275 	void *vaddr;
276 
277 	switch (type) {
278 	default:
279 		MISSING_CASE(type);
280 		fallthrough;	/* to use PAGE_KERNEL anyway */
281 	case I915_MAP_WB:
282 		/*
283 		 * On 32b, highmem using a finite set of indirect PTE (i.e.
284 		 * vmap) to provide virtual mappings of the high pages.
285 		 * As these are finite, map_new_virtual() must wait for some
286 		 * other kmap() to finish when it runs out. If we map a large
287 		 * number of objects, there is no method for it to tell us
288 		 * to release the mappings, and we deadlock.
289 		 *
290 		 * However, if we make an explicit vmap of the page, that
291 		 * uses a larger vmalloc arena, and also has the ability
292 		 * to tell us to release unwanted mappings. Most importantly,
293 		 * it will fail and propagate an error instead of waiting
294 		 * forever.
295 		 *
296 		 * So if the page is beyond the 32b boundary, make an explicit
297 		 * vmap.
298 		 */
299 		if (n_pages == 1 && !PageHighMem(sg_page(obj->mm.pages->sgl)))
300 			return page_address(sg_page(obj->mm.pages->sgl));
301 		pgprot = PAGE_KERNEL;
302 		break;
303 	case I915_MAP_WC:
304 		pgprot = pgprot_writecombine(PAGE_KERNEL_IO);
305 		break;
306 	}
307 
308 	if (n_pages > ARRAY_SIZE(stack)) {
309 		/* Too big for stack -- allocate temporary array instead */
310 		pages = kvmalloc_array(n_pages, sizeof(*pages), GFP_KERNEL);
311 		if (!pages)
312 			return ERR_PTR(-ENOMEM);
313 	}
314 
315 	i = 0;
316 	for_each_sgt_page(page, iter, obj->mm.pages)
317 		pages[i++] = page;
318 	vaddr = vmap(pages, n_pages, 0, pgprot);
319 	if (pages != stack)
320 		kvfree(pages);
321 
322 	return vaddr ?: ERR_PTR(-ENOMEM);
323 }
324 
325 static void *i915_gem_object_map_pfn(struct drm_i915_gem_object *obj,
326 				     enum i915_map_type type)
327 {
328 	resource_size_t iomap = obj->mm.region->iomap.base -
329 		obj->mm.region->region.start;
330 	unsigned long n_pfn = obj->base.size >> PAGE_SHIFT;
331 	unsigned long stack[32], *pfns = stack, i;
332 	struct sgt_iter iter;
333 	dma_addr_t addr;
334 	void *vaddr;
335 
336 	GEM_BUG_ON(type != I915_MAP_WC);
337 
338 	if (n_pfn > ARRAY_SIZE(stack)) {
339 		/* Too big for stack -- allocate temporary array instead */
340 		pfns = kvmalloc_array(n_pfn, sizeof(*pfns), GFP_KERNEL);
341 		if (!pfns)
342 			return ERR_PTR(-ENOMEM);
343 	}
344 
345 	i = 0;
346 	for_each_sgt_daddr(addr, iter, obj->mm.pages)
347 		pfns[i++] = (iomap + addr) >> PAGE_SHIFT;
348 	vaddr = vmap_pfn(pfns, n_pfn, pgprot_writecombine(PAGE_KERNEL_IO));
349 	if (pfns != stack)
350 		kvfree(pfns);
351 
352 	return vaddr ?: ERR_PTR(-ENOMEM);
353 }
354 
355 /* get, pin, and map the pages of the object into kernel space */
356 void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj,
357 			      enum i915_map_type type)
358 {
359 	enum i915_map_type has_type;
360 	bool pinned;
361 	void *ptr;
362 	int err;
363 
364 	if (!i915_gem_object_has_struct_page(obj) &&
365 	    !i915_gem_object_has_iomem(obj))
366 		return ERR_PTR(-ENXIO);
367 
368 	if (WARN_ON_ONCE(obj->flags & I915_BO_ALLOC_GPU_ONLY))
369 		return ERR_PTR(-EINVAL);
370 
371 	assert_object_held(obj);
372 
373 	pinned = !(type & I915_MAP_OVERRIDE);
374 	type &= ~I915_MAP_OVERRIDE;
375 
376 	if (!atomic_inc_not_zero(&obj->mm.pages_pin_count)) {
377 		if (unlikely(!i915_gem_object_has_pages(obj))) {
378 			GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj));
379 
380 			err = ____i915_gem_object_get_pages(obj);
381 			if (err)
382 				return ERR_PTR(err);
383 
384 			smp_mb__before_atomic();
385 		}
386 		atomic_inc(&obj->mm.pages_pin_count);
387 		pinned = false;
388 	}
389 	GEM_BUG_ON(!i915_gem_object_has_pages(obj));
390 
391 	/*
392 	 * For discrete our CPU mappings needs to be consistent in order to
393 	 * function correctly on !x86. When mapping things through TTM, we use
394 	 * the same rules to determine the caching type.
395 	 *
396 	 * The caching rules, starting from DG1:
397 	 *
398 	 *	- If the object can be placed in device local-memory, then the
399 	 *	  pages should be allocated and mapped as write-combined only.
400 	 *
401 	 *	- Everything else is always allocated and mapped as write-back,
402 	 *	  with the guarantee that everything is also coherent with the
403 	 *	  GPU.
404 	 *
405 	 * Internal users of lmem are already expected to get this right, so no
406 	 * fudging needed there.
407 	 */
408 	if (i915_gem_object_placement_possible(obj, INTEL_MEMORY_LOCAL)) {
409 		if (type != I915_MAP_WC && !obj->mm.n_placements) {
410 			ptr = ERR_PTR(-ENODEV);
411 			goto err_unpin;
412 		}
413 
414 		type = I915_MAP_WC;
415 	} else if (IS_DGFX(to_i915(obj->base.dev))) {
416 		type = I915_MAP_WB;
417 	}
418 
419 	ptr = page_unpack_bits(obj->mm.mapping, &has_type);
420 	if (ptr && has_type != type) {
421 		if (pinned) {
422 			ptr = ERR_PTR(-EBUSY);
423 			goto err_unpin;
424 		}
425 
426 		unmap_object(obj, ptr);
427 
428 		ptr = obj->mm.mapping = NULL;
429 	}
430 
431 	if (!ptr) {
432 		err = i915_gem_object_wait_moving_fence(obj, true);
433 		if (err) {
434 			ptr = ERR_PTR(err);
435 			goto err_unpin;
436 		}
437 
438 		if (GEM_WARN_ON(type == I915_MAP_WC && !pat_enabled()))
439 			ptr = ERR_PTR(-ENODEV);
440 		else if (i915_gem_object_has_struct_page(obj))
441 			ptr = i915_gem_object_map_page(obj, type);
442 		else
443 			ptr = i915_gem_object_map_pfn(obj, type);
444 		if (IS_ERR(ptr))
445 			goto err_unpin;
446 
447 		obj->mm.mapping = page_pack_bits(ptr, type);
448 	}
449 
450 	return ptr;
451 
452 err_unpin:
453 	atomic_dec(&obj->mm.pages_pin_count);
454 	return ptr;
455 }
456 
457 void *i915_gem_object_pin_map_unlocked(struct drm_i915_gem_object *obj,
458 				       enum i915_map_type type)
459 {
460 	void *ret;
461 
462 	i915_gem_object_lock(obj, NULL);
463 	ret = i915_gem_object_pin_map(obj, type);
464 	i915_gem_object_unlock(obj);
465 
466 	return ret;
467 }
468 
469 void __i915_gem_object_flush_map(struct drm_i915_gem_object *obj,
470 				 unsigned long offset,
471 				 unsigned long size)
472 {
473 	enum i915_map_type has_type;
474 	void *ptr;
475 
476 	GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
477 	GEM_BUG_ON(range_overflows_t(typeof(obj->base.size),
478 				     offset, size, obj->base.size));
479 
480 	wmb(); /* let all previous writes be visible to coherent partners */
481 	obj->mm.dirty = true;
482 
483 	if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE)
484 		return;
485 
486 	ptr = page_unpack_bits(obj->mm.mapping, &has_type);
487 	if (has_type == I915_MAP_WC)
488 		return;
489 
490 	drm_clflush_virt_range(ptr + offset, size);
491 	if (size == obj->base.size) {
492 		obj->write_domain &= ~I915_GEM_DOMAIN_CPU;
493 		obj->cache_dirty = false;
494 	}
495 }
496 
497 void __i915_gem_object_release_map(struct drm_i915_gem_object *obj)
498 {
499 	GEM_BUG_ON(!obj->mm.mapping);
500 
501 	/*
502 	 * We allow removing the mapping from underneath pinned pages!
503 	 *
504 	 * Furthermore, since this is an unsafe operation reserved only
505 	 * for construction time manipulation, we ignore locking prudence.
506 	 */
507 	unmap_object(obj, page_mask_bits(fetch_and_zero(&obj->mm.mapping)));
508 
509 	i915_gem_object_unpin_map(obj);
510 }
511 
512 struct scatterlist *
513 __i915_gem_object_get_sg(struct drm_i915_gem_object *obj,
514 			 struct i915_gem_object_page_iter *iter,
515 			 unsigned int n,
516 			 unsigned int *offset,
517 			 bool dma)
518 {
519 	struct scatterlist *sg;
520 	unsigned int idx, count;
521 
522 	might_sleep();
523 	GEM_BUG_ON(n >= obj->base.size >> PAGE_SHIFT);
524 	if (!i915_gem_object_has_pinned_pages(obj))
525 		assert_object_held(obj);
526 
527 	/* As we iterate forward through the sg, we record each entry in a
528 	 * radixtree for quick repeated (backwards) lookups. If we have seen
529 	 * this index previously, we will have an entry for it.
530 	 *
531 	 * Initial lookup is O(N), but this is amortized to O(1) for
532 	 * sequential page access (where each new request is consecutive
533 	 * to the previous one). Repeated lookups are O(lg(obj->base.size)),
534 	 * i.e. O(1) with a large constant!
535 	 */
536 	if (n < READ_ONCE(iter->sg_idx))
537 		goto lookup;
538 
539 	mutex_lock(&iter->lock);
540 
541 	/* We prefer to reuse the last sg so that repeated lookup of this
542 	 * (or the subsequent) sg are fast - comparing against the last
543 	 * sg is faster than going through the radixtree.
544 	 */
545 
546 	sg = iter->sg_pos;
547 	idx = iter->sg_idx;
548 	count = dma ? __sg_dma_page_count(sg) : __sg_page_count(sg);
549 
550 	while (idx + count <= n) {
551 		void *entry;
552 		unsigned long i;
553 		int ret;
554 
555 		/* If we cannot allocate and insert this entry, or the
556 		 * individual pages from this range, cancel updating the
557 		 * sg_idx so that on this lookup we are forced to linearly
558 		 * scan onwards, but on future lookups we will try the
559 		 * insertion again (in which case we need to be careful of
560 		 * the error return reporting that we have already inserted
561 		 * this index).
562 		 */
563 		ret = radix_tree_insert(&iter->radix, idx, sg);
564 		if (ret && ret != -EEXIST)
565 			goto scan;
566 
567 		entry = xa_mk_value(idx);
568 		for (i = 1; i < count; i++) {
569 			ret = radix_tree_insert(&iter->radix, idx + i, entry);
570 			if (ret && ret != -EEXIST)
571 				goto scan;
572 		}
573 
574 		idx += count;
575 		sg = ____sg_next(sg);
576 		count = dma ? __sg_dma_page_count(sg) : __sg_page_count(sg);
577 	}
578 
579 scan:
580 	iter->sg_pos = sg;
581 	iter->sg_idx = idx;
582 
583 	mutex_unlock(&iter->lock);
584 
585 	if (unlikely(n < idx)) /* insertion completed by another thread */
586 		goto lookup;
587 
588 	/* In case we failed to insert the entry into the radixtree, we need
589 	 * to look beyond the current sg.
590 	 */
591 	while (idx + count <= n) {
592 		idx += count;
593 		sg = ____sg_next(sg);
594 		count = dma ? __sg_dma_page_count(sg) : __sg_page_count(sg);
595 	}
596 
597 	*offset = n - idx;
598 	return sg;
599 
600 lookup:
601 	rcu_read_lock();
602 
603 	sg = radix_tree_lookup(&iter->radix, n);
604 	GEM_BUG_ON(!sg);
605 
606 	/* If this index is in the middle of multi-page sg entry,
607 	 * the radix tree will contain a value entry that points
608 	 * to the start of that range. We will return the pointer to
609 	 * the base page and the offset of this page within the
610 	 * sg entry's range.
611 	 */
612 	*offset = 0;
613 	if (unlikely(xa_is_value(sg))) {
614 		unsigned long base = xa_to_value(sg);
615 
616 		sg = radix_tree_lookup(&iter->radix, base);
617 		GEM_BUG_ON(!sg);
618 
619 		*offset = n - base;
620 	}
621 
622 	rcu_read_unlock();
623 
624 	return sg;
625 }
626 
627 struct page *
628 i915_gem_object_get_page(struct drm_i915_gem_object *obj, unsigned int n)
629 {
630 	struct scatterlist *sg;
631 	unsigned int offset;
632 
633 	GEM_BUG_ON(!i915_gem_object_has_struct_page(obj));
634 
635 	sg = i915_gem_object_get_sg(obj, n, &offset);
636 	return nth_page(sg_page(sg), offset);
637 }
638 
639 /* Like i915_gem_object_get_page(), but mark the returned page dirty */
640 struct page *
641 i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj,
642 			       unsigned int n)
643 {
644 	struct page *page;
645 
646 	page = i915_gem_object_get_page(obj, n);
647 	if (!obj->mm.dirty)
648 		set_page_dirty(page);
649 
650 	return page;
651 }
652 
653 dma_addr_t
654 i915_gem_object_get_dma_address_len(struct drm_i915_gem_object *obj,
655 				    unsigned long n,
656 				    unsigned int *len)
657 {
658 	struct scatterlist *sg;
659 	unsigned int offset;
660 
661 	sg = i915_gem_object_get_sg_dma(obj, n, &offset);
662 
663 	if (len)
664 		*len = sg_dma_len(sg) - (offset << PAGE_SHIFT);
665 
666 	return sg_dma_address(sg) + (offset << PAGE_SHIFT);
667 }
668 
669 dma_addr_t
670 i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj,
671 				unsigned long n)
672 {
673 	return i915_gem_object_get_dma_address_len(obj, n, NULL);
674 }
675