1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2019 Intel Corporation 5 */ 6 7 #include "gem/i915_gem_object.h" 8 #include "i915_drv.h" 9 #include "i915_vma.h" 10 #include "intel_engine.h" 11 #include "intel_ring.h" 12 #include "intel_timeline.h" 13 14 unsigned int intel_ring_update_space(struct intel_ring *ring) 15 { 16 unsigned int space; 17 18 space = __intel_ring_space(ring->head, ring->emit, ring->size); 19 20 ring->space = space; 21 return space; 22 } 23 24 int intel_ring_pin(struct intel_ring *ring) 25 { 26 struct i915_vma *vma = ring->vma; 27 unsigned int flags; 28 void *addr; 29 int ret; 30 31 if (atomic_fetch_inc(&ring->pin_count)) 32 return 0; 33 34 flags = PIN_GLOBAL; 35 36 /* Ring wraparound at offset 0 sometimes hangs. No idea why. */ 37 flags |= PIN_OFFSET_BIAS | i915_ggtt_pin_bias(vma); 38 39 if (vma->obj->stolen) 40 flags |= PIN_MAPPABLE; 41 else 42 flags |= PIN_HIGH; 43 44 ret = i915_vma_pin(vma, 0, 0, flags); 45 if (unlikely(ret)) 46 goto err_unpin; 47 48 if (i915_vma_is_map_and_fenceable(vma)) 49 addr = (void __force *)i915_vma_pin_iomap(vma); 50 else 51 addr = i915_gem_object_pin_map(vma->obj, 52 i915_coherent_map_type(vma->vm->i915)); 53 if (IS_ERR(addr)) { 54 ret = PTR_ERR(addr); 55 goto err_ring; 56 } 57 58 i915_vma_make_unshrinkable(vma); 59 60 /* Discard any unused bytes beyond that submitted to hw. */ 61 intel_ring_reset(ring, ring->emit); 62 63 ring->vaddr = addr; 64 return 0; 65 66 err_ring: 67 i915_vma_unpin(vma); 68 err_unpin: 69 atomic_dec(&ring->pin_count); 70 return ret; 71 } 72 73 void intel_ring_reset(struct intel_ring *ring, u32 tail) 74 { 75 tail = intel_ring_wrap(ring, tail); 76 ring->tail = tail; 77 ring->head = tail; 78 ring->emit = tail; 79 intel_ring_update_space(ring); 80 } 81 82 void intel_ring_unpin(struct intel_ring *ring) 83 { 84 struct i915_vma *vma = ring->vma; 85 86 if (!atomic_dec_and_test(&ring->pin_count)) 87 return; 88 89 i915_vma_unset_ggtt_write(vma); 90 if (i915_vma_is_map_and_fenceable(vma)) 91 i915_vma_unpin_iomap(vma); 92 else 93 i915_gem_object_unpin_map(vma->obj); 94 95 i915_vma_make_purgeable(vma); 96 i915_vma_unpin(vma); 97 } 98 99 static struct i915_vma *create_ring_vma(struct i915_ggtt *ggtt, int size) 100 { 101 struct i915_address_space *vm = &ggtt->vm; 102 struct drm_i915_private *i915 = vm->i915; 103 struct drm_i915_gem_object *obj; 104 struct i915_vma *vma; 105 106 obj = ERR_PTR(-ENODEV); 107 if (i915_ggtt_has_aperture(ggtt)) 108 obj = i915_gem_object_create_stolen(i915, size); 109 if (IS_ERR(obj)) 110 obj = i915_gem_object_create_internal(i915, size); 111 if (IS_ERR(obj)) 112 return ERR_CAST(obj); 113 114 /* 115 * Mark ring buffers as read-only from GPU side (so no stray overwrites) 116 * if supported by the platform's GGTT. 117 */ 118 if (vm->has_read_only) 119 i915_gem_object_set_readonly(obj); 120 121 vma = i915_vma_instance(obj, vm, NULL); 122 if (IS_ERR(vma)) 123 goto err; 124 125 return vma; 126 127 err: 128 i915_gem_object_put(obj); 129 return vma; 130 } 131 132 struct intel_ring * 133 intel_engine_create_ring(struct intel_engine_cs *engine, int size) 134 { 135 struct drm_i915_private *i915 = engine->i915; 136 struct intel_ring *ring; 137 struct i915_vma *vma; 138 139 GEM_BUG_ON(!is_power_of_2(size)); 140 GEM_BUG_ON(RING_CTL_SIZE(size) & ~RING_NR_PAGES); 141 142 ring = kzalloc(sizeof(*ring), GFP_KERNEL); 143 if (!ring) 144 return ERR_PTR(-ENOMEM); 145 146 kref_init(&ring->ref); 147 ring->size = size; 148 ring->wrap = BITS_PER_TYPE(ring->size) - ilog2(size); 149 150 /* 151 * Workaround an erratum on the i830 which causes a hang if 152 * the TAIL pointer points to within the last 2 cachelines 153 * of the buffer. 154 */ 155 ring->effective_size = size; 156 if (IS_I830(i915) || IS_I845G(i915)) 157 ring->effective_size -= 2 * CACHELINE_BYTES; 158 159 intel_ring_update_space(ring); 160 161 vma = create_ring_vma(engine->gt->ggtt, size); 162 if (IS_ERR(vma)) { 163 kfree(ring); 164 return ERR_CAST(vma); 165 } 166 ring->vma = vma; 167 168 return ring; 169 } 170 171 void intel_ring_free(struct kref *ref) 172 { 173 struct intel_ring *ring = container_of(ref, typeof(*ring), ref); 174 175 i915_vma_put(ring->vma); 176 kfree(ring); 177 } 178 179 static noinline int 180 wait_for_space(struct intel_ring *ring, 181 struct intel_timeline *tl, 182 unsigned int bytes) 183 { 184 struct i915_request *target; 185 long timeout; 186 187 if (intel_ring_update_space(ring) >= bytes) 188 return 0; 189 190 GEM_BUG_ON(list_empty(&tl->requests)); 191 list_for_each_entry(target, &tl->requests, link) { 192 if (target->ring != ring) 193 continue; 194 195 /* Would completion of this request free enough space? */ 196 if (bytes <= __intel_ring_space(target->postfix, 197 ring->emit, ring->size)) 198 break; 199 } 200 201 if (GEM_WARN_ON(&target->link == &tl->requests)) 202 return -ENOSPC; 203 204 timeout = i915_request_wait(target, 205 I915_WAIT_INTERRUPTIBLE, 206 MAX_SCHEDULE_TIMEOUT); 207 if (timeout < 0) 208 return timeout; 209 210 i915_request_retire_upto(target); 211 212 intel_ring_update_space(ring); 213 GEM_BUG_ON(ring->space < bytes); 214 return 0; 215 } 216 217 u32 *intel_ring_begin(struct i915_request *rq, unsigned int num_dwords) 218 { 219 struct intel_ring *ring = rq->ring; 220 const unsigned int remain_usable = ring->effective_size - ring->emit; 221 const unsigned int bytes = num_dwords * sizeof(u32); 222 unsigned int need_wrap = 0; 223 unsigned int total_bytes; 224 u32 *cs; 225 226 /* Packets must be qword aligned. */ 227 GEM_BUG_ON(num_dwords & 1); 228 229 total_bytes = bytes + rq->reserved_space; 230 GEM_BUG_ON(total_bytes > ring->effective_size); 231 232 if (unlikely(total_bytes > remain_usable)) { 233 const int remain_actual = ring->size - ring->emit; 234 235 if (bytes > remain_usable) { 236 /* 237 * Not enough space for the basic request. So need to 238 * flush out the remainder and then wait for 239 * base + reserved. 240 */ 241 total_bytes += remain_actual; 242 need_wrap = remain_actual | 1; 243 } else { 244 /* 245 * The base request will fit but the reserved space 246 * falls off the end. So we don't need an immediate 247 * wrap and only need to effectively wait for the 248 * reserved size from the start of ringbuffer. 249 */ 250 total_bytes = rq->reserved_space + remain_actual; 251 } 252 } 253 254 if (unlikely(total_bytes > ring->space)) { 255 int ret; 256 257 /* 258 * Space is reserved in the ringbuffer for finalising the 259 * request, as that cannot be allowed to fail. During request 260 * finalisation, reserved_space is set to 0 to stop the 261 * overallocation and the assumption is that then we never need 262 * to wait (which has the risk of failing with EINTR). 263 * 264 * See also i915_request_alloc() and i915_request_add(). 265 */ 266 GEM_BUG_ON(!rq->reserved_space); 267 268 ret = wait_for_space(ring, 269 i915_request_timeline(rq), 270 total_bytes); 271 if (unlikely(ret)) 272 return ERR_PTR(ret); 273 } 274 275 if (unlikely(need_wrap)) { 276 need_wrap &= ~1; 277 GEM_BUG_ON(need_wrap > ring->space); 278 GEM_BUG_ON(ring->emit + need_wrap > ring->size); 279 GEM_BUG_ON(!IS_ALIGNED(need_wrap, sizeof(u64))); 280 281 /* Fill the tail with MI_NOOP */ 282 memset64(ring->vaddr + ring->emit, 0, need_wrap / sizeof(u64)); 283 ring->space -= need_wrap; 284 ring->emit = 0; 285 } 286 287 GEM_BUG_ON(ring->emit > ring->size - bytes); 288 GEM_BUG_ON(ring->space < bytes); 289 cs = ring->vaddr + ring->emit; 290 GEM_DEBUG_EXEC(memset32(cs, POISON_INUSE, bytes / sizeof(*cs))); 291 ring->emit += bytes; 292 ring->space -= bytes; 293 294 return cs; 295 } 296 297 /* Align the ring tail to a cacheline boundary */ 298 int intel_ring_cacheline_align(struct i915_request *rq) 299 { 300 int num_dwords; 301 void *cs; 302 303 num_dwords = (rq->ring->emit & (CACHELINE_BYTES - 1)) / sizeof(u32); 304 if (num_dwords == 0) 305 return 0; 306 307 num_dwords = CACHELINE_DWORDS - num_dwords; 308 GEM_BUG_ON(num_dwords & 1); 309 310 cs = intel_ring_begin(rq, num_dwords); 311 if (IS_ERR(cs)) 312 return PTR_ERR(cs); 313 314 memset64(cs, (u64)MI_NOOP << 32 | MI_NOOP, num_dwords / 2); 315 intel_ring_advance(rq, cs + num_dwords); 316 317 GEM_BUG_ON(rq->ring->emit & (CACHELINE_BYTES - 1)); 318 return 0; 319 } 320