xref: /openbmc/linux/drivers/usb/cdns3/cdnsp-mem.c (revision 7d94bc8e)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Cadence CDNSP DRD Driver.
4  *
5  * Copyright (C) 2020 Cadence.
6  *
7  * Author: Pawel Laszczak <pawell@cadence.com>
8  *
9  * Code based on Linux XHCI driver.
10  * Origin: Copyright (C) 2008 Intel Corp.
11  */
12 
13 #include <linux/dma-mapping.h>
14 #include <linux/dmapool.h>
15 #include <linux/slab.h>
16 #include <linux/usb.h>
17 
18 #include "cdnsp-gadget.h"
19 #include "cdnsp-trace.h"
20 
21 static void cdnsp_free_stream_info(struct cdnsp_device *pdev,
22 				   struct cdnsp_ep *pep);
23 /*
24  * Allocates a generic ring segment from the ring pool, sets the dma address,
25  * initializes the segment to zero, and sets the private next pointer to NULL.
26  *
27  * "All components of all Command and Transfer TRBs shall be initialized to '0'"
28  */
cdnsp_segment_alloc(struct cdnsp_device * pdev,unsigned int cycle_state,unsigned int max_packet,gfp_t flags)29 static struct cdnsp_segment *cdnsp_segment_alloc(struct cdnsp_device *pdev,
30 						 unsigned int cycle_state,
31 						 unsigned int max_packet,
32 						 gfp_t flags)
33 {
34 	struct cdnsp_segment *seg;
35 	dma_addr_t dma;
36 	int i;
37 
38 	seg = kzalloc(sizeof(*seg), flags);
39 	if (!seg)
40 		return NULL;
41 
42 	seg->trbs = dma_pool_zalloc(pdev->segment_pool, flags, &dma);
43 	if (!seg->trbs) {
44 		kfree(seg);
45 		return NULL;
46 	}
47 
48 	if (max_packet) {
49 		seg->bounce_buf = kzalloc(max_packet, flags | GFP_DMA);
50 		if (!seg->bounce_buf)
51 			goto free_dma;
52 	}
53 
54 	/* If the cycle state is 0, set the cycle bit to 1 for all the TRBs. */
55 	if (cycle_state == 0) {
56 		for (i = 0; i < TRBS_PER_SEGMENT; i++)
57 			seg->trbs[i].link.control |= cpu_to_le32(TRB_CYCLE);
58 	}
59 	seg->dma = dma;
60 	seg->next = NULL;
61 
62 	return seg;
63 
64 free_dma:
65 	dma_pool_free(pdev->segment_pool, seg->trbs, dma);
66 	kfree(seg);
67 
68 	return NULL;
69 }
70 
cdnsp_segment_free(struct cdnsp_device * pdev,struct cdnsp_segment * seg)71 static void cdnsp_segment_free(struct cdnsp_device *pdev,
72 			       struct cdnsp_segment *seg)
73 {
74 	if (seg->trbs)
75 		dma_pool_free(pdev->segment_pool, seg->trbs, seg->dma);
76 
77 	kfree(seg->bounce_buf);
78 	kfree(seg);
79 }
80 
cdnsp_free_segments_for_ring(struct cdnsp_device * pdev,struct cdnsp_segment * first)81 static void cdnsp_free_segments_for_ring(struct cdnsp_device *pdev,
82 					 struct cdnsp_segment *first)
83 {
84 	struct cdnsp_segment *seg;
85 
86 	seg = first->next;
87 
88 	while (seg != first) {
89 		struct cdnsp_segment *next = seg->next;
90 
91 		cdnsp_segment_free(pdev, seg);
92 		seg = next;
93 	}
94 
95 	cdnsp_segment_free(pdev, first);
96 }
97 
98 /*
99  * Make the prev segment point to the next segment.
100  *
101  * Change the last TRB in the prev segment to be a Link TRB which points to the
102  * DMA address of the next segment. The caller needs to set any Link TRB
103  * related flags, such as End TRB, Toggle Cycle, and no snoop.
104  */
cdnsp_link_segments(struct cdnsp_device * pdev,struct cdnsp_segment * prev,struct cdnsp_segment * next,enum cdnsp_ring_type type)105 static void cdnsp_link_segments(struct cdnsp_device *pdev,
106 				struct cdnsp_segment *prev,
107 				struct cdnsp_segment *next,
108 				enum cdnsp_ring_type type)
109 {
110 	struct cdnsp_link_trb *link;
111 	u32 val;
112 
113 	if (!prev || !next)
114 		return;
115 
116 	prev->next = next;
117 	if (type != TYPE_EVENT) {
118 		link = &prev->trbs[TRBS_PER_SEGMENT - 1].link;
119 		link->segment_ptr = cpu_to_le64(next->dma);
120 
121 		/*
122 		 * Set the last TRB in the segment to have a TRB type ID
123 		 * of Link TRB
124 		 */
125 		val = le32_to_cpu(link->control);
126 		val &= ~TRB_TYPE_BITMASK;
127 		val |= TRB_TYPE(TRB_LINK);
128 		link->control = cpu_to_le32(val);
129 	}
130 }
131 
132 /*
133  * Link the ring to the new segments.
134  * Set Toggle Cycle for the new ring if needed.
135  */
cdnsp_link_rings(struct cdnsp_device * pdev,struct cdnsp_ring * ring,struct cdnsp_segment * first,struct cdnsp_segment * last,unsigned int num_segs)136 static void cdnsp_link_rings(struct cdnsp_device *pdev,
137 			     struct cdnsp_ring *ring,
138 			     struct cdnsp_segment *first,
139 			     struct cdnsp_segment *last,
140 			     unsigned int num_segs)
141 {
142 	struct cdnsp_segment *next;
143 
144 	if (!ring || !first || !last)
145 		return;
146 
147 	next = ring->enq_seg->next;
148 	cdnsp_link_segments(pdev, ring->enq_seg, first, ring->type);
149 	cdnsp_link_segments(pdev, last, next, ring->type);
150 	ring->num_segs += num_segs;
151 	ring->num_trbs_free += (TRBS_PER_SEGMENT - 1) * num_segs;
152 
153 	if (ring->type != TYPE_EVENT && ring->enq_seg == ring->last_seg) {
154 		ring->last_seg->trbs[TRBS_PER_SEGMENT - 1].link.control &=
155 			~cpu_to_le32(LINK_TOGGLE);
156 		last->trbs[TRBS_PER_SEGMENT - 1].link.control |=
157 			cpu_to_le32(LINK_TOGGLE);
158 		ring->last_seg = last;
159 	}
160 }
161 
162 /*
163  * We need a radix tree for mapping physical addresses of TRBs to which stream
164  * ID they belong to. We need to do this because the device controller won't
165  * tell us which stream ring the TRB came from. We could store the stream ID
166  * in an event data TRB, but that doesn't help us for the cancellation case,
167  * since the endpoint may stop before it reaches that event data TRB.
168  *
169  * The radix tree maps the upper portion of the TRB DMA address to a ring
170  * segment that has the same upper portion of DMA addresses. For example,
171  * say I have segments of size 1KB, that are always 1KB aligned. A segment may
172  * start at 0x10c91000 and end at 0x10c913f0. If I use the upper 10 bits, the
173  * key to the stream ID is 0x43244. I can use the DMA address of the TRB to
174  * pass the radix tree a key to get the right stream ID:
175  *
176  *	0x10c90fff >> 10 = 0x43243
177  *	0x10c912c0 >> 10 = 0x43244
178  *	0x10c91400 >> 10 = 0x43245
179  *
180  * Obviously, only those TRBs with DMA addresses that are within the segment
181  * will make the radix tree return the stream ID for that ring.
182  *
183  * Caveats for the radix tree:
184  *
185  * The radix tree uses an unsigned long as a key pair. On 32-bit systems, an
186  * unsigned long will be 32-bits; on a 64-bit system an unsigned long will be
187  * 64-bits. Since we only request 32-bit DMA addresses, we can use that as the
188  * key on 32-bit or 64-bit systems (it would also be fine if we asked for 64-bit
189  * PCI DMA addresses on a 64-bit system). There might be a problem on 32-bit
190  * extended systems (where the DMA address can be bigger than 32-bits),
191  * if we allow the PCI dma mask to be bigger than 32-bits. So don't do that.
192  */
cdnsp_insert_segment_mapping(struct radix_tree_root * trb_address_map,struct cdnsp_ring * ring,struct cdnsp_segment * seg,gfp_t mem_flags)193 static int cdnsp_insert_segment_mapping(struct radix_tree_root *trb_address_map,
194 					struct cdnsp_ring *ring,
195 					struct cdnsp_segment *seg,
196 					gfp_t mem_flags)
197 {
198 	unsigned long key;
199 	int ret;
200 
201 	key = (unsigned long)(seg->dma >> TRB_SEGMENT_SHIFT);
202 
203 	/* Skip any segments that were already added. */
204 	if (radix_tree_lookup(trb_address_map, key))
205 		return 0;
206 
207 	ret = radix_tree_maybe_preload(mem_flags);
208 	if (ret)
209 		return ret;
210 
211 	ret = radix_tree_insert(trb_address_map, key, ring);
212 	radix_tree_preload_end();
213 
214 	return ret;
215 }
216 
cdnsp_remove_segment_mapping(struct radix_tree_root * trb_address_map,struct cdnsp_segment * seg)217 static void cdnsp_remove_segment_mapping(struct radix_tree_root *trb_address_map,
218 					 struct cdnsp_segment *seg)
219 {
220 	unsigned long key;
221 
222 	key = (unsigned long)(seg->dma >> TRB_SEGMENT_SHIFT);
223 	if (radix_tree_lookup(trb_address_map, key))
224 		radix_tree_delete(trb_address_map, key);
225 }
226 
cdnsp_update_stream_segment_mapping(struct radix_tree_root * trb_address_map,struct cdnsp_ring * ring,struct cdnsp_segment * first_seg,struct cdnsp_segment * last_seg,gfp_t mem_flags)227 static int cdnsp_update_stream_segment_mapping(struct radix_tree_root *trb_address_map,
228 					       struct cdnsp_ring *ring,
229 					       struct cdnsp_segment *first_seg,
230 					       struct cdnsp_segment *last_seg,
231 					       gfp_t mem_flags)
232 {
233 	struct cdnsp_segment *failed_seg;
234 	struct cdnsp_segment *seg;
235 	int ret;
236 
237 	seg = first_seg;
238 	do {
239 		ret = cdnsp_insert_segment_mapping(trb_address_map, ring, seg,
240 						   mem_flags);
241 		if (ret)
242 			goto remove_streams;
243 		if (seg == last_seg)
244 			return 0;
245 		seg = seg->next;
246 	} while (seg != first_seg);
247 
248 	return 0;
249 
250 remove_streams:
251 	failed_seg = seg;
252 	seg = first_seg;
253 	do {
254 		cdnsp_remove_segment_mapping(trb_address_map, seg);
255 		if (seg == failed_seg)
256 			return ret;
257 		seg = seg->next;
258 	} while (seg != first_seg);
259 
260 	return ret;
261 }
262 
cdnsp_remove_stream_mapping(struct cdnsp_ring * ring)263 static void cdnsp_remove_stream_mapping(struct cdnsp_ring *ring)
264 {
265 	struct cdnsp_segment *seg;
266 
267 	seg = ring->first_seg;
268 	do {
269 		cdnsp_remove_segment_mapping(ring->trb_address_map, seg);
270 		seg = seg->next;
271 	} while (seg != ring->first_seg);
272 }
273 
cdnsp_update_stream_mapping(struct cdnsp_ring * ring)274 static int cdnsp_update_stream_mapping(struct cdnsp_ring *ring)
275 {
276 	return cdnsp_update_stream_segment_mapping(ring->trb_address_map, ring,
277 			ring->first_seg, ring->last_seg, GFP_ATOMIC);
278 }
279 
cdnsp_ring_free(struct cdnsp_device * pdev,struct cdnsp_ring * ring)280 static void cdnsp_ring_free(struct cdnsp_device *pdev, struct cdnsp_ring *ring)
281 {
282 	if (!ring)
283 		return;
284 
285 	trace_cdnsp_ring_free(ring);
286 
287 	if (ring->first_seg) {
288 		if (ring->type == TYPE_STREAM)
289 			cdnsp_remove_stream_mapping(ring);
290 
291 		cdnsp_free_segments_for_ring(pdev, ring->first_seg);
292 	}
293 
294 	kfree(ring);
295 }
296 
cdnsp_initialize_ring_info(struct cdnsp_ring * ring)297 void cdnsp_initialize_ring_info(struct cdnsp_ring *ring)
298 {
299 	ring->enqueue = ring->first_seg->trbs;
300 	ring->enq_seg = ring->first_seg;
301 	ring->dequeue = ring->enqueue;
302 	ring->deq_seg = ring->first_seg;
303 
304 	/*
305 	 * The ring is initialized to 0. The producer must write 1 to the cycle
306 	 * bit to handover ownership of the TRB, so PCS = 1. The consumer must
307 	 * compare CCS to the cycle bit to check ownership, so CCS = 1.
308 	 *
309 	 * New rings are initialized with cycle state equal to 1; if we are
310 	 * handling ring expansion, set the cycle state equal to the old ring.
311 	 */
312 	ring->cycle_state = 1;
313 
314 	/*
315 	 * Each segment has a link TRB, and leave an extra TRB for SW
316 	 * accounting purpose
317 	 */
318 	ring->num_trbs_free = ring->num_segs * (TRBS_PER_SEGMENT - 1) - 1;
319 }
320 
321 /* Allocate segments and link them for a ring. */
cdnsp_alloc_segments_for_ring(struct cdnsp_device * pdev,struct cdnsp_segment ** first,struct cdnsp_segment ** last,unsigned int num_segs,unsigned int cycle_state,enum cdnsp_ring_type type,unsigned int max_packet,gfp_t flags)322 static int cdnsp_alloc_segments_for_ring(struct cdnsp_device *pdev,
323 					 struct cdnsp_segment **first,
324 					 struct cdnsp_segment **last,
325 					 unsigned int num_segs,
326 					 unsigned int cycle_state,
327 					 enum cdnsp_ring_type type,
328 					 unsigned int max_packet,
329 					 gfp_t flags)
330 {
331 	struct cdnsp_segment *prev;
332 
333 	/* Allocate first segment. */
334 	prev = cdnsp_segment_alloc(pdev, cycle_state, max_packet, flags);
335 	if (!prev)
336 		return -ENOMEM;
337 
338 	num_segs--;
339 	*first = prev;
340 
341 	/* Allocate all other segments. */
342 	while (num_segs > 0) {
343 		struct cdnsp_segment	*next;
344 
345 		next = cdnsp_segment_alloc(pdev, cycle_state,
346 					   max_packet, flags);
347 		if (!next) {
348 			cdnsp_free_segments_for_ring(pdev, *first);
349 			return -ENOMEM;
350 		}
351 
352 		cdnsp_link_segments(pdev, prev, next, type);
353 
354 		prev = next;
355 		num_segs--;
356 	}
357 
358 	cdnsp_link_segments(pdev, prev, *first, type);
359 	*last = prev;
360 
361 	return 0;
362 }
363 
364 /*
365  * Create a new ring with zero or more segments.
366  *
367  * Link each segment together into a ring.
368  * Set the end flag and the cycle toggle bit on the last segment.
369  */
cdnsp_ring_alloc(struct cdnsp_device * pdev,unsigned int num_segs,enum cdnsp_ring_type type,unsigned int max_packet,gfp_t flags)370 static struct cdnsp_ring *cdnsp_ring_alloc(struct cdnsp_device *pdev,
371 					   unsigned int num_segs,
372 					   enum cdnsp_ring_type type,
373 					   unsigned int max_packet,
374 					   gfp_t flags)
375 {
376 	struct cdnsp_ring *ring;
377 	int ret;
378 
379 	ring = kzalloc(sizeof *(ring), flags);
380 	if (!ring)
381 		return NULL;
382 
383 	ring->num_segs = num_segs;
384 	ring->bounce_buf_len = max_packet;
385 	INIT_LIST_HEAD(&ring->td_list);
386 	ring->type = type;
387 
388 	if (num_segs == 0)
389 		return ring;
390 
391 	ret = cdnsp_alloc_segments_for_ring(pdev, &ring->first_seg,
392 					    &ring->last_seg, num_segs,
393 					    1, type, max_packet, flags);
394 	if (ret)
395 		goto fail;
396 
397 	/* Only event ring does not use link TRB. */
398 	if (type != TYPE_EVENT)
399 		ring->last_seg->trbs[TRBS_PER_SEGMENT - 1].link.control |=
400 			cpu_to_le32(LINK_TOGGLE);
401 
402 	cdnsp_initialize_ring_info(ring);
403 	trace_cdnsp_ring_alloc(ring);
404 	return ring;
405 fail:
406 	kfree(ring);
407 	return NULL;
408 }
409 
cdnsp_free_endpoint_rings(struct cdnsp_device * pdev,struct cdnsp_ep * pep)410 void cdnsp_free_endpoint_rings(struct cdnsp_device *pdev, struct cdnsp_ep *pep)
411 {
412 	cdnsp_ring_free(pdev, pep->ring);
413 	pep->ring = NULL;
414 	cdnsp_free_stream_info(pdev, pep);
415 }
416 
417 /*
418  * Expand an existing ring.
419  * Allocate a new ring which has same segment numbers and link the two rings.
420  */
cdnsp_ring_expansion(struct cdnsp_device * pdev,struct cdnsp_ring * ring,unsigned int num_trbs,gfp_t flags)421 int cdnsp_ring_expansion(struct cdnsp_device *pdev,
422 			 struct cdnsp_ring *ring,
423 			 unsigned int num_trbs,
424 			 gfp_t flags)
425 {
426 	unsigned int num_segs_needed;
427 	struct cdnsp_segment *first;
428 	struct cdnsp_segment *last;
429 	unsigned int num_segs;
430 	int ret;
431 
432 	num_segs_needed = (num_trbs + (TRBS_PER_SEGMENT - 1) - 1) /
433 			(TRBS_PER_SEGMENT - 1);
434 
435 	/* Allocate number of segments we needed, or double the ring size. */
436 	num_segs = max(ring->num_segs, num_segs_needed);
437 
438 	ret = cdnsp_alloc_segments_for_ring(pdev, &first, &last, num_segs,
439 					    ring->cycle_state, ring->type,
440 					    ring->bounce_buf_len, flags);
441 	if (ret)
442 		return -ENOMEM;
443 
444 	if (ring->type == TYPE_STREAM)
445 		ret = cdnsp_update_stream_segment_mapping(ring->trb_address_map,
446 							  ring, first,
447 							  last, flags);
448 
449 	if (ret) {
450 		cdnsp_free_segments_for_ring(pdev, first);
451 
452 		return ret;
453 	}
454 
455 	cdnsp_link_rings(pdev, ring, first, last, num_segs);
456 	trace_cdnsp_ring_expansion(ring);
457 
458 	return 0;
459 }
460 
cdnsp_init_device_ctx(struct cdnsp_device * pdev)461 static int cdnsp_init_device_ctx(struct cdnsp_device *pdev)
462 {
463 	int size = HCC_64BYTE_CONTEXT(pdev->hcc_params) ? 2048 : 1024;
464 
465 	pdev->out_ctx.type = CDNSP_CTX_TYPE_DEVICE;
466 	pdev->out_ctx.size = size;
467 	pdev->out_ctx.ctx_size = CTX_SIZE(pdev->hcc_params);
468 	pdev->out_ctx.bytes = dma_pool_zalloc(pdev->device_pool, GFP_ATOMIC,
469 					      &pdev->out_ctx.dma);
470 
471 	if (!pdev->out_ctx.bytes)
472 		return -ENOMEM;
473 
474 	pdev->in_ctx.type = CDNSP_CTX_TYPE_INPUT;
475 	pdev->in_ctx.ctx_size = pdev->out_ctx.ctx_size;
476 	pdev->in_ctx.size = size + pdev->out_ctx.ctx_size;
477 	pdev->in_ctx.bytes = dma_pool_zalloc(pdev->device_pool, GFP_ATOMIC,
478 					     &pdev->in_ctx.dma);
479 
480 	if (!pdev->in_ctx.bytes) {
481 		dma_pool_free(pdev->device_pool, pdev->out_ctx.bytes,
482 			      pdev->out_ctx.dma);
483 		return -ENOMEM;
484 	}
485 
486 	return 0;
487 }
488 
489 struct cdnsp_input_control_ctx
cdnsp_get_input_control_ctx(struct cdnsp_container_ctx * ctx)490 	*cdnsp_get_input_control_ctx(struct cdnsp_container_ctx *ctx)
491 {
492 	if (ctx->type != CDNSP_CTX_TYPE_INPUT)
493 		return NULL;
494 
495 	return (struct cdnsp_input_control_ctx *)ctx->bytes;
496 }
497 
cdnsp_get_slot_ctx(struct cdnsp_container_ctx * ctx)498 struct cdnsp_slot_ctx *cdnsp_get_slot_ctx(struct cdnsp_container_ctx *ctx)
499 {
500 	if (ctx->type == CDNSP_CTX_TYPE_DEVICE)
501 		return (struct cdnsp_slot_ctx *)ctx->bytes;
502 
503 	return (struct cdnsp_slot_ctx *)(ctx->bytes + ctx->ctx_size);
504 }
505 
cdnsp_get_ep_ctx(struct cdnsp_container_ctx * ctx,unsigned int ep_index)506 struct cdnsp_ep_ctx *cdnsp_get_ep_ctx(struct cdnsp_container_ctx *ctx,
507 				      unsigned int ep_index)
508 {
509 	/* Increment ep index by offset of start of ep ctx array. */
510 	ep_index++;
511 	if (ctx->type == CDNSP_CTX_TYPE_INPUT)
512 		ep_index++;
513 
514 	return (struct cdnsp_ep_ctx *)(ctx->bytes + (ep_index * ctx->ctx_size));
515 }
516 
cdnsp_free_stream_ctx(struct cdnsp_device * pdev,struct cdnsp_ep * pep)517 static void cdnsp_free_stream_ctx(struct cdnsp_device *pdev,
518 				  struct cdnsp_ep *pep)
519 {
520 	dma_pool_free(pdev->device_pool, pep->stream_info.stream_ctx_array,
521 		      pep->stream_info.ctx_array_dma);
522 }
523 
524 /* The stream context array must be a power of 2. */
525 static struct cdnsp_stream_ctx
cdnsp_alloc_stream_ctx(struct cdnsp_device * pdev,struct cdnsp_ep * pep)526 	*cdnsp_alloc_stream_ctx(struct cdnsp_device *pdev, struct cdnsp_ep *pep)
527 {
528 	size_t size = sizeof(struct cdnsp_stream_ctx) *
529 		      pep->stream_info.num_stream_ctxs;
530 
531 	if (size > CDNSP_CTX_SIZE)
532 		return NULL;
533 
534 	/**
535 	 * Driver uses intentionally the device_pool to allocated stream
536 	 * context array. Device Pool has 2048 bytes of size what gives us
537 	 * 128 entries.
538 	 */
539 	return dma_pool_zalloc(pdev->device_pool, GFP_DMA32 | GFP_ATOMIC,
540 			       &pep->stream_info.ctx_array_dma);
541 }
542 
cdnsp_dma_to_transfer_ring(struct cdnsp_ep * pep,u64 address)543 struct cdnsp_ring *cdnsp_dma_to_transfer_ring(struct cdnsp_ep *pep, u64 address)
544 {
545 	if (pep->ep_state & EP_HAS_STREAMS)
546 		return radix_tree_lookup(&pep->stream_info.trb_address_map,
547 					 address >> TRB_SEGMENT_SHIFT);
548 
549 	return pep->ring;
550 }
551 
552 /*
553  * Change an endpoint's internal structure so it supports stream IDs.
554  * The number of requested streams includes stream 0, which cannot be used by
555  * driver.
556  *
557  * The number of stream contexts in the stream context array may be bigger than
558  * the number of streams the driver wants to use. This is because the number of
559  * stream context array entries must be a power of two.
560  */
cdnsp_alloc_stream_info(struct cdnsp_device * pdev,struct cdnsp_ep * pep,unsigned int num_stream_ctxs,unsigned int num_streams)561 int cdnsp_alloc_stream_info(struct cdnsp_device *pdev,
562 			    struct cdnsp_ep *pep,
563 			    unsigned int num_stream_ctxs,
564 			    unsigned int num_streams)
565 {
566 	struct cdnsp_stream_info *stream_info;
567 	struct cdnsp_ring *cur_ring;
568 	u32 cur_stream;
569 	u64 addr;
570 	int ret;
571 	int mps;
572 
573 	stream_info = &pep->stream_info;
574 	stream_info->num_streams = num_streams;
575 	stream_info->num_stream_ctxs = num_stream_ctxs;
576 
577 	/* Initialize the array of virtual pointers to stream rings. */
578 	stream_info->stream_rings = kcalloc(num_streams,
579 					    sizeof(struct cdnsp_ring *),
580 					    GFP_ATOMIC);
581 	if (!stream_info->stream_rings)
582 		return -ENOMEM;
583 
584 	/* Initialize the array of DMA addresses for stream rings for the HW. */
585 	stream_info->stream_ctx_array = cdnsp_alloc_stream_ctx(pdev, pep);
586 	if (!stream_info->stream_ctx_array)
587 		goto cleanup_stream_rings;
588 
589 	memset(stream_info->stream_ctx_array, 0,
590 	       sizeof(struct cdnsp_stream_ctx) * num_stream_ctxs);
591 	INIT_RADIX_TREE(&stream_info->trb_address_map, GFP_ATOMIC);
592 	mps = usb_endpoint_maxp(pep->endpoint.desc);
593 
594 	/*
595 	 * Allocate rings for all the streams that the driver will use,
596 	 * and add their segment DMA addresses to the radix tree.
597 	 * Stream 0 is reserved.
598 	 */
599 	for (cur_stream = 1; cur_stream < num_streams; cur_stream++) {
600 		cur_ring = cdnsp_ring_alloc(pdev, 2, TYPE_STREAM, mps,
601 					    GFP_ATOMIC);
602 		stream_info->stream_rings[cur_stream] = cur_ring;
603 
604 		if (!cur_ring)
605 			goto cleanup_rings;
606 
607 		cur_ring->stream_id = cur_stream;
608 		cur_ring->trb_address_map = &stream_info->trb_address_map;
609 
610 		/* Set deq ptr, cycle bit, and stream context type. */
611 		addr = cur_ring->first_seg->dma | SCT_FOR_CTX(SCT_PRI_TR) |
612 		       cur_ring->cycle_state;
613 
614 		stream_info->stream_ctx_array[cur_stream].stream_ring =
615 			cpu_to_le64(addr);
616 
617 		trace_cdnsp_set_stream_ring(cur_ring);
618 
619 		ret = cdnsp_update_stream_mapping(cur_ring);
620 		if (ret)
621 			goto cleanup_rings;
622 	}
623 
624 	return 0;
625 
626 cleanup_rings:
627 	for (cur_stream = 1; cur_stream < num_streams; cur_stream++) {
628 		cur_ring = stream_info->stream_rings[cur_stream];
629 		if (cur_ring) {
630 			cdnsp_ring_free(pdev, cur_ring);
631 			stream_info->stream_rings[cur_stream] = NULL;
632 		}
633 	}
634 
635 cleanup_stream_rings:
636 	kfree(pep->stream_info.stream_rings);
637 
638 	return -ENOMEM;
639 }
640 
641 /* Frees all stream contexts associated with the endpoint. */
cdnsp_free_stream_info(struct cdnsp_device * pdev,struct cdnsp_ep * pep)642 static void cdnsp_free_stream_info(struct cdnsp_device *pdev,
643 				   struct cdnsp_ep *pep)
644 {
645 	struct cdnsp_stream_info *stream_info = &pep->stream_info;
646 	struct cdnsp_ring *cur_ring;
647 	int cur_stream;
648 
649 	if (!(pep->ep_state & EP_HAS_STREAMS))
650 		return;
651 
652 	for (cur_stream = 1; cur_stream < stream_info->num_streams;
653 	     cur_stream++) {
654 		cur_ring = stream_info->stream_rings[cur_stream];
655 		if (cur_ring) {
656 			cdnsp_ring_free(pdev, cur_ring);
657 			stream_info->stream_rings[cur_stream] = NULL;
658 		}
659 	}
660 
661 	if (stream_info->stream_ctx_array)
662 		cdnsp_free_stream_ctx(pdev, pep);
663 
664 	kfree(stream_info->stream_rings);
665 	pep->ep_state &= ~EP_HAS_STREAMS;
666 }
667 
668 /* All the cdnsp_tds in the ring's TD list should be freed at this point.*/
cdnsp_free_priv_device(struct cdnsp_device * pdev)669 static void cdnsp_free_priv_device(struct cdnsp_device *pdev)
670 {
671 	pdev->dcbaa->dev_context_ptrs[1] = 0;
672 
673 	cdnsp_free_endpoint_rings(pdev, &pdev->eps[0]);
674 
675 	if (pdev->in_ctx.bytes)
676 		dma_pool_free(pdev->device_pool, pdev->in_ctx.bytes,
677 			      pdev->in_ctx.dma);
678 
679 	if (pdev->out_ctx.bytes)
680 		dma_pool_free(pdev->device_pool, pdev->out_ctx.bytes,
681 			      pdev->out_ctx.dma);
682 
683 	pdev->in_ctx.bytes = NULL;
684 	pdev->out_ctx.bytes = NULL;
685 }
686 
cdnsp_alloc_priv_device(struct cdnsp_device * pdev)687 static int cdnsp_alloc_priv_device(struct cdnsp_device *pdev)
688 {
689 	int ret;
690 
691 	ret = cdnsp_init_device_ctx(pdev);
692 	if (ret)
693 		return ret;
694 
695 	/* Allocate endpoint 0 ring. */
696 	pdev->eps[0].ring = cdnsp_ring_alloc(pdev, 2, TYPE_CTRL, 0, GFP_ATOMIC);
697 	if (!pdev->eps[0].ring)
698 		goto fail;
699 
700 	/* Point to output device context in dcbaa. */
701 	pdev->dcbaa->dev_context_ptrs[1] = cpu_to_le64(pdev->out_ctx.dma);
702 	pdev->cmd.in_ctx = &pdev->in_ctx;
703 
704 	trace_cdnsp_alloc_priv_device(pdev);
705 	return 0;
706 fail:
707 	dma_pool_free(pdev->device_pool, pdev->out_ctx.bytes,
708 		      pdev->out_ctx.dma);
709 	dma_pool_free(pdev->device_pool, pdev->in_ctx.bytes,
710 		      pdev->in_ctx.dma);
711 
712 	return ret;
713 }
714 
cdnsp_copy_ep0_dequeue_into_input_ctx(struct cdnsp_device * pdev)715 void cdnsp_copy_ep0_dequeue_into_input_ctx(struct cdnsp_device *pdev)
716 {
717 	struct cdnsp_ep_ctx *ep0_ctx = pdev->eps[0].in_ctx;
718 	struct cdnsp_ring *ep_ring = pdev->eps[0].ring;
719 	dma_addr_t dma;
720 
721 	dma = cdnsp_trb_virt_to_dma(ep_ring->enq_seg, ep_ring->enqueue);
722 	ep0_ctx->deq = cpu_to_le64(dma | ep_ring->cycle_state);
723 }
724 
725 /* Setup an controller private device for a Set Address command. */
cdnsp_setup_addressable_priv_dev(struct cdnsp_device * pdev)726 int cdnsp_setup_addressable_priv_dev(struct cdnsp_device *pdev)
727 {
728 	struct cdnsp_slot_ctx *slot_ctx;
729 	struct cdnsp_ep_ctx *ep0_ctx;
730 	u32 max_packets, port;
731 
732 	ep0_ctx = cdnsp_get_ep_ctx(&pdev->in_ctx, 0);
733 	slot_ctx = cdnsp_get_slot_ctx(&pdev->in_ctx);
734 
735 	/* Only the control endpoint is valid - one endpoint context. */
736 	slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(1));
737 
738 	switch (pdev->gadget.speed) {
739 	case USB_SPEED_SUPER_PLUS:
740 		slot_ctx->dev_info |= cpu_to_le32(SLOT_SPEED_SSP);
741 		max_packets = MAX_PACKET(512);
742 		break;
743 	case USB_SPEED_SUPER:
744 		slot_ctx->dev_info |= cpu_to_le32(SLOT_SPEED_SS);
745 		max_packets = MAX_PACKET(512);
746 		break;
747 	case USB_SPEED_HIGH:
748 		slot_ctx->dev_info |= cpu_to_le32(SLOT_SPEED_HS);
749 		max_packets = MAX_PACKET(64);
750 		break;
751 	case USB_SPEED_FULL:
752 		slot_ctx->dev_info |= cpu_to_le32(SLOT_SPEED_FS);
753 		max_packets = MAX_PACKET(64);
754 		break;
755 	default:
756 		/* Speed was not set , this shouldn't happen. */
757 		return -EINVAL;
758 	}
759 
760 	port = DEV_PORT(pdev->active_port->port_num);
761 	slot_ctx->dev_port |= cpu_to_le32(port);
762 	slot_ctx->dev_state = cpu_to_le32((pdev->device_address &
763 					   DEV_ADDR_MASK));
764 	ep0_ctx->tx_info = cpu_to_le32(EP_AVG_TRB_LENGTH(0x8));
765 	ep0_ctx->ep_info2 = cpu_to_le32(EP_TYPE(CTRL_EP));
766 	ep0_ctx->ep_info2 |= cpu_to_le32(MAX_BURST(0) | ERROR_COUNT(3) |
767 					 max_packets);
768 
769 	ep0_ctx->deq = cpu_to_le64(pdev->eps[0].ring->first_seg->dma |
770 				   pdev->eps[0].ring->cycle_state);
771 
772 	trace_cdnsp_setup_addressable_priv_device(pdev);
773 
774 	return 0;
775 }
776 
777 /*
778  * Convert interval expressed as 2^(bInterval - 1) == interval into
779  * straight exponent value 2^n == interval.
780  */
cdnsp_parse_exponent_interval(struct usb_gadget * g,struct cdnsp_ep * pep)781 static unsigned int cdnsp_parse_exponent_interval(struct usb_gadget *g,
782 						  struct cdnsp_ep *pep)
783 {
784 	unsigned int interval;
785 
786 	interval = clamp_val(pep->endpoint.desc->bInterval, 1, 16) - 1;
787 	if (interval != pep->endpoint.desc->bInterval - 1)
788 		dev_warn(&g->dev, "ep %s - rounding interval to %d %sframes\n",
789 			 pep->name, 1 << interval,
790 			 g->speed == USB_SPEED_FULL ? "" : "micro");
791 
792 	/*
793 	 * Full speed isoc endpoints specify interval in frames,
794 	 * not microframes. We are using microframes everywhere,
795 	 * so adjust accordingly.
796 	 */
797 	if (g->speed == USB_SPEED_FULL)
798 		interval += 3;	/* 1 frame = 2^3 uframes */
799 
800 	/* Controller handles only up to 512ms (2^12). */
801 	if (interval > 12)
802 		interval = 12;
803 
804 	return interval;
805 }
806 
807 /*
808  * Convert bInterval expressed in microframes (in 1-255 range) to exponent of
809  * microframes, rounded down to nearest power of 2.
810  */
cdnsp_microframes_to_exponent(struct usb_gadget * g,struct cdnsp_ep * pep,unsigned int desc_interval,unsigned int min_exponent,unsigned int max_exponent)811 static unsigned int cdnsp_microframes_to_exponent(struct usb_gadget *g,
812 						  struct cdnsp_ep *pep,
813 						  unsigned int desc_interval,
814 						  unsigned int min_exponent,
815 						  unsigned int max_exponent)
816 {
817 	unsigned int interval;
818 
819 	interval = fls(desc_interval) - 1;
820 	return clamp_val(interval, min_exponent, max_exponent);
821 }
822 
823 /*
824  * Return the polling interval.
825  *
826  * The polling interval is expressed in "microframes". If controllers's Interval
827  * field is set to N, it will service the endpoint every 2^(Interval)*125us.
828  */
cdnsp_get_endpoint_interval(struct usb_gadget * g,struct cdnsp_ep * pep)829 static unsigned int cdnsp_get_endpoint_interval(struct usb_gadget *g,
830 						struct cdnsp_ep *pep)
831 {
832 	unsigned int interval = 0;
833 
834 	switch (g->speed) {
835 	case USB_SPEED_HIGH:
836 	case USB_SPEED_SUPER_PLUS:
837 	case USB_SPEED_SUPER:
838 		if (usb_endpoint_xfer_int(pep->endpoint.desc) ||
839 		    usb_endpoint_xfer_isoc(pep->endpoint.desc))
840 			interval = cdnsp_parse_exponent_interval(g, pep);
841 		break;
842 	case USB_SPEED_FULL:
843 		if (usb_endpoint_xfer_isoc(pep->endpoint.desc)) {
844 			interval = cdnsp_parse_exponent_interval(g, pep);
845 		} else if (usb_endpoint_xfer_int(pep->endpoint.desc)) {
846 			interval = pep->endpoint.desc->bInterval << 3;
847 			interval = cdnsp_microframes_to_exponent(g, pep,
848 								 interval,
849 								 3, 10);
850 		}
851 
852 		break;
853 	default:
854 		WARN_ON(1);
855 	}
856 
857 	return interval;
858 }
859 
860 /*
861  * The "Mult" field in the endpoint context is only set for SuperSpeed isoc eps.
862  * High speed endpoint descriptors can define "the number of additional
863  * transaction opportunities per microframe", but that goes in the Max Burst
864  * endpoint context field.
865  */
cdnsp_get_endpoint_mult(struct usb_gadget * g,struct cdnsp_ep * pep)866 static u32 cdnsp_get_endpoint_mult(struct usb_gadget *g, struct cdnsp_ep *pep)
867 {
868 	if (g->speed < USB_SPEED_SUPER ||
869 	    !usb_endpoint_xfer_isoc(pep->endpoint.desc))
870 		return 0;
871 
872 	return pep->endpoint.comp_desc->bmAttributes;
873 }
874 
cdnsp_get_endpoint_max_burst(struct usb_gadget * g,struct cdnsp_ep * pep)875 static u32 cdnsp_get_endpoint_max_burst(struct usb_gadget *g,
876 					struct cdnsp_ep *pep)
877 {
878 	/* Super speed and Plus have max burst in ep companion desc */
879 	if (g->speed >= USB_SPEED_SUPER)
880 		return pep->endpoint.comp_desc->bMaxBurst;
881 
882 	if (g->speed == USB_SPEED_HIGH &&
883 	    (usb_endpoint_xfer_isoc(pep->endpoint.desc) ||
884 	     usb_endpoint_xfer_int(pep->endpoint.desc)))
885 		return usb_endpoint_maxp_mult(pep->endpoint.desc) - 1;
886 
887 	return 0;
888 }
889 
cdnsp_get_endpoint_type(const struct usb_endpoint_descriptor * desc)890 static u32 cdnsp_get_endpoint_type(const struct usb_endpoint_descriptor *desc)
891 {
892 	int in;
893 
894 	in = usb_endpoint_dir_in(desc);
895 
896 	switch (usb_endpoint_type(desc)) {
897 	case USB_ENDPOINT_XFER_CONTROL:
898 		return CTRL_EP;
899 	case USB_ENDPOINT_XFER_BULK:
900 		return in ? BULK_IN_EP : BULK_OUT_EP;
901 	case USB_ENDPOINT_XFER_ISOC:
902 		return in ? ISOC_IN_EP : ISOC_OUT_EP;
903 	case USB_ENDPOINT_XFER_INT:
904 		return in ? INT_IN_EP : INT_OUT_EP;
905 	}
906 
907 	return 0;
908 }
909 
910 /*
911  * Return the maximum endpoint service interval time (ESIT) payload.
912  * Basically, this is the maxpacket size, multiplied by the burst size
913  * and mult size.
914  */
cdnsp_get_max_esit_payload(struct usb_gadget * g,struct cdnsp_ep * pep)915 static u32 cdnsp_get_max_esit_payload(struct usb_gadget *g,
916 				      struct cdnsp_ep *pep)
917 {
918 	int max_packet;
919 	int max_burst;
920 
921 	/* Only applies for interrupt or isochronous endpoints*/
922 	if (usb_endpoint_xfer_control(pep->endpoint.desc) ||
923 	    usb_endpoint_xfer_bulk(pep->endpoint.desc))
924 		return 0;
925 
926 	/* SuperSpeedPlus Isoc ep sending over 48k per EIST. */
927 	if (g->speed >= USB_SPEED_SUPER_PLUS &&
928 	    USB_SS_SSP_ISOC_COMP(pep->endpoint.desc->bmAttributes))
929 		return le16_to_cpu(pep->endpoint.comp_desc->wBytesPerInterval);
930 	/* SuperSpeed or SuperSpeedPlus Isoc ep with less than 48k per esit */
931 	else if (g->speed >= USB_SPEED_SUPER)
932 		return le16_to_cpu(pep->endpoint.comp_desc->wBytesPerInterval);
933 
934 	max_packet = usb_endpoint_maxp(pep->endpoint.desc);
935 	max_burst = usb_endpoint_maxp_mult(pep->endpoint.desc);
936 
937 	/* A 0 in max burst means 1 transfer per ESIT */
938 	return max_packet * max_burst;
939 }
940 
cdnsp_endpoint_init(struct cdnsp_device * pdev,struct cdnsp_ep * pep,gfp_t mem_flags)941 int cdnsp_endpoint_init(struct cdnsp_device *pdev,
942 			struct cdnsp_ep *pep,
943 			gfp_t mem_flags)
944 {
945 	enum cdnsp_ring_type ring_type;
946 	struct cdnsp_ep_ctx *ep_ctx;
947 	unsigned int err_count = 0;
948 	unsigned int avg_trb_len;
949 	unsigned int max_packet;
950 	unsigned int max_burst;
951 	unsigned int interval;
952 	u32 max_esit_payload;
953 	unsigned int mult;
954 	u32 endpoint_type;
955 	int ret;
956 
957 	ep_ctx = pep->in_ctx;
958 
959 	endpoint_type = cdnsp_get_endpoint_type(pep->endpoint.desc);
960 	if (!endpoint_type)
961 		return -EINVAL;
962 
963 	ring_type = usb_endpoint_type(pep->endpoint.desc);
964 
965 	/*
966 	 * Get values to fill the endpoint context, mostly from ep descriptor.
967 	 * The average TRB buffer length for bulk endpoints is unclear as we
968 	 * have no clue on scatter gather list entry size. For Isoc and Int,
969 	 * set it to max available.
970 	 */
971 	max_esit_payload = cdnsp_get_max_esit_payload(&pdev->gadget, pep);
972 	interval = cdnsp_get_endpoint_interval(&pdev->gadget, pep);
973 	mult = cdnsp_get_endpoint_mult(&pdev->gadget, pep);
974 	max_packet = usb_endpoint_maxp(pep->endpoint.desc);
975 	max_burst = cdnsp_get_endpoint_max_burst(&pdev->gadget, pep);
976 	avg_trb_len = max_esit_payload;
977 
978 	/* Allow 3 retries for everything but isoc, set CErr = 3. */
979 	if (!usb_endpoint_xfer_isoc(pep->endpoint.desc))
980 		err_count = 3;
981 	if (usb_endpoint_xfer_bulk(pep->endpoint.desc) &&
982 	    pdev->gadget.speed == USB_SPEED_HIGH)
983 		max_packet = 512;
984 	/* Controller spec indicates that ctrl ep avg TRB Length should be 8. */
985 	if (usb_endpoint_xfer_control(pep->endpoint.desc))
986 		avg_trb_len = 8;
987 
988 	/* Set up the endpoint ring. */
989 	pep->ring = cdnsp_ring_alloc(pdev, 2, ring_type, max_packet, mem_flags);
990 	if (!pep->ring)
991 		return -ENOMEM;
992 
993 	pep->skip = false;
994 
995 	/* Fill the endpoint context */
996 	ep_ctx->ep_info = cpu_to_le32(EP_MAX_ESIT_PAYLOAD_HI(max_esit_payload) |
997 				EP_INTERVAL(interval) | EP_MULT(mult));
998 	ep_ctx->ep_info2 = cpu_to_le32(EP_TYPE(endpoint_type) |
999 				MAX_PACKET(max_packet) | MAX_BURST(max_burst) |
1000 				ERROR_COUNT(err_count));
1001 	ep_ctx->deq = cpu_to_le64(pep->ring->first_seg->dma |
1002 				  pep->ring->cycle_state);
1003 
1004 	ep_ctx->tx_info = cpu_to_le32(EP_MAX_ESIT_PAYLOAD_LO(max_esit_payload) |
1005 				EP_AVG_TRB_LENGTH(avg_trb_len));
1006 
1007 	if (usb_endpoint_xfer_bulk(pep->endpoint.desc) &&
1008 	    pdev->gadget.speed > USB_SPEED_HIGH) {
1009 		ret = cdnsp_alloc_streams(pdev, pep);
1010 		if (ret < 0)
1011 			return ret;
1012 	}
1013 
1014 	return 0;
1015 }
1016 
cdnsp_endpoint_zero(struct cdnsp_device * pdev,struct cdnsp_ep * pep)1017 void cdnsp_endpoint_zero(struct cdnsp_device *pdev, struct cdnsp_ep *pep)
1018 {
1019 	pep->in_ctx->ep_info = 0;
1020 	pep->in_ctx->ep_info2 = 0;
1021 	pep->in_ctx->deq = 0;
1022 	pep->in_ctx->tx_info = 0;
1023 }
1024 
cdnsp_alloc_erst(struct cdnsp_device * pdev,struct cdnsp_ring * evt_ring,struct cdnsp_erst * erst)1025 static int cdnsp_alloc_erst(struct cdnsp_device *pdev,
1026 			    struct cdnsp_ring *evt_ring,
1027 			    struct cdnsp_erst *erst)
1028 {
1029 	struct cdnsp_erst_entry *entry;
1030 	struct cdnsp_segment *seg;
1031 	unsigned int val;
1032 	size_t size;
1033 
1034 	size = sizeof(struct cdnsp_erst_entry) * evt_ring->num_segs;
1035 	erst->entries = dma_alloc_coherent(pdev->dev, size,
1036 					   &erst->erst_dma_addr, GFP_KERNEL);
1037 	if (!erst->entries)
1038 		return -ENOMEM;
1039 
1040 	erst->num_entries = evt_ring->num_segs;
1041 
1042 	seg = evt_ring->first_seg;
1043 	for (val = 0; val < evt_ring->num_segs; val++) {
1044 		entry = &erst->entries[val];
1045 		entry->seg_addr = cpu_to_le64(seg->dma);
1046 		entry->seg_size = cpu_to_le32(TRBS_PER_SEGMENT);
1047 		entry->rsvd = 0;
1048 		seg = seg->next;
1049 	}
1050 
1051 	return 0;
1052 }
1053 
cdnsp_free_erst(struct cdnsp_device * pdev,struct cdnsp_erst * erst)1054 static void cdnsp_free_erst(struct cdnsp_device *pdev, struct cdnsp_erst *erst)
1055 {
1056 	size_t size = sizeof(struct cdnsp_erst_entry) * (erst->num_entries);
1057 	struct device *dev = pdev->dev;
1058 
1059 	if (erst->entries)
1060 		dma_free_coherent(dev, size, erst->entries,
1061 				  erst->erst_dma_addr);
1062 
1063 	erst->entries = NULL;
1064 }
1065 
cdnsp_mem_cleanup(struct cdnsp_device * pdev)1066 void cdnsp_mem_cleanup(struct cdnsp_device *pdev)
1067 {
1068 	struct device *dev = pdev->dev;
1069 
1070 	cdnsp_free_priv_device(pdev);
1071 	cdnsp_free_erst(pdev, &pdev->erst);
1072 
1073 	if (pdev->event_ring)
1074 		cdnsp_ring_free(pdev, pdev->event_ring);
1075 
1076 	pdev->event_ring = NULL;
1077 
1078 	if (pdev->cmd_ring)
1079 		cdnsp_ring_free(pdev, pdev->cmd_ring);
1080 
1081 	pdev->cmd_ring = NULL;
1082 
1083 	dma_pool_destroy(pdev->segment_pool);
1084 	pdev->segment_pool = NULL;
1085 	dma_pool_destroy(pdev->device_pool);
1086 	pdev->device_pool = NULL;
1087 
1088 	dma_free_coherent(dev, sizeof(*pdev->dcbaa),
1089 			  pdev->dcbaa, pdev->dcbaa->dma);
1090 
1091 	pdev->dcbaa = NULL;
1092 
1093 	pdev->usb2_port.exist = 0;
1094 	pdev->usb3_port.exist = 0;
1095 	pdev->usb2_port.port_num = 0;
1096 	pdev->usb3_port.port_num = 0;
1097 	pdev->active_port = NULL;
1098 }
1099 
cdnsp_set_event_deq(struct cdnsp_device * pdev)1100 static void cdnsp_set_event_deq(struct cdnsp_device *pdev)
1101 {
1102 	dma_addr_t deq;
1103 	u64 temp;
1104 
1105 	deq = cdnsp_trb_virt_to_dma(pdev->event_ring->deq_seg,
1106 				    pdev->event_ring->dequeue);
1107 
1108 	/* Update controller event ring dequeue pointer */
1109 	temp = cdnsp_read_64(&pdev->ir_set->erst_dequeue);
1110 	temp &= ERST_PTR_MASK;
1111 
1112 	/*
1113 	 * Don't clear the EHB bit (which is RW1C) because
1114 	 * there might be more events to service.
1115 	 */
1116 	temp &= ~ERST_EHB;
1117 
1118 	cdnsp_write_64(((u64)deq & (u64)~ERST_PTR_MASK) | temp,
1119 		       &pdev->ir_set->erst_dequeue);
1120 }
1121 
cdnsp_add_in_port(struct cdnsp_device * pdev,struct cdnsp_port * port,__le32 __iomem * addr)1122 static void cdnsp_add_in_port(struct cdnsp_device *pdev,
1123 			      struct cdnsp_port *port,
1124 			      __le32 __iomem *addr)
1125 {
1126 	u32 temp, port_offset, port_count;
1127 
1128 	temp = readl(addr);
1129 	port->maj_rev = CDNSP_EXT_PORT_MAJOR(temp);
1130 	port->min_rev = CDNSP_EXT_PORT_MINOR(temp);
1131 
1132 	/* Port offset and count in the third dword.*/
1133 	temp = readl(addr + 2);
1134 	port_offset = CDNSP_EXT_PORT_OFF(temp);
1135 	port_count = CDNSP_EXT_PORT_COUNT(temp);
1136 
1137 	trace_cdnsp_port_info(addr, port_offset, port_count, port->maj_rev);
1138 
1139 	port->port_num = port_offset;
1140 	port->exist = 1;
1141 }
1142 
1143 /*
1144  * Scan the Extended Capabilities for the "Supported Protocol Capabilities" that
1145  * specify what speeds each port is supposed to be.
1146  */
cdnsp_setup_port_arrays(struct cdnsp_device * pdev)1147 static int cdnsp_setup_port_arrays(struct cdnsp_device *pdev)
1148 {
1149 	void __iomem *base;
1150 	u32 offset;
1151 	int i;
1152 
1153 	base = &pdev->cap_regs->hc_capbase;
1154 	offset = cdnsp_find_next_ext_cap(base, 0,
1155 					 EXT_CAP_CFG_DEV_20PORT_CAP_ID);
1156 	pdev->port20_regs = base + offset;
1157 
1158 	offset = cdnsp_find_next_ext_cap(base, 0, D_XEC_CFG_3XPORT_CAP);
1159 	pdev->port3x_regs =  base + offset;
1160 
1161 	offset = 0;
1162 	base = &pdev->cap_regs->hc_capbase;
1163 
1164 	/* Driver expects max 2 extended protocol capability. */
1165 	for (i = 0; i < 2; i++) {
1166 		u32 temp;
1167 
1168 		offset = cdnsp_find_next_ext_cap(base, offset,
1169 						 EXT_CAPS_PROTOCOL);
1170 		temp = readl(base + offset);
1171 
1172 		if (CDNSP_EXT_PORT_MAJOR(temp) == 0x03 &&
1173 		    !pdev->usb3_port.port_num)
1174 			cdnsp_add_in_port(pdev, &pdev->usb3_port,
1175 					  base + offset);
1176 
1177 		if (CDNSP_EXT_PORT_MAJOR(temp) == 0x02 &&
1178 		    !pdev->usb2_port.port_num)
1179 			cdnsp_add_in_port(pdev, &pdev->usb2_port,
1180 					  base + offset);
1181 	}
1182 
1183 	if (!pdev->usb2_port.exist || !pdev->usb3_port.exist) {
1184 		dev_err(pdev->dev, "Error: Only one port detected\n");
1185 		return -ENODEV;
1186 	}
1187 
1188 	trace_cdnsp_init("Found USB 2.0 ports and  USB 3.0 ports.");
1189 
1190 	pdev->usb2_port.regs = (struct cdnsp_port_regs __iomem *)
1191 			       (&pdev->op_regs->port_reg_base + NUM_PORT_REGS *
1192 				(pdev->usb2_port.port_num - 1));
1193 
1194 	pdev->usb3_port.regs = (struct cdnsp_port_regs __iomem *)
1195 			       (&pdev->op_regs->port_reg_base + NUM_PORT_REGS *
1196 				(pdev->usb3_port.port_num - 1));
1197 
1198 	return 0;
1199 }
1200 
1201 /*
1202  * Initialize memory for CDNSP (one-time init).
1203  *
1204  * Program the PAGESIZE register, initialize the device context array, create
1205  * device contexts, set up a command ring segment, create event
1206  * ring (one for now).
1207  */
cdnsp_mem_init(struct cdnsp_device * pdev)1208 int cdnsp_mem_init(struct cdnsp_device *pdev)
1209 {
1210 	struct device *dev = pdev->dev;
1211 	int ret = -ENOMEM;
1212 	unsigned int val;
1213 	dma_addr_t dma;
1214 	u32 page_size;
1215 	u64 val_64;
1216 
1217 	/*
1218 	 * Use 4K pages, since that's common and the minimum the
1219 	 * controller supports
1220 	 */
1221 	page_size = 1 << 12;
1222 
1223 	val = readl(&pdev->op_regs->config_reg);
1224 	val |= ((val & ~MAX_DEVS) | CDNSP_DEV_MAX_SLOTS) | CONFIG_U3E;
1225 	writel(val, &pdev->op_regs->config_reg);
1226 
1227 	/*
1228 	 * Doorbell array must be physically contiguous
1229 	 * and 64-byte (cache line) aligned.
1230 	 */
1231 	pdev->dcbaa = dma_alloc_coherent(dev, sizeof(*pdev->dcbaa),
1232 					 &dma, GFP_KERNEL);
1233 	if (!pdev->dcbaa)
1234 		return -ENOMEM;
1235 
1236 	pdev->dcbaa->dma = dma;
1237 
1238 	cdnsp_write_64(dma, &pdev->op_regs->dcbaa_ptr);
1239 
1240 	/*
1241 	 * Initialize the ring segment pool.  The ring must be a contiguous
1242 	 * structure comprised of TRBs. The TRBs must be 16 byte aligned,
1243 	 * however, the command ring segment needs 64-byte aligned segments
1244 	 * and our use of dma addresses in the trb_address_map radix tree needs
1245 	 * TRB_SEGMENT_SIZE alignment, so driver pick the greater alignment
1246 	 * need.
1247 	 */
1248 	pdev->segment_pool = dma_pool_create("CDNSP ring segments", dev,
1249 					     TRB_SEGMENT_SIZE, TRB_SEGMENT_SIZE,
1250 					     page_size);
1251 	if (!pdev->segment_pool)
1252 		goto release_dcbaa;
1253 
1254 	pdev->device_pool = dma_pool_create("CDNSP input/output contexts", dev,
1255 					    CDNSP_CTX_SIZE, 64, page_size);
1256 	if (!pdev->device_pool)
1257 		goto destroy_segment_pool;
1258 
1259 
1260 	/* Set up the command ring to have one segments for now. */
1261 	pdev->cmd_ring = cdnsp_ring_alloc(pdev, 1, TYPE_COMMAND, 0, GFP_KERNEL);
1262 	if (!pdev->cmd_ring)
1263 		goto destroy_device_pool;
1264 
1265 	/* Set the address in the Command Ring Control register */
1266 	val_64 = cdnsp_read_64(&pdev->op_regs->cmd_ring);
1267 	val_64 = (val_64 & (u64)CMD_RING_RSVD_BITS) |
1268 		 (pdev->cmd_ring->first_seg->dma & (u64)~CMD_RING_RSVD_BITS) |
1269 		 pdev->cmd_ring->cycle_state;
1270 	cdnsp_write_64(val_64, &pdev->op_regs->cmd_ring);
1271 
1272 	val = readl(&pdev->cap_regs->db_off);
1273 	val &= DBOFF_MASK;
1274 	pdev->dba = (void __iomem *)pdev->cap_regs + val;
1275 
1276 	/* Set ir_set to interrupt register set 0 */
1277 	pdev->ir_set = &pdev->run_regs->ir_set[0];
1278 
1279 	/*
1280 	 * Event ring setup: Allocate a normal ring, but also setup
1281 	 * the event ring segment table (ERST).
1282 	 */
1283 	pdev->event_ring = cdnsp_ring_alloc(pdev, ERST_NUM_SEGS, TYPE_EVENT,
1284 					    0, GFP_KERNEL);
1285 	if (!pdev->event_ring)
1286 		goto free_cmd_ring;
1287 
1288 	ret = cdnsp_alloc_erst(pdev, pdev->event_ring, &pdev->erst);
1289 	if (ret)
1290 		goto free_event_ring;
1291 
1292 	/* Set ERST count with the number of entries in the segment table. */
1293 	val = readl(&pdev->ir_set->erst_size);
1294 	val &= ERST_SIZE_MASK;
1295 	val |= ERST_NUM_SEGS;
1296 	writel(val, &pdev->ir_set->erst_size);
1297 
1298 	/* Set the segment table base address. */
1299 	val_64 = cdnsp_read_64(&pdev->ir_set->erst_base);
1300 	val_64 &= ERST_PTR_MASK;
1301 	val_64 |= (pdev->erst.erst_dma_addr & (u64)~ERST_PTR_MASK);
1302 	cdnsp_write_64(val_64, &pdev->ir_set->erst_base);
1303 
1304 	/* Set the event ring dequeue address. */
1305 	cdnsp_set_event_deq(pdev);
1306 
1307 	ret = cdnsp_setup_port_arrays(pdev);
1308 	if (ret)
1309 		goto free_erst;
1310 
1311 	ret = cdnsp_alloc_priv_device(pdev);
1312 	if (ret) {
1313 		dev_err(pdev->dev,
1314 			"Could not allocate cdnsp_device data structures\n");
1315 		goto free_erst;
1316 	}
1317 
1318 	return 0;
1319 
1320 free_erst:
1321 	cdnsp_free_erst(pdev, &pdev->erst);
1322 free_event_ring:
1323 	cdnsp_ring_free(pdev, pdev->event_ring);
1324 free_cmd_ring:
1325 	cdnsp_ring_free(pdev, pdev->cmd_ring);
1326 destroy_device_pool:
1327 	dma_pool_destroy(pdev->device_pool);
1328 destroy_segment_pool:
1329 	dma_pool_destroy(pdev->segment_pool);
1330 release_dcbaa:
1331 	dma_free_coherent(dev, sizeof(*pdev->dcbaa), pdev->dcbaa,
1332 			  pdev->dcbaa->dma);
1333 
1334 	cdnsp_reset(pdev);
1335 
1336 	return ret;
1337 }
1338