Loading...
Note: File does not exist in v3.15.
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Cadence CDNSP DRD Driver.
4 *
5 * Copyright (C) 2020 Cadence.
6 *
7 * Author: Pawel Laszczak <pawell@cadence.com>
8 *
9 * Code based on Linux XHCI driver.
10 * Origin: Copyright (C) 2008 Intel Corp.
11 */
12
13#include <linux/dma-mapping.h>
14#include <linux/dmapool.h>
15#include <linux/slab.h>
16#include <linux/usb.h>
17
18#include "cdnsp-gadget.h"
19#include "cdnsp-trace.h"
20
21static void cdnsp_free_stream_info(struct cdnsp_device *pdev,
22 struct cdnsp_ep *pep);
23/*
24 * Allocates a generic ring segment from the ring pool, sets the dma address,
25 * initializes the segment to zero, and sets the private next pointer to NULL.
26 *
27 * "All components of all Command and Transfer TRBs shall be initialized to '0'"
28 */
29static struct cdnsp_segment *cdnsp_segment_alloc(struct cdnsp_device *pdev,
30 unsigned int cycle_state,
31 unsigned int max_packet,
32 gfp_t flags)
33{
34 struct cdnsp_segment *seg;
35 dma_addr_t dma;
36 int i;
37
38 seg = kzalloc(sizeof(*seg), flags);
39 if (!seg)
40 return NULL;
41
42 seg->trbs = dma_pool_zalloc(pdev->segment_pool, flags, &dma);
43 if (!seg->trbs) {
44 kfree(seg);
45 return NULL;
46 }
47
48 if (max_packet) {
49 seg->bounce_buf = kzalloc(max_packet, flags | GFP_DMA);
50 if (!seg->bounce_buf)
51 goto free_dma;
52 }
53
54 /* If the cycle state is 0, set the cycle bit to 1 for all the TRBs. */
55 if (cycle_state == 0) {
56 for (i = 0; i < TRBS_PER_SEGMENT; i++)
57 seg->trbs[i].link.control |= cpu_to_le32(TRB_CYCLE);
58 }
59 seg->dma = dma;
60 seg->next = NULL;
61
62 return seg;
63
64free_dma:
65 dma_pool_free(pdev->segment_pool, seg->trbs, dma);
66 kfree(seg);
67
68 return NULL;
69}
70
71static void cdnsp_segment_free(struct cdnsp_device *pdev,
72 struct cdnsp_segment *seg)
73{
74 if (seg->trbs)
75 dma_pool_free(pdev->segment_pool, seg->trbs, seg->dma);
76
77 kfree(seg->bounce_buf);
78 kfree(seg);
79}
80
81static void cdnsp_free_segments_for_ring(struct cdnsp_device *pdev,
82 struct cdnsp_segment *first)
83{
84 struct cdnsp_segment *seg;
85
86 seg = first->next;
87
88 while (seg != first) {
89 struct cdnsp_segment *next = seg->next;
90
91 cdnsp_segment_free(pdev, seg);
92 seg = next;
93 }
94
95 cdnsp_segment_free(pdev, first);
96}
97
98/*
99 * Make the prev segment point to the next segment.
100 *
101 * Change the last TRB in the prev segment to be a Link TRB which points to the
102 * DMA address of the next segment. The caller needs to set any Link TRB
103 * related flags, such as End TRB, Toggle Cycle, and no snoop.
104 */
105static void cdnsp_link_segments(struct cdnsp_device *pdev,
106 struct cdnsp_segment *prev,
107 struct cdnsp_segment *next,
108 enum cdnsp_ring_type type)
109{
110 struct cdnsp_link_trb *link;
111 u32 val;
112
113 if (!prev || !next)
114 return;
115
116 prev->next = next;
117 if (type != TYPE_EVENT) {
118 link = &prev->trbs[TRBS_PER_SEGMENT - 1].link;
119 link->segment_ptr = cpu_to_le64(next->dma);
120
121 /*
122 * Set the last TRB in the segment to have a TRB type ID
123 * of Link TRB
124 */
125 val = le32_to_cpu(link->control);
126 val &= ~TRB_TYPE_BITMASK;
127 val |= TRB_TYPE(TRB_LINK);
128 link->control = cpu_to_le32(val);
129 }
130}
131
132/*
133 * Link the ring to the new segments.
134 * Set Toggle Cycle for the new ring if needed.
135 */
136static void cdnsp_link_rings(struct cdnsp_device *pdev,
137 struct cdnsp_ring *ring,
138 struct cdnsp_segment *first,
139 struct cdnsp_segment *last,
140 unsigned int num_segs)
141{
142 struct cdnsp_segment *next;
143
144 if (!ring || !first || !last)
145 return;
146
147 next = ring->enq_seg->next;
148 cdnsp_link_segments(pdev, ring->enq_seg, first, ring->type);
149 cdnsp_link_segments(pdev, last, next, ring->type);
150 ring->num_segs += num_segs;
151 ring->num_trbs_free += (TRBS_PER_SEGMENT - 1) * num_segs;
152
153 if (ring->type != TYPE_EVENT && ring->enq_seg == ring->last_seg) {
154 ring->last_seg->trbs[TRBS_PER_SEGMENT - 1].link.control &=
155 ~cpu_to_le32(LINK_TOGGLE);
156 last->trbs[TRBS_PER_SEGMENT - 1].link.control |=
157 cpu_to_le32(LINK_TOGGLE);
158 ring->last_seg = last;
159 }
160}
161
162/*
163 * We need a radix tree for mapping physical addresses of TRBs to which stream
164 * ID they belong to. We need to do this because the device controller won't
165 * tell us which stream ring the TRB came from. We could store the stream ID
166 * in an event data TRB, but that doesn't help us for the cancellation case,
167 * since the endpoint may stop before it reaches that event data TRB.
168 *
169 * The radix tree maps the upper portion of the TRB DMA address to a ring
170 * segment that has the same upper portion of DMA addresses. For example,
171 * say I have segments of size 1KB, that are always 1KB aligned. A segment may
172 * start at 0x10c91000 and end at 0x10c913f0. If I use the upper 10 bits, the
173 * key to the stream ID is 0x43244. I can use the DMA address of the TRB to
174 * pass the radix tree a key to get the right stream ID:
175 *
176 * 0x10c90fff >> 10 = 0x43243
177 * 0x10c912c0 >> 10 = 0x43244
178 * 0x10c91400 >> 10 = 0x43245
179 *
180 * Obviously, only those TRBs with DMA addresses that are within the segment
181 * will make the radix tree return the stream ID for that ring.
182 *
183 * Caveats for the radix tree:
184 *
185 * The radix tree uses an unsigned long as a key pair. On 32-bit systems, an
186 * unsigned long will be 32-bits; on a 64-bit system an unsigned long will be
187 * 64-bits. Since we only request 32-bit DMA addresses, we can use that as the
188 * key on 32-bit or 64-bit systems (it would also be fine if we asked for 64-bit
189 * PCI DMA addresses on a 64-bit system). There might be a problem on 32-bit
190 * extended systems (where the DMA address can be bigger than 32-bits),
191 * if we allow the PCI dma mask to be bigger than 32-bits. So don't do that.
192 */
193static int cdnsp_insert_segment_mapping(struct radix_tree_root *trb_address_map,
194 struct cdnsp_ring *ring,
195 struct cdnsp_segment *seg,
196 gfp_t mem_flags)
197{
198 unsigned long key;
199 int ret;
200
201 key = (unsigned long)(seg->dma >> TRB_SEGMENT_SHIFT);
202
203 /* Skip any segments that were already added. */
204 if (radix_tree_lookup(trb_address_map, key))
205 return 0;
206
207 ret = radix_tree_maybe_preload(mem_flags);
208 if (ret)
209 return ret;
210
211 ret = radix_tree_insert(trb_address_map, key, ring);
212 radix_tree_preload_end();
213
214 return ret;
215}
216
217static void cdnsp_remove_segment_mapping(struct radix_tree_root *trb_address_map,
218 struct cdnsp_segment *seg)
219{
220 unsigned long key;
221
222 key = (unsigned long)(seg->dma >> TRB_SEGMENT_SHIFT);
223 if (radix_tree_lookup(trb_address_map, key))
224 radix_tree_delete(trb_address_map, key);
225}
226
227static int cdnsp_update_stream_segment_mapping(struct radix_tree_root *trb_address_map,
228 struct cdnsp_ring *ring,
229 struct cdnsp_segment *first_seg,
230 struct cdnsp_segment *last_seg,
231 gfp_t mem_flags)
232{
233 struct cdnsp_segment *failed_seg;
234 struct cdnsp_segment *seg;
235 int ret;
236
237 seg = first_seg;
238 do {
239 ret = cdnsp_insert_segment_mapping(trb_address_map, ring, seg,
240 mem_flags);
241 if (ret)
242 goto remove_streams;
243 if (seg == last_seg)
244 return 0;
245 seg = seg->next;
246 } while (seg != first_seg);
247
248 return 0;
249
250remove_streams:
251 failed_seg = seg;
252 seg = first_seg;
253 do {
254 cdnsp_remove_segment_mapping(trb_address_map, seg);
255 if (seg == failed_seg)
256 return ret;
257 seg = seg->next;
258 } while (seg != first_seg);
259
260 return ret;
261}
262
263static void cdnsp_remove_stream_mapping(struct cdnsp_ring *ring)
264{
265 struct cdnsp_segment *seg;
266
267 seg = ring->first_seg;
268 do {
269 cdnsp_remove_segment_mapping(ring->trb_address_map, seg);
270 seg = seg->next;
271 } while (seg != ring->first_seg);
272}
273
274static int cdnsp_update_stream_mapping(struct cdnsp_ring *ring)
275{
276 return cdnsp_update_stream_segment_mapping(ring->trb_address_map, ring,
277 ring->first_seg, ring->last_seg, GFP_ATOMIC);
278}
279
280static void cdnsp_ring_free(struct cdnsp_device *pdev, struct cdnsp_ring *ring)
281{
282 if (!ring)
283 return;
284
285 trace_cdnsp_ring_free(ring);
286
287 if (ring->first_seg) {
288 if (ring->type == TYPE_STREAM)
289 cdnsp_remove_stream_mapping(ring);
290
291 cdnsp_free_segments_for_ring(pdev, ring->first_seg);
292 }
293
294 kfree(ring);
295}
296
297void cdnsp_initialize_ring_info(struct cdnsp_ring *ring)
298{
299 ring->enqueue = ring->first_seg->trbs;
300 ring->enq_seg = ring->first_seg;
301 ring->dequeue = ring->enqueue;
302 ring->deq_seg = ring->first_seg;
303
304 /*
305 * The ring is initialized to 0. The producer must write 1 to the cycle
306 * bit to handover ownership of the TRB, so PCS = 1. The consumer must
307 * compare CCS to the cycle bit to check ownership, so CCS = 1.
308 *
309 * New rings are initialized with cycle state equal to 1; if we are
310 * handling ring expansion, set the cycle state equal to the old ring.
311 */
312 ring->cycle_state = 1;
313
314 /*
315 * Each segment has a link TRB, and leave an extra TRB for SW
316 * accounting purpose
317 */
318 ring->num_trbs_free = ring->num_segs * (TRBS_PER_SEGMENT - 1) - 1;
319}
320
321/* Allocate segments and link them for a ring. */
322static int cdnsp_alloc_segments_for_ring(struct cdnsp_device *pdev,
323 struct cdnsp_segment **first,
324 struct cdnsp_segment **last,
325 unsigned int num_segs,
326 unsigned int cycle_state,
327 enum cdnsp_ring_type type,
328 unsigned int max_packet,
329 gfp_t flags)
330{
331 struct cdnsp_segment *prev;
332
333 /* Allocate first segment. */
334 prev = cdnsp_segment_alloc(pdev, cycle_state, max_packet, flags);
335 if (!prev)
336 return -ENOMEM;
337
338 num_segs--;
339 *first = prev;
340
341 /* Allocate all other segments. */
342 while (num_segs > 0) {
343 struct cdnsp_segment *next;
344
345 next = cdnsp_segment_alloc(pdev, cycle_state,
346 max_packet, flags);
347 if (!next) {
348 cdnsp_free_segments_for_ring(pdev, *first);
349 return -ENOMEM;
350 }
351
352 cdnsp_link_segments(pdev, prev, next, type);
353
354 prev = next;
355 num_segs--;
356 }
357
358 cdnsp_link_segments(pdev, prev, *first, type);
359 *last = prev;
360
361 return 0;
362}
363
364/*
365 * Create a new ring with zero or more segments.
366 *
367 * Link each segment together into a ring.
368 * Set the end flag and the cycle toggle bit on the last segment.
369 */
370static struct cdnsp_ring *cdnsp_ring_alloc(struct cdnsp_device *pdev,
371 unsigned int num_segs,
372 enum cdnsp_ring_type type,
373 unsigned int max_packet,
374 gfp_t flags)
375{
376 struct cdnsp_ring *ring;
377 int ret;
378
379 ring = kzalloc(sizeof *(ring), flags);
380 if (!ring)
381 return NULL;
382
383 ring->num_segs = num_segs;
384 ring->bounce_buf_len = max_packet;
385 INIT_LIST_HEAD(&ring->td_list);
386 ring->type = type;
387
388 if (num_segs == 0)
389 return ring;
390
391 ret = cdnsp_alloc_segments_for_ring(pdev, &ring->first_seg,
392 &ring->last_seg, num_segs,
393 1, type, max_packet, flags);
394 if (ret)
395 goto fail;
396
397 /* Only event ring does not use link TRB. */
398 if (type != TYPE_EVENT)
399 ring->last_seg->trbs[TRBS_PER_SEGMENT - 1].link.control |=
400 cpu_to_le32(LINK_TOGGLE);
401
402 cdnsp_initialize_ring_info(ring);
403 trace_cdnsp_ring_alloc(ring);
404 return ring;
405fail:
406 kfree(ring);
407 return NULL;
408}
409
410void cdnsp_free_endpoint_rings(struct cdnsp_device *pdev, struct cdnsp_ep *pep)
411{
412 cdnsp_ring_free(pdev, pep->ring);
413 pep->ring = NULL;
414 cdnsp_free_stream_info(pdev, pep);
415}
416
417/*
418 * Expand an existing ring.
419 * Allocate a new ring which has same segment numbers and link the two rings.
420 */
421int cdnsp_ring_expansion(struct cdnsp_device *pdev,
422 struct cdnsp_ring *ring,
423 unsigned int num_trbs,
424 gfp_t flags)
425{
426 unsigned int num_segs_needed;
427 struct cdnsp_segment *first;
428 struct cdnsp_segment *last;
429 unsigned int num_segs;
430 int ret;
431
432 num_segs_needed = (num_trbs + (TRBS_PER_SEGMENT - 1) - 1) /
433 (TRBS_PER_SEGMENT - 1);
434
435 /* Allocate number of segments we needed, or double the ring size. */
436 num_segs = max(ring->num_segs, num_segs_needed);
437
438 ret = cdnsp_alloc_segments_for_ring(pdev, &first, &last, num_segs,
439 ring->cycle_state, ring->type,
440 ring->bounce_buf_len, flags);
441 if (ret)
442 return -ENOMEM;
443
444 if (ring->type == TYPE_STREAM)
445 ret = cdnsp_update_stream_segment_mapping(ring->trb_address_map,
446 ring, first,
447 last, flags);
448
449 if (ret) {
450 cdnsp_free_segments_for_ring(pdev, first);
451
452 return ret;
453 }
454
455 cdnsp_link_rings(pdev, ring, first, last, num_segs);
456 trace_cdnsp_ring_expansion(ring);
457
458 return 0;
459}
460
461static int cdnsp_init_device_ctx(struct cdnsp_device *pdev)
462{
463 int size = HCC_64BYTE_CONTEXT(pdev->hcc_params) ? 2048 : 1024;
464
465 pdev->out_ctx.type = CDNSP_CTX_TYPE_DEVICE;
466 pdev->out_ctx.size = size;
467 pdev->out_ctx.ctx_size = CTX_SIZE(pdev->hcc_params);
468 pdev->out_ctx.bytes = dma_pool_zalloc(pdev->device_pool, GFP_ATOMIC,
469 &pdev->out_ctx.dma);
470
471 if (!pdev->out_ctx.bytes)
472 return -ENOMEM;
473
474 pdev->in_ctx.type = CDNSP_CTX_TYPE_INPUT;
475 pdev->in_ctx.ctx_size = pdev->out_ctx.ctx_size;
476 pdev->in_ctx.size = size + pdev->out_ctx.ctx_size;
477 pdev->in_ctx.bytes = dma_pool_zalloc(pdev->device_pool, GFP_ATOMIC,
478 &pdev->in_ctx.dma);
479
480 if (!pdev->in_ctx.bytes) {
481 dma_pool_free(pdev->device_pool, pdev->out_ctx.bytes,
482 pdev->out_ctx.dma);
483 return -ENOMEM;
484 }
485
486 return 0;
487}
488
489struct cdnsp_input_control_ctx
490 *cdnsp_get_input_control_ctx(struct cdnsp_container_ctx *ctx)
491{
492 if (ctx->type != CDNSP_CTX_TYPE_INPUT)
493 return NULL;
494
495 return (struct cdnsp_input_control_ctx *)ctx->bytes;
496}
497
498struct cdnsp_slot_ctx *cdnsp_get_slot_ctx(struct cdnsp_container_ctx *ctx)
499{
500 if (ctx->type == CDNSP_CTX_TYPE_DEVICE)
501 return (struct cdnsp_slot_ctx *)ctx->bytes;
502
503 return (struct cdnsp_slot_ctx *)(ctx->bytes + ctx->ctx_size);
504}
505
506struct cdnsp_ep_ctx *cdnsp_get_ep_ctx(struct cdnsp_container_ctx *ctx,
507 unsigned int ep_index)
508{
509 /* Increment ep index by offset of start of ep ctx array. */
510 ep_index++;
511 if (ctx->type == CDNSP_CTX_TYPE_INPUT)
512 ep_index++;
513
514 return (struct cdnsp_ep_ctx *)(ctx->bytes + (ep_index * ctx->ctx_size));
515}
516
517static void cdnsp_free_stream_ctx(struct cdnsp_device *pdev,
518 struct cdnsp_ep *pep)
519{
520 dma_pool_free(pdev->device_pool, pep->stream_info.stream_ctx_array,
521 pep->stream_info.ctx_array_dma);
522}
523
524/* The stream context array must be a power of 2. */
525static struct cdnsp_stream_ctx
526 *cdnsp_alloc_stream_ctx(struct cdnsp_device *pdev, struct cdnsp_ep *pep)
527{
528 size_t size = sizeof(struct cdnsp_stream_ctx) *
529 pep->stream_info.num_stream_ctxs;
530
531 if (size > CDNSP_CTX_SIZE)
532 return NULL;
533
534 /**
535 * Driver uses intentionally the device_pool to allocated stream
536 * context array. Device Pool has 2048 bytes of size what gives us
537 * 128 entries.
538 */
539 return dma_pool_zalloc(pdev->device_pool, GFP_DMA32 | GFP_ATOMIC,
540 &pep->stream_info.ctx_array_dma);
541}
542
543struct cdnsp_ring *cdnsp_dma_to_transfer_ring(struct cdnsp_ep *pep, u64 address)
544{
545 if (pep->ep_state & EP_HAS_STREAMS)
546 return radix_tree_lookup(&pep->stream_info.trb_address_map,
547 address >> TRB_SEGMENT_SHIFT);
548
549 return pep->ring;
550}
551
552/*
553 * Change an endpoint's internal structure so it supports stream IDs.
554 * The number of requested streams includes stream 0, which cannot be used by
555 * driver.
556 *
557 * The number of stream contexts in the stream context array may be bigger than
558 * the number of streams the driver wants to use. This is because the number of
559 * stream context array entries must be a power of two.
560 */
561int cdnsp_alloc_stream_info(struct cdnsp_device *pdev,
562 struct cdnsp_ep *pep,
563 unsigned int num_stream_ctxs,
564 unsigned int num_streams)
565{
566 struct cdnsp_stream_info *stream_info;
567 struct cdnsp_ring *cur_ring;
568 u32 cur_stream;
569 u64 addr;
570 int ret;
571 int mps;
572
573 stream_info = &pep->stream_info;
574 stream_info->num_streams = num_streams;
575 stream_info->num_stream_ctxs = num_stream_ctxs;
576
577 /* Initialize the array of virtual pointers to stream rings. */
578 stream_info->stream_rings = kcalloc(num_streams,
579 sizeof(struct cdnsp_ring *),
580 GFP_ATOMIC);
581 if (!stream_info->stream_rings)
582 return -ENOMEM;
583
584 /* Initialize the array of DMA addresses for stream rings for the HW. */
585 stream_info->stream_ctx_array = cdnsp_alloc_stream_ctx(pdev, pep);
586 if (!stream_info->stream_ctx_array)
587 goto cleanup_stream_rings;
588
589 memset(stream_info->stream_ctx_array, 0,
590 sizeof(struct cdnsp_stream_ctx) * num_stream_ctxs);
591 INIT_RADIX_TREE(&stream_info->trb_address_map, GFP_ATOMIC);
592 mps = usb_endpoint_maxp(pep->endpoint.desc);
593
594 /*
595 * Allocate rings for all the streams that the driver will use,
596 * and add their segment DMA addresses to the radix tree.
597 * Stream 0 is reserved.
598 */
599 for (cur_stream = 1; cur_stream < num_streams; cur_stream++) {
600 cur_ring = cdnsp_ring_alloc(pdev, 2, TYPE_STREAM, mps,
601 GFP_ATOMIC);
602 stream_info->stream_rings[cur_stream] = cur_ring;
603
604 if (!cur_ring)
605 goto cleanup_rings;
606
607 cur_ring->stream_id = cur_stream;
608 cur_ring->trb_address_map = &stream_info->trb_address_map;
609
610 /* Set deq ptr, cycle bit, and stream context type. */
611 addr = cur_ring->first_seg->dma | SCT_FOR_CTX(SCT_PRI_TR) |
612 cur_ring->cycle_state;
613
614 stream_info->stream_ctx_array[cur_stream].stream_ring =
615 cpu_to_le64(addr);
616
617 trace_cdnsp_set_stream_ring(cur_ring);
618
619 ret = cdnsp_update_stream_mapping(cur_ring);
620 if (ret)
621 goto cleanup_rings;
622 }
623
624 return 0;
625
626cleanup_rings:
627 for (cur_stream = 1; cur_stream < num_streams; cur_stream++) {
628 cur_ring = stream_info->stream_rings[cur_stream];
629 if (cur_ring) {
630 cdnsp_ring_free(pdev, cur_ring);
631 stream_info->stream_rings[cur_stream] = NULL;
632 }
633 }
634
635cleanup_stream_rings:
636 kfree(pep->stream_info.stream_rings);
637
638 return -ENOMEM;
639}
640
641/* Frees all stream contexts associated with the endpoint. */
642static void cdnsp_free_stream_info(struct cdnsp_device *pdev,
643 struct cdnsp_ep *pep)
644{
645 struct cdnsp_stream_info *stream_info = &pep->stream_info;
646 struct cdnsp_ring *cur_ring;
647 int cur_stream;
648
649 if (!(pep->ep_state & EP_HAS_STREAMS))
650 return;
651
652 for (cur_stream = 1; cur_stream < stream_info->num_streams;
653 cur_stream++) {
654 cur_ring = stream_info->stream_rings[cur_stream];
655 if (cur_ring) {
656 cdnsp_ring_free(pdev, cur_ring);
657 stream_info->stream_rings[cur_stream] = NULL;
658 }
659 }
660
661 if (stream_info->stream_ctx_array)
662 cdnsp_free_stream_ctx(pdev, pep);
663
664 kfree(stream_info->stream_rings);
665 pep->ep_state &= ~EP_HAS_STREAMS;
666}
667
668/* All the cdnsp_tds in the ring's TD list should be freed at this point.*/
669static void cdnsp_free_priv_device(struct cdnsp_device *pdev)
670{
671 pdev->dcbaa->dev_context_ptrs[1] = 0;
672
673 cdnsp_free_endpoint_rings(pdev, &pdev->eps[0]);
674
675 if (pdev->in_ctx.bytes)
676 dma_pool_free(pdev->device_pool, pdev->in_ctx.bytes,
677 pdev->in_ctx.dma);
678
679 if (pdev->out_ctx.bytes)
680 dma_pool_free(pdev->device_pool, pdev->out_ctx.bytes,
681 pdev->out_ctx.dma);
682
683 pdev->in_ctx.bytes = NULL;
684 pdev->out_ctx.bytes = NULL;
685}
686
687static int cdnsp_alloc_priv_device(struct cdnsp_device *pdev)
688{
689 int ret;
690
691 ret = cdnsp_init_device_ctx(pdev);
692 if (ret)
693 return ret;
694
695 /* Allocate endpoint 0 ring. */
696 pdev->eps[0].ring = cdnsp_ring_alloc(pdev, 2, TYPE_CTRL, 0, GFP_ATOMIC);
697 if (!pdev->eps[0].ring)
698 goto fail;
699
700 /* Point to output device context in dcbaa. */
701 pdev->dcbaa->dev_context_ptrs[1] = cpu_to_le64(pdev->out_ctx.dma);
702 pdev->cmd.in_ctx = &pdev->in_ctx;
703
704 trace_cdnsp_alloc_priv_device(pdev);
705 return 0;
706fail:
707 dma_pool_free(pdev->device_pool, pdev->out_ctx.bytes,
708 pdev->out_ctx.dma);
709 dma_pool_free(pdev->device_pool, pdev->in_ctx.bytes,
710 pdev->in_ctx.dma);
711
712 return ret;
713}
714
715void cdnsp_copy_ep0_dequeue_into_input_ctx(struct cdnsp_device *pdev)
716{
717 struct cdnsp_ep_ctx *ep0_ctx = pdev->eps[0].in_ctx;
718 struct cdnsp_ring *ep_ring = pdev->eps[0].ring;
719 dma_addr_t dma;
720
721 dma = cdnsp_trb_virt_to_dma(ep_ring->enq_seg, ep_ring->enqueue);
722 ep0_ctx->deq = cpu_to_le64(dma | ep_ring->cycle_state);
723}
724
725/* Setup an controller private device for a Set Address command. */
726int cdnsp_setup_addressable_priv_dev(struct cdnsp_device *pdev)
727{
728 struct cdnsp_slot_ctx *slot_ctx;
729 struct cdnsp_ep_ctx *ep0_ctx;
730 u32 max_packets, port;
731
732 ep0_ctx = cdnsp_get_ep_ctx(&pdev->in_ctx, 0);
733 slot_ctx = cdnsp_get_slot_ctx(&pdev->in_ctx);
734
735 /* Only the control endpoint is valid - one endpoint context. */
736 slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(1));
737
738 switch (pdev->gadget.speed) {
739 case USB_SPEED_SUPER_PLUS:
740 slot_ctx->dev_info |= cpu_to_le32(SLOT_SPEED_SSP);
741 max_packets = MAX_PACKET(512);
742 break;
743 case USB_SPEED_SUPER:
744 slot_ctx->dev_info |= cpu_to_le32(SLOT_SPEED_SS);
745 max_packets = MAX_PACKET(512);
746 break;
747 case USB_SPEED_HIGH:
748 slot_ctx->dev_info |= cpu_to_le32(SLOT_SPEED_HS);
749 max_packets = MAX_PACKET(64);
750 break;
751 case USB_SPEED_FULL:
752 slot_ctx->dev_info |= cpu_to_le32(SLOT_SPEED_FS);
753 max_packets = MAX_PACKET(64);
754 break;
755 default:
756 /* Speed was not set , this shouldn't happen. */
757 return -EINVAL;
758 }
759
760 port = DEV_PORT(pdev->active_port->port_num);
761 slot_ctx->dev_port |= cpu_to_le32(port);
762 slot_ctx->dev_state = cpu_to_le32((pdev->device_address &
763 DEV_ADDR_MASK));
764 ep0_ctx->tx_info = cpu_to_le32(EP_AVG_TRB_LENGTH(0x8));
765 ep0_ctx->ep_info2 = cpu_to_le32(EP_TYPE(CTRL_EP));
766 ep0_ctx->ep_info2 |= cpu_to_le32(MAX_BURST(0) | ERROR_COUNT(3) |
767 max_packets);
768
769 ep0_ctx->deq = cpu_to_le64(pdev->eps[0].ring->first_seg->dma |
770 pdev->eps[0].ring->cycle_state);
771
772 trace_cdnsp_setup_addressable_priv_device(pdev);
773
774 return 0;
775}
776
777/*
778 * Convert interval expressed as 2^(bInterval - 1) == interval into
779 * straight exponent value 2^n == interval.
780 */
781static unsigned int cdnsp_parse_exponent_interval(struct usb_gadget *g,
782 struct cdnsp_ep *pep)
783{
784 unsigned int interval;
785
786 interval = clamp_val(pep->endpoint.desc->bInterval, 1, 16) - 1;
787 if (interval != pep->endpoint.desc->bInterval - 1)
788 dev_warn(&g->dev, "ep %s - rounding interval to %d %sframes\n",
789 pep->name, 1 << interval,
790 g->speed == USB_SPEED_FULL ? "" : "micro");
791
792 /*
793 * Full speed isoc endpoints specify interval in frames,
794 * not microframes. We are using microframes everywhere,
795 * so adjust accordingly.
796 */
797 if (g->speed == USB_SPEED_FULL)
798 interval += 3; /* 1 frame = 2^3 uframes */
799
800 /* Controller handles only up to 512ms (2^12). */
801 if (interval > 12)
802 interval = 12;
803
804 return interval;
805}
806
807/*
808 * Convert bInterval expressed in microframes (in 1-255 range) to exponent of
809 * microframes, rounded down to nearest power of 2.
810 */
811static unsigned int cdnsp_microframes_to_exponent(struct usb_gadget *g,
812 struct cdnsp_ep *pep,
813 unsigned int desc_interval,
814 unsigned int min_exponent,
815 unsigned int max_exponent)
816{
817 unsigned int interval;
818
819 interval = fls(desc_interval) - 1;
820 return clamp_val(interval, min_exponent, max_exponent);
821}
822
823/*
824 * Return the polling interval.
825 *
826 * The polling interval is expressed in "microframes". If controllers's Interval
827 * field is set to N, it will service the endpoint every 2^(Interval)*125us.
828 */
829static unsigned int cdnsp_get_endpoint_interval(struct usb_gadget *g,
830 struct cdnsp_ep *pep)
831{
832 unsigned int interval = 0;
833
834 switch (g->speed) {
835 case USB_SPEED_HIGH:
836 case USB_SPEED_SUPER_PLUS:
837 case USB_SPEED_SUPER:
838 if (usb_endpoint_xfer_int(pep->endpoint.desc) ||
839 usb_endpoint_xfer_isoc(pep->endpoint.desc))
840 interval = cdnsp_parse_exponent_interval(g, pep);
841 break;
842 case USB_SPEED_FULL:
843 if (usb_endpoint_xfer_isoc(pep->endpoint.desc)) {
844 interval = cdnsp_parse_exponent_interval(g, pep);
845 } else if (usb_endpoint_xfer_int(pep->endpoint.desc)) {
846 interval = pep->endpoint.desc->bInterval << 3;
847 interval = cdnsp_microframes_to_exponent(g, pep,
848 interval,
849 3, 10);
850 }
851
852 break;
853 default:
854 WARN_ON(1);
855 }
856
857 return interval;
858}
859
860/*
861 * The "Mult" field in the endpoint context is only set for SuperSpeed isoc eps.
862 * High speed endpoint descriptors can define "the number of additional
863 * transaction opportunities per microframe", but that goes in the Max Burst
864 * endpoint context field.
865 */
866static u32 cdnsp_get_endpoint_mult(struct usb_gadget *g, struct cdnsp_ep *pep)
867{
868 if (g->speed < USB_SPEED_SUPER ||
869 !usb_endpoint_xfer_isoc(pep->endpoint.desc))
870 return 0;
871
872 return pep->endpoint.comp_desc->bmAttributes;
873}
874
875static u32 cdnsp_get_endpoint_max_burst(struct usb_gadget *g,
876 struct cdnsp_ep *pep)
877{
878 /* Super speed and Plus have max burst in ep companion desc */
879 if (g->speed >= USB_SPEED_SUPER)
880 return pep->endpoint.comp_desc->bMaxBurst;
881
882 if (g->speed == USB_SPEED_HIGH &&
883 (usb_endpoint_xfer_isoc(pep->endpoint.desc) ||
884 usb_endpoint_xfer_int(pep->endpoint.desc)))
885 return usb_endpoint_maxp_mult(pep->endpoint.desc) - 1;
886
887 return 0;
888}
889
890static u32 cdnsp_get_endpoint_type(const struct usb_endpoint_descriptor *desc)
891{
892 int in;
893
894 in = usb_endpoint_dir_in(desc);
895
896 switch (usb_endpoint_type(desc)) {
897 case USB_ENDPOINT_XFER_CONTROL:
898 return CTRL_EP;
899 case USB_ENDPOINT_XFER_BULK:
900 return in ? BULK_IN_EP : BULK_OUT_EP;
901 case USB_ENDPOINT_XFER_ISOC:
902 return in ? ISOC_IN_EP : ISOC_OUT_EP;
903 case USB_ENDPOINT_XFER_INT:
904 return in ? INT_IN_EP : INT_OUT_EP;
905 }
906
907 return 0;
908}
909
910/*
911 * Return the maximum endpoint service interval time (ESIT) payload.
912 * Basically, this is the maxpacket size, multiplied by the burst size
913 * and mult size.
914 */
915static u32 cdnsp_get_max_esit_payload(struct usb_gadget *g,
916 struct cdnsp_ep *pep)
917{
918 int max_packet;
919 int max_burst;
920
921 /* Only applies for interrupt or isochronous endpoints*/
922 if (usb_endpoint_xfer_control(pep->endpoint.desc) ||
923 usb_endpoint_xfer_bulk(pep->endpoint.desc))
924 return 0;
925
926 /* SuperSpeedPlus Isoc ep sending over 48k per EIST. */
927 if (g->speed >= USB_SPEED_SUPER_PLUS &&
928 USB_SS_SSP_ISOC_COMP(pep->endpoint.desc->bmAttributes))
929 return le16_to_cpu(pep->endpoint.comp_desc->wBytesPerInterval);
930 /* SuperSpeed or SuperSpeedPlus Isoc ep with less than 48k per esit */
931 else if (g->speed >= USB_SPEED_SUPER)
932 return le16_to_cpu(pep->endpoint.comp_desc->wBytesPerInterval);
933
934 max_packet = usb_endpoint_maxp(pep->endpoint.desc);
935 max_burst = usb_endpoint_maxp_mult(pep->endpoint.desc);
936
937 /* A 0 in max burst means 1 transfer per ESIT */
938 return max_packet * max_burst;
939}
940
941int cdnsp_endpoint_init(struct cdnsp_device *pdev,
942 struct cdnsp_ep *pep,
943 gfp_t mem_flags)
944{
945 enum cdnsp_ring_type ring_type;
946 struct cdnsp_ep_ctx *ep_ctx;
947 unsigned int err_count = 0;
948 unsigned int avg_trb_len;
949 unsigned int max_packet;
950 unsigned int max_burst;
951 unsigned int interval;
952 u32 max_esit_payload;
953 unsigned int mult;
954 u32 endpoint_type;
955 int ret;
956
957 ep_ctx = pep->in_ctx;
958
959 endpoint_type = cdnsp_get_endpoint_type(pep->endpoint.desc);
960 if (!endpoint_type)
961 return -EINVAL;
962
963 ring_type = usb_endpoint_type(pep->endpoint.desc);
964
965 /*
966 * Get values to fill the endpoint context, mostly from ep descriptor.
967 * The average TRB buffer length for bulk endpoints is unclear as we
968 * have no clue on scatter gather list entry size. For Isoc and Int,
969 * set it to max available.
970 */
971 max_esit_payload = cdnsp_get_max_esit_payload(&pdev->gadget, pep);
972 interval = cdnsp_get_endpoint_interval(&pdev->gadget, pep);
973 mult = cdnsp_get_endpoint_mult(&pdev->gadget, pep);
974 max_packet = usb_endpoint_maxp(pep->endpoint.desc);
975 max_burst = cdnsp_get_endpoint_max_burst(&pdev->gadget, pep);
976 avg_trb_len = max_esit_payload;
977
978 /* Allow 3 retries for everything but isoc, set CErr = 3. */
979 if (!usb_endpoint_xfer_isoc(pep->endpoint.desc))
980 err_count = 3;
981 if (usb_endpoint_xfer_bulk(pep->endpoint.desc) &&
982 pdev->gadget.speed == USB_SPEED_HIGH)
983 max_packet = 512;
984 /* Controller spec indicates that ctrl ep avg TRB Length should be 8. */
985 if (usb_endpoint_xfer_control(pep->endpoint.desc))
986 avg_trb_len = 8;
987
988 /* Set up the endpoint ring. */
989 pep->ring = cdnsp_ring_alloc(pdev, 2, ring_type, max_packet, mem_flags);
990 if (!pep->ring)
991 return -ENOMEM;
992
993 pep->skip = false;
994
995 /* Fill the endpoint context */
996 ep_ctx->ep_info = cpu_to_le32(EP_MAX_ESIT_PAYLOAD_HI(max_esit_payload) |
997 EP_INTERVAL(interval) | EP_MULT(mult));
998 ep_ctx->ep_info2 = cpu_to_le32(EP_TYPE(endpoint_type) |
999 MAX_PACKET(max_packet) | MAX_BURST(max_burst) |
1000 ERROR_COUNT(err_count));
1001 ep_ctx->deq = cpu_to_le64(pep->ring->first_seg->dma |
1002 pep->ring->cycle_state);
1003
1004 ep_ctx->tx_info = cpu_to_le32(EP_MAX_ESIT_PAYLOAD_LO(max_esit_payload) |
1005 EP_AVG_TRB_LENGTH(avg_trb_len));
1006
1007 if (usb_endpoint_xfer_bulk(pep->endpoint.desc) &&
1008 pdev->gadget.speed > USB_SPEED_HIGH) {
1009 ret = cdnsp_alloc_streams(pdev, pep);
1010 if (ret < 0)
1011 return ret;
1012 }
1013
1014 return 0;
1015}
1016
1017void cdnsp_endpoint_zero(struct cdnsp_device *pdev, struct cdnsp_ep *pep)
1018{
1019 pep->in_ctx->ep_info = 0;
1020 pep->in_ctx->ep_info2 = 0;
1021 pep->in_ctx->deq = 0;
1022 pep->in_ctx->tx_info = 0;
1023}
1024
1025static int cdnsp_alloc_erst(struct cdnsp_device *pdev,
1026 struct cdnsp_ring *evt_ring,
1027 struct cdnsp_erst *erst)
1028{
1029 struct cdnsp_erst_entry *entry;
1030 struct cdnsp_segment *seg;
1031 unsigned int val;
1032 size_t size;
1033
1034 size = sizeof(struct cdnsp_erst_entry) * evt_ring->num_segs;
1035 erst->entries = dma_alloc_coherent(pdev->dev, size,
1036 &erst->erst_dma_addr, GFP_KERNEL);
1037 if (!erst->entries)
1038 return -ENOMEM;
1039
1040 erst->num_entries = evt_ring->num_segs;
1041
1042 seg = evt_ring->first_seg;
1043 for (val = 0; val < evt_ring->num_segs; val++) {
1044 entry = &erst->entries[val];
1045 entry->seg_addr = cpu_to_le64(seg->dma);
1046 entry->seg_size = cpu_to_le32(TRBS_PER_SEGMENT);
1047 entry->rsvd = 0;
1048 seg = seg->next;
1049 }
1050
1051 return 0;
1052}
1053
1054static void cdnsp_free_erst(struct cdnsp_device *pdev, struct cdnsp_erst *erst)
1055{
1056 size_t size = sizeof(struct cdnsp_erst_entry) * (erst->num_entries);
1057 struct device *dev = pdev->dev;
1058
1059 if (erst->entries)
1060 dma_free_coherent(dev, size, erst->entries,
1061 erst->erst_dma_addr);
1062
1063 erst->entries = NULL;
1064}
1065
1066void cdnsp_mem_cleanup(struct cdnsp_device *pdev)
1067{
1068 struct device *dev = pdev->dev;
1069
1070 cdnsp_free_priv_device(pdev);
1071 cdnsp_free_erst(pdev, &pdev->erst);
1072
1073 if (pdev->event_ring)
1074 cdnsp_ring_free(pdev, pdev->event_ring);
1075
1076 pdev->event_ring = NULL;
1077
1078 if (pdev->cmd_ring)
1079 cdnsp_ring_free(pdev, pdev->cmd_ring);
1080
1081 pdev->cmd_ring = NULL;
1082
1083 dma_pool_destroy(pdev->segment_pool);
1084 pdev->segment_pool = NULL;
1085 dma_pool_destroy(pdev->device_pool);
1086 pdev->device_pool = NULL;
1087
1088 dma_free_coherent(dev, sizeof(*pdev->dcbaa),
1089 pdev->dcbaa, pdev->dcbaa->dma);
1090
1091 pdev->dcbaa = NULL;
1092
1093 pdev->usb2_port.exist = 0;
1094 pdev->usb3_port.exist = 0;
1095 pdev->usb2_port.port_num = 0;
1096 pdev->usb3_port.port_num = 0;
1097 pdev->active_port = NULL;
1098}
1099
1100static void cdnsp_set_event_deq(struct cdnsp_device *pdev)
1101{
1102 dma_addr_t deq;
1103 u64 temp;
1104
1105 deq = cdnsp_trb_virt_to_dma(pdev->event_ring->deq_seg,
1106 pdev->event_ring->dequeue);
1107
1108 /* Update controller event ring dequeue pointer */
1109 temp = cdnsp_read_64(&pdev->ir_set->erst_dequeue);
1110 temp &= ERST_PTR_MASK;
1111
1112 /*
1113 * Don't clear the EHB bit (which is RW1C) because
1114 * there might be more events to service.
1115 */
1116 temp &= ~ERST_EHB;
1117
1118 cdnsp_write_64(((u64)deq & (u64)~ERST_PTR_MASK) | temp,
1119 &pdev->ir_set->erst_dequeue);
1120}
1121
1122static void cdnsp_add_in_port(struct cdnsp_device *pdev,
1123 struct cdnsp_port *port,
1124 __le32 __iomem *addr)
1125{
1126 u32 temp, port_offset, port_count;
1127
1128 temp = readl(addr);
1129 port->maj_rev = CDNSP_EXT_PORT_MAJOR(temp);
1130 port->min_rev = CDNSP_EXT_PORT_MINOR(temp);
1131
1132 /* Port offset and count in the third dword.*/
1133 temp = readl(addr + 2);
1134 port_offset = CDNSP_EXT_PORT_OFF(temp);
1135 port_count = CDNSP_EXT_PORT_COUNT(temp);
1136
1137 trace_cdnsp_port_info(addr, port_offset, port_count, port->maj_rev);
1138
1139 port->port_num = port_offset;
1140 port->exist = 1;
1141}
1142
1143/*
1144 * Scan the Extended Capabilities for the "Supported Protocol Capabilities" that
1145 * specify what speeds each port is supposed to be.
1146 */
1147static int cdnsp_setup_port_arrays(struct cdnsp_device *pdev)
1148{
1149 void __iomem *base;
1150 u32 offset;
1151 int i;
1152
1153 base = &pdev->cap_regs->hc_capbase;
1154 offset = cdnsp_find_next_ext_cap(base, 0,
1155 EXT_CAP_CFG_DEV_20PORT_CAP_ID);
1156 pdev->port20_regs = base + offset;
1157
1158 offset = cdnsp_find_next_ext_cap(base, 0, D_XEC_CFG_3XPORT_CAP);
1159 pdev->port3x_regs = base + offset;
1160
1161 offset = 0;
1162 base = &pdev->cap_regs->hc_capbase;
1163
1164 /* Driver expects max 2 extended protocol capability. */
1165 for (i = 0; i < 2; i++) {
1166 u32 temp;
1167
1168 offset = cdnsp_find_next_ext_cap(base, offset,
1169 EXT_CAPS_PROTOCOL);
1170 temp = readl(base + offset);
1171
1172 if (CDNSP_EXT_PORT_MAJOR(temp) == 0x03 &&
1173 !pdev->usb3_port.port_num)
1174 cdnsp_add_in_port(pdev, &pdev->usb3_port,
1175 base + offset);
1176
1177 if (CDNSP_EXT_PORT_MAJOR(temp) == 0x02 &&
1178 !pdev->usb2_port.port_num)
1179 cdnsp_add_in_port(pdev, &pdev->usb2_port,
1180 base + offset);
1181 }
1182
1183 if (!pdev->usb2_port.exist || !pdev->usb3_port.exist) {
1184 dev_err(pdev->dev, "Error: Only one port detected\n");
1185 return -ENODEV;
1186 }
1187
1188 trace_cdnsp_init("Found USB 2.0 ports and USB 3.0 ports.");
1189
1190 pdev->usb2_port.regs = (struct cdnsp_port_regs __iomem *)
1191 (&pdev->op_regs->port_reg_base + NUM_PORT_REGS *
1192 (pdev->usb2_port.port_num - 1));
1193
1194 pdev->usb3_port.regs = (struct cdnsp_port_regs __iomem *)
1195 (&pdev->op_regs->port_reg_base + NUM_PORT_REGS *
1196 (pdev->usb3_port.port_num - 1));
1197
1198 return 0;
1199}
1200
1201/*
1202 * Initialize memory for CDNSP (one-time init).
1203 *
1204 * Program the PAGESIZE register, initialize the device context array, create
1205 * device contexts, set up a command ring segment, create event
1206 * ring (one for now).
1207 */
1208int cdnsp_mem_init(struct cdnsp_device *pdev)
1209{
1210 struct device *dev = pdev->dev;
1211 int ret = -ENOMEM;
1212 unsigned int val;
1213 dma_addr_t dma;
1214 u32 page_size;
1215 u64 val_64;
1216
1217 /*
1218 * Use 4K pages, since that's common and the minimum the
1219 * controller supports
1220 */
1221 page_size = 1 << 12;
1222
1223 val = readl(&pdev->op_regs->config_reg);
1224 val |= ((val & ~MAX_DEVS) | CDNSP_DEV_MAX_SLOTS) | CONFIG_U3E;
1225 writel(val, &pdev->op_regs->config_reg);
1226
1227 /*
1228 * Doorbell array must be physically contiguous
1229 * and 64-byte (cache line) aligned.
1230 */
1231 pdev->dcbaa = dma_alloc_coherent(dev, sizeof(*pdev->dcbaa),
1232 &dma, GFP_KERNEL);
1233 if (!pdev->dcbaa)
1234 return -ENOMEM;
1235
1236 pdev->dcbaa->dma = dma;
1237
1238 cdnsp_write_64(dma, &pdev->op_regs->dcbaa_ptr);
1239
1240 /*
1241 * Initialize the ring segment pool. The ring must be a contiguous
1242 * structure comprised of TRBs. The TRBs must be 16 byte aligned,
1243 * however, the command ring segment needs 64-byte aligned segments
1244 * and our use of dma addresses in the trb_address_map radix tree needs
1245 * TRB_SEGMENT_SIZE alignment, so driver pick the greater alignment
1246 * need.
1247 */
1248 pdev->segment_pool = dma_pool_create("CDNSP ring segments", dev,
1249 TRB_SEGMENT_SIZE, TRB_SEGMENT_SIZE,
1250 page_size);
1251 if (!pdev->segment_pool)
1252 goto release_dcbaa;
1253
1254 pdev->device_pool = dma_pool_create("CDNSP input/output contexts", dev,
1255 CDNSP_CTX_SIZE, 64, page_size);
1256 if (!pdev->device_pool)
1257 goto destroy_segment_pool;
1258
1259
1260 /* Set up the command ring to have one segments for now. */
1261 pdev->cmd_ring = cdnsp_ring_alloc(pdev, 1, TYPE_COMMAND, 0, GFP_KERNEL);
1262 if (!pdev->cmd_ring)
1263 goto destroy_device_pool;
1264
1265 /* Set the address in the Command Ring Control register */
1266 val_64 = cdnsp_read_64(&pdev->op_regs->cmd_ring);
1267 val_64 = (val_64 & (u64)CMD_RING_RSVD_BITS) |
1268 (pdev->cmd_ring->first_seg->dma & (u64)~CMD_RING_RSVD_BITS) |
1269 pdev->cmd_ring->cycle_state;
1270 cdnsp_write_64(val_64, &pdev->op_regs->cmd_ring);
1271
1272 val = readl(&pdev->cap_regs->db_off);
1273 val &= DBOFF_MASK;
1274 pdev->dba = (void __iomem *)pdev->cap_regs + val;
1275
1276 /* Set ir_set to interrupt register set 0 */
1277 pdev->ir_set = &pdev->run_regs->ir_set[0];
1278
1279 /*
1280 * Event ring setup: Allocate a normal ring, but also setup
1281 * the event ring segment table (ERST).
1282 */
1283 pdev->event_ring = cdnsp_ring_alloc(pdev, ERST_NUM_SEGS, TYPE_EVENT,
1284 0, GFP_KERNEL);
1285 if (!pdev->event_ring)
1286 goto free_cmd_ring;
1287
1288 ret = cdnsp_alloc_erst(pdev, pdev->event_ring, &pdev->erst);
1289 if (ret)
1290 goto free_event_ring;
1291
1292 /* Set ERST count with the number of entries in the segment table. */
1293 val = readl(&pdev->ir_set->erst_size);
1294 val &= ERST_SIZE_MASK;
1295 val |= ERST_NUM_SEGS;
1296 writel(val, &pdev->ir_set->erst_size);
1297
1298 /* Set the segment table base address. */
1299 val_64 = cdnsp_read_64(&pdev->ir_set->erst_base);
1300 val_64 &= ERST_PTR_MASK;
1301 val_64 |= (pdev->erst.erst_dma_addr & (u64)~ERST_PTR_MASK);
1302 cdnsp_write_64(val_64, &pdev->ir_set->erst_base);
1303
1304 /* Set the event ring dequeue address. */
1305 cdnsp_set_event_deq(pdev);
1306
1307 ret = cdnsp_setup_port_arrays(pdev);
1308 if (ret)
1309 goto free_erst;
1310
1311 ret = cdnsp_alloc_priv_device(pdev);
1312 if (ret) {
1313 dev_err(pdev->dev,
1314 "Could not allocate cdnsp_device data structures\n");
1315 goto free_erst;
1316 }
1317
1318 return 0;
1319
1320free_erst:
1321 cdnsp_free_erst(pdev, &pdev->erst);
1322free_event_ring:
1323 cdnsp_ring_free(pdev, pdev->event_ring);
1324free_cmd_ring:
1325 cdnsp_ring_free(pdev, pdev->cmd_ring);
1326destroy_device_pool:
1327 dma_pool_destroy(pdev->device_pool);
1328destroy_segment_pool:
1329 dma_pool_destroy(pdev->segment_pool);
1330release_dcbaa:
1331 dma_free_coherent(dev, sizeof(*pdev->dcbaa), pdev->dcbaa,
1332 pdev->dcbaa->dma);
1333
1334 cdnsp_reset(pdev);
1335
1336 return ret;
1337}