Loading...
Note: File does not exist in v6.8.
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Cadence USBSS DRD Driver - gadget side.
4 *
5 * Copyright (C) 2018-2019 Cadence Design Systems.
6 * Copyright (C) 2017-2018 NXP
7 *
8 * Authors: Pawel Jez <pjez@cadence.com>,
9 * Pawel Laszczak <pawell@cadence.com>
10 * Peter Chen <peter.chen@nxp.com>
11 */
12
13/*
14 * Work around 1:
15 * At some situations, the controller may get stale data address in TRB
16 * at below sequences:
17 * 1. Controller read TRB includes data address
18 * 2. Software updates TRBs includes data address and Cycle bit
19 * 3. Controller read TRB which includes Cycle bit
20 * 4. DMA run with stale data address
21 *
22 * To fix this problem, driver needs to make the first TRB in TD as invalid.
23 * After preparing all TRBs driver needs to check the position of DMA and
24 * if the DMA point to the first just added TRB and doorbell is 1,
25 * then driver must defer making this TRB as valid. This TRB will be make
26 * as valid during adding next TRB only if DMA is stopped or at TRBERR
27 * interrupt.
28 *
29 * Issue has been fixed in DEV_VER_V3 version of controller.
30 *
31 * Work around 2:
32 * Controller for OUT endpoints has shared on-chip buffers for all incoming
33 * packets, including ep0out. It's FIFO buffer, so packets must be handle by DMA
34 * in correct order. If the first packet in the buffer will not be handled,
35 * then the following packets directed for other endpoints and functions
36 * will be blocked.
37 * Additionally the packets directed to one endpoint can block entire on-chip
38 * buffers. In this case transfer to other endpoints also will blocked.
39 *
40 * To resolve this issue after raising the descriptor missing interrupt
41 * driver prepares internal usb_request object and use it to arm DMA transfer.
42 *
43 * The problematic situation was observed in case when endpoint has been enabled
44 * but no usb_request were queued. Driver try detects such endpoints and will
45 * use this workaround only for these endpoint.
46 *
47 * Driver use limited number of buffer. This number can be set by macro
48 * CDNS3_WA2_NUM_BUFFERS.
49 *
50 * Such blocking situation was observed on ACM gadget. For this function
51 * host send OUT data packet but ACM function is not prepared for this packet.
52 * It's cause that buffer placed in on chip memory block transfer to other
53 * endpoints.
54 *
55 * Issue has been fixed in DEV_VER_V2 version of controller.
56 *
57 */
58
59#include <linux/dma-mapping.h>
60#include <linux/usb/gadget.h>
61#include <linux/module.h>
62#include <linux/iopoll.h>
63
64#include "core.h"
65#include "gadget-export.h"
66#include "gadget.h"
67#include "trace.h"
68#include "drd.h"
69
70static int __cdns3_gadget_ep_queue(struct usb_ep *ep,
71 struct usb_request *request,
72 gfp_t gfp_flags);
73
74/**
75 * cdns3_set_register_bit - set bit in given register.
76 * @ptr: address of device controller register to be read and changed
77 * @mask: bits requested to set
78 */
79void cdns3_set_register_bit(void __iomem *ptr, u32 mask)
80{
81 mask = readl(ptr) | mask;
82 writel(mask, ptr);
83}
84
85/**
86 * cdns3_ep_addr_to_index - Macro converts endpoint address to
87 * index of endpoint object in cdns3_device.eps[] container
88 * @ep_addr: endpoint address for which endpoint object is required
89 *
90 */
91u8 cdns3_ep_addr_to_index(u8 ep_addr)
92{
93 return (((ep_addr & 0x7F)) + ((ep_addr & USB_DIR_IN) ? 16 : 0));
94}
95
96static int cdns3_get_dma_pos(struct cdns3_device *priv_dev,
97 struct cdns3_endpoint *priv_ep)
98{
99 int dma_index;
100
101 dma_index = readl(&priv_dev->regs->ep_traddr) - priv_ep->trb_pool_dma;
102
103 return dma_index / TRB_SIZE;
104}
105
106/**
107 * cdns3_next_request - returns next request from list
108 * @list: list containing requests
109 *
110 * Returns request or NULL if no requests in list
111 */
112struct usb_request *cdns3_next_request(struct list_head *list)
113{
114 return list_first_entry_or_null(list, struct usb_request, list);
115}
116
117/**
118 * cdns3_next_align_buf - returns next buffer from list
119 * @list: list containing buffers
120 *
121 * Returns buffer or NULL if no buffers in list
122 */
123struct cdns3_aligned_buf *cdns3_next_align_buf(struct list_head *list)
124{
125 return list_first_entry_or_null(list, struct cdns3_aligned_buf, list);
126}
127
128/**
129 * cdns3_next_priv_request - returns next request from list
130 * @list: list containing requests
131 *
132 * Returns request or NULL if no requests in list
133 */
134struct cdns3_request *cdns3_next_priv_request(struct list_head *list)
135{
136 return list_first_entry_or_null(list, struct cdns3_request, list);
137}
138
139/**
140 * select_ep - selects endpoint
141 * @priv_dev: extended gadget object
142 * @ep: endpoint address
143 */
144void cdns3_select_ep(struct cdns3_device *priv_dev, u32 ep)
145{
146 if (priv_dev->selected_ep == ep)
147 return;
148
149 priv_dev->selected_ep = ep;
150 writel(ep, &priv_dev->regs->ep_sel);
151}
152
153dma_addr_t cdns3_trb_virt_to_dma(struct cdns3_endpoint *priv_ep,
154 struct cdns3_trb *trb)
155{
156 u32 offset = (char *)trb - (char *)priv_ep->trb_pool;
157
158 return priv_ep->trb_pool_dma + offset;
159}
160
161int cdns3_ring_size(struct cdns3_endpoint *priv_ep)
162{
163 switch (priv_ep->type) {
164 case USB_ENDPOINT_XFER_ISOC:
165 return TRB_ISO_RING_SIZE;
166 case USB_ENDPOINT_XFER_CONTROL:
167 return TRB_CTRL_RING_SIZE;
168 default:
169 return TRB_RING_SIZE;
170 }
171}
172
173/**
174 * cdns3_allocate_trb_pool - Allocates TRB's pool for selected endpoint
175 * @priv_ep: endpoint object
176 *
177 * Function will return 0 on success or -ENOMEM on allocation error
178 */
179int cdns3_allocate_trb_pool(struct cdns3_endpoint *priv_ep)
180{
181 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
182 int ring_size = cdns3_ring_size(priv_ep);
183 struct cdns3_trb *link_trb;
184
185 if (!priv_ep->trb_pool) {
186 priv_ep->trb_pool = dma_alloc_coherent(priv_dev->sysdev,
187 ring_size,
188 &priv_ep->trb_pool_dma,
189 GFP_DMA32 | GFP_ATOMIC);
190 if (!priv_ep->trb_pool)
191 return -ENOMEM;
192 } else {
193 memset(priv_ep->trb_pool, 0, ring_size);
194 }
195
196 if (!priv_ep->num)
197 return 0;
198
199 priv_ep->num_trbs = ring_size / TRB_SIZE;
200 /* Initialize the last TRB as Link TRB. */
201 link_trb = (priv_ep->trb_pool + (priv_ep->num_trbs - 1));
202 link_trb->buffer = TRB_BUFFER(priv_ep->trb_pool_dma);
203 link_trb->control = TRB_CYCLE | TRB_TYPE(TRB_LINK) | TRB_TOGGLE;
204
205 return 0;
206}
207
208static void cdns3_free_trb_pool(struct cdns3_endpoint *priv_ep)
209{
210 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
211
212 if (priv_ep->trb_pool) {
213 dma_free_coherent(priv_dev->sysdev,
214 cdns3_ring_size(priv_ep),
215 priv_ep->trb_pool, priv_ep->trb_pool_dma);
216 priv_ep->trb_pool = NULL;
217 }
218}
219
220/**
221 * cdns3_ep_stall_flush - Stalls and flushes selected endpoint
222 * @priv_ep: endpoint object
223 *
224 * Endpoint must be selected before call to this function
225 */
226static void cdns3_ep_stall_flush(struct cdns3_endpoint *priv_ep)
227{
228 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
229 int val;
230
231 trace_cdns3_halt(priv_ep, 1, 1);
232
233 writel(EP_CMD_DFLUSH | EP_CMD_ERDY | EP_CMD_SSTALL,
234 &priv_dev->regs->ep_cmd);
235
236 /* wait for DFLUSH cleared */
237 readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
238 !(val & EP_CMD_DFLUSH), 1, 1000);
239 priv_ep->flags |= EP_STALLED;
240 priv_ep->flags &= ~EP_STALL_PENDING;
241}
242
243/**
244 * cdns3_hw_reset_eps_config - reset endpoints configuration kept by controller.
245 * @priv_dev: extended gadget object
246 */
247void cdns3_hw_reset_eps_config(struct cdns3_device *priv_dev)
248{
249 writel(USB_CONF_CFGRST, &priv_dev->regs->usb_conf);
250
251 cdns3_allow_enable_l1(priv_dev, 0);
252 priv_dev->hw_configured_flag = 0;
253 priv_dev->onchip_used_size = 0;
254 priv_dev->out_mem_is_allocated = 0;
255 priv_dev->wait_for_setup = 0;
256}
257
258/**
259 * cdns3_ep_inc_trb - increment a trb index.
260 * @index: Pointer to the TRB index to increment.
261 * @cs: Cycle state
262 * @trb_in_seg: number of TRBs in segment
263 *
264 * The index should never point to the link TRB. After incrementing,
265 * if it is point to the link TRB, wrap around to the beginning and revert
266 * cycle state bit The
267 * link TRB is always at the last TRB entry.
268 */
269static void cdns3_ep_inc_trb(int *index, u8 *cs, int trb_in_seg)
270{
271 (*index)++;
272 if (*index == (trb_in_seg - 1)) {
273 *index = 0;
274 *cs ^= 1;
275 }
276}
277
278/**
279 * cdns3_ep_inc_enq - increment endpoint's enqueue pointer
280 * @priv_ep: The endpoint whose enqueue pointer we're incrementing
281 */
282static void cdns3_ep_inc_enq(struct cdns3_endpoint *priv_ep)
283{
284 priv_ep->free_trbs--;
285 cdns3_ep_inc_trb(&priv_ep->enqueue, &priv_ep->pcs, priv_ep->num_trbs);
286}
287
288/**
289 * cdns3_ep_inc_deq - increment endpoint's dequeue pointer
290 * @priv_ep: The endpoint whose dequeue pointer we're incrementing
291 */
292static void cdns3_ep_inc_deq(struct cdns3_endpoint *priv_ep)
293{
294 priv_ep->free_trbs++;
295 cdns3_ep_inc_trb(&priv_ep->dequeue, &priv_ep->ccs, priv_ep->num_trbs);
296}
297
298void cdns3_move_deq_to_next_trb(struct cdns3_request *priv_req)
299{
300 struct cdns3_endpoint *priv_ep = priv_req->priv_ep;
301 int current_trb = priv_req->start_trb;
302
303 while (current_trb != priv_req->end_trb) {
304 cdns3_ep_inc_deq(priv_ep);
305 current_trb = priv_ep->dequeue;
306 }
307
308 cdns3_ep_inc_deq(priv_ep);
309}
310
311/**
312 * cdns3_allow_enable_l1 - enable/disable permits to transition to L1.
313 * @priv_dev: Extended gadget object
314 * @enable: Enable/disable permit to transition to L1.
315 *
316 * If bit USB_CONF_L1EN is set and device receive Extended Token packet,
317 * then controller answer with ACK handshake.
318 * If bit USB_CONF_L1DS is set and device receive Extended Token packet,
319 * then controller answer with NYET handshake.
320 */
321void cdns3_allow_enable_l1(struct cdns3_device *priv_dev, int enable)
322{
323 if (enable)
324 writel(USB_CONF_L1EN, &priv_dev->regs->usb_conf);
325 else
326 writel(USB_CONF_L1DS, &priv_dev->regs->usb_conf);
327}
328
329enum usb_device_speed cdns3_get_speed(struct cdns3_device *priv_dev)
330{
331 u32 reg;
332
333 reg = readl(&priv_dev->regs->usb_sts);
334
335 if (DEV_SUPERSPEED(reg))
336 return USB_SPEED_SUPER;
337 else if (DEV_HIGHSPEED(reg))
338 return USB_SPEED_HIGH;
339 else if (DEV_FULLSPEED(reg))
340 return USB_SPEED_FULL;
341 else if (DEV_LOWSPEED(reg))
342 return USB_SPEED_LOW;
343 return USB_SPEED_UNKNOWN;
344}
345
346/**
347 * cdns3_start_all_request - add to ring all request not started
348 * @priv_dev: Extended gadget object
349 * @priv_ep: The endpoint for whom request will be started.
350 *
351 * Returns return ENOMEM if transfer ring i not enough TRBs to start
352 * all requests.
353 */
354static int cdns3_start_all_request(struct cdns3_device *priv_dev,
355 struct cdns3_endpoint *priv_ep)
356{
357 struct usb_request *request;
358 int ret = 0;
359
360 while (!list_empty(&priv_ep->deferred_req_list)) {
361 request = cdns3_next_request(&priv_ep->deferred_req_list);
362
363 ret = cdns3_ep_run_transfer(priv_ep, request);
364 if (ret)
365 return ret;
366
367 list_del(&request->list);
368 list_add_tail(&request->list,
369 &priv_ep->pending_req_list);
370 }
371
372 priv_ep->flags &= ~EP_RING_FULL;
373 return ret;
374}
375
376/*
377 * WA2: Set flag for all not ISOC OUT endpoints. If this flag is set
378 * driver try to detect whether endpoint need additional internal
379 * buffer for unblocking on-chip FIFO buffer. This flag will be cleared
380 * if before first DESCMISS interrupt the DMA will be armed.
381 */
382#define cdns3_wa2_enable_detection(priv_dev, ep_priv, reg) do { \
383 if (!priv_ep->dir && priv_ep->type != USB_ENDPOINT_XFER_ISOC) { \
384 priv_ep->flags |= EP_QUIRK_EXTRA_BUF_DET; \
385 (reg) |= EP_STS_EN_DESCMISEN; \
386 } } while (0)
387
388/**
389 * cdns3_wa2_descmiss_copy_data copy data from internal requests to
390 * request queued by class driver.
391 * @priv_ep: extended endpoint object
392 * @request: request object
393 */
394static void cdns3_wa2_descmiss_copy_data(struct cdns3_endpoint *priv_ep,
395 struct usb_request *request)
396{
397 struct usb_request *descmiss_req;
398 struct cdns3_request *descmiss_priv_req;
399
400 while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
401 int chunk_end;
402 int length;
403
404 descmiss_priv_req =
405 cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
406 descmiss_req = &descmiss_priv_req->request;
407
408 /* driver can't touch pending request */
409 if (descmiss_priv_req->flags & REQUEST_PENDING)
410 break;
411
412 chunk_end = descmiss_priv_req->flags & REQUEST_INTERNAL_CH;
413 length = request->actual + descmiss_req->actual;
414
415 request->status = descmiss_req->status;
416
417 if (length <= request->length) {
418 memcpy(&((u8 *)request->buf)[request->actual],
419 descmiss_req->buf,
420 descmiss_req->actual);
421 request->actual = length;
422 } else {
423 /* It should never occures */
424 request->status = -ENOMEM;
425 }
426
427 list_del_init(&descmiss_priv_req->list);
428
429 kfree(descmiss_req->buf);
430 cdns3_gadget_ep_free_request(&priv_ep->endpoint, descmiss_req);
431 --priv_ep->wa2_counter;
432
433 if (!chunk_end)
434 break;
435 }
436}
437
438struct usb_request *cdns3_wa2_gadget_giveback(struct cdns3_device *priv_dev,
439 struct cdns3_endpoint *priv_ep,
440 struct cdns3_request *priv_req)
441{
442 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN &&
443 priv_req->flags & REQUEST_INTERNAL) {
444 struct usb_request *req;
445
446 req = cdns3_next_request(&priv_ep->deferred_req_list);
447
448 priv_ep->descmis_req = NULL;
449
450 if (!req)
451 return NULL;
452
453 cdns3_wa2_descmiss_copy_data(priv_ep, req);
454 if (!(priv_ep->flags & EP_QUIRK_END_TRANSFER) &&
455 req->length != req->actual) {
456 /* wait for next part of transfer */
457 return NULL;
458 }
459
460 if (req->status == -EINPROGRESS)
461 req->status = 0;
462
463 list_del_init(&req->list);
464 cdns3_start_all_request(priv_dev, priv_ep);
465 return req;
466 }
467
468 return &priv_req->request;
469}
470
471int cdns3_wa2_gadget_ep_queue(struct cdns3_device *priv_dev,
472 struct cdns3_endpoint *priv_ep,
473 struct cdns3_request *priv_req)
474{
475 int deferred = 0;
476
477 /*
478 * If transfer was queued before DESCMISS appear than we
479 * can disable handling of DESCMISS interrupt. Driver assumes that it
480 * can disable special treatment for this endpoint.
481 */
482 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) {
483 u32 reg;
484
485 cdns3_select_ep(priv_dev, priv_ep->num | priv_ep->dir);
486 priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET;
487 reg = readl(&priv_dev->regs->ep_sts_en);
488 reg &= ~EP_STS_EN_DESCMISEN;
489 trace_cdns3_wa2(priv_ep, "workaround disabled\n");
490 writel(reg, &priv_dev->regs->ep_sts_en);
491 }
492
493 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) {
494 u8 pending_empty = list_empty(&priv_ep->pending_req_list);
495 u8 descmiss_empty = list_empty(&priv_ep->wa2_descmiss_req_list);
496
497 /*
498 * DESCMISS transfer has been finished, so data will be
499 * directly copied from internal allocated usb_request
500 * objects.
501 */
502 if (pending_empty && !descmiss_empty &&
503 !(priv_req->flags & REQUEST_INTERNAL)) {
504 cdns3_wa2_descmiss_copy_data(priv_ep,
505 &priv_req->request);
506
507 trace_cdns3_wa2(priv_ep, "get internal stored data");
508
509 list_add_tail(&priv_req->request.list,
510 &priv_ep->pending_req_list);
511 cdns3_gadget_giveback(priv_ep, priv_req,
512 priv_req->request.status);
513
514 /*
515 * Intentionally driver returns positive value as
516 * correct value. It informs that transfer has
517 * been finished.
518 */
519 return EINPROGRESS;
520 }
521
522 /*
523 * Driver will wait for completion DESCMISS transfer,
524 * before starts new, not DESCMISS transfer.
525 */
526 if (!pending_empty && !descmiss_empty) {
527 trace_cdns3_wa2(priv_ep, "wait for pending transfer\n");
528 deferred = 1;
529 }
530
531 if (priv_req->flags & REQUEST_INTERNAL)
532 list_add_tail(&priv_req->list,
533 &priv_ep->wa2_descmiss_req_list);
534 }
535
536 return deferred;
537}
538
539static void cdns3_wa2_remove_old_request(struct cdns3_endpoint *priv_ep)
540{
541 struct cdns3_request *priv_req;
542
543 while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
544 u8 chain;
545
546 priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
547 chain = !!(priv_req->flags & REQUEST_INTERNAL_CH);
548
549 trace_cdns3_wa2(priv_ep, "removes eldest request");
550
551 kfree(priv_req->request.buf);
552 cdns3_gadget_ep_free_request(&priv_ep->endpoint,
553 &priv_req->request);
554 list_del_init(&priv_req->list);
555 --priv_ep->wa2_counter;
556
557 if (!chain)
558 break;
559 }
560}
561
562/**
563 * cdns3_wa2_descmissing_packet - handles descriptor missing event.
564 * @priv_dev: extended gadget object
565 *
566 * This function is used only for WA2. For more information see Work around 2
567 * description.
568 */
569static void cdns3_wa2_descmissing_packet(struct cdns3_endpoint *priv_ep)
570{
571 struct cdns3_request *priv_req;
572 struct usb_request *request;
573
574 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) {
575 priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET;
576 priv_ep->flags |= EP_QUIRK_EXTRA_BUF_EN;
577 }
578
579 trace_cdns3_wa2(priv_ep, "Description Missing detected\n");
580
581 if (priv_ep->wa2_counter >= CDNS3_WA2_NUM_BUFFERS)
582 cdns3_wa2_remove_old_request(priv_ep);
583
584 request = cdns3_gadget_ep_alloc_request(&priv_ep->endpoint,
585 GFP_ATOMIC);
586 if (!request)
587 goto err;
588
589 priv_req = to_cdns3_request(request);
590 priv_req->flags |= REQUEST_INTERNAL;
591
592 /* if this field is still assigned it indicate that transfer related
593 * with this request has not been finished yet. Driver in this
594 * case simply allocate next request and assign flag REQUEST_INTERNAL_CH
595 * flag to previous one. It will indicate that current request is
596 * part of the previous one.
597 */
598 if (priv_ep->descmis_req)
599 priv_ep->descmis_req->flags |= REQUEST_INTERNAL_CH;
600
601 priv_req->request.buf = kzalloc(CDNS3_DESCMIS_BUF_SIZE,
602 GFP_ATOMIC);
603 priv_ep->wa2_counter++;
604
605 if (!priv_req->request.buf) {
606 cdns3_gadget_ep_free_request(&priv_ep->endpoint, request);
607 goto err;
608 }
609
610 priv_req->request.length = CDNS3_DESCMIS_BUF_SIZE;
611 priv_ep->descmis_req = priv_req;
612
613 __cdns3_gadget_ep_queue(&priv_ep->endpoint,
614 &priv_ep->descmis_req->request,
615 GFP_ATOMIC);
616
617 return;
618
619err:
620 dev_err(priv_ep->cdns3_dev->dev,
621 "Failed: No sufficient memory for DESCMIS\n");
622}
623
624/**
625 * cdns3_gadget_giveback - call struct usb_request's ->complete callback
626 * @priv_ep: The endpoint to whom the request belongs to
627 * @priv_req: The request we're giving back
628 * @status: completion code for the request
629 *
630 * Must be called with controller's lock held and interrupts disabled. This
631 * function will unmap @req and call its ->complete() callback to notify upper
632 * layers that it has completed.
633 */
634void cdns3_gadget_giveback(struct cdns3_endpoint *priv_ep,
635 struct cdns3_request *priv_req,
636 int status)
637{
638 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
639 struct usb_request *request = &priv_req->request;
640
641 list_del_init(&request->list);
642
643 if (request->status == -EINPROGRESS)
644 request->status = status;
645
646 usb_gadget_unmap_request_by_dev(priv_dev->sysdev, request,
647 priv_ep->dir);
648
649 if ((priv_req->flags & REQUEST_UNALIGNED) &&
650 priv_ep->dir == USB_DIR_OUT && !request->status)
651 memcpy(request->buf, priv_req->aligned_buf->buf,
652 request->length);
653
654 priv_req->flags &= ~(REQUEST_PENDING | REQUEST_UNALIGNED);
655 trace_cdns3_gadget_giveback(priv_req);
656
657 if (priv_dev->dev_ver < DEV_VER_V2) {
658 request = cdns3_wa2_gadget_giveback(priv_dev, priv_ep,
659 priv_req);
660 if (!request)
661 return;
662 }
663
664 if (request->complete) {
665 spin_unlock(&priv_dev->lock);
666 usb_gadget_giveback_request(&priv_ep->endpoint,
667 request);
668 spin_lock(&priv_dev->lock);
669 }
670
671 if (request->buf == priv_dev->zlp_buf)
672 cdns3_gadget_ep_free_request(&priv_ep->endpoint, request);
673}
674
675void cdns3_wa1_restore_cycle_bit(struct cdns3_endpoint *priv_ep)
676{
677 /* Work around for stale data address in TRB*/
678 if (priv_ep->wa1_set) {
679 trace_cdns3_wa1(priv_ep, "restore cycle bit");
680
681 priv_ep->wa1_set = 0;
682 priv_ep->wa1_trb_index = 0xFFFF;
683 if (priv_ep->wa1_cycle_bit) {
684 priv_ep->wa1_trb->control =
685 priv_ep->wa1_trb->control | 0x1;
686 } else {
687 priv_ep->wa1_trb->control =
688 priv_ep->wa1_trb->control & ~0x1;
689 }
690 }
691}
692
693static void cdns3_free_aligned_request_buf(struct work_struct *work)
694{
695 struct cdns3_device *priv_dev = container_of(work, struct cdns3_device,
696 aligned_buf_wq);
697 struct cdns3_aligned_buf *buf, *tmp;
698 unsigned long flags;
699
700 spin_lock_irqsave(&priv_dev->lock, flags);
701
702 list_for_each_entry_safe(buf, tmp, &priv_dev->aligned_buf_list, list) {
703 if (!buf->in_use) {
704 list_del(&buf->list);
705
706 /*
707 * Re-enable interrupts to free DMA capable memory.
708 * Driver can't free this memory with disabled
709 * interrupts.
710 */
711 spin_unlock_irqrestore(&priv_dev->lock, flags);
712 dma_free_coherent(priv_dev->sysdev, buf->size,
713 buf->buf, buf->dma);
714 kfree(buf);
715 spin_lock_irqsave(&priv_dev->lock, flags);
716 }
717 }
718
719 spin_unlock_irqrestore(&priv_dev->lock, flags);
720}
721
722static int cdns3_prepare_aligned_request_buf(struct cdns3_request *priv_req)
723{
724 struct cdns3_endpoint *priv_ep = priv_req->priv_ep;
725 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
726 struct cdns3_aligned_buf *buf;
727
728 /* check if buffer is aligned to 8. */
729 if (!((uintptr_t)priv_req->request.buf & 0x7))
730 return 0;
731
732 buf = priv_req->aligned_buf;
733
734 if (!buf || priv_req->request.length > buf->size) {
735 buf = kzalloc(sizeof(*buf), GFP_ATOMIC);
736 if (!buf)
737 return -ENOMEM;
738
739 buf->size = priv_req->request.length;
740
741 buf->buf = dma_alloc_coherent(priv_dev->sysdev,
742 buf->size,
743 &buf->dma,
744 GFP_ATOMIC);
745 if (!buf->buf) {
746 kfree(buf);
747 return -ENOMEM;
748 }
749
750 if (priv_req->aligned_buf) {
751 trace_cdns3_free_aligned_request(priv_req);
752 priv_req->aligned_buf->in_use = 0;
753 queue_work(system_freezable_wq,
754 &priv_dev->aligned_buf_wq);
755 }
756
757 buf->in_use = 1;
758 priv_req->aligned_buf = buf;
759
760 list_add_tail(&buf->list,
761 &priv_dev->aligned_buf_list);
762 }
763
764 if (priv_ep->dir == USB_DIR_IN) {
765 memcpy(buf->buf, priv_req->request.buf,
766 priv_req->request.length);
767 }
768
769 priv_req->flags |= REQUEST_UNALIGNED;
770 trace_cdns3_prepare_aligned_request(priv_req);
771
772 return 0;
773}
774
775static int cdns3_wa1_update_guard(struct cdns3_endpoint *priv_ep,
776 struct cdns3_trb *trb)
777{
778 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
779
780 if (!priv_ep->wa1_set) {
781 u32 doorbell;
782
783 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
784
785 if (doorbell) {
786 priv_ep->wa1_cycle_bit = priv_ep->pcs ? TRB_CYCLE : 0;
787 priv_ep->wa1_set = 1;
788 priv_ep->wa1_trb = trb;
789 priv_ep->wa1_trb_index = priv_ep->enqueue;
790 trace_cdns3_wa1(priv_ep, "set guard");
791 return 0;
792 }
793 }
794 return 1;
795}
796
797static void cdns3_wa1_tray_restore_cycle_bit(struct cdns3_device *priv_dev,
798 struct cdns3_endpoint *priv_ep)
799{
800 int dma_index;
801 u32 doorbell;
802
803 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
804 dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);
805
806 if (!doorbell || dma_index != priv_ep->wa1_trb_index)
807 cdns3_wa1_restore_cycle_bit(priv_ep);
808}
809
810/**
811 * cdns3_ep_run_transfer - start transfer on no-default endpoint hardware
812 * @priv_ep: endpoint object
813 *
814 * Returns zero on success or negative value on failure
815 */
816int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep,
817 struct usb_request *request)
818{
819 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
820 struct cdns3_request *priv_req;
821 struct cdns3_trb *trb;
822 dma_addr_t trb_dma;
823 u32 togle_pcs = 1;
824 int sg_iter = 0;
825 int num_trb;
826 int address;
827 u32 control;
828 int pcs;
829
830 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC)
831 num_trb = priv_ep->interval;
832 else
833 num_trb = request->num_sgs ? request->num_sgs : 1;
834
835 if (num_trb > priv_ep->free_trbs) {
836 priv_ep->flags |= EP_RING_FULL;
837 return -ENOBUFS;
838 }
839
840 priv_req = to_cdns3_request(request);
841 address = priv_ep->endpoint.desc->bEndpointAddress;
842
843 priv_ep->flags |= EP_PENDING_REQUEST;
844
845 /* must allocate buffer aligned to 8 */
846 if (priv_req->flags & REQUEST_UNALIGNED)
847 trb_dma = priv_req->aligned_buf->dma;
848 else
849 trb_dma = request->dma;
850
851 trb = priv_ep->trb_pool + priv_ep->enqueue;
852 priv_req->start_trb = priv_ep->enqueue;
853 priv_req->trb = trb;
854
855 cdns3_select_ep(priv_ep->cdns3_dev, address);
856
857 /* prepare ring */
858 if ((priv_ep->enqueue + num_trb) >= (priv_ep->num_trbs - 1)) {
859 struct cdns3_trb *link_trb;
860 int doorbell, dma_index;
861 u32 ch_bit = 0;
862
863 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
864 dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);
865
866 /* Driver can't update LINK TRB if it is current processed. */
867 if (doorbell && dma_index == priv_ep->num_trbs - 1) {
868 priv_ep->flags |= EP_DEFERRED_DRDY;
869 return -ENOBUFS;
870 }
871
872 /*updating C bt in Link TRB before starting DMA*/
873 link_trb = priv_ep->trb_pool + (priv_ep->num_trbs - 1);
874 /*
875 * For TRs size equal 2 enabling TRB_CHAIN for epXin causes
876 * that DMA stuck at the LINK TRB.
877 * On the other hand, removing TRB_CHAIN for longer TRs for
878 * epXout cause that DMA stuck after handling LINK TRB.
879 * To eliminate this strange behavioral driver set TRB_CHAIN
880 * bit only for TR size > 2.
881 */
882 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC ||
883 TRBS_PER_SEGMENT > 2)
884 ch_bit = TRB_CHAIN;
885
886 link_trb->control = ((priv_ep->pcs) ? TRB_CYCLE : 0) |
887 TRB_TYPE(TRB_LINK) | TRB_TOGGLE | ch_bit;
888 }
889
890 if (priv_dev->dev_ver <= DEV_VER_V2)
891 togle_pcs = cdns3_wa1_update_guard(priv_ep, trb);
892
893 /* set incorrect Cycle Bit for first trb*/
894 control = priv_ep->pcs ? 0 : TRB_CYCLE;
895
896 do {
897 u32 length;
898 u16 td_size = 0;
899
900 /* fill TRB */
901 control |= TRB_TYPE(TRB_NORMAL);
902 trb->buffer = TRB_BUFFER(request->num_sgs == 0
903 ? trb_dma : request->sg[sg_iter].dma_address);
904
905 if (likely(!request->num_sgs))
906 length = request->length;
907 else
908 length = request->sg[sg_iter].length;
909
910 if (likely(priv_dev->dev_ver >= DEV_VER_V2))
911 td_size = DIV_ROUND_UP(length,
912 priv_ep->endpoint.maxpacket);
913
914 trb->length = TRB_BURST_LEN(priv_ep->trb_burst_size) |
915 TRB_LEN(length);
916 if (priv_dev->gadget.speed == USB_SPEED_SUPER)
917 trb->length |= TRB_TDL_SS_SIZE(td_size);
918 else
919 control |= TRB_TDL_HS_SIZE(td_size);
920
921 pcs = priv_ep->pcs ? TRB_CYCLE : 0;
922
923 /*
924 * first trb should be prepared as last to avoid processing
925 * transfer to early
926 */
927 if (sg_iter != 0)
928 control |= pcs;
929
930 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir) {
931 control |= TRB_IOC | TRB_ISP;
932 } else {
933 /* for last element in TD or in SG list */
934 if (sg_iter == (num_trb - 1) && sg_iter != 0)
935 control |= pcs | TRB_IOC | TRB_ISP;
936 }
937
938 if (sg_iter)
939 trb->control = control;
940 else
941 priv_req->trb->control = control;
942
943 control = 0;
944 ++sg_iter;
945 priv_req->end_trb = priv_ep->enqueue;
946 cdns3_ep_inc_enq(priv_ep);
947 trb = priv_ep->trb_pool + priv_ep->enqueue;
948 } while (sg_iter < num_trb);
949
950 trb = priv_req->trb;
951
952 priv_req->flags |= REQUEST_PENDING;
953
954 if (sg_iter == 1)
955 trb->control |= TRB_IOC | TRB_ISP;
956
957 /*
958 * Memory barrier - cycle bit must be set before other filds in trb.
959 */
960 wmb();
961
962 /* give the TD to the consumer*/
963 if (togle_pcs)
964 trb->control = trb->control ^ 1;
965
966 if (priv_dev->dev_ver <= DEV_VER_V2)
967 cdns3_wa1_tray_restore_cycle_bit(priv_dev, priv_ep);
968
969 trace_cdns3_prepare_trb(priv_ep, priv_req->trb);
970
971 /*
972 * Memory barrier - Cycle Bit must be set before trb->length and
973 * trb->buffer fields.
974 */
975 wmb();
976
977 /*
978 * For DMULT mode we can set address to transfer ring only once after
979 * enabling endpoint.
980 */
981 if (priv_ep->flags & EP_UPDATE_EP_TRBADDR) {
982 /*
983 * Until SW is not ready to handle the OUT transfer the ISO OUT
984 * Endpoint should be disabled (EP_CFG.ENABLE = 0).
985 * EP_CFG_ENABLE must be set before updating ep_traddr.
986 */
987 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir &&
988 !(priv_ep->flags & EP_QUIRK_ISO_OUT_EN)) {
989 priv_ep->flags |= EP_QUIRK_ISO_OUT_EN;
990 cdns3_set_register_bit(&priv_dev->regs->ep_cfg,
991 EP_CFG_ENABLE);
992 }
993
994 writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma +
995 priv_req->start_trb * TRB_SIZE),
996 &priv_dev->regs->ep_traddr);
997
998 priv_ep->flags &= ~EP_UPDATE_EP_TRBADDR;
999 }
1000
1001 if (!priv_ep->wa1_set && !(priv_ep->flags & EP_STALLED)) {
1002 trace_cdns3_ring(priv_ep);
1003 /*clearing TRBERR and EP_STS_DESCMIS before seting DRDY*/
1004 writel(EP_STS_TRBERR | EP_STS_DESCMIS, &priv_dev->regs->ep_sts);
1005 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
1006 trace_cdns3_doorbell_epx(priv_ep->name,
1007 readl(&priv_dev->regs->ep_traddr));
1008 }
1009
1010 /* WORKAROUND for transition to L0 */
1011 __cdns3_gadget_wakeup(priv_dev);
1012
1013 return 0;
1014}
1015
1016void cdns3_set_hw_configuration(struct cdns3_device *priv_dev)
1017{
1018 struct cdns3_endpoint *priv_ep;
1019 struct usb_ep *ep;
1020 int val;
1021
1022 if (priv_dev->hw_configured_flag)
1023 return;
1024
1025 writel(USB_CONF_CFGSET, &priv_dev->regs->usb_conf);
1026 writel(EP_CMD_ERDY | EP_CMD_REQ_CMPL, &priv_dev->regs->ep_cmd);
1027
1028 cdns3_set_register_bit(&priv_dev->regs->usb_conf,
1029 USB_CONF_U1EN | USB_CONF_U2EN);
1030
1031 /* wait until configuration set */
1032 readl_poll_timeout_atomic(&priv_dev->regs->usb_sts, val,
1033 val & USB_STS_CFGSTS_MASK, 1, 100);
1034
1035 priv_dev->hw_configured_flag = 1;
1036
1037 list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
1038 if (ep->enabled) {
1039 priv_ep = ep_to_cdns3_ep(ep);
1040 cdns3_start_all_request(priv_dev, priv_ep);
1041 }
1042 }
1043}
1044
1045/**
1046 * cdns3_request_handled - check whether request has been handled by DMA
1047 *
1048 * @priv_ep: extended endpoint object.
1049 * @priv_req: request object for checking
1050 *
1051 * Endpoint must be selected before invoking this function.
1052 *
1053 * Returns false if request has not been handled by DMA, else returns true.
1054 *
1055 * SR - start ring
1056 * ER - end ring
1057 * DQ = priv_ep->dequeue - dequeue position
1058 * EQ = priv_ep->enqueue - enqueue position
1059 * ST = priv_req->start_trb - index of first TRB in transfer ring
1060 * ET = priv_req->end_trb - index of last TRB in transfer ring
1061 * CI = current_index - index of processed TRB by DMA.
1062 *
1063 * As first step, function checks if cycle bit for priv_req->start_trb is
1064 * correct.
1065 *
1066 * some rules:
1067 * 1. priv_ep->dequeue never exceed current_index.
1068 * 2 priv_ep->enqueue never exceed priv_ep->dequeue
1069 * 3. exception: priv_ep->enqueue == priv_ep->dequeue
1070 * and priv_ep->free_trbs is zero.
1071 * This case indicate that TR is full.
1072 *
1073 * Then We can split recognition into two parts:
1074 * Case 1 - priv_ep->dequeue < current_index
1075 * SR ... EQ ... DQ ... CI ... ER
1076 * SR ... DQ ... CI ... EQ ... ER
1077 *
1078 * Request has been handled by DMA if ST and ET is between DQ and CI.
1079 *
1080 * Case 2 - priv_ep->dequeue > current_index
1081 * This situation take place when CI go through the LINK TRB at the end of
1082 * transfer ring.
1083 * SR ... CI ... EQ ... DQ ... ER
1084 *
1085 * Request has been handled by DMA if ET is less then CI or
1086 * ET is greater or equal DQ.
1087 */
1088static bool cdns3_request_handled(struct cdns3_endpoint *priv_ep,
1089 struct cdns3_request *priv_req)
1090{
1091 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1092 struct cdns3_trb *trb = priv_req->trb;
1093 int current_index = 0;
1094 int handled = 0;
1095 int doorbell;
1096
1097 current_index = cdns3_get_dma_pos(priv_dev, priv_ep);
1098 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
1099
1100 trb = &priv_ep->trb_pool[priv_req->start_trb];
1101
1102 if ((trb->control & TRB_CYCLE) != priv_ep->ccs)
1103 goto finish;
1104
1105 if (doorbell == 1 && current_index == priv_ep->dequeue)
1106 goto finish;
1107
1108 /* The corner case for TRBS_PER_SEGMENT equal 2). */
1109 if (TRBS_PER_SEGMENT == 2 && priv_ep->type != USB_ENDPOINT_XFER_ISOC) {
1110 handled = 1;
1111 goto finish;
1112 }
1113
1114 if (priv_ep->enqueue == priv_ep->dequeue &&
1115 priv_ep->free_trbs == 0) {
1116 handled = 1;
1117 } else if (priv_ep->dequeue < current_index) {
1118 if ((current_index == (priv_ep->num_trbs - 1)) &&
1119 !priv_ep->dequeue)
1120 goto finish;
1121
1122 if (priv_req->end_trb >= priv_ep->dequeue &&
1123 priv_req->end_trb < current_index)
1124 handled = 1;
1125 } else if (priv_ep->dequeue > current_index) {
1126 if (priv_req->end_trb < current_index ||
1127 priv_req->end_trb >= priv_ep->dequeue)
1128 handled = 1;
1129 }
1130
1131finish:
1132 trace_cdns3_request_handled(priv_req, current_index, handled);
1133
1134 return handled;
1135}
1136
1137static void cdns3_transfer_completed(struct cdns3_device *priv_dev,
1138 struct cdns3_endpoint *priv_ep)
1139{
1140 struct cdns3_request *priv_req;
1141 struct usb_request *request;
1142 struct cdns3_trb *trb;
1143
1144 while (!list_empty(&priv_ep->pending_req_list)) {
1145 request = cdns3_next_request(&priv_ep->pending_req_list);
1146 priv_req = to_cdns3_request(request);
1147
1148 trb = priv_ep->trb_pool + priv_ep->dequeue;
1149
1150 /* Request was dequeued and TRB was changed to TRB_LINK. */
1151 if (TRB_FIELD_TO_TYPE(trb->control) == TRB_LINK) {
1152 trace_cdns3_complete_trb(priv_ep, trb);
1153 cdns3_move_deq_to_next_trb(priv_req);
1154 }
1155
1156 /* Re-select endpoint. It could be changed by other CPU during
1157 * handling usb_gadget_giveback_request.
1158 */
1159 cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
1160
1161 if (!cdns3_request_handled(priv_ep, priv_req))
1162 goto prepare_next_td;
1163
1164 trb = priv_ep->trb_pool + priv_ep->dequeue;
1165 trace_cdns3_complete_trb(priv_ep, trb);
1166
1167 if (trb != priv_req->trb)
1168 dev_warn(priv_dev->dev,
1169 "request_trb=0x%p, queue_trb=0x%p\n",
1170 priv_req->trb, trb);
1171
1172 request->actual = TRB_LEN(le32_to_cpu(trb->length));
1173 cdns3_move_deq_to_next_trb(priv_req);
1174 cdns3_gadget_giveback(priv_ep, priv_req, 0);
1175
1176 if (priv_ep->type != USB_ENDPOINT_XFER_ISOC &&
1177 TRBS_PER_SEGMENT == 2)
1178 break;
1179 }
1180 priv_ep->flags &= ~EP_PENDING_REQUEST;
1181
1182prepare_next_td:
1183 if (!(priv_ep->flags & EP_STALLED) &&
1184 !(priv_ep->flags & EP_STALL_PENDING))
1185 cdns3_start_all_request(priv_dev, priv_ep);
1186}
1187
1188void cdns3_rearm_transfer(struct cdns3_endpoint *priv_ep, u8 rearm)
1189{
1190 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1191
1192 cdns3_wa1_restore_cycle_bit(priv_ep);
1193
1194 if (rearm) {
1195 trace_cdns3_ring(priv_ep);
1196
1197 /* Cycle Bit must be updated before arming DMA. */
1198 wmb();
1199 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
1200
1201 __cdns3_gadget_wakeup(priv_dev);
1202
1203 trace_cdns3_doorbell_epx(priv_ep->name,
1204 readl(&priv_dev->regs->ep_traddr));
1205 }
1206}
1207
1208/**
1209 * cdns3_check_ep_interrupt_proceed - Processes interrupt related to endpoint
1210 * @priv_ep: endpoint object
1211 *
1212 * Returns 0
1213 */
1214static int cdns3_check_ep_interrupt_proceed(struct cdns3_endpoint *priv_ep)
1215{
1216 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1217 u32 ep_sts_reg;
1218
1219 cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
1220
1221 trace_cdns3_epx_irq(priv_dev, priv_ep);
1222
1223 ep_sts_reg = readl(&priv_dev->regs->ep_sts);
1224 writel(ep_sts_reg, &priv_dev->regs->ep_sts);
1225
1226 if (ep_sts_reg & EP_STS_TRBERR) {
1227 if (priv_ep->flags & EP_STALL_PENDING &&
1228 !(ep_sts_reg & EP_STS_DESCMIS &&
1229 priv_dev->dev_ver < DEV_VER_V2)) {
1230 cdns3_ep_stall_flush(priv_ep);
1231 }
1232
1233 /*
1234 * For isochronous transfer driver completes request on
1235 * IOC or on TRBERR. IOC appears only when device receive
1236 * OUT data packet. If host disable stream or lost some packet
1237 * then the only way to finish all queued transfer is to do it
1238 * on TRBERR event.
1239 */
1240 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC &&
1241 !priv_ep->wa1_set) {
1242 if (!priv_ep->dir) {
1243 u32 ep_cfg = readl(&priv_dev->regs->ep_cfg);
1244
1245 ep_cfg &= ~EP_CFG_ENABLE;
1246 writel(ep_cfg, &priv_dev->regs->ep_cfg);
1247 priv_ep->flags &= ~EP_QUIRK_ISO_OUT_EN;
1248 }
1249 cdns3_transfer_completed(priv_dev, priv_ep);
1250 } else if (!(priv_ep->flags & EP_STALLED) &&
1251 !(priv_ep->flags & EP_STALL_PENDING)) {
1252 if (priv_ep->flags & EP_DEFERRED_DRDY) {
1253 priv_ep->flags &= ~EP_DEFERRED_DRDY;
1254 cdns3_start_all_request(priv_dev, priv_ep);
1255 } else {
1256 cdns3_rearm_transfer(priv_ep,
1257 priv_ep->wa1_set);
1258 }
1259 }
1260 }
1261
1262 if ((ep_sts_reg & EP_STS_IOC) || (ep_sts_reg & EP_STS_ISP)) {
1263 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) {
1264 if (ep_sts_reg & EP_STS_ISP)
1265 priv_ep->flags |= EP_QUIRK_END_TRANSFER;
1266 else
1267 priv_ep->flags &= ~EP_QUIRK_END_TRANSFER;
1268 }
1269
1270 cdns3_transfer_completed(priv_dev, priv_ep);
1271 }
1272
1273 /*
1274 * WA2: this condition should only be meet when
1275 * priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET or
1276 * priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN.
1277 * In other cases this interrupt will be disabled/
1278 */
1279 if (ep_sts_reg & EP_STS_DESCMIS && priv_dev->dev_ver < DEV_VER_V2 &&
1280 !(priv_ep->flags & EP_STALLED))
1281 cdns3_wa2_descmissing_packet(priv_ep);
1282
1283 return 0;
1284}
1285
1286static void cdns3_disconnect_gadget(struct cdns3_device *priv_dev)
1287{
1288 if (priv_dev->gadget_driver && priv_dev->gadget_driver->disconnect) {
1289 spin_unlock(&priv_dev->lock);
1290 priv_dev->gadget_driver->disconnect(&priv_dev->gadget);
1291 spin_lock(&priv_dev->lock);
1292 }
1293}
1294
1295/**
1296 * cdns3_check_usb_interrupt_proceed - Processes interrupt related to device
1297 * @priv_dev: extended gadget object
1298 * @usb_ists: bitmap representation of device's reported interrupts
1299 * (usb_ists register value)
1300 */
1301static void cdns3_check_usb_interrupt_proceed(struct cdns3_device *priv_dev,
1302 u32 usb_ists)
1303{
1304 int speed = 0;
1305
1306 trace_cdns3_usb_irq(priv_dev, usb_ists);
1307 if (usb_ists & USB_ISTS_L1ENTI) {
1308 /*
1309 * WORKAROUND: CDNS3 controller has issue with hardware resuming
1310 * from L1. To fix it, if any DMA transfer is pending driver
1311 * must starts driving resume signal immediately.
1312 */
1313 if (readl(&priv_dev->regs->drbl))
1314 __cdns3_gadget_wakeup(priv_dev);
1315 }
1316
1317 /* Connection detected */
1318 if (usb_ists & (USB_ISTS_CON2I | USB_ISTS_CONI)) {
1319 speed = cdns3_get_speed(priv_dev);
1320 priv_dev->gadget.speed = speed;
1321 usb_gadget_set_state(&priv_dev->gadget, USB_STATE_POWERED);
1322 cdns3_ep0_config(priv_dev);
1323 }
1324
1325 /* Disconnection detected */
1326 if (usb_ists & (USB_ISTS_DIS2I | USB_ISTS_DISI)) {
1327 cdns3_disconnect_gadget(priv_dev);
1328 priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
1329 usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED);
1330 cdns3_hw_reset_eps_config(priv_dev);
1331 }
1332
1333 if (usb_ists & (USB_ISTS_L2ENTI | USB_ISTS_U3ENTI)) {
1334 if (priv_dev->gadget_driver &&
1335 priv_dev->gadget_driver->suspend) {
1336 spin_unlock(&priv_dev->lock);
1337 priv_dev->gadget_driver->suspend(&priv_dev->gadget);
1338 spin_lock(&priv_dev->lock);
1339 }
1340 }
1341
1342 if (usb_ists & (USB_ISTS_L2EXTI | USB_ISTS_U3EXTI)) {
1343 if (priv_dev->gadget_driver &&
1344 priv_dev->gadget_driver->resume) {
1345 spin_unlock(&priv_dev->lock);
1346 priv_dev->gadget_driver->resume(&priv_dev->gadget);
1347 spin_lock(&priv_dev->lock);
1348 }
1349 }
1350
1351 /* reset*/
1352 if (usb_ists & (USB_ISTS_UWRESI | USB_ISTS_UHRESI | USB_ISTS_U2RESI)) {
1353 if (priv_dev->gadget_driver) {
1354 spin_unlock(&priv_dev->lock);
1355 usb_gadget_udc_reset(&priv_dev->gadget,
1356 priv_dev->gadget_driver);
1357 spin_lock(&priv_dev->lock);
1358
1359 /*read again to check the actual speed*/
1360 speed = cdns3_get_speed(priv_dev);
1361 priv_dev->gadget.speed = speed;
1362 cdns3_hw_reset_eps_config(priv_dev);
1363 cdns3_ep0_config(priv_dev);
1364 }
1365 }
1366}
1367
1368/**
1369 * cdns3_device_irq_handler- interrupt handler for device part of controller
1370 *
1371 * @irq: irq number for cdns3 core device
1372 * @data: structure of cdns3
1373 *
1374 * Returns IRQ_HANDLED or IRQ_NONE
1375 */
1376static irqreturn_t cdns3_device_irq_handler(int irq, void *data)
1377{
1378 struct cdns3_device *priv_dev;
1379 struct cdns3 *cdns = data;
1380 irqreturn_t ret = IRQ_NONE;
1381 u32 reg;
1382
1383 priv_dev = cdns->gadget_dev;
1384
1385 /* check USB device interrupt */
1386 reg = readl(&priv_dev->regs->usb_ists);
1387 if (reg) {
1388 /* After masking interrupts the new interrupts won't be
1389 * reported in usb_ists/ep_ists. In order to not lose some
1390 * of them driver disables only detected interrupts.
1391 * They will be enabled ASAP after clearing source of
1392 * interrupt. This an unusual behavior only applies to
1393 * usb_ists register.
1394 */
1395 reg = ~reg & readl(&priv_dev->regs->usb_ien);
1396 /* mask deferred interrupt. */
1397 writel(reg, &priv_dev->regs->usb_ien);
1398 ret = IRQ_WAKE_THREAD;
1399 }
1400
1401 /* check endpoint interrupt */
1402 reg = readl(&priv_dev->regs->ep_ists);
1403 if (reg) {
1404 writel(0, &priv_dev->regs->ep_ien);
1405 ret = IRQ_WAKE_THREAD;
1406 }
1407
1408 return ret;
1409}
1410
1411/**
1412 * cdns3_device_thread_irq_handler- interrupt handler for device part
1413 * of controller
1414 *
1415 * @irq: irq number for cdns3 core device
1416 * @data: structure of cdns3
1417 *
1418 * Returns IRQ_HANDLED or IRQ_NONE
1419 */
1420static irqreturn_t cdns3_device_thread_irq_handler(int irq, void *data)
1421{
1422 struct cdns3_device *priv_dev;
1423 struct cdns3 *cdns = data;
1424 irqreturn_t ret = IRQ_NONE;
1425 unsigned long flags;
1426 int bit;
1427 u32 reg;
1428
1429 priv_dev = cdns->gadget_dev;
1430 spin_lock_irqsave(&priv_dev->lock, flags);
1431
1432 reg = readl(&priv_dev->regs->usb_ists);
1433 if (reg) {
1434 writel(reg, &priv_dev->regs->usb_ists);
1435 writel(USB_IEN_INIT, &priv_dev->regs->usb_ien);
1436 cdns3_check_usb_interrupt_proceed(priv_dev, reg);
1437 ret = IRQ_HANDLED;
1438 }
1439
1440 reg = readl(&priv_dev->regs->ep_ists);
1441
1442 /* handle default endpoint OUT */
1443 if (reg & EP_ISTS_EP_OUT0) {
1444 cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_OUT);
1445 ret = IRQ_HANDLED;
1446 }
1447
1448 /* handle default endpoint IN */
1449 if (reg & EP_ISTS_EP_IN0) {
1450 cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_IN);
1451 ret = IRQ_HANDLED;
1452 }
1453
1454 /* check if interrupt from non default endpoint, if no exit */
1455 reg &= ~(EP_ISTS_EP_OUT0 | EP_ISTS_EP_IN0);
1456 if (!reg)
1457 goto irqend;
1458
1459 for_each_set_bit(bit, (unsigned long *)®,
1460 sizeof(u32) * BITS_PER_BYTE) {
1461 cdns3_check_ep_interrupt_proceed(priv_dev->eps[bit]);
1462 ret = IRQ_HANDLED;
1463 }
1464
1465irqend:
1466 writel(~0, &priv_dev->regs->ep_ien);
1467 spin_unlock_irqrestore(&priv_dev->lock, flags);
1468
1469 return ret;
1470}
1471
1472/**
1473 * cdns3_ep_onchip_buffer_reserve - Try to reserve onchip buf for EP
1474 *
1475 * The real reservation will occur during write to EP_CFG register,
1476 * this function is used to check if the 'size' reservation is allowed.
1477 *
1478 * @priv_dev: extended gadget object
1479 * @size: the size (KB) for EP would like to allocate
1480 * @is_in: endpoint direction
1481 *
1482 * Return 0 if the required size can met or negative value on failure
1483 */
1484static int cdns3_ep_onchip_buffer_reserve(struct cdns3_device *priv_dev,
1485 int size, int is_in)
1486{
1487 int remained;
1488
1489 /* 2KB are reserved for EP0*/
1490 remained = priv_dev->onchip_buffers - priv_dev->onchip_used_size - 2;
1491
1492 if (is_in) {
1493 if (remained < size)
1494 return -EPERM;
1495
1496 priv_dev->onchip_used_size += size;
1497 } else {
1498 int required;
1499
1500 /**
1501 * ALL OUT EPs are shared the same chunk onchip memory, so
1502 * driver checks if it already has assigned enough buffers
1503 */
1504 if (priv_dev->out_mem_is_allocated >= size)
1505 return 0;
1506
1507 required = size - priv_dev->out_mem_is_allocated;
1508
1509 if (required > remained)
1510 return -EPERM;
1511
1512 priv_dev->out_mem_is_allocated += required;
1513 priv_dev->onchip_used_size += required;
1514 }
1515
1516 return 0;
1517}
1518
1519void cdns3_configure_dmult(struct cdns3_device *priv_dev,
1520 struct cdns3_endpoint *priv_ep)
1521{
1522 struct cdns3_usb_regs __iomem *regs = priv_dev->regs;
1523
1524 /* For dev_ver > DEV_VER_V2 DMULT is configured per endpoint */
1525 if (priv_dev->dev_ver <= DEV_VER_V2)
1526 writel(USB_CONF_DMULT, ®s->usb_conf);
1527
1528 if (priv_dev->dev_ver == DEV_VER_V2)
1529 writel(USB_CONF2_EN_TDL_TRB, ®s->usb_conf2);
1530
1531 if (priv_dev->dev_ver >= DEV_VER_V3 && priv_ep) {
1532 u32 mask;
1533
1534 if (priv_ep->dir)
1535 mask = BIT(priv_ep->num + 16);
1536 else
1537 mask = BIT(priv_ep->num);
1538
1539 if (priv_ep->type != USB_ENDPOINT_XFER_ISOC) {
1540 cdns3_set_register_bit(®s->tdl_from_trb, mask);
1541 cdns3_set_register_bit(®s->tdl_beh, mask);
1542 cdns3_set_register_bit(®s->tdl_beh2, mask);
1543 cdns3_set_register_bit(®s->dma_adv_td, mask);
1544 }
1545
1546 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir)
1547 cdns3_set_register_bit(®s->tdl_from_trb, mask);
1548
1549 cdns3_set_register_bit(®s->dtrans, mask);
1550 }
1551}
1552
1553/**
1554 * cdns3_ep_config Configure hardware endpoint
1555 * @priv_ep: extended endpoint object
1556 */
1557void cdns3_ep_config(struct cdns3_endpoint *priv_ep)
1558{
1559 bool is_iso_ep = (priv_ep->type == USB_ENDPOINT_XFER_ISOC);
1560 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1561 u32 bEndpointAddress = priv_ep->num | priv_ep->dir;
1562 u32 max_packet_size = 0;
1563 u8 maxburst = 0;
1564 u32 ep_cfg = 0;
1565 u8 buffering;
1566 u8 mult = 0;
1567 int ret;
1568
1569 buffering = CDNS3_EP_BUF_SIZE - 1;
1570
1571 cdns3_configure_dmult(priv_dev, priv_ep);
1572
1573 switch (priv_ep->type) {
1574 case USB_ENDPOINT_XFER_INT:
1575 ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_INT);
1576
1577 if ((priv_dev->dev_ver == DEV_VER_V2 && !priv_ep->dir) ||
1578 priv_dev->dev_ver > DEV_VER_V2)
1579 ep_cfg |= EP_CFG_TDL_CHK;
1580 break;
1581 case USB_ENDPOINT_XFER_BULK:
1582 ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_BULK);
1583
1584 if ((priv_dev->dev_ver == DEV_VER_V2 && !priv_ep->dir) ||
1585 priv_dev->dev_ver > DEV_VER_V2)
1586 ep_cfg |= EP_CFG_TDL_CHK;
1587 break;
1588 default:
1589 ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_ISOC);
1590 mult = CDNS3_EP_ISO_HS_MULT - 1;
1591 buffering = mult + 1;
1592 }
1593
1594 switch (priv_dev->gadget.speed) {
1595 case USB_SPEED_FULL:
1596 max_packet_size = is_iso_ep ? 1023 : 64;
1597 break;
1598 case USB_SPEED_HIGH:
1599 max_packet_size = is_iso_ep ? 1024 : 512;
1600 break;
1601 case USB_SPEED_SUPER:
1602 /* It's limitation that driver assumes in driver. */
1603 mult = 0;
1604 max_packet_size = 1024;
1605 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC) {
1606 maxburst = CDNS3_EP_ISO_SS_BURST - 1;
1607 buffering = (mult + 1) *
1608 (maxburst + 1);
1609
1610 if (priv_ep->interval > 1)
1611 buffering++;
1612 } else {
1613 maxburst = CDNS3_EP_BUF_SIZE - 1;
1614 }
1615 break;
1616 default:
1617 /* all other speed are not supported */
1618 return;
1619 }
1620
1621 if (max_packet_size == 1024)
1622 priv_ep->trb_burst_size = 128;
1623 else if (max_packet_size >= 512)
1624 priv_ep->trb_burst_size = 64;
1625 else
1626 priv_ep->trb_burst_size = 16;
1627
1628 ret = cdns3_ep_onchip_buffer_reserve(priv_dev, buffering + 1,
1629 !!priv_ep->dir);
1630 if (ret) {
1631 dev_err(priv_dev->dev, "onchip mem is full, ep is invalid\n");
1632 return;
1633 }
1634
1635 ep_cfg |= EP_CFG_MAXPKTSIZE(max_packet_size) |
1636 EP_CFG_MULT(mult) |
1637 EP_CFG_BUFFERING(buffering) |
1638 EP_CFG_MAXBURST(maxburst);
1639
1640 cdns3_select_ep(priv_dev, bEndpointAddress);
1641 writel(ep_cfg, &priv_dev->regs->ep_cfg);
1642
1643 dev_dbg(priv_dev->dev, "Configure %s: with val %08x\n",
1644 priv_ep->name, ep_cfg);
1645}
1646
1647/* Find correct direction for HW endpoint according to description */
1648static int cdns3_ep_dir_is_correct(struct usb_endpoint_descriptor *desc,
1649 struct cdns3_endpoint *priv_ep)
1650{
1651 return (priv_ep->endpoint.caps.dir_in && usb_endpoint_dir_in(desc)) ||
1652 (priv_ep->endpoint.caps.dir_out && usb_endpoint_dir_out(desc));
1653}
1654
1655static struct
1656cdns3_endpoint *cdns3_find_available_ep(struct cdns3_device *priv_dev,
1657 struct usb_endpoint_descriptor *desc)
1658{
1659 struct usb_ep *ep;
1660 struct cdns3_endpoint *priv_ep;
1661
1662 list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
1663 unsigned long num;
1664 int ret;
1665 /* ep name pattern likes epXin or epXout */
1666 char c[2] = {ep->name[2], '\0'};
1667
1668 ret = kstrtoul(c, 10, &num);
1669 if (ret)
1670 return ERR_PTR(ret);
1671
1672 priv_ep = ep_to_cdns3_ep(ep);
1673 if (cdns3_ep_dir_is_correct(desc, priv_ep)) {
1674 if (!(priv_ep->flags & EP_CLAIMED)) {
1675 priv_ep->num = num;
1676 return priv_ep;
1677 }
1678 }
1679 }
1680
1681 return ERR_PTR(-ENOENT);
1682}
1683
1684/*
1685 * Cadence IP has one limitation that all endpoints must be configured
1686 * (Type & MaxPacketSize) before setting configuration through hardware
1687 * register, it means we can't change endpoints configuration after
1688 * set_configuration.
1689 *
1690 * This function set EP_CLAIMED flag which is added when the gadget driver
1691 * uses usb_ep_autoconfig to configure specific endpoint;
1692 * When the udc driver receives set_configurion request,
1693 * it goes through all claimed endpoints, and configure all endpoints
1694 * accordingly.
1695 *
1696 * At usb_ep_ops.enable/disable, we only enable and disable endpoint through
1697 * ep_cfg register which can be changed after set_configuration, and do
1698 * some software operation accordingly.
1699 */
1700static struct
1701usb_ep *cdns3_gadget_match_ep(struct usb_gadget *gadget,
1702 struct usb_endpoint_descriptor *desc,
1703 struct usb_ss_ep_comp_descriptor *comp_desc)
1704{
1705 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
1706 struct cdns3_endpoint *priv_ep;
1707 unsigned long flags;
1708
1709 priv_ep = cdns3_find_available_ep(priv_dev, desc);
1710 if (IS_ERR(priv_ep)) {
1711 dev_err(priv_dev->dev, "no available ep\n");
1712 return NULL;
1713 }
1714
1715 dev_dbg(priv_dev->dev, "match endpoint: %s\n", priv_ep->name);
1716
1717 spin_lock_irqsave(&priv_dev->lock, flags);
1718 priv_ep->endpoint.desc = desc;
1719 priv_ep->dir = usb_endpoint_dir_in(desc) ? USB_DIR_IN : USB_DIR_OUT;
1720 priv_ep->type = usb_endpoint_type(desc);
1721 priv_ep->flags |= EP_CLAIMED;
1722 priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
1723
1724 spin_unlock_irqrestore(&priv_dev->lock, flags);
1725 return &priv_ep->endpoint;
1726}
1727
1728/**
1729 * cdns3_gadget_ep_alloc_request Allocates request
1730 * @ep: endpoint object associated with request
1731 * @gfp_flags: gfp flags
1732 *
1733 * Returns allocated request address, NULL on allocation error
1734 */
1735struct usb_request *cdns3_gadget_ep_alloc_request(struct usb_ep *ep,
1736 gfp_t gfp_flags)
1737{
1738 struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
1739 struct cdns3_request *priv_req;
1740
1741 priv_req = kzalloc(sizeof(*priv_req), gfp_flags);
1742 if (!priv_req)
1743 return NULL;
1744
1745 priv_req->priv_ep = priv_ep;
1746
1747 trace_cdns3_alloc_request(priv_req);
1748 return &priv_req->request;
1749}
1750
1751/**
1752 * cdns3_gadget_ep_free_request Free memory occupied by request
1753 * @ep: endpoint object associated with request
1754 * @request: request to free memory
1755 */
1756void cdns3_gadget_ep_free_request(struct usb_ep *ep,
1757 struct usb_request *request)
1758{
1759 struct cdns3_request *priv_req = to_cdns3_request(request);
1760
1761 if (priv_req->aligned_buf)
1762 priv_req->aligned_buf->in_use = 0;
1763
1764 trace_cdns3_free_request(priv_req);
1765 kfree(priv_req);
1766}
1767
1768/**
1769 * cdns3_gadget_ep_enable Enable endpoint
1770 * @ep: endpoint object
1771 * @desc: endpoint descriptor
1772 *
1773 * Returns 0 on success, error code elsewhere
1774 */
1775static int cdns3_gadget_ep_enable(struct usb_ep *ep,
1776 const struct usb_endpoint_descriptor *desc)
1777{
1778 struct cdns3_endpoint *priv_ep;
1779 struct cdns3_device *priv_dev;
1780 u32 reg = EP_STS_EN_TRBERREN;
1781 u32 bEndpointAddress;
1782 unsigned long flags;
1783 int enable = 1;
1784 int ret;
1785 int val;
1786
1787 priv_ep = ep_to_cdns3_ep(ep);
1788 priv_dev = priv_ep->cdns3_dev;
1789
1790 if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
1791 dev_dbg(priv_dev->dev, "usbss: invalid parameters\n");
1792 return -EINVAL;
1793 }
1794
1795 if (!desc->wMaxPacketSize) {
1796 dev_err(priv_dev->dev, "usbss: missing wMaxPacketSize\n");
1797 return -EINVAL;
1798 }
1799
1800 if (dev_WARN_ONCE(priv_dev->dev, priv_ep->flags & EP_ENABLED,
1801 "%s is already enabled\n", priv_ep->name))
1802 return 0;
1803
1804 spin_lock_irqsave(&priv_dev->lock, flags);
1805
1806 priv_ep->endpoint.desc = desc;
1807 priv_ep->type = usb_endpoint_type(desc);
1808 priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
1809
1810 if (priv_ep->interval > ISO_MAX_INTERVAL &&
1811 priv_ep->type == USB_ENDPOINT_XFER_ISOC) {
1812 dev_err(priv_dev->dev, "Driver is limited to %d period\n",
1813 ISO_MAX_INTERVAL);
1814
1815 ret = -EINVAL;
1816 goto exit;
1817 }
1818
1819 ret = cdns3_allocate_trb_pool(priv_ep);
1820
1821 if (ret)
1822 goto exit;
1823
1824 bEndpointAddress = priv_ep->num | priv_ep->dir;
1825 cdns3_select_ep(priv_dev, bEndpointAddress);
1826
1827 trace_cdns3_gadget_ep_enable(priv_ep);
1828
1829 writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
1830
1831 ret = readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
1832 !(val & (EP_CMD_CSTALL | EP_CMD_EPRST)),
1833 1, 1000);
1834
1835 if (unlikely(ret)) {
1836 cdns3_free_trb_pool(priv_ep);
1837 ret = -EINVAL;
1838 goto exit;
1839 }
1840
1841 /* enable interrupt for selected endpoint */
1842 cdns3_set_register_bit(&priv_dev->regs->ep_ien,
1843 BIT(cdns3_ep_addr_to_index(bEndpointAddress)));
1844
1845 if (priv_dev->dev_ver < DEV_VER_V2)
1846 cdns3_wa2_enable_detection(priv_dev, priv_ep, reg);
1847
1848 writel(reg, &priv_dev->regs->ep_sts_en);
1849
1850 /*
1851 * For some versions of controller at some point during ISO OUT traffic
1852 * DMA reads Transfer Ring for the EP which has never got doorbell.
1853 * This issue was detected only on simulation, but to avoid this issue
1854 * driver add protection against it. To fix it driver enable ISO OUT
1855 * endpoint before setting DRBL. This special treatment of ISO OUT
1856 * endpoints are recommended by controller specification.
1857 */
1858 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir)
1859 enable = 0;
1860
1861 if (enable)
1862 cdns3_set_register_bit(&priv_dev->regs->ep_cfg, EP_CFG_ENABLE);
1863
1864 ep->desc = desc;
1865 priv_ep->flags &= ~(EP_PENDING_REQUEST | EP_STALLED | EP_STALL_PENDING |
1866 EP_QUIRK_ISO_OUT_EN | EP_QUIRK_EXTRA_BUF_EN);
1867 priv_ep->flags |= EP_ENABLED | EP_UPDATE_EP_TRBADDR;
1868 priv_ep->wa1_set = 0;
1869 priv_ep->enqueue = 0;
1870 priv_ep->dequeue = 0;
1871 reg = readl(&priv_dev->regs->ep_sts);
1872 priv_ep->pcs = !!EP_STS_CCS(reg);
1873 priv_ep->ccs = !!EP_STS_CCS(reg);
1874 /* one TRB is reserved for link TRB used in DMULT mode*/
1875 priv_ep->free_trbs = priv_ep->num_trbs - 1;
1876exit:
1877 spin_unlock_irqrestore(&priv_dev->lock, flags);
1878
1879 return ret;
1880}
1881
1882/**
1883 * cdns3_gadget_ep_disable Disable endpoint
1884 * @ep: endpoint object
1885 *
1886 * Returns 0 on success, error code elsewhere
1887 */
1888static int cdns3_gadget_ep_disable(struct usb_ep *ep)
1889{
1890 struct cdns3_endpoint *priv_ep;
1891 struct cdns3_request *priv_req;
1892 struct cdns3_device *priv_dev;
1893 struct usb_request *request;
1894 unsigned long flags;
1895 int ret = 0;
1896 u32 ep_cfg;
1897 int val;
1898
1899 if (!ep) {
1900 pr_err("usbss: invalid parameters\n");
1901 return -EINVAL;
1902 }
1903
1904 priv_ep = ep_to_cdns3_ep(ep);
1905 priv_dev = priv_ep->cdns3_dev;
1906
1907 if (dev_WARN_ONCE(priv_dev->dev, !(priv_ep->flags & EP_ENABLED),
1908 "%s is already disabled\n", priv_ep->name))
1909 return 0;
1910
1911 spin_lock_irqsave(&priv_dev->lock, flags);
1912
1913 trace_cdns3_gadget_ep_disable(priv_ep);
1914
1915 cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
1916
1917 ep_cfg = readl(&priv_dev->regs->ep_cfg);
1918 ep_cfg &= ~EP_CFG_ENABLE;
1919 writel(ep_cfg, &priv_dev->regs->ep_cfg);
1920
1921 /**
1922 * Driver needs some time before resetting endpoint.
1923 * It need waits for clearing DBUSY bit or for timeout expired.
1924 * 10us is enough time for controller to stop transfer.
1925 */
1926 readl_poll_timeout_atomic(&priv_dev->regs->ep_sts, val,
1927 !(val & EP_STS_DBUSY), 1, 10);
1928 writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
1929
1930 readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
1931 !(val & (EP_CMD_CSTALL | EP_CMD_EPRST)),
1932 1, 1000);
1933 if (unlikely(ret))
1934 dev_err(priv_dev->dev, "Timeout: %s resetting failed.\n",
1935 priv_ep->name);
1936
1937 while (!list_empty(&priv_ep->pending_req_list)) {
1938 request = cdns3_next_request(&priv_ep->pending_req_list);
1939
1940 cdns3_gadget_giveback(priv_ep, to_cdns3_request(request),
1941 -ESHUTDOWN);
1942 }
1943
1944 while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
1945 priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
1946
1947 kfree(priv_req->request.buf);
1948 cdns3_gadget_ep_free_request(&priv_ep->endpoint,
1949 &priv_req->request);
1950 list_del_init(&priv_req->list);
1951 --priv_ep->wa2_counter;
1952 }
1953
1954 while (!list_empty(&priv_ep->deferred_req_list)) {
1955 request = cdns3_next_request(&priv_ep->deferred_req_list);
1956
1957 cdns3_gadget_giveback(priv_ep, to_cdns3_request(request),
1958 -ESHUTDOWN);
1959 }
1960
1961 priv_ep->descmis_req = NULL;
1962
1963 ep->desc = NULL;
1964 priv_ep->flags &= ~EP_ENABLED;
1965
1966 spin_unlock_irqrestore(&priv_dev->lock, flags);
1967
1968 return ret;
1969}
1970
1971/**
1972 * cdns3_gadget_ep_queue Transfer data on endpoint
1973 * @ep: endpoint object
1974 * @request: request object
1975 * @gfp_flags: gfp flags
1976 *
1977 * Returns 0 on success, error code elsewhere
1978 */
1979static int __cdns3_gadget_ep_queue(struct usb_ep *ep,
1980 struct usb_request *request,
1981 gfp_t gfp_flags)
1982{
1983 struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
1984 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1985 struct cdns3_request *priv_req;
1986 int ret = 0;
1987
1988 request->actual = 0;
1989 request->status = -EINPROGRESS;
1990 priv_req = to_cdns3_request(request);
1991 trace_cdns3_ep_queue(priv_req);
1992
1993 if (priv_dev->dev_ver < DEV_VER_V2) {
1994 ret = cdns3_wa2_gadget_ep_queue(priv_dev, priv_ep,
1995 priv_req);
1996
1997 if (ret == EINPROGRESS)
1998 return 0;
1999 }
2000
2001 ret = cdns3_prepare_aligned_request_buf(priv_req);
2002 if (ret < 0)
2003 return ret;
2004
2005 ret = usb_gadget_map_request_by_dev(priv_dev->sysdev, request,
2006 usb_endpoint_dir_in(ep->desc));
2007 if (ret)
2008 return ret;
2009
2010 list_add_tail(&request->list, &priv_ep->deferred_req_list);
2011
2012 /*
2013 * If hardware endpoint configuration has not been set yet then
2014 * just queue request in deferred list. Transfer will be started in
2015 * cdns3_set_hw_configuration.
2016 */
2017 if (priv_dev->hw_configured_flag && !(priv_ep->flags & EP_STALLED) &&
2018 !(priv_ep->flags & EP_STALL_PENDING))
2019 cdns3_start_all_request(priv_dev, priv_ep);
2020
2021 return 0;
2022}
2023
2024static int cdns3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
2025 gfp_t gfp_flags)
2026{
2027 struct usb_request *zlp_request;
2028 struct cdns3_endpoint *priv_ep;
2029 struct cdns3_device *priv_dev;
2030 unsigned long flags;
2031 int ret;
2032
2033 if (!request || !ep)
2034 return -EINVAL;
2035
2036 priv_ep = ep_to_cdns3_ep(ep);
2037 priv_dev = priv_ep->cdns3_dev;
2038
2039 spin_lock_irqsave(&priv_dev->lock, flags);
2040
2041 ret = __cdns3_gadget_ep_queue(ep, request, gfp_flags);
2042
2043 if (ret == 0 && request->zero && request->length &&
2044 (request->length % ep->maxpacket == 0)) {
2045 struct cdns3_request *priv_req;
2046
2047 zlp_request = cdns3_gadget_ep_alloc_request(ep, GFP_ATOMIC);
2048 zlp_request->buf = priv_dev->zlp_buf;
2049 zlp_request->length = 0;
2050
2051 priv_req = to_cdns3_request(zlp_request);
2052 priv_req->flags |= REQUEST_ZLP;
2053
2054 dev_dbg(priv_dev->dev, "Queuing ZLP for endpoint: %s\n",
2055 priv_ep->name);
2056 ret = __cdns3_gadget_ep_queue(ep, zlp_request, gfp_flags);
2057 }
2058
2059 spin_unlock_irqrestore(&priv_dev->lock, flags);
2060 return ret;
2061}
2062
2063/**
2064 * cdns3_gadget_ep_dequeue Remove request from transfer queue
2065 * @ep: endpoint object associated with request
2066 * @request: request object
2067 *
2068 * Returns 0 on success, error code elsewhere
2069 */
2070int cdns3_gadget_ep_dequeue(struct usb_ep *ep,
2071 struct usb_request *request)
2072{
2073 struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2074 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2075 struct usb_request *req, *req_temp;
2076 struct cdns3_request *priv_req;
2077 struct cdns3_trb *link_trb;
2078 u8 req_on_hw_ring = 0;
2079 unsigned long flags;
2080 int ret = 0;
2081
2082 if (!ep || !request || !ep->desc)
2083 return -EINVAL;
2084
2085 spin_lock_irqsave(&priv_dev->lock, flags);
2086
2087 priv_req = to_cdns3_request(request);
2088
2089 trace_cdns3_ep_dequeue(priv_req);
2090
2091 cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
2092
2093 list_for_each_entry_safe(req, req_temp, &priv_ep->pending_req_list,
2094 list) {
2095 if (request == req) {
2096 req_on_hw_ring = 1;
2097 goto found;
2098 }
2099 }
2100
2101 list_for_each_entry_safe(req, req_temp, &priv_ep->deferred_req_list,
2102 list) {
2103 if (request == req)
2104 goto found;
2105 }
2106
2107 goto not_found;
2108
2109found:
2110 link_trb = priv_req->trb;
2111
2112 /* Update ring only if removed request is on pending_req_list list */
2113 if (req_on_hw_ring) {
2114 link_trb->buffer = TRB_BUFFER(priv_ep->trb_pool_dma +
2115 (priv_req->start_trb * TRB_SIZE));
2116 link_trb->control = (link_trb->control & TRB_CYCLE) |
2117 TRB_TYPE(TRB_LINK) | TRB_CHAIN;
2118
2119 if (priv_ep->wa1_trb == priv_req->trb)
2120 cdns3_wa1_restore_cycle_bit(priv_ep);
2121 }
2122
2123 cdns3_gadget_giveback(priv_ep, priv_req, -ECONNRESET);
2124
2125not_found:
2126 spin_unlock_irqrestore(&priv_dev->lock, flags);
2127 return ret;
2128}
2129
2130/**
2131 * __cdns3_gadget_ep_set_halt Sets stall on selected endpoint
2132 * Should be called after acquiring spin_lock and selecting ep
2133 * @ep: endpoint object to set stall on.
2134 */
2135void __cdns3_gadget_ep_set_halt(struct cdns3_endpoint *priv_ep)
2136{
2137 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2138
2139 trace_cdns3_halt(priv_ep, 1, 0);
2140
2141 if (!(priv_ep->flags & EP_STALLED)) {
2142 u32 ep_sts_reg = readl(&priv_dev->regs->ep_sts);
2143
2144 if (!(ep_sts_reg & EP_STS_DBUSY))
2145 cdns3_ep_stall_flush(priv_ep);
2146 else
2147 priv_ep->flags |= EP_STALL_PENDING;
2148 }
2149}
2150
2151/**
2152 * __cdns3_gadget_ep_clear_halt Clears stall on selected endpoint
2153 * Should be called after acquiring spin_lock and selecting ep
2154 * @ep: endpoint object to clear stall on
2155 */
2156int __cdns3_gadget_ep_clear_halt(struct cdns3_endpoint *priv_ep)
2157{
2158 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2159 struct usb_request *request;
2160 int ret;
2161 int val;
2162
2163 trace_cdns3_halt(priv_ep, 0, 0);
2164
2165 writel(EP_CMD_CSTALL | EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2166
2167 /* wait for EPRST cleared */
2168 ret = readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2169 !(val & EP_CMD_EPRST), 1, 100);
2170 if (ret)
2171 return -EINVAL;
2172
2173 priv_ep->flags &= ~(EP_STALLED | EP_STALL_PENDING);
2174
2175 request = cdns3_next_request(&priv_ep->pending_req_list);
2176
2177 if (request)
2178 cdns3_rearm_transfer(priv_ep, 1);
2179
2180 cdns3_start_all_request(priv_dev, priv_ep);
2181 return ret;
2182}
2183
2184/**
2185 * cdns3_gadget_ep_set_halt Sets/clears stall on selected endpoint
2186 * @ep: endpoint object to set/clear stall on
2187 * @value: 1 for set stall, 0 for clear stall
2188 *
2189 * Returns 0 on success, error code elsewhere
2190 */
2191int cdns3_gadget_ep_set_halt(struct usb_ep *ep, int value)
2192{
2193 struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2194 struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2195 unsigned long flags;
2196 int ret = 0;
2197
2198 if (!(priv_ep->flags & EP_ENABLED))
2199 return -EPERM;
2200
2201 spin_lock_irqsave(&priv_dev->lock, flags);
2202
2203 cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
2204
2205 if (!value) {
2206 priv_ep->flags &= ~EP_WEDGE;
2207 ret = __cdns3_gadget_ep_clear_halt(priv_ep);
2208 } else {
2209 __cdns3_gadget_ep_set_halt(priv_ep);
2210 }
2211
2212 spin_unlock_irqrestore(&priv_dev->lock, flags);
2213
2214 return ret;
2215}
2216
2217extern const struct usb_ep_ops cdns3_gadget_ep0_ops;
2218
2219static const struct usb_ep_ops cdns3_gadget_ep_ops = {
2220 .enable = cdns3_gadget_ep_enable,
2221 .disable = cdns3_gadget_ep_disable,
2222 .alloc_request = cdns3_gadget_ep_alloc_request,
2223 .free_request = cdns3_gadget_ep_free_request,
2224 .queue = cdns3_gadget_ep_queue,
2225 .dequeue = cdns3_gadget_ep_dequeue,
2226 .set_halt = cdns3_gadget_ep_set_halt,
2227 .set_wedge = cdns3_gadget_ep_set_wedge,
2228};
2229
2230/**
2231 * cdns3_gadget_get_frame Returns number of actual ITP frame
2232 * @gadget: gadget object
2233 *
2234 * Returns number of actual ITP frame
2235 */
2236static int cdns3_gadget_get_frame(struct usb_gadget *gadget)
2237{
2238 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2239
2240 return readl(&priv_dev->regs->usb_itpn);
2241}
2242
2243int __cdns3_gadget_wakeup(struct cdns3_device *priv_dev)
2244{
2245 enum usb_device_speed speed;
2246
2247 speed = cdns3_get_speed(priv_dev);
2248
2249 if (speed >= USB_SPEED_SUPER)
2250 return 0;
2251
2252 /* Start driving resume signaling to indicate remote wakeup. */
2253 writel(USB_CONF_LGO_L0, &priv_dev->regs->usb_conf);
2254
2255 return 0;
2256}
2257
2258static int cdns3_gadget_wakeup(struct usb_gadget *gadget)
2259{
2260 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2261 unsigned long flags;
2262 int ret = 0;
2263
2264 spin_lock_irqsave(&priv_dev->lock, flags);
2265 ret = __cdns3_gadget_wakeup(priv_dev);
2266 spin_unlock_irqrestore(&priv_dev->lock, flags);
2267 return ret;
2268}
2269
2270static int cdns3_gadget_set_selfpowered(struct usb_gadget *gadget,
2271 int is_selfpowered)
2272{
2273 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2274 unsigned long flags;
2275
2276 spin_lock_irqsave(&priv_dev->lock, flags);
2277 priv_dev->is_selfpowered = !!is_selfpowered;
2278 spin_unlock_irqrestore(&priv_dev->lock, flags);
2279 return 0;
2280}
2281
2282static int cdns3_gadget_pullup(struct usb_gadget *gadget, int is_on)
2283{
2284 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2285
2286 if (is_on)
2287 writel(USB_CONF_DEVEN, &priv_dev->regs->usb_conf);
2288 else
2289 writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf);
2290
2291 return 0;
2292}
2293
2294static void cdns3_gadget_config(struct cdns3_device *priv_dev)
2295{
2296 struct cdns3_usb_regs __iomem *regs = priv_dev->regs;
2297 u32 reg;
2298
2299 cdns3_ep0_config(priv_dev);
2300
2301 /* enable interrupts for endpoint 0 (in and out) */
2302 writel(EP_IEN_EP_OUT0 | EP_IEN_EP_IN0, ®s->ep_ien);
2303
2304 /*
2305 * Driver needs to modify LFPS minimal U1 Exit time for DEV_VER_TI_V1
2306 * revision of controller.
2307 */
2308 if (priv_dev->dev_ver == DEV_VER_TI_V1) {
2309 reg = readl(®s->dbg_link1);
2310
2311 reg &= ~DBG_LINK1_LFPS_MIN_GEN_U1_EXIT_MASK;
2312 reg |= DBG_LINK1_LFPS_MIN_GEN_U1_EXIT(0x55) |
2313 DBG_LINK1_LFPS_MIN_GEN_U1_EXIT_SET;
2314 writel(reg, ®s->dbg_link1);
2315 }
2316
2317 /*
2318 * By default some platforms has set protected access to memory.
2319 * This cause problem with cache, so driver restore non-secure
2320 * access to memory.
2321 */
2322 reg = readl(®s->dma_axi_ctrl);
2323 reg |= DMA_AXI_CTRL_MARPROT(DMA_AXI_CTRL_NON_SECURE) |
2324 DMA_AXI_CTRL_MAWPROT(DMA_AXI_CTRL_NON_SECURE);
2325 writel(reg, ®s->dma_axi_ctrl);
2326
2327 /* enable generic interrupt*/
2328 writel(USB_IEN_INIT, ®s->usb_ien);
2329 writel(USB_CONF_CLK2OFFDS | USB_CONF_L1DS, ®s->usb_conf);
2330
2331 cdns3_configure_dmult(priv_dev, NULL);
2332}
2333
2334/**
2335 * cdns3_gadget_udc_start Gadget start
2336 * @gadget: gadget object
2337 * @driver: driver which operates on this gadget
2338 *
2339 * Returns 0 on success, error code elsewhere
2340 */
2341static int cdns3_gadget_udc_start(struct usb_gadget *gadget,
2342 struct usb_gadget_driver *driver)
2343{
2344 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2345 unsigned long flags;
2346 enum usb_device_speed max_speed = driver->max_speed;
2347
2348 spin_lock_irqsave(&priv_dev->lock, flags);
2349 priv_dev->gadget_driver = driver;
2350
2351 /* limit speed if necessary */
2352 max_speed = min(driver->max_speed, gadget->max_speed);
2353
2354 switch (max_speed) {
2355 case USB_SPEED_FULL:
2356 writel(USB_CONF_SFORCE_FS, &priv_dev->regs->usb_conf);
2357 writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf);
2358 break;
2359 case USB_SPEED_HIGH:
2360 writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf);
2361 break;
2362 case USB_SPEED_SUPER:
2363 break;
2364 default:
2365 dev_err(priv_dev->dev,
2366 "invalid maximum_speed parameter %d\n",
2367 max_speed);
2368 /* fall through */
2369 case USB_SPEED_UNKNOWN:
2370 /* default to superspeed */
2371 max_speed = USB_SPEED_SUPER;
2372 break;
2373 }
2374
2375 cdns3_gadget_config(priv_dev);
2376 spin_unlock_irqrestore(&priv_dev->lock, flags);
2377 return 0;
2378}
2379
2380/**
2381 * cdns3_gadget_udc_stop Stops gadget
2382 * @gadget: gadget object
2383 *
2384 * Returns 0
2385 */
2386static int cdns3_gadget_udc_stop(struct usb_gadget *gadget)
2387{
2388 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2389 struct cdns3_endpoint *priv_ep;
2390 u32 bEndpointAddress;
2391 struct usb_ep *ep;
2392 int ret = 0;
2393 int val;
2394
2395 priv_dev->gadget_driver = NULL;
2396
2397 priv_dev->onchip_used_size = 0;
2398 priv_dev->out_mem_is_allocated = 0;
2399 priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
2400
2401 list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
2402 priv_ep = ep_to_cdns3_ep(ep);
2403 bEndpointAddress = priv_ep->num | priv_ep->dir;
2404 cdns3_select_ep(priv_dev, bEndpointAddress);
2405 writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2406 readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2407 !(val & EP_CMD_EPRST), 1, 100);
2408
2409 priv_ep->flags &= ~EP_CLAIMED;
2410 }
2411
2412 /* disable interrupt for device */
2413 writel(0, &priv_dev->regs->usb_ien);
2414 writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf);
2415
2416 return ret;
2417}
2418
2419static const struct usb_gadget_ops cdns3_gadget_ops = {
2420 .get_frame = cdns3_gadget_get_frame,
2421 .wakeup = cdns3_gadget_wakeup,
2422 .set_selfpowered = cdns3_gadget_set_selfpowered,
2423 .pullup = cdns3_gadget_pullup,
2424 .udc_start = cdns3_gadget_udc_start,
2425 .udc_stop = cdns3_gadget_udc_stop,
2426 .match_ep = cdns3_gadget_match_ep,
2427};
2428
2429static void cdns3_free_all_eps(struct cdns3_device *priv_dev)
2430{
2431 int i;
2432
2433 /* ep0 OUT point to ep0 IN. */
2434 priv_dev->eps[16] = NULL;
2435
2436 for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++)
2437 if (priv_dev->eps[i]) {
2438 cdns3_free_trb_pool(priv_dev->eps[i]);
2439 devm_kfree(priv_dev->dev, priv_dev->eps[i]);
2440 }
2441}
2442
2443/**
2444 * cdns3_init_eps Initializes software endpoints of gadget
2445 * @cdns3: extended gadget object
2446 *
2447 * Returns 0 on success, error code elsewhere
2448 */
2449static int cdns3_init_eps(struct cdns3_device *priv_dev)
2450{
2451 u32 ep_enabled_reg, iso_ep_reg;
2452 struct cdns3_endpoint *priv_ep;
2453 int ep_dir, ep_number;
2454 u32 ep_mask;
2455 int ret = 0;
2456 int i;
2457
2458 /* Read it from USB_CAP3 to USB_CAP5 */
2459 ep_enabled_reg = readl(&priv_dev->regs->usb_cap3);
2460 iso_ep_reg = readl(&priv_dev->regs->usb_cap4);
2461
2462 dev_dbg(priv_dev->dev, "Initializing non-zero endpoints\n");
2463
2464 for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++) {
2465 ep_dir = i >> 4; /* i div 16 */
2466 ep_number = i & 0xF; /* i % 16 */
2467 ep_mask = BIT(i);
2468
2469 if (!(ep_enabled_reg & ep_mask))
2470 continue;
2471
2472 if (ep_dir && !ep_number) {
2473 priv_dev->eps[i] = priv_dev->eps[0];
2474 continue;
2475 }
2476
2477 priv_ep = devm_kzalloc(priv_dev->dev, sizeof(*priv_ep),
2478 GFP_KERNEL);
2479 if (!priv_ep)
2480 goto err;
2481
2482 /* set parent of endpoint object */
2483 priv_ep->cdns3_dev = priv_dev;
2484 priv_dev->eps[i] = priv_ep;
2485 priv_ep->num = ep_number;
2486 priv_ep->dir = ep_dir ? USB_DIR_IN : USB_DIR_OUT;
2487
2488 if (!ep_number) {
2489 ret = cdns3_init_ep0(priv_dev, priv_ep);
2490 if (ret) {
2491 dev_err(priv_dev->dev, "Failed to init ep0\n");
2492 goto err;
2493 }
2494 } else {
2495 snprintf(priv_ep->name, sizeof(priv_ep->name), "ep%d%s",
2496 ep_number, !!ep_dir ? "in" : "out");
2497 priv_ep->endpoint.name = priv_ep->name;
2498
2499 usb_ep_set_maxpacket_limit(&priv_ep->endpoint,
2500 CDNS3_EP_MAX_PACKET_LIMIT);
2501 priv_ep->endpoint.max_streams = CDNS3_EP_MAX_STREAMS;
2502 priv_ep->endpoint.ops = &cdns3_gadget_ep_ops;
2503 if (ep_dir)
2504 priv_ep->endpoint.caps.dir_in = 1;
2505 else
2506 priv_ep->endpoint.caps.dir_out = 1;
2507
2508 if (iso_ep_reg & ep_mask)
2509 priv_ep->endpoint.caps.type_iso = 1;
2510
2511 priv_ep->endpoint.caps.type_bulk = 1;
2512 priv_ep->endpoint.caps.type_int = 1;
2513
2514 list_add_tail(&priv_ep->endpoint.ep_list,
2515 &priv_dev->gadget.ep_list);
2516 }
2517
2518 priv_ep->flags = 0;
2519
2520 dev_info(priv_dev->dev, "Initialized %s support: %s %s\n",
2521 priv_ep->name,
2522 priv_ep->endpoint.caps.type_bulk ? "BULK, INT" : "",
2523 priv_ep->endpoint.caps.type_iso ? "ISO" : "");
2524
2525 INIT_LIST_HEAD(&priv_ep->pending_req_list);
2526 INIT_LIST_HEAD(&priv_ep->deferred_req_list);
2527 INIT_LIST_HEAD(&priv_ep->wa2_descmiss_req_list);
2528 }
2529
2530 return 0;
2531err:
2532 cdns3_free_all_eps(priv_dev);
2533 return -ENOMEM;
2534}
2535
2536void cdns3_gadget_exit(struct cdns3 *cdns)
2537{
2538 struct cdns3_device *priv_dev;
2539
2540 priv_dev = cdns->gadget_dev;
2541
2542 devm_free_irq(cdns->dev, cdns->dev_irq, cdns);
2543
2544 pm_runtime_mark_last_busy(cdns->dev);
2545 pm_runtime_put_autosuspend(cdns->dev);
2546
2547 usb_del_gadget_udc(&priv_dev->gadget);
2548
2549 cdns3_free_all_eps(priv_dev);
2550
2551 while (!list_empty(&priv_dev->aligned_buf_list)) {
2552 struct cdns3_aligned_buf *buf;
2553
2554 buf = cdns3_next_align_buf(&priv_dev->aligned_buf_list);
2555 dma_free_coherent(priv_dev->sysdev, buf->size,
2556 buf->buf,
2557 buf->dma);
2558
2559 list_del(&buf->list);
2560 kfree(buf);
2561 }
2562
2563 dma_free_coherent(priv_dev->sysdev, 8, priv_dev->setup_buf,
2564 priv_dev->setup_dma);
2565
2566 kfree(priv_dev->zlp_buf);
2567 kfree(priv_dev);
2568 cdns->gadget_dev = NULL;
2569 cdns3_drd_switch_gadget(cdns, 0);
2570}
2571
2572static int cdns3_gadget_start(struct cdns3 *cdns)
2573{
2574 struct cdns3_device *priv_dev;
2575 u32 max_speed;
2576 int ret;
2577
2578 priv_dev = kzalloc(sizeof(*priv_dev), GFP_KERNEL);
2579 if (!priv_dev)
2580 return -ENOMEM;
2581
2582 cdns->gadget_dev = priv_dev;
2583 priv_dev->sysdev = cdns->dev;
2584 priv_dev->dev = cdns->dev;
2585 priv_dev->regs = cdns->dev_regs;
2586
2587 device_property_read_u16(priv_dev->dev, "cdns,on-chip-buff-size",
2588 &priv_dev->onchip_buffers);
2589
2590 if (priv_dev->onchip_buffers <= 0) {
2591 u32 reg = readl(&priv_dev->regs->usb_cap2);
2592
2593 priv_dev->onchip_buffers = USB_CAP2_ACTUAL_MEM_SIZE(reg);
2594 }
2595
2596 if (!priv_dev->onchip_buffers)
2597 priv_dev->onchip_buffers = 256;
2598
2599 max_speed = usb_get_maximum_speed(cdns->dev);
2600
2601 /* Check the maximum_speed parameter */
2602 switch (max_speed) {
2603 case USB_SPEED_FULL:
2604 case USB_SPEED_HIGH:
2605 case USB_SPEED_SUPER:
2606 break;
2607 default:
2608 dev_err(cdns->dev, "invalid maximum_speed parameter %d\n",
2609 max_speed);
2610 /* fall through */
2611 case USB_SPEED_UNKNOWN:
2612 /* default to superspeed */
2613 max_speed = USB_SPEED_SUPER;
2614 break;
2615 }
2616
2617 /* fill gadget fields */
2618 priv_dev->gadget.max_speed = max_speed;
2619 priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
2620 priv_dev->gadget.ops = &cdns3_gadget_ops;
2621 priv_dev->gadget.name = "usb-ss-gadget";
2622 priv_dev->gadget.sg_supported = 1;
2623 priv_dev->gadget.quirk_avoids_skb_reserve = 1;
2624
2625 spin_lock_init(&priv_dev->lock);
2626 INIT_WORK(&priv_dev->pending_status_wq,
2627 cdns3_pending_setup_status_handler);
2628
2629 INIT_WORK(&priv_dev->aligned_buf_wq,
2630 cdns3_free_aligned_request_buf);
2631
2632 /* initialize endpoint container */
2633 INIT_LIST_HEAD(&priv_dev->gadget.ep_list);
2634 INIT_LIST_HEAD(&priv_dev->aligned_buf_list);
2635
2636 ret = cdns3_init_eps(priv_dev);
2637 if (ret) {
2638 dev_err(priv_dev->dev, "Failed to create endpoints\n");
2639 goto err1;
2640 }
2641
2642 /* allocate memory for setup packet buffer */
2643 priv_dev->setup_buf = dma_alloc_coherent(priv_dev->sysdev, 8,
2644 &priv_dev->setup_dma, GFP_DMA);
2645 if (!priv_dev->setup_buf) {
2646 ret = -ENOMEM;
2647 goto err2;
2648 }
2649
2650 priv_dev->dev_ver = readl(&priv_dev->regs->usb_cap6);
2651
2652 dev_dbg(priv_dev->dev, "Device Controller version: %08x\n",
2653 readl(&priv_dev->regs->usb_cap6));
2654 dev_dbg(priv_dev->dev, "USB Capabilities:: %08x\n",
2655 readl(&priv_dev->regs->usb_cap1));
2656 dev_dbg(priv_dev->dev, "On-Chip memory configuration: %08x\n",
2657 readl(&priv_dev->regs->usb_cap2));
2658
2659 priv_dev->dev_ver = GET_DEV_BASE_VERSION(priv_dev->dev_ver);
2660
2661 priv_dev->zlp_buf = kzalloc(CDNS3_EP_ZLP_BUF_SIZE, GFP_KERNEL);
2662 if (!priv_dev->zlp_buf) {
2663 ret = -ENOMEM;
2664 goto err3;
2665 }
2666
2667 /* add USB gadget device */
2668 ret = usb_add_gadget_udc(priv_dev->dev, &priv_dev->gadget);
2669 if (ret < 0) {
2670 dev_err(priv_dev->dev,
2671 "Failed to register USB device controller\n");
2672 goto err4;
2673 }
2674
2675 return 0;
2676err4:
2677 kfree(priv_dev->zlp_buf);
2678err3:
2679 dma_free_coherent(priv_dev->sysdev, 8, priv_dev->setup_buf,
2680 priv_dev->setup_dma);
2681err2:
2682 cdns3_free_all_eps(priv_dev);
2683err1:
2684 cdns->gadget_dev = NULL;
2685 return ret;
2686}
2687
2688static int __cdns3_gadget_init(struct cdns3 *cdns)
2689{
2690 int ret = 0;
2691
2692 /* Ensure 32-bit DMA Mask in case we switched back from Host mode */
2693 ret = dma_set_mask_and_coherent(cdns->dev, DMA_BIT_MASK(32));
2694 if (ret) {
2695 dev_err(cdns->dev, "Failed to set dma mask: %d\n", ret);
2696 return ret;
2697 }
2698
2699 cdns3_drd_switch_gadget(cdns, 1);
2700 pm_runtime_get_sync(cdns->dev);
2701
2702 ret = cdns3_gadget_start(cdns);
2703 if (ret)
2704 return ret;
2705
2706 /*
2707 * Because interrupt line can be shared with other components in
2708 * driver it can't use IRQF_ONESHOT flag here.
2709 */
2710 ret = devm_request_threaded_irq(cdns->dev, cdns->dev_irq,
2711 cdns3_device_irq_handler,
2712 cdns3_device_thread_irq_handler,
2713 IRQF_SHARED, dev_name(cdns->dev), cdns);
2714
2715 if (ret)
2716 goto err0;
2717
2718 return 0;
2719err0:
2720 cdns3_gadget_exit(cdns);
2721 return ret;
2722}
2723
2724static int cdns3_gadget_suspend(struct cdns3 *cdns, bool do_wakeup)
2725{
2726 struct cdns3_device *priv_dev = cdns->gadget_dev;
2727
2728 cdns3_disconnect_gadget(priv_dev);
2729
2730 priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
2731 usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED);
2732 cdns3_hw_reset_eps_config(priv_dev);
2733
2734 /* disable interrupt for device */
2735 writel(0, &priv_dev->regs->usb_ien);
2736
2737 return 0;
2738}
2739
2740static int cdns3_gadget_resume(struct cdns3 *cdns, bool hibernated)
2741{
2742 struct cdns3_device *priv_dev = cdns->gadget_dev;
2743
2744 if (!priv_dev->gadget_driver)
2745 return 0;
2746
2747 cdns3_gadget_config(priv_dev);
2748
2749 return 0;
2750}
2751
2752/**
2753 * cdns3_gadget_init - initialize device structure
2754 *
2755 * cdns: cdns3 instance
2756 *
2757 * This function initializes the gadget.
2758 */
2759int cdns3_gadget_init(struct cdns3 *cdns)
2760{
2761 struct cdns3_role_driver *rdrv;
2762
2763 rdrv = devm_kzalloc(cdns->dev, sizeof(*rdrv), GFP_KERNEL);
2764 if (!rdrv)
2765 return -ENOMEM;
2766
2767 rdrv->start = __cdns3_gadget_init;
2768 rdrv->stop = cdns3_gadget_exit;
2769 rdrv->suspend = cdns3_gadget_suspend;
2770 rdrv->resume = cdns3_gadget_resume;
2771 rdrv->state = CDNS3_ROLE_STATE_INACTIVE;
2772 rdrv->name = "gadget";
2773 cdns->roles[USB_ROLE_DEVICE] = rdrv;
2774
2775 return 0;
2776}